Checksums: Work on QIODevice*s

Needed for cfapi where we want to feed data through a custom device
which retrieves data from the windows api.

diff --git a/src/common/checksums.cpp b/src/common/checksums.cpp
index 49ac2eafb886d79b0d3965e74cbfd19b3b9012e4..3776c601fe7714fc7e301940298ee6c87be42db5 100644 (file)
--- a/src/common/checksums.cpp
+++ b/src/common/checksums.cpp
@@ -89,45 +89,38 @@ Q_LOGGING_CATEGORY(lcChecksums, "nextcloud.sync.checksums", QtInfoMsg)
 
 #define BUFSIZE qint64(500 * 1024) // 500 KiB
 
-static QByteArray calcCryptoHash( const QString& filename, QCryptographicHash::Algorithm algo )
+static QByteArray calcCryptoHash(QIODevice *device, QCryptographicHash::Algorithm algo)
 {
-     QFile file(filename);
      QByteArray arr;
      QCryptographicHash crypto( algo );
 
-     if (file.open(QIODevice::ReadOnly)) {
-         if (crypto.addData(&file)) {
-             arr = crypto.result().toHex();
-         }
+     if (crypto.addData(device)) {
+         arr = crypto.result().toHex();
      }
      return arr;
  }
 
-QByteArray calcMd5(const QString &filename)
+QByteArray calcMd5(QIODevice *device)
 {
-    return calcCryptoHash(filename, QCryptographicHash::Md5);
+    return calcCryptoHash(device, QCryptographicHash::Md5);
 }
 
-QByteArray calcSha1(const QString &filename)
+QByteArray calcSha1(QIODevice *device)
 {
-    return calcCryptoHash(filename, QCryptographicHash::Sha1);
+    return calcCryptoHash(device, QCryptographicHash::Sha1);
 }
 
 #ifdef ZLIB_FOUND
-QByteArray calcAdler32(const QString &filename)
+QByteArray calcAdler32(QIODevice *device)
 {
-    QFile file(filename);
-    const qint64 bufSize = qMin(BUFSIZE, file.size() + 1);
-    QByteArray buf(bufSize, Qt::Uninitialized);
+    QByteArray buf(BUFSIZE, Qt::Uninitialized);
 
     unsigned int adler = adler32(0L, Z_NULL, 0);
-    if (file.open(QIODevice::ReadOnly)) {
-        qint64 size;
-        while (!file.atEnd()) {
-            size = file.read(buf.data(), bufSize);
-            if (size > 0)
-                adler = adler32(adler, (const Bytef *)buf.data(), size);
-        }
+    qint64 size;
+    while (!device->atEnd()) {
+        size = device->read(buf.data(), BUFSIZE);
+        if (size > 0)
+            adler = adler32(adler, (const Bytef *)buf.data(), size);
     }
 
     return QByteArray::number(adler, 16);
@@ -228,15 +221,38 @@ QByteArray ComputeChecksum::checksumType() const
 void ComputeChecksum::start(const QString &filePath)
 {
     qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread";
+    _file = new QFile(filePath, this);
+    if (!_file->open(QIODevice::ReadOnly)) {
+        qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading to compute a checksum" << _file->errorString();
+        emit done(QByteArray(), QByteArray());
+        return;
+    }
+    start(_file);
+}
+
+void ComputeChecksum::start(QIODevice *device)
+{
+    qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of iodevice in a thread";
 
     // Calculate the checksum in a different thread first.
     connect(&_watcher, &QFutureWatcherBase::finished,
         this, &ComputeChecksum::slotCalculationDone,
         Qt::UniqueConnection);
-    _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType()));
+    _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, device, checksumType()));
 }
 
-QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray &checksumType)
+QByteArray ComputeChecksum::computeNowOnFile(const QString &filePath, const QByteArray &checksumType)
+{
+    QFile file(filePath);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading and computing checksum" << file.errorString();
+        return QByteArray();
+    }
+
+    return computeNow(&file, checksumType);
+}
+
+QByteArray ComputeChecksum::computeNow(QIODevice *device, const QByteArray &checksumType)
 {
     if (!checksumComputationEnabled()) {
         qCWarning(lcChecksums) << "Checksum computation disabled by environment variable";
@@ -244,20 +260,20 @@ QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray
     }
 
     if (checksumType == checkSumMD5C) {
-        return calcMd5(filePath);
+        return calcMd5(device);
     } else if (checksumType == checkSumSHA1C) {
-        return calcSha1(filePath);
+        return calcSha1(device);
     } else if (checksumType == checkSumSHA2C) {
-        return calcCryptoHash(filePath, QCryptographicHash::Sha256);
+        return calcCryptoHash(device, QCryptographicHash::Sha256);
     }
 #if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
     else if (checksumType == checkSumSHA3C) {
-        return calcCryptoHash(filePath, QCryptographicHash::Sha3_256);
+        return calcCryptoHash(device, QCryptographicHash::Sha3_256);
     }
 #endif
 #ifdef ZLIB_FOUND
     else if (checksumType == checkSumAdlerC) {
-        return calcAdler32(filePath);
+        return calcAdler32(device);
     }
 #endif
     // for an unknown checksum or no checksum, we're done right now
@@ -269,6 +285,10 @@ QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray
 
 void ComputeChecksum::slotCalculationDone()
 {
+    // Close the file and delete the instance
+    if (_file)
+        delete _file;
+
     QByteArray checksum = _watcher.future().result();
     if (!checksum.isNull()) {
         emit done(_checksumType, checksum);
@@ -283,25 +303,37 @@ ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent)
 {
 }
 
-void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader)
+ComputeChecksum *ValidateChecksumHeader::prepareStart(const QByteArray &checksumHeader)
 {
     // If the incoming header is empty no validation can happen. Just continue.
     if (checksumHeader.isEmpty()) {
         emit validated(QByteArray(), QByteArray());
-        return;
+        return nullptr;
     }
 
     if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) {
         qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader;
         emit validationFailed(tr("The checksum header is malformed."));
-        return;
+        return nullptr;
     }
 
     auto calculator = new ComputeChecksum(this);
     calculator->setChecksumType(_expectedChecksumType);
     connect(calculator, &ComputeChecksum::done,
         this, &ValidateChecksumHeader::slotChecksumCalculated);
-    calculator->start(filePath);
+    return calculator;
+}
+
+void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader)
+{
+    if (auto calculator = prepareStart(checksumHeader))
+        calculator->start(filePath);
+}
+
+void ValidateChecksumHeader::start(QIODevice *device, const QByteArray &checksumHeader)
+{
+    if (auto calculator = prepareStart(checksumHeader))
+        calculator->start(device);
 }
 
 void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType,
@@ -327,7 +359,7 @@ QByteArray CSyncChecksumHook::hook(const QByteArray &path, const QByteArray &oth
         return nullptr;
 
     qCInfo(lcChecksums) << "Computing" << type << "checksum of" << path << "in the csync hook";
-    QByteArray checksum = ComputeChecksum::computeNow(QString::fromUtf8(path), type);
+    QByteArray checksum = ComputeChecksum::computeNowOnFile(QString::fromUtf8(path), type);
     if (checksum.isNull()) {
         qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path;
         return nullptr;

diff --git a/src/common/checksums.h b/src/common/checksums.h
index f00a31cf86184c375081babcd8cb1312d357af00..754e1ae5a39c0186cf13306658f80d0d39da2eea 100644 (file)
--- a/src/common/checksums.h
+++ b/src/common/checksums.h
@@ -25,6 +25,8 @@
 #include <QByteArray>
 #include <QFutureWatcher>
 
+class QFile;
+
 namespace OCC {
 
 /**
@@ -65,10 +67,10 @@ OCSYNC_EXPORT bool uploadChecksumEnabled();
 OCSYNC_EXPORT QByteArray contentChecksumType();
 
 // Exported functions for the tests.
-QByteArray OCSYNC_EXPORT calcMd5(const QString &fileName);
-QByteArray OCSYNC_EXPORT calcSha1(const QString &fileName);
+QByteArray OCSYNC_EXPORT calcMd5(QIODevice *device);
+QByteArray OCSYNC_EXPORT calcSha1(QIODevice *device);
 #ifdef ZLIB_FOUND
-QByteArray OCSYNC_EXPORT calcAdler32(const QString &fileName);
+QByteArray OCSYNC_EXPORT calcAdler32(QIODevice *device);
 #endif
 
 /**
@@ -88,17 +90,34 @@ public:
 
     QByteArray checksumType() const;
 
+    /**
+     * Computes the checksum for given device.
+     *
+     * done() is emitted when the calculation finishes.
+     *
+     * Does not take ownership of the device.
+     * Does not call open() on the device.
+     */
+    void start(QIODevice *device);
+
     /**
      * Computes the checksum for the given file path.
      *
      * done() is emitted when the calculation finishes.
+     *
+     * Convenience wrapper for start(QIODevice*) above.
      */
     void start(const QString &filePath);
 
     /**
      * Computes the checksum synchronously.
      */
-    static QByteArray computeNow(const QString &filePath, const QByteArray &checksumType);
+    static QByteArray computeNow(QIODevice *device, const QByteArray &checksumType);
+
+    /**
+     * Computes the checksum synchronously on file. Convenience wrapper for computeNow().
+     */
+    static QByteArray computeNowOnFile(const QString &filePath, const QByteArray &checksumType);
 
 signals:
     void done(const QByteArray &checksumType, const QByteArray &checksum);
@@ -109,6 +128,9 @@ private slots:
 private:
     QByteArray _checksumType;
 
+    // The convenience wrapper may open a file and must close it too
+    QFile *_file = nullptr;
+
     // watcher for the checksum calculation thread
     QFutureWatcher<QByteArray> _watcher;
 };
@@ -124,11 +146,21 @@ public:
     explicit ValidateChecksumHeader(QObject *parent = nullptr);
 
     /**
-     * Check a file's actual checksum against the provided checksumHeader
+     * Check a device's actual checksum against the provided checksumHeader
      *
      * If no checksum is there, or if a correct checksum is there, the signal validated()
      * will be emitted. In case of any kind of error, the signal validationFailed() will
      * be emitted.
+     *
+     * Does not take ownership of the device.
+     * Does not call open() on the device.
+     */
+    void start(QIODevice *device, const QByteArray &checksumHeader);
+
+    /**
+     * Same as above but opening a file by path.
+     *
+     * Convenience function for start(QIODevice*) above
      */
     void start(const QString &filePath, const QByteArray &checksumHeader);
 
@@ -140,6 +172,8 @@ private slots:
     void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum);
 
 private:
+    ComputeChecksum *prepareStart(const QByteArray &checksumHeader);
+
     QByteArray _expectedChecksumType;
     QByteArray _expectedChecksum;
 };

diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp
index a0949c330d81a67ead6b5ffc71f0221872b5a5b0..9ca95d07ec710a2121d34f07d5be8655bee5bf32 100644 (file)
--- a/src/libsync/discovery.cpp
+++ b/src/libsync/discovery.cpp
@@ -313,7 +313,7 @@ static bool computeLocalChecksum(const QByteArray &header, const QString &path,
     auto type = parseChecksumHeaderType(header);
     if (!type.isEmpty()) {
         // TODO: compute async?
-        QByteArray checksum = ComputeChecksum::computeNow(path, type);
+        QByteArray checksum = ComputeChecksum::computeNowOnFile(path, type);
         if (!checksum.isEmpty()) {
             item->_checksumHeader = makeChecksumHeader(type, checksum);
             return true;

diff --git a/test/testchecksumvalidator.cpp b/test/testchecksumvalidator.cpp
index c7f181d2cd845dda24211e2b367b6025b39c5da5..827686600eea2d38beca2327f0fed3bf1fdc0e33 100644 (file)
--- a/test/testchecksumvalidator.cpp
+++ b/test/testchecksumvalidator.cpp
@@ -77,7 +77,11 @@ using namespace OCC::Utility;
         QVERIFY(writeRandomFile(file));
         QFileInfo fi(file);
         QVERIFY(fi.exists());
-        QByteArray sum = calcMd5(file);
+
+        QFile fileDevice(file);
+        fileDevice.open(QIODevice::ReadOnly);
+        QByteArray sum = calcMd5(&fileDevice);
+        fileDevice.close();
 
         QByteArray sSum = shellSum("md5sum", file);
         if (sSum.isEmpty())
@@ -93,7 +97,11 @@ using namespace OCC::Utility;
         writeRandomFile(file);
         QFileInfo fi(file);
         QVERIFY(fi.exists());
-        QByteArray sum = calcSha1(file);
+
+        QFile fileDevice(file);
+        fileDevice.open(QIODevice::ReadOnly);
+        QByteArray sum = calcSha1(&fileDevice);
+        fileDevice.close();
 
         QByteArray sSum = shellSum("sha1sum", file);
         if (sSum.isEmpty())
@@ -113,7 +121,9 @@ using namespace OCC::Utility;
 
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = calcAdler32( _testfile );
+        auto file = new QFile(_testfile, vali);
+        file->open(QIODevice::ReadOnly);
+        _expected = calcAdler32(file);
         qDebug() << "XX Expected Checksum: " << _expected;
         vali->start(_testfile);
 
@@ -132,7 +142,9 @@ using namespace OCC::Utility;
         vali->setChecksumType(_expectedType);
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = calcMd5( _testfile );
+        auto file = new QFile(_testfile, vali);
+        file->open(QIODevice::ReadOnly);
+        _expected = calcMd5(file);
         vali->start(_testfile);
 
         QEventLoop loop;
@@ -149,7 +161,9 @@ using namespace OCC::Utility;
         vali->setChecksumType(_expectedType);
         connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
 
-        _expected = calcSha1( _testfile );
+        auto file = new QFile(_testfile, vali);
+        file->open(QIODevice::ReadOnly);
+        _expected = calcSha1(file);
 
         vali->start(_testfile);
 
@@ -164,26 +178,34 @@ using namespace OCC::Utility;
 #ifndef ZLIB_FOUND
         QSKIP("ZLIB not found.", SkipSingle);
 #else
-        QByteArray adler =  checkSumAdlerC;
-        adler.append(":");
-        adler.append(calcAdler32( _testfile ));
-        _successDown = false;
-
         auto *vali = new ValidateChecksumHeader(this);
         connect(vali, SIGNAL(validated(QByteArray,QByteArray)), this, SLOT(slotDownValidated()));
         connect(vali, SIGNAL(validationFailed(QString)), this, SLOT(slotDownError(QString)));
-        vali->start(_testfile, adler);
+
+        auto file = new QFile(_testfile, vali);
+        file->open(QIODevice::ReadOnly);
+        _expected = calcAdler32(file);
+
+        QByteArray adler = checkSumAdlerC;
+        adler.append(":");
+        adler.append(_expected);
+
+        file->seek(0);
+        _successDown = false;
+        vali->start(file, adler);
 
         QTRY_VERIFY(_successDown);
 
         _expectedError = QLatin1String("The downloaded file does not match the checksum, it will be resumed.");
         _errorSeen = false;
-        vali->start(_testfile, "Adler32:543345");
+        file->seek(0);
+        vali->start(file, "Adler32:543345");
         QTRY_VERIFY(_errorSeen);
 
         _expectedError = QLatin1String("The checksum header contained an unknown checksum type 'Klaas32'");
         _errorSeen = false;
-        vali->start(_testfile, "Klaas32:543345");
+        file->seek(0);
+        vali->start(file, "Klaas32:543345");
         QTRY_VERIFY(_errorSeen);
 
         delete vali;