--- /dev/null
+/*
+ * Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#include "checksumcalculator.h"
+
+#include <zlib.h>
+
+#include <QCryptographicHash>
+#include <QFile>
+#include <QLoggingCategory>
+
+namespace
+{
+constexpr qint64 bufSize = 500 * 1024;
+}
+
+namespace OCC {
+
+Q_LOGGING_CATEGORY(lcChecksumCalculator, "nextcloud.common.checksumcalculator", QtInfoMsg)
+
+static QCryptographicHash::Algorithm algorithmTypeToQCryptoHashAlgorithm(ChecksumCalculator::AlgorithmType algorithmType)
+{
+ switch (algorithmType) {
+ case ChecksumCalculator::AlgorithmType::Undefined:
+ case ChecksumCalculator::AlgorithmType::Adler32:
+ qCWarning(lcChecksumCalculator) << "Invalid algorithm type" << static_cast<int>(algorithmType);
+ return static_cast<QCryptographicHash::Algorithm>(-1);
+ case ChecksumCalculator::AlgorithmType::MD5:
+ return QCryptographicHash::Algorithm::Md5;
+ case ChecksumCalculator::AlgorithmType::SHA1:
+ return QCryptographicHash::Algorithm::Sha1;
+ case ChecksumCalculator::AlgorithmType::SHA256:
+ return QCryptographicHash::Algorithm::Sha256;
+ case ChecksumCalculator::AlgorithmType::SHA3_256:
+ return QCryptographicHash::Algorithm::Sha3_256;
+ }
+ return static_cast<QCryptographicHash::Algorithm>(-1);
+}
+
+ChecksumCalculator::ChecksumCalculator(QSharedPointer<QIODevice> sharedDevice, const QByteArray &checksumTypeName)
+ : _device(sharedDevice)
+{
+ if (checksumTypeName == checkSumMD5C) {
+ _algorithmType = AlgorithmType::MD5;
+ } else if (checksumTypeName == checkSumSHA1C) {
+ _algorithmType = AlgorithmType::SHA1;
+ } else if (checksumTypeName == checkSumSHA2C) {
+ _algorithmType = AlgorithmType::SHA256;
+ } else if (checksumTypeName == checkSumSHA3C) {
+ _algorithmType = AlgorithmType::SHA3_256;
+ } else if (checksumTypeName == checkSumAdlerC) {
+ _algorithmType = AlgorithmType::Adler32;
+ }
+
+ initChecksumAlgorithm();
+}
+
+ChecksumCalculator::~ChecksumCalculator()
+{
+ QMutexLocker locker(&_deviceMutex);
+ if (_device && _device->isOpen()) {
+ _device->close();
+ }
+}
+
+QByteArray ChecksumCalculator::calculate()
+{
+ QByteArray result;
+
+ if (!_isInitialized) {
+ return result;
+ }
+
+ Q_ASSERT(!_device->isOpen());
+ if (_device->isOpen()) {
+ qCWarning(lcChecksumCalculator) << "Device already open. Ignoring.";
+ }
+
+ if (!_device->isOpen() && !_device->open(QIODevice::ReadOnly)) {
+ if (auto file = qobject_cast<QFile *>(_device.data())) {
+ qCWarning(lcChecksumCalculator) << "Could not open file" << file->fileName() << "for reading to compute a checksum" << file->errorString();
+ } else {
+ qCWarning(lcChecksumCalculator) << "Could not open device" << _device.data() << "for reading to compute a checksum" << _device->errorString();
+ }
+ return result;
+ }
+
+ bool isAnyChunkAdded = false;
+
+ for (;;) {
+ QMutexLocker locker(&_deviceMutex);
+ if (!_device->isOpen() || _device->atEnd()) {
+ break;
+ }
+ const auto toRead = qMin(_device->bytesAvailable(), bufSize);
+ if (toRead <= 0) {
+ break;
+ }
+ QByteArray buf(toRead, Qt::Uninitialized);
+ const auto sizeRead = _device->read(buf.data(), toRead);
+ if (sizeRead <= 0) {
+ break;
+ }
+ if (!addChunk(buf, sizeRead)) {
+ break;
+ }
+ isAnyChunkAdded = true;
+ }
+
+ {
+ QMutexLocker locker(&_deviceMutex);
+ if (!_device->isOpen()) {
+ return result;
+ }
+ }
+
+ if (isAnyChunkAdded) {
+ if (_algorithmType == AlgorithmType::Adler32) {
+ result = QByteArray::number(_adlerHash, 16);
+ } else {
+ Q_ASSERT(_cryptographicHash);
+ if (_cryptographicHash) {
+ result = _cryptographicHash->result().toHex();
+ }
+ }
+ }
+
+ {
+ QMutexLocker locker(&_deviceMutex);
+ if (_device->isOpen()) {
+ _device->close();
+ }
+ }
+
+ return result;
+}
+
+void ChecksumCalculator::initChecksumAlgorithm()
+{
+ if (_algorithmType == AlgorithmType::Undefined) {
+ qCWarning(lcChecksumCalculator) << "_algorithmType is Undefined, impossible to init Checksum Algorithm";
+ return;
+ }
+
+ {
+ QMutexLocker locker(&_deviceMutex);
+ if (_device->size() == 0) {
+ return;
+ }
+ }
+
+ if (_algorithmType == AlgorithmType::Adler32) {
+ _adlerHash = adler32(0L, Z_NULL, 0);
+ } else {
+ _cryptographicHash.reset(new QCryptographicHash(algorithmTypeToQCryptoHashAlgorithm(_algorithmType)));
+ }
+
+ _isInitialized = true;
+}
+
+bool ChecksumCalculator::addChunk(const QByteArray &chunk, const qint64 size)
+{
+ Q_ASSERT(_algorithmType != AlgorithmType::Undefined);
+ if (_algorithmType == AlgorithmType::Undefined) {
+ qCWarning(lcChecksumCalculator) << "_algorithmType is Undefined, impossible to add a chunk!";
+ return false;
+ }
+
+ if (_algorithmType == AlgorithmType::Adler32) {
+ _adlerHash = adler32(_adlerHash, (const Bytef *)chunk.data(), size);
+ return true;
+ } else {
+ Q_ASSERT(_cryptographicHash);
+ if (_cryptographicHash) {
+ _cryptographicHash->addData(chunk.data(), size);
+ return true;
+ }
+ }
+ return false;
+}
+
+}
--- /dev/null
+/*
+ * Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#pragma once
+
+#include "ocsynclib.h"
+#include "config.h"
+#include "checksumconsts.h"
+
+#include <QObject>
+#include <QByteArray>
+#include <QFutureWatcher>
+#include <QMutex>
+
+#include <memory>
+
+class QCryptographicHash;
+
+namespace OCC {
+class OCSYNC_EXPORT ChecksumCalculator
+{
+ Q_DISABLE_COPY(ChecksumCalculator)
+
+public:
+ enum class AlgorithmType {
+ Undefined = -1,
+ MD5,
+ SHA1,
+ SHA256,
+ SHA3_256,
+ Adler32,
+ };
+
+ ChecksumCalculator(QSharedPointer<QIODevice> sharedDevice, const QByteArray &checksumTypeName);
+ ~ChecksumCalculator();
+ [[nodiscard]] QByteArray calculate();
+
+private:
+ void initChecksumAlgorithm();
+ bool addChunk(const QByteArray &chunk, const qint64 size);
+ QSharedPointer<QIODevice> _device;
+ QScopedPointer<QCryptographicHash> _cryptographicHash;
+ unsigned int _adlerHash = 0;
+ bool _isInitialized = false;
+ AlgorithmType _algorithmType = AlgorithmType::Undefined;
+ QMutex _deviceMutex;
+};
+}
--- /dev/null
+/*
+ * Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#pragma once
+
+namespace OCC
+{
+/**
+ * Tags for checksum headers values.
+ * They are here for being shared between Upload- and Download Job
+ */
+static constexpr auto checkSumMD5C = "MD5";
+static constexpr auto checkSumSHA1C = "SHA1";
+static constexpr auto checkSumSHA2C = "SHA256";
+static constexpr auto checkSumSHA3C = "SHA3-256";
+static constexpr auto checkSumAdlerC = "Adler32";
+}
#include "config.h"
#include "filesystembase.h"
#include "common/checksums.h"
+#include "checksumcalculator.h"
#include "asserts.h"
#include <QLoggingCategory>
#define BUFSIZE qint64(500 * 1024) // 500 KiB
-static QByteArray calcCryptoHash(QIODevice *device, QCryptographicHash::Algorithm algo)
-{
- QByteArray arr;
- QCryptographicHash crypto( algo );
-
- if (crypto.addData(device)) {
- arr = crypto.result().toHex();
- }
- return arr;
- }
-
-QByteArray calcMd5(QIODevice *device)
-{
- return calcCryptoHash(device, QCryptographicHash::Md5);
-}
-
-QByteArray calcSha1(QIODevice *device)
-{
- return calcCryptoHash(device, QCryptographicHash::Sha1);
-}
-
-#ifdef ZLIB_FOUND
-QByteArray calcAdler32(QIODevice *device)
-{
- if (device->size() == 0)
- {
- return QByteArray();
- }
- QByteArray buf(BUFSIZE, Qt::Uninitialized);
-
- unsigned int adler = adler32(0L, Z_NULL, 0);
- qint64 size = 0;
- while (!device->atEnd()) {
- size = device->read(buf.data(), BUFSIZE);
- if (size > 0)
- adler = adler32(adler, (const Bytef *)buf.data(), size);
- }
-
- return QByteArray::number(adler, 16);
-}
-#endif
-
QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum)
{
if (checksumType.isEmpty() || checksum.isEmpty())
void ComputeChecksum::start(const QString &filePath)
{
qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread";
- startImpl(std::make_unique<QFile>(filePath));
+ startImpl(QSharedPointer<QFile>::create(filePath));
}
-void ComputeChecksum::start(std::unique_ptr<QIODevice> device)
+void ComputeChecksum::start(QSharedPointer<QIODevice> device)
{
ENFORCE(device);
qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of device" << device.get() << "in a thread";
ASSERT(!device->parent());
- startImpl(std::move(device));
+ startImpl(device);
}
-void ComputeChecksum::startImpl(std::unique_ptr<QIODevice> device)
+void ComputeChecksum::startImpl(QSharedPointer<QIODevice> device)
{
connect(&_watcher, &QFutureWatcherBase::finished,
this, &ComputeChecksum::slotCalculationDone,
Qt::UniqueConnection);
- // We'd prefer to move the unique_ptr into the lambda, but that's
- // awkward with the C++ standard we're on
- auto sharedDevice = QSharedPointer<QIODevice>(device.release());
-
- // Bug: The thread will keep running even if ComputeChecksum is deleted.
- auto type = checksumType();
- _watcher.setFuture(QtConcurrent::run([sharedDevice, type]() {
- if (!sharedDevice->open(QIODevice::ReadOnly)) {
- if (auto file = qobject_cast<QFile *>(sharedDevice.data())) {
- qCWarning(lcChecksums) << "Could not open file" << file->fileName()
- << "for reading to compute a checksum" << file->errorString();
- } else {
- qCWarning(lcChecksums) << "Could not open device" << sharedDevice.data()
- << "for reading to compute a checksum" << sharedDevice->errorString();
- }
- return QByteArray();
- }
- auto result = ComputeChecksum::computeNow(sharedDevice.data(), type);
- sharedDevice->close();
- return result;
+ _checksumCalculator.reset(new ChecksumCalculator(device, _checksumType));
+ _watcher.setFuture(QtConcurrent::run([this]() {
+ return _checksumCalculator->calculate();
}));
}
QByteArray ComputeChecksum::computeNowOnFile(const QString &filePath, const QByteArray &checksumType)
{
- QFile file(filePath);
- if (!file.open(QIODevice::ReadOnly)) {
- qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading and computing checksum" << file.errorString();
- return QByteArray();
- }
-
- return computeNow(&file, checksumType);
+ return computeNow(QSharedPointer<QFile>::create(filePath), checksumType);
}
-QByteArray ComputeChecksum::computeNow(QIODevice *device, const QByteArray &checksumType)
+QByteArray ComputeChecksum::computeNow(QSharedPointer<QIODevice> device, const QByteArray &checksumType)
{
if (!checksumComputationEnabled()) {
qCWarning(lcChecksums) << "Checksum computation disabled by environment variable";
return QByteArray();
}
- if (checksumType == checkSumMD5C) {
- return calcMd5(device);
- } else if (checksumType == checkSumSHA1C) {
- return calcSha1(device);
- } else if (checksumType == checkSumSHA2C) {
- return calcCryptoHash(device, QCryptographicHash::Sha256);
- }
-#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
- else if (checksumType == checkSumSHA3C) {
- return calcCryptoHash(device, QCryptographicHash::Sha3_256);
- }
-#endif
-#ifdef ZLIB_FOUND
- else if (checksumType == checkSumAdlerC) {
- return calcAdler32(device);
- }
-#endif
- // for an unknown checksum or no checksum, we're done right now
- if (!checksumType.isEmpty()) {
- qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType;
- }
- return QByteArray();
+ ChecksumCalculator checksumCalculator(device, checksumType);
+ return checksumCalculator.calculate();
}
void ComputeChecksum::slotCalculationDone()
calculator->start(filePath);
}
-void ValidateChecksumHeader::start(std::unique_ptr<QIODevice> device, const QByteArray &checksumHeader)
+void ValidateChecksumHeader::start(QSharedPointer<QIODevice> device, const QByteArray &checksumHeader)
{
- if (auto calculator = prepareStart(checksumHeader))
- calculator->start(std::move(device));
+ if (auto calculator = prepareStart(checksumHeader)) {
+ calculator->start(device);
+ }
}
QByteArray ValidateChecksumHeader::calculatedChecksumType() const
namespace OCC {
-/**
- * Tags for checksum headers values.
- * They are here for being shared between Upload- and Download Job
- */
-static const char checkSumMD5C[] = "MD5";
-static const char checkSumSHA1C[] = "SHA1";
-static const char checkSumSHA2C[] = "SHA256";
-static const char checkSumSHA3C[] = "SHA3-256";
-static const char checkSumAdlerC[] = "Adler32";
-
+class ChecksumCalculator;
class SyncJournalDb;
/**
/// Checks OWNCLOUD_DISABLE_CHECKSUM_UPLOAD
OCSYNC_EXPORT bool uploadChecksumEnabled();
-// Exported functions for the tests.
-QByteArray OCSYNC_EXPORT calcMd5(QIODevice *device);
-QByteArray OCSYNC_EXPORT calcSha1(QIODevice *device);
-#ifdef ZLIB_FOUND
-QByteArray OCSYNC_EXPORT calcAdler32(QIODevice *device);
-#endif
-
/**
* Computes the checksum of a file.
* \ingroup libsync
* The device ownership transfers into the thread that
* will compute the checksum. It must not have a parent.
*/
- void start(std::unique_ptr<QIODevice> device);
+ void start(QSharedPointer<QIODevice> device);
/**
* Computes the checksum synchronously.
*/
- static QByteArray computeNow(QIODevice *device, const QByteArray &checksumType);
+ static QByteArray computeNow(QSharedPointer<QIODevice> device, const QByteArray &checksumType);
/**
* Computes the checksum synchronously on file. Convenience wrapper for computeNow().
void slotCalculationDone();
private:
- void startImpl(std::unique_ptr<QIODevice> device);
+ void startImpl(QSharedPointer<QIODevice> device);
QByteArray _checksumType;
// watcher for the checksum calculation thread
QFutureWatcher<QByteArray> _watcher;
+
+ QScopedPointer<ChecksumCalculator> _checksumCalculator;
};
/**
* The device ownership transfers into the thread that
* will compute the checksum. It must not have a parent.
*/
- void start(std::unique_ptr<QIODevice> device, const QByteArray &checksumHeader);
+ void start(QSharedPointer<QIODevice> device, const QByteArray &checksumHeader);
[[nodiscard]] QByteArray calculatedChecksumType() const;
[[nodiscard]] QByteArray calculatedChecksum() const;
# help keep track of the different code licenses.
set(common_SOURCES
${CMAKE_CURRENT_LIST_DIR}/checksums.cpp
+ ${CMAKE_CURRENT_LIST_DIR}/checksumcalculator.cpp
${CMAKE_CURRENT_LIST_DIR}/filesystembase.cpp
${CMAKE_CURRENT_LIST_DIR}/ownsql.cpp
${CMAKE_CURRENT_LIST_DIR}/preparedsqlquerymanager.cpp
#include "common/checksums.h"
#include "networkjobs.h"
+#include "common/checksumcalculator.h"
+#include "common/checksumconsts.h"
#include "common/utility.h"
#include "filesystem.h"
#include "propagatorjobs.h"
QFileInfo fi(file);
QVERIFY(fi.exists());
- QFile fileDevice(file);
- fileDevice.open(QIODevice::ReadOnly);
- QByteArray sum = calcMd5(&fileDevice);
- fileDevice.close();
+ auto sharedFile(QSharedPointer<QFile>::create(file));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumMD5C);
+
+ const auto sum = checksumCalculator.calculate();
QByteArray sSum = shellSum("md5sum", file);
if (sSum.isEmpty())
QFileInfo fi(file);
QVERIFY(fi.exists());
- QFile fileDevice(file);
- fileDevice.open(QIODevice::ReadOnly);
- QByteArray sum = calcSha1(&fileDevice);
- fileDevice.close();
+ auto sharedFile(QSharedPointer<QFile>::create(file));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumSHA1C);
+
+ const auto sum = checksumCalculator.calculate();
QByteArray sSum = shellSum("sha1sum", file);
if (sSum.isEmpty())
connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated);
- auto file = new QFile(_testfile, vali);
- file->open(QIODevice::ReadOnly);
- _expected = calcAdler32(file);
+ auto sharedFile(QSharedPointer<QFile>::create(_testfile));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumAdlerC);
+
+ _expected = checksumCalculator.calculate();
+
qDebug() << "XX Expected Checksum: " << _expected;
vali->start(_testfile);
vali->setChecksumType(_expectedType);
connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated);
- auto file = new QFile(_testfile, vali);
- file->open(QIODevice::ReadOnly);
- _expected = calcMd5(file);
+ auto sharedFile(QSharedPointer<QFile>::create(_testfile));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumMD5C);
+
+ _expected = checksumCalculator.calculate();
vali->start(_testfile);
QEventLoop loop;
vali->setChecksumType(_expectedType);
connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated);
- auto file = new QFile(_testfile, vali);
- file->open(QIODevice::ReadOnly);
- _expected = calcSha1(file);
+ auto sharedFile(QSharedPointer<QFile>::create(_testfile));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumSHA1C);
+ _expected = checksumCalculator.calculate();
vali->start(_testfile);
connect(vali, &ValidateChecksumHeader::validated, this, &TestChecksumValidator::slotDownValidated);
connect(vali, &ValidateChecksumHeader::validationFailed, this, &TestChecksumValidator::slotDownError);
- auto file = new QFile(_testfile, vali);
- file->open(QIODevice::ReadOnly);
- _expected = calcAdler32(file);
+ auto sharedFile(QSharedPointer<QFile>::create(_testfile));
+ ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumAdlerC);
+ _expected = checksumCalculator.calculate();
QByteArray adler = checkSumAdlerC;
adler.append(":");
adler.append(_expected);
- file->seek(0);
+ sharedFile->open(QIODevice::ReadOnly);
+ sharedFile->seek(0);
_successDown = false;
vali->start(_testfile, adler);
_expectedError = QStringLiteral("The downloaded file does not match the checksum, it will be resumed. \"543345\" != \"%1\"").arg(QString::fromUtf8(_expected));
_expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumMismatch;
_errorSeen = false;
- file->seek(0);
+ sharedFile->seek(0);
vali->start(_testfile, "Adler32:543345");
QTRY_VERIFY(_errorSeen);
_expectedError = QLatin1String("The checksum header contained an unknown checksum type \"Klaas32\"");
_expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumTypeUnknown;
_errorSeen = false;
- file->seek(0);
+ sharedFile->seek(0);
vali->start(_testfile, "Klaas32:543345");
QTRY_VERIFY(_errorSeen);
projects / nextcloud-desktop.git / commitdiff