From 3701fbcbfeab4bc4d4a725948c66d6eb49947fc2 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Tue, 12 May 2015 16:36:40 +0200 Subject: [PATCH] PropagateUpload: Add checksum calculation if required by config. If the config file has an transmissionChecksum entry, a checksum is added to the PUT requests in a header. --- src/libsync/networkjobs.cpp | 1 - src/libsync/propagateupload.cpp | 90 +++++++++++++++++++++++++++++++++ src/libsync/propagateupload.h | 13 +++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 8a43e5cea..4b423789f 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -43,7 +43,6 @@ namespace OCC { AbstractNetworkJob::AbstractNetworkJob(AccountPtr account, const QString &path, QObject *parent) : QObject(parent) - , _duration(0) , _timedout(false) , _followRedirects(false) , _ignoreCredentialFailure(false) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 887731eb1..eb5346cb3 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -21,12 +21,15 @@ #include "utility.h" #include "filesystem.h" #include "propagatorjobs.h" +#include "configfile.h" + #include #include #include #include #include #include +#include #ifdef USE_NEON #include "propagator_legacy.h" @@ -40,6 +43,14 @@ const char owncloudShouldSoftCancelPropertyName[] = "owncloud-should-soft-cancel namespace OCC { +/** + * Tags for checksum headers. + */ +static const char checkSumMD5C[] = "MD5"; +static const char checkSumSHA1C[] = "SHA1"; +static const char checkSumAdlerC[] = "Adler32"; +static const char checkSumAdlerUpperC[] = "ADLER32"; + /** * We do not want to upload files that are currently being modified. * To avoid that, we don't upload files that have a modification time @@ -196,16 +207,87 @@ void PropagateUploadFileQNAM::start() if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) { return; } + ConfigFile cfg; // FIXME: Do not open it for each and every propagation. + + const QString filePath = _propagator->getFilePath(_item._file); + _item._modtime = FileSystem::getModTime(filePath); + // calculate the files checksum + const QString transChecksum = cfg.transmissionChecksum(); + _stopWatch.start(); + + if( transChecksum.isEmpty() ) { + // no checksumming required, jump to really start the uplaod + startUpload(); + } else { + // Calculate the checksum in a different thread first. + connect( &_watcher, SIGNAL(finished()), + this, SLOT(slotChecksumCalculated())); + bool haveFuture = true; + if( transChecksum == checkSumMD5C ) { + _item._checksum = checkSumMD5C; + _item._checksum += ":"; + _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5Worker,filePath)); + + } else if( transChecksum == checkSumSHA1C ) { + _item._checksum = checkSumSHA1C; + _item._checksum += ":"; + _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1Worker, filePath)); + } +#ifdef ZLIB_FOUND + else if( transChecksum == checkSumAdlerC) { + _item._checksum = checkSumAdlerC; + _item._checksum += ":"; + _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32Worker, filePath)); + } +#endif + else { + haveFuture = false; + } + // in case there is a wrong checksum header, let continue without + if( !haveFuture ) { + startUpload(); + } + } + +} + +void PropagateUploadFileQNAM::slotChecksumCalculated( ) +{ + QByteArray checksum = _watcher.future().result(); + + if( !checksum.isEmpty() ) { + _item._checksum.append(checksum); + } else { + _item._checksum.clear(); + } + + startUpload(); +} + +void PropagateUploadFileQNAM::startUpload() +{ const QString fullFilePath(_propagator->getFilePath(_item._file)); if (!FileSystem::fileExists(fullFilePath)) { done(SyncFileItem::SoftError, tr("File Removed")); return; } + _stopWatch.addLapTime(QLatin1String("Checksum")); + + // Update the mtime and size, it might have changed since discovery. + time_t prevModtime = _item._modtime; // the value was set in PropagateUploadFileQNAM::start() + // but a potential checksum calculation could have taken some time during which the file could + // have been changed again, so better check again here. // Update the mtime and size, it might have changed since discovery. _item._modtime = FileSystem::getModTime(fullFilePath); + if( prevModtime != _item._modtime ) { + _propagator->_anotherSyncNeeded = true; + done(SyncFileItem::SoftError, tr("Local filei changed while calculating the checksum.")); + return; + } + quint64 fileSize = FileSystem::getSize(fullFilePath); _item._size = fileSize; @@ -420,6 +502,9 @@ void PropagateUploadFileQNAM::startNextChunk() if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size. currentChunkSize = chunkSize(); } + if( !_item._checksum.isEmpty() ) { + headers["OC-Checksum"] = _item._checksum; + } } } @@ -640,6 +725,11 @@ void PropagateUploadFileQNAM::slotPutFinished() // Well, the mtime was not set #endif } + + // performance logging + _item._requestDuration = _stopWatch.stop(); + qDebug() << "*==* duration UPLOAD" << _item._size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item._requestDuration; + finalize(_item); } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 4bfbe3036..1e3b6f2e1 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -19,6 +19,8 @@ #include #include #include +#include + namespace OCC { class BandwidthManager; @@ -75,6 +77,8 @@ protected slots: class PUTFileJob : public AbstractNetworkJob { Q_OBJECT + +private: QScopedPointer _device; QMap _headers; QString _errorString; @@ -146,6 +150,7 @@ signals: class PropagateUploadFileQNAM : public PropagateItemJob { Q_OBJECT +private: /** * That's the start chunk that was stored in the database for resuming. * In the non-resuming case it is 0. @@ -163,6 +168,12 @@ class PropagateUploadFileQNAM : public PropagateItemJob { QElapsedTimer _duration; QVector _jobs; /// network jobs that are currently in transit bool _finished; // Tells that all the jobs have been finished + + // watcher for the checksum calculation thread + QFutureWatcher _watcher; + + // measure the performance of checksum calc and upload + Utility::StopWatch _stopWatch; public: PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItem& item) : PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {} @@ -175,6 +186,8 @@ private slots: void startNextChunk(); void finalize(const SyncFileItem&); void slotJobDestroyed(QObject *job); + void startUpload(); + void slotChecksumCalculated(); private: void startPollJob(const QString& path); void abortWithError(SyncFileItem::Status status, const QString &error); -- 2.30.2