Enforce Chunk V2 API chunk size limits at the sync options level

Signed-off-by: Claudio Cambra <claudio.cambra@nextcloud.com>

diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp
index 6b62183cd1e08bcf631089e36ece338b04b2fda5..a0889f5c3dae1ad8d598683bfe66c18812d84ac2 100644 (file)
--- a/src/gui/folder.cpp
+++ b/src/gui/folder.cpp
@@ -1037,9 +1037,9 @@ SyncOptions Folder::initializeSyncOptions() const
     opt._parallelNetworkJobs = _accountState->account()->isHttp2Supported() ? 20 : 6;
 
     // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller
-    opt._minChunkSize = qMin(cfgFile.minChunkSize(), 5LL * 1000LL * 1000LL);
-    opt._maxChunkSize = qMax(cfgFile.maxChunkSize(), 5LL * 1000LL * 1000LL);
-    opt._initialChunkSize = qBound(opt._minChunkSize, cfgFile.chunkSize(), opt._maxChunkSize);
+    opt.setMinChunkSize(cfgFile.minChunkSize());
+    opt.setMaxChunkSize(cfgFile.maxChunkSize());
+    opt._initialChunkSize = ::qBound(opt.minChunkSize(), cfgFile.chunkSize(), opt.maxChunkSize());
     opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration();
 
     opt.fillFromEnvironmentVariables();

diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp
index f273c1704af1feab96660335fed3b4ff9f603b05..8b340562fa48494694550800874be3d02e37a541 100644 (file)
--- a/src/libsync/owncloudpropagator.cpp
+++ b/src/libsync/owncloudpropagator.cpp
@@ -1046,7 +1046,8 @@ bool OwncloudPropagator::isDelayedUploadItem(const SyncFileItemPtr &item) const
         return true;
     };
 
-    return account()->capabilities().bulkUpload() && !_scheduleDelayedTasks && !item->isEncrypted() && _syncOptions._minChunkSize > item->_size && !isInBulkUploadBlackList(item->_file) && !checkFileShouldBeEncrypted(item);
+    return account()->capabilities().bulkUpload() && !_scheduleDelayedTasks && !item->isEncrypted() && _syncOptions.minChunkSize() > item->_size
+        && !isInBulkUploadBlackList(item->_file) && !checkFileShouldBeEncrypted(item);
 }
 
 void OwncloudPropagator::setScheduleDelayedTasks(bool active)

diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp
index ff316164f402f81f69a95165f429c3d403b8a619..4c3ce698308febb13f46c0bc877be16599767b81 100644 (file)
--- a/src/libsync/propagateuploadng.cpp
+++ b/src/libsync/propagateuploadng.cpp
@@ -427,10 +427,7 @@ void PropagateUploadFileNG::slotPutFinished()
         qint64 targetSize = propagator()->_chunkSize / 2 + predictedGoodSize / 2;
 
         // Adjust the dynamic chunk size _chunkSize used for sizing of the item's chunks to be send
-        propagator()->_chunkSize = qBound(
-            propagator()->syncOptions()._minChunkSize,
-            targetSize,
-            propagator()->syncOptions()._maxChunkSize);
+        propagator()->_chunkSize = ::qBound(propagator()->syncOptions().minChunkSize(), targetSize, propagator()->syncOptions().maxChunkSize());
 
         qCInfo(lcPropagateUploadNG) << "Chunked upload of" << _currentChunkSize << "bytes took" << uploadTime.count()
                                   << "ms, desired is" << targetDuration.count() << "ms, expected good chunk size is"

diff --git a/src/libsync/syncoptions.cpp b/src/libsync/syncoptions.cpp
index c6312d4b41be320c80821d5642e19a3d3c6d27b8..c11d3a9bba2b157c5e505126ccb10d83bdbcd128 100644 (file)
--- a/src/libsync/syncoptions.cpp
+++ b/src/libsync/syncoptions.cpp
@@ -26,6 +26,26 @@ SyncOptions::SyncOptions()
 
 SyncOptions::~SyncOptions() = default;
 
+qint64 SyncOptions::minChunkSize() const
+{
+    return _minChunkSize;
+}
+
+void SyncOptions::setMinChunkSize(const qint64 minChunkSize)
+{
+    _minChunkSize = ::qBound(_minChunkSize, minChunkSize, _maxChunkSize);
+}
+
+qint64 SyncOptions::maxChunkSize() const
+{
+    return _maxChunkSize;
+}
+
+void SyncOptions::setMaxChunkSize(const qint64 maxChunkSize)
+{
+    _maxChunkSize = ::qBound(_minChunkSize, maxChunkSize, _maxChunkSize);
+}
+
 void SyncOptions::fillFromEnvironmentVariables()
 {
     QByteArray chunkSizeEnv = qgetenv("OWNCLOUD_CHUNK_SIZE");

diff --git a/src/libsync/syncoptions.h b/src/libsync/syncoptions.h
index 7d23574204d7dfba77dd34e78777de2f26690c43..59fd233dc3401cc73ac484f4934de8bfa0690100 100644 (file)
--- a/src/libsync/syncoptions.h
+++ b/src/libsync/syncoptions.h
@@ -57,12 +57,6 @@ public:
      */
     qint64 _initialChunkSize = 10 * 1000 * 1000; // 10MB
 
-    /** The minimum chunk size in bytes for chunked uploads */
-    qint64 _minChunkSize = 1 * 1000 * 1000; // 1MB
-
-    /** The maximum chunk size in bytes for chunked uploads */
-    qint64 _maxChunkSize = 1000 * 1000 * 1000; // 1000MB
-
     /** The target duration of chunk uploads for dynamic chunk sizing.
      *
      * Set to 0 it will disable dynamic chunk sizing.
@@ -72,6 +66,17 @@ public:
     /** The maximum number of active jobs in parallel  */
     int _parallelNetworkJobs = 6;
 
+    static constexpr auto chunkV2MinChunkSize = 5LL * 1000LL * 1000LL; // 5 MB
+    static constexpr auto chunkV2MaxChunkSize = 5LL * 1000LL * 1000LL * 1000LL; // 5 GB
+
+    /** The minimum chunk size in bytes for chunked uploads */
+    qint64 minChunkSize() const;
+    void setMinChunkSize(const qint64 minChunkSize);
+
+    /** The maximum chunk size in bytes for chunked uploads */
+    qint64 maxChunkSize() const;
+    void setMaxChunkSize(const qint64 maxChunkSize);
+
     /** Reads settings from env vars where available.
      *
      * Currently reads _initialChunkSize, _minChunkSize, _maxChunkSize,
@@ -110,6 +115,9 @@ private:
      * Invalid pattern by default.
      */
     QRegularExpression _fileRegex = QRegularExpression(QStringLiteral("("));
+
+    qint64 _minChunkSize = chunkV2MinChunkSize;
+    qint64 _maxChunkSize = chunkV2MaxChunkSize;
 };
 
 }

diff --git a/test/testasyncop.cpp b/test/testasyncop.cpp
index ca590bc749fb16e3fe15b82fb79b2bf89d276afd..6b9e8a1ae06b87bd64a0311cb8c38ba189f26155 100644 (file)
--- a/test/testasyncop.cpp
+++ b/test/testasyncop.cpp
@@ -54,7 +54,7 @@ private slots:
         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" } } } });
         // Reduce max chunk size a bit so we get more chunks
         SyncOptions options;
-        options._maxChunkSize = 20 * 1000;
+        options.setMaxChunkSize(20 * 1000);
         fakeFolder.syncEngine().setSyncOptions(options);
         int nGET = 0;
 
@@ -144,16 +144,15 @@ private slots:
             testCases[file] = { std::move(cb) };
         };
         fakeFolder.localModifier().mkdir("success");
-        insertFile("success/chunked_success", options._maxChunkSize * 3, successCallback);
+        insertFile("success/chunked_success", options.maxChunkSize() * 3, successCallback);
         insertFile("success/single_success", 300, successCallback);
-        insertFile("success/chunked_patience", options._maxChunkSize * 3,
-            waitAndChain(waitAndChain(successCallback)));
+        insertFile("success/chunked_patience", options.maxChunkSize() * 3, waitAndChain(waitAndChain(successCallback)));
         insertFile("success/single_patience", 300,
             waitAndChain(waitAndChain(successCallback)));
         fakeFolder.localModifier().mkdir("err");
-        insertFile("err/chunked_error", options._maxChunkSize * 3, errorCallback);
+        insertFile("err/chunked_error", options.maxChunkSize() * 3, errorCallback);
         insertFile("err/single_error", 300, errorCallback);
-        insertFile("err/chunked_error2", options._maxChunkSize * 3, waitAndChain(errorCallback));
+        insertFile("err/chunked_error2", options.maxChunkSize() * 3, waitAndChain(errorCallback));
         insertFile("err/single_error2", 300, waitAndChain(errorCallback));
 
         // First sync should finish by itself.
@@ -171,11 +170,10 @@ private slots:
         fakeFolder.localModifier().mkdir("waiting");
         insertFile("waiting/small", 300, waitForeverCallback);
         insertFile("waiting/willNotConflict", 300, waitForeverCallback);
-        insertFile("waiting/big", options._maxChunkSize * 3,
-            waitAndChain(waitAndChain([&](TestCase *tc, const QNetworkRequest &request) {
-                QTimer::singleShot(0, &fakeFolder.syncEngine(), &SyncEngine::abort);
-                return waitAndChain(waitForeverCallback)(tc, request);
-            })));
+        insertFile("waiting/big", options.maxChunkSize() * 3, waitAndChain(waitAndChain([&](TestCase *tc, const QNetworkRequest &request) {
+                       QTimer::singleShot(0, &fakeFolder.syncEngine(), &SyncEngine::abort);
+                       return waitAndChain(waitForeverCallback)(tc, request);
+                   })));
 
         QVERIFY(fakeFolder.syncJournal().wipeErrorBlacklist() != -1);
 

diff --git a/test/testchunkingng.cpp b/test/testchunkingng.cpp
index fbd8d93bb7012d69a483051aba182d9fe3b60614..f760051840e6d0eb1efb6d3c248fda2306d6ae73 100644 (file)
--- a/test/testchunkingng.cpp
+++ b/test/testchunkingng.cpp
@@ -43,9 +43,9 @@ static void partialUpload(FakeFolder &fakeFolder, const QString &name, qint64 si
 static void setChunkSize(SyncEngine &engine, qint64 size)
 {
     SyncOptions options;
-    options._maxChunkSize = size;
+    options.setMaxChunkSize(size);
+    options.setMinChunkSize(size);
     options._initialChunkSize = size;
-    options._minChunkSize = size;
     engine.setSyncOptions(options);
 }
 
@@ -607,6 +607,13 @@ private slots:
     void testVeryBigFiles() {
         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
+
+        // Dynamic chunk sizing tries to go for biggest chunks possible depending on upload speed.
+        // In the tests this is immediate, so we need to give a file larger than the max chunk size
+        // and cap the max chunk size a bit
+        auto opts = fakeFolder.syncEngine().syncOptions();
+        opts.setMaxChunkSize(500LL * 1000LL * 1000LL); // 500MB
+        fakeFolder.syncEngine().setSyncOptions(opts);
         const qint64 size = 2.5 * 1024 * 1024 * 1024; // 2.5 GiB
 
         // Partial upload of big files
@@ -623,7 +630,6 @@ private slots:
         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
         QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
 
-
         // Upload another file again, this time without interruption
         fakeFolder.localModifier().appendByte("A/a0");
         QVERIFY(fakeFolder.syncOnce());

diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp
index 24cb3e75113ede0e1fab5b39c8a9452452fc7dd1..c7408f9f9dc68b10062322e3ca619dddc8ca17c8 100644 (file)
--- a/test/testsyncengine.cpp
+++ b/test/testsyncengine.cpp
@@ -851,8 +851,8 @@ private slots:
         FakeFolder fakeFolder{ FileInfo{} };
         SyncOptions options;
         options._initialChunkSize = 10;
-        options._maxChunkSize = 10;
-        options._minChunkSize = 10;
+        options.setMaxChunkSize(10);
+        options.setMinChunkSize(10);
         fakeFolder.syncEngine().setSyncOptions(options);
 
         QObject parent;