dynamic chunking prototype

diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp
index cee5f1aebe2521bbc37c181e19c444f3bd0253e4..0c57eab1eae992c278abfb2a5205e8184b8e4d7a 100644 (file)
--- a/src/libsync/capabilities.cpp
+++ b/src/libsync/capabilities.cpp
@@ -130,4 +130,12 @@ QList<int> Capabilities::httpErrorCodesThatResetFailingChunkedUploads() const
     return list;
 }
 
+quint64 Capabilities::requestMaxDurationDC() const
+{
+    QByteArray requestMaxDurationDC = _capabilities["dav"].toMap()["max_single_upload_request_duration_msec"].toByteArray();
+    if (!requestMaxDurationDC.isEmpty())
+        return requestMaxDurationDC.toLongLong();
+    return 0;
+}
+
 }

diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h
index e3d8c013c238be6afc9a40236a3780e8117758ac..0c28efa614516e150f0fd49f5e7aeeae6dc17d25 100644 (file)
--- a/src/libsync/capabilities.h
+++ b/src/libsync/capabilities.h
@@ -41,6 +41,7 @@ public:
     int  sharePublicLinkExpireDateDays() const;
     bool shareResharing() const;
     bool chunkingNg() const;
+    quint64 requestMaxDurationDC() const;
 
     /// disable parallel upload in chunking
     bool chunkingParallelUploadDisabled() const;

diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp
index 2132ced728a7db0e86936f08367d2ab29d576821..bad02f49ddd3a5fadd46b269cd3d2fb44c4d5497 100644 (file)
--- a/src/libsync/configfile.cpp
+++ b/src/libsync/configfile.cpp
@@ -53,6 +53,7 @@ static const char updateCheckIntervalC[] = "updateCheckInterval";
 static const char geometryC[] = "geometry";
 static const char timeoutC[] = "timeout";
 static const char chunkSizeC[] = "chunkSize";
+static const char maxChunkSizeC[] = "maxChunkSizeC";
 
 static const char proxyHostC[] = "Proxy/host";
 static const char proxyTypeC[] = "Proxy/type";
@@ -130,6 +131,18 @@ quint64 ConfigFile::chunkSize() const
     return settings.value(QLatin1String(chunkSizeC), 10*1000*1000).toLongLong(); // default to 10 MB
 }
 
+quint64 ConfigFile::maxChunkSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(maxChunkSizeC), 50*1000*1000).toLongLong(); // default to 50 MB
+}
+
+quint64 ConfigFile::minChunkSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(maxChunkSizeC), 1000*1000).toLongLong(); // default to 1 MB
+}
+
 void ConfigFile::setOptionalDesktopNotifications(bool show)
 {
     QSettings settings(configFile(), QSettings::IniFormat);

diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h
index 0fe3b3cd06394633c0fe94e7b40a218e29f9d6b7..8191ab13e6e121a71700ccd8160d65a1e9384874 100644 (file)
--- a/src/libsync/configfile.h
+++ b/src/libsync/configfile.h
@@ -115,6 +115,8 @@ public:
 
     int timeout() const;
     quint64 chunkSize() const;
+    quint64 maxChunkSize() const;
+    quint64 minChunkSize() const;
 
     void saveGeometry(QWidget *w);
     void restoreGeometry(QWidget *w);

diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp
index 300b7778af4075c88bc107e5829c3e088ac26c79..3c79a87f3e592f05c68472edb2dac630aefff9c5 100644 (file)
--- a/src/libsync/owncloudpropagator.cpp
+++ b/src/libsync/owncloudpropagator.cpp
@@ -370,7 +370,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItemPtr &item) {
             } else {
                 PropagateUploadFileCommon *job = 0;
                 if (item->_size > chunkSize() && account()->capabilities().chunkingNg()) {
-                    job = new PropagateUploadFileNG(this, item);
+                    job = new PropagateUploadFileNG(this, item, account()->capabilities().requestMaxDurationDC());
                 } else {
                     job = new PropagateUploadFileV1(this, item);
                 }
@@ -547,6 +547,18 @@ quint64 OwncloudPropagator::chunkSize()
     return chunkSize;
 }
 
+quint64 OwncloudPropagator::maxChunkSize()
+{
+    static uint chunkSize;
+    if (!chunkSize) {
+        chunkSize = qgetenv("OWNCLOUD_MAX_CHUNK_SIZE").toUInt();
+        if (chunkSize == 0) {
+            ConfigFile cfg;
+            chunkSize = cfg.maxChunkSize();
+        }
+    }
+    return chunkSize;
+}
 
 bool OwncloudPropagator::localFileNameClash( const QString& relFile )
 {

diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h
index 0cb949e39b843552ef650662c72e364681ae7f9d..be913b75386b4cfa336103596d4b5551fdd8ea3f 100644 (file)
--- a/src/libsync/owncloudpropagator.h
+++ b/src/libsync/owncloudpropagator.h
@@ -279,7 +279,6 @@ public:
     SyncJournalDb * const _journal;
     bool _finishedEmited; // used to ensure that finished is only emitted once
 
-
 public:
     OwncloudPropagator(AccountPtr account, const QString &localDir,
                        const QString &remoteFolder, SyncJournalDb *progressDb)
@@ -357,6 +356,7 @@ public:
 
     /** returns the size of chunks in bytes  */
     static quint64 chunkSize();
+    static quint64 maxChunkSize();
 
     AccountPtr account() const;
 

diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h
index 3a7224a7019bde4caeffb14c2f36d3bcd13ff871..cf6e5b7956fbb8595458083cca1b645a13437323 100644 (file)
--- a/src/libsync/propagateupload.h
+++ b/src/libsync/propagateupload.h
@@ -304,6 +304,24 @@ private:
     uint _transferId; /// transfer id (part of the url)
     int _currentChunk; /// Id of the next chunk that will be sent
     bool _removeJobError; /// If not null, there was an error removing the job
+    quint64 _lastChunkSize; /// current chunk size
+
+    /*
+    * This is value in ms obtained from the server.
+    *
+    * Dynamic Chunking attribute the maximum number of miliseconds that single request below chunk size can take
+    * This value should be based on heuristics with default value 10000ms, time it takes to transfer 10MB chunk on 1MB/s upload link.
+    *
+    * Suggested solution will be to evaluate max(SNR, MORD) where:
+    * > SNR - Slow network request, so time it will take to transmit default chunking sized request at specific low upload bandwidth
+    * > MORD - Maximum observed request time, so double the time of maximum observed RTT of the very small PUT request (e.g. 1kB) to the system
+    *
+    * Exemplary, syncing 100MB files, with chunking size 10MB, will cause sync of 10 PUT requests which max evaluation was set to <max_single_upload_request_duration_msec>
+    *
+    * Dynamic chunking client algorithm is specified in the ownCloud documentation and uses <max_single_upload_request_duration_msec> to estimate if given
+    * bandwidth allows higher chunk sizes (because of high goodput)
+    */
+    quint64 _requestMaxDuration;
 
     // Map chunk number with its size  from the PROPFIND on resume.
     // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.)
@@ -311,6 +329,12 @@ private:
     QMap<int, ServerChunkInfo> _serverChunks;
 
     quint64 chunkSize() const { return propagator()->chunkSize(); }
+    quint64 maxChunkSize() const { return propagator()->maxChunkSize(); }
+
+    quint64 getRequestMaxDurationDC(){
+        return _requestMaxDuration;
+    }
+
     /**
      * Return the URL of a chunk.
      * If chunk == -1, returns the URL of the parent folder containing the chunks
@@ -318,10 +342,11 @@ private:
     QUrl chunkUrl(int chunk = -1);
 
 public:
-    PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item) :
-        PropagateUploadFileCommon(propagator,item) {}
+    PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item, const quint64& requestMaxDuration) :
+        PropagateUploadFileCommon(propagator,item), _lastChunkSize(0), _requestMaxDuration(requestMaxDuration) {}
 
     void doStartUpload() Q_DECL_OVERRIDE;
+
 private:
     void startNewUpload();
     void startNextChunk();

diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp
index 9803385be1008436c02108452abbad24b37fd7a5..63d71d3c4d9a4399b87043c538abed489593e6db 100644 (file)
--- a/src/libsync/propagateuploadng.cpp
+++ b/src/libsync/propagateuploadng.cpp
@@ -32,7 +32,7 @@
 #include <QDir>
 #include <cmath>
 #include <cstring>
-
+#include <cmath>
 namespace OCC {
 
 QUrl PropagateUploadFileNG::chunkUrl(int chunk)
@@ -272,7 +272,41 @@ void PropagateUploadFileNG::startNextChunk()
     quint64 fileSize = _item->_size;
     ENFORCE(fileSize >= _sent, "Sent data exceeds file size");
 
-    quint64 currentChunkSize = qMin(chunkSize(), fileSize - _sent);
+    quint64 currentChunkSize = chunkSize();
+
+    // this will check if getRequestMaxDurationDC is set to 0 or not
+    double requestMaxDurationDC = (double) getRequestMaxDurationDC();
+    if (requestMaxDurationDC != 0) {
+        // this if first chunked file request, so it can start with default size of chunkSize()
+        // if _lastChunkSize != 0 it means that we already have send one request
+        if(_lastChunkSize != 0){
+            //TODO: this is done step by step for debugging purposes
+
+            //get last request timestamp
+            double lastChunkLap = (double) _stopWatch.durationOfLap(QLatin1String("ChunkDuration"));
+
+            //get duration of the request
+            double requestDuration = (double) _stopWatch.addLapTime(QLatin1String("ChunkDuration")) - lastChunkLap;
+
+            // calculate natural logarithm
+            double correctionParameter = log(requestMaxDurationDC / requestDuration) - 1;
+
+            // If logarithm is smaller or equal zero, it means that we exceeded max request duration
+            // If exceeded it will use currentChunkSize = chunkSize()
+            // If did not exceeded, we will increase the chunk size
+            // motivation for logarithm is specified in the dynamic chunking documentation
+            // TODO: give link to documentation
+            if (correctionParameter>0){
+                currentChunkSize = qMin(_lastChunkSize + (qint64) correctionParameter*chunkSize(), maxChunkSize());
+            }
+        }
+
+        //remember the value of last chunk size
+        _lastChunkSize = currentChunkSize;
+    }
+
+    // prevent situation that chunk size is bigger then required one to send
+    currentChunkSize = qMin(currentChunkSize, fileSize - _sent);
 
     if (currentChunkSize == 0) {
         ASSERT(_jobs.isEmpty());