Upload: Read file chunks gradually #7226

Instead of all at once, to reduce peak memory use.

Changing UploadDevice in this way requires keeping the file open for the
duration of the upload. It also means changes to open(), seek(), close()
to ensure that uses of the device work right when a request needs to
be resent.

diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp
index 77f4f73f72bba3a924db241b6c5807a0443eb835..2b043f3581a75349a7e804757fa8c14a7d84f7e3 100644 (file)
--- a/src/libsync/propagateupload.cpp
+++ b/src/libsync/propagateupload.cpp
@@ -443,8 +443,11 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh
     doStartUpload();
 }
 
-UploadDevice::UploadDevice(BandwidthManager *bwm)
-    : _read(0)
+UploadDevice::UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm)
+    : _file(fileName)
+    , _start(start)
+    , _size(size)
+    , _read(0)
     , _bandwidthManager(bwm)
     , _bandwidthQuota(0)
     , _readWithProgress(0)
@@ -462,29 +465,28 @@ UploadDevice::~UploadDevice()
     }
 }
 
-bool UploadDevice::prepareAndOpen(const QString &fileName, qint64 start, qint64 size)
+bool UploadDevice::open(QIODevice::OpenMode mode)
 {
-    _data.clear();
-    _read = 0;
+    if (mode & QIODevice::WriteOnly)
+        return false;
 
-    QFile file(fileName);
     QString openError;
-    if (!FileSystem::openAndSeekFileSharedRead(&file, &openError, start)) {
+    if (!FileSystem::openAndSeekFileSharedRead(&_file, &openError, _start)) {
         setErrorString(openError);
         return false;
     }
 
-    size = qBound(0ll, size, FileSystem::getSize(fileName) - start);
-    _data.resize(size);
-    auto read = file.read(_data.data(), size);
-    if (read != size) {
-        setErrorString(file.errorString());
-        return false;
-    }
+    _size = qBound(0ll, _size, FileSystem::getSize(_file.fileName()) - _start);
+    _read = 0;
 
-    return QIODevice::open(QIODevice::ReadOnly);
+    return QIODevice::open(mode);
 }
 
+void UploadDevice::close()
+{
+    _file.close();
+    QIODevice::close();
+}
 
 qint64 UploadDevice::writeData(const char *, qint64)
 {
@@ -494,15 +496,15 @@ qint64 UploadDevice::writeData(const char *, qint64)
 
 qint64 UploadDevice::readData(char *data, qint64 maxlen)
 {
-    if (_data.size() - _read <= 0) {
+    if (_size - _read <= 0) {
         // at end
         if (_bandwidthManager) {
             _bandwidthManager->unregisterUploadDevice(this);
         }
         return -1;
     }
-    maxlen = qMin(maxlen, _data.size() - _read);
-    if (maxlen == 0) {
+    maxlen = qMin(maxlen, _size - _read);
+    if (maxlen <= 0) {
         return 0;
     }
     if (isChoked()) {
@@ -515,9 +517,14 @@ qint64 UploadDevice::readData(char *data, qint64 maxlen)
         }
         _bandwidthQuota -= maxlen;
     }
-    std::memcpy(data, _data.data() + _read, maxlen);
-    _read += maxlen;
-    return maxlen;
+
+    auto c = _file.read(data, maxlen);
+    if (c < 0) {
+        setErrorString(_file.errorString());
+        return -1;
+    }
+    _read += c;
+    return c;
 }
 
 void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
@@ -530,17 +537,17 @@ void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t)
 
 bool UploadDevice::atEnd() const
 {
-    return _read >= _data.size();
+    return _read >= _size;
 }
 
 qint64 UploadDevice::size() const
 {
-    return _data.size();
+    return _size;
 }
 
 qint64 UploadDevice::bytesAvailable() const
 {
-    return _data.size() - _read + QIODevice::bytesAvailable();
+    return _size - _read + QIODevice::bytesAvailable();
 }
 
 // random access, we can seek
@@ -554,10 +561,11 @@ bool UploadDevice::seek(qint64 pos)
     if (!QIODevice::seek(pos)) {
         return false;
     }
-    if (pos < 0 || pos > _data.size()) {
+    if (pos < 0 || pos > _size) {
         return false;
     }
     _read = pos;
+    _file.seek(_start + pos);
     return true;
 }
 

diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h
index dd6680e707b0b9d3c0b384d5ecff8a8943264870..229a0b222a45eec090e986cffaa2a72f3ac79da9 100644 (file)
--- a/src/libsync/propagateupload.h
+++ b/src/libsync/propagateupload.h
@@ -36,11 +36,11 @@ class UploadDevice : public QIODevice
 {
     Q_OBJECT
 public:
-    UploadDevice(BandwidthManager *bwm);
+    UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm);
     ~UploadDevice();
 
-    /** Reads the data from the file and opens the device */
-    bool prepareAndOpen(const QString &fileName, qint64 start, qint64 size);
+    bool open(QIODevice::OpenMode mode) override;
+    void close() override;
 
     qint64 writeData(const char *, qint64) override;
     qint64 readData(char *data, qint64 maxlen) override;
@@ -59,10 +59,15 @@ public:
 signals:
 
 private:
-    // The file data
-    QByteArray _data;
-    // Position in the data
-    qint64 _read;
+    /// The local file to read data from
+    QFile _file;
+
+    /// Start of the file data to use
+    qint64 _start = 0;
+    /// Amount of file data after _start to use
+    qint64 _size = 0;
+    /// Position between _start and _start+_size
+    qint64 _read = 0;
 
     // Bandwidth manager related
     QPointer<BandwidthManager> _bandwidthManager;

diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp
index 09bfc5e5ea107f9c727949798fc9e79f97923c3e..fa79633d1bfdbeadfe7e9c273023b64813b55276 100644 (file)
--- a/src/libsync/propagateuploadng.cpp
+++ b/src/libsync/propagateuploadng.cpp
@@ -316,10 +316,10 @@ void PropagateUploadFileNG::startNextChunk()
         return;
     }
 
-    auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
     const QString fileName = _fileToUpload._path;
-
-    if (!device->prepareAndOpen(fileName, _sent, _currentChunkSize)) {
+    auto device = std::make_unique<UploadDevice>(
+            fileName, _currentChunk, _currentChunkSize, &propagator()->_bandwidthManager);
+    if (!device->open(QIODevice::ReadOnly)) {
         qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
 
         // If the file is currently locked, we want to retry the sync

diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp
index 6135e715282c4da0fe1a6d8dc0153560c6246a82..e1ae8b5faa7a7dfdb05b447b4725cf45e0c3fa54 100644 (file)
--- a/src/libsync/propagateuploadv1.cpp
+++ b/src/libsync/propagateuploadv1.cpp
@@ -90,7 +90,6 @@ void PropagateUploadFileV1::startNextChunk()
 
     QString path = _fileToUpload._file;
 
-    auto device = std::make_unique<UploadDevice>(&propagator()->_bandwidthManager);
     qint64 chunkStart = 0;
     qint64 currentChunkSize = fileSize;
     bool isFinalChunk = false;
@@ -124,8 +123,9 @@ void PropagateUploadFileV1::startNextChunk()
     }
 
     const QString fileName = _fileToUpload._path;
-    qDebug() << "Trying to upload" << fileName;
-    if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) {
+    auto device = std::make_unique<UploadDevice>(
+            fileName, chunkStart, currentChunkSize, &propagator()->_bandwidthManager);
+    if (!device->open(QIODevice::ReadOnly)) {
         qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString();
 
         // If the file is currently locked, we want to retry the sync