From: Christian Kamm Date: Thu, 6 Jun 2019 10:22:02 +0000 (+0200) Subject: Upload: Read file chunks gradually #7226 X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~230 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=452ed56571ae18614a37e0266a42f9037e09b5c4;p=nextcloud-desktop.git Upload: Read file chunks gradually #7226 Instead of all at once, to reduce peak memory use. Changing UploadDevice in this way requires keeping the file open for the duration of the upload. It also means changes to open(), seek(), close() to ensure that uses of the device work right when a request needs to be resent. --- diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 77f4f73f7..2b043f358 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -443,8 +443,11 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh doStartUpload(); } -UploadDevice::UploadDevice(BandwidthManager *bwm) - : _read(0) +UploadDevice::UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm) + : _file(fileName) + , _start(start) + , _size(size) + , _read(0) , _bandwidthManager(bwm) , _bandwidthQuota(0) , _readWithProgress(0) @@ -462,29 +465,28 @@ UploadDevice::~UploadDevice() } } -bool UploadDevice::prepareAndOpen(const QString &fileName, qint64 start, qint64 size) +bool UploadDevice::open(QIODevice::OpenMode mode) { - _data.clear(); - _read = 0; + if (mode & QIODevice::WriteOnly) + return false; - QFile file(fileName); QString openError; - if (!FileSystem::openAndSeekFileSharedRead(&file, &openError, start)) { + if (!FileSystem::openAndSeekFileSharedRead(&_file, &openError, _start)) { setErrorString(openError); return false; } - size = qBound(0ll, size, FileSystem::getSize(fileName) - start); - _data.resize(size); - auto read = file.read(_data.data(), size); - if (read != size) { - setErrorString(file.errorString()); - return false; - } + _size = qBound(0ll, _size, FileSystem::getSize(_file.fileName()) - _start); + _read = 0; - return QIODevice::open(QIODevice::ReadOnly); + return QIODevice::open(mode); } +void UploadDevice::close() +{ + _file.close(); + QIODevice::close(); +} qint64 UploadDevice::writeData(const char *, qint64) { @@ -494,15 +496,15 @@ qint64 UploadDevice::writeData(const char *, qint64) qint64 UploadDevice::readData(char *data, qint64 maxlen) { - if (_data.size() - _read <= 0) { + if (_size - _read <= 0) { // at end if (_bandwidthManager) { _bandwidthManager->unregisterUploadDevice(this); } return -1; } - maxlen = qMin(maxlen, _data.size() - _read); - if (maxlen == 0) { + maxlen = qMin(maxlen, _size - _read); + if (maxlen <= 0) { return 0; } if (isChoked()) { @@ -515,9 +517,14 @@ qint64 UploadDevice::readData(char *data, qint64 maxlen) } _bandwidthQuota -= maxlen; } - std::memcpy(data, _data.data() + _read, maxlen); - _read += maxlen; - return maxlen; + + auto c = _file.read(data, maxlen); + if (c < 0) { + setErrorString(_file.errorString()); + return -1; + } + _read += c; + return c; } void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t) @@ -530,17 +537,17 @@ void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t) bool UploadDevice::atEnd() const { - return _read >= _data.size(); + return _read >= _size; } qint64 UploadDevice::size() const { - return _data.size(); + return _size; } qint64 UploadDevice::bytesAvailable() const { - return _data.size() - _read + QIODevice::bytesAvailable(); + return _size - _read + QIODevice::bytesAvailable(); } // random access, we can seek @@ -554,10 +561,11 @@ bool UploadDevice::seek(qint64 pos) if (!QIODevice::seek(pos)) { return false; } - if (pos < 0 || pos > _data.size()) { + if (pos < 0 || pos > _size) { return false; } _read = pos; + _file.seek(_start + pos); return true; } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index dd6680e70..229a0b222 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -36,11 +36,11 @@ class UploadDevice : public QIODevice { Q_OBJECT public: - UploadDevice(BandwidthManager *bwm); + UploadDevice(const QString &fileName, qint64 start, qint64 size, BandwidthManager *bwm); ~UploadDevice(); - /** Reads the data from the file and opens the device */ - bool prepareAndOpen(const QString &fileName, qint64 start, qint64 size); + bool open(QIODevice::OpenMode mode) override; + void close() override; qint64 writeData(const char *, qint64) override; qint64 readData(char *data, qint64 maxlen) override; @@ -59,10 +59,15 @@ public: signals: private: - // The file data - QByteArray _data; - // Position in the data - qint64 _read; + /// The local file to read data from + QFile _file; + + /// Start of the file data to use + qint64 _start = 0; + /// Amount of file data after _start to use + qint64 _size = 0; + /// Position between _start and _start+_size + qint64 _read = 0; // Bandwidth manager related QPointer _bandwidthManager; diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index 09bfc5e5e..fa79633d1 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -316,10 +316,10 @@ void PropagateUploadFileNG::startNextChunk() return; } - auto device = std::make_unique(&propagator()->_bandwidthManager); const QString fileName = _fileToUpload._path; - - if (!device->prepareAndOpen(fileName, _sent, _currentChunkSize)) { + auto device = std::make_unique( + fileName, _currentChunk, _currentChunkSize, &propagator()->_bandwidthManager); + if (!device->open(QIODevice::ReadOnly)) { qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString(); // If the file is currently locked, we want to retry the sync diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index 6135e7152..e1ae8b5fa 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -90,7 +90,6 @@ void PropagateUploadFileV1::startNextChunk() QString path = _fileToUpload._file; - auto device = std::make_unique(&propagator()->_bandwidthManager); qint64 chunkStart = 0; qint64 currentChunkSize = fileSize; bool isFinalChunk = false; @@ -124,8 +123,9 @@ void PropagateUploadFileV1::startNextChunk() } const QString fileName = _fileToUpload._path; - qDebug() << "Trying to upload" << fileName; - if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) { + auto device = std::make_unique( + fileName, chunkStart, currentChunkSize, &propagator()->_bandwidthManager); + if (!device->open(QIODevice::ReadOnly)) { qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString(); // If the file is currently locked, we want to retry the sync