From: Olivier Goffart Date: Fri, 13 Jul 2018 13:33:54 +0000 (+0200) Subject: New Discovery algorithm: Check that the original file is still on the server while... X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~556 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=f43d07dc052cb7e314f34e96576924e0796179b6;p=nextcloud-desktop.git New Discovery algorithm: Check that the original file is still on the server while renaming --- diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index f7f0f89e4..86741882a 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -65,8 +65,8 @@ void ProcessDirectoryJob::start() qCInfo(lcDisco) << "STARTING" << _currentFolder._server << _queryServer << _currentFolder._local << _queryLocal; if (_queryServer == NormalQuery) { - _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder._server, this); - connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) { + auto serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder._server, this); + connect(serverJob, &DiscoverServerJob::finished, this, [this](const auto &results) { if (results) { _serverEntries = *results; _hasServerEntries = true; @@ -77,7 +77,7 @@ void ProcessDirectoryJob::start() qFatal("TODO: ERROR HANDLING"); } }); - _serverJob->start(); + serverJob->start(); } else { _hasServerEntries = true; } @@ -358,6 +358,20 @@ void ProcessDirectoryJob::processFile(PathTuple path, item->_modtime = serverEntry.modtime; item->_size = serverEntry.size; + auto postProcessNew = [item, this, path, serverEntry] { + if (item->isDirectory()) { + if (_discoveryData->checkSelectiveSyncNewFolder(path._server, serverEntry.remotePerm)) { + return; + } + } + + // Turn new remote files into virtual files if the option is enabled. + if (_discoveryData->_syncOptions._newFilesAreVirtual && item->_type == ItemTypeFile) { + item->_type = ItemTypeVirtualFile; + item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix); + } + }; + if (!localEntry.isValid()) { // Check for renames (if there is a file with the same file id) bool done = false; @@ -426,48 +440,111 @@ void ProcessDirectoryJob::processFile(PathTuple path, } } + bool wasDeletedOnServer = false; auto it = _discoveryData->_deletedItem.find(originalPath); if (it != _discoveryData->_deletedItem.end()) { - if ((*it)->_instruction != CSYNC_INSTRUCTION_REMOVE) - return; + ASSERT((*it)->_instruction == CSYNC_INSTRUCTION_REMOVE); (*it)->_instruction = CSYNC_INSTRUCTION_NONE; + wasDeletedOnServer = true; + } + auto otherJob = _discoveryData->_queuedDeletedDirectories.take(originalPath); + if (otherJob) { + delete otherJob; + wasDeletedOnServer = true; } - delete _discoveryData->_queuedDeletedDirectories.take(originalPath); - _discoveryData->_renamedItems.insert(originalPath); - - item->_modtime = base._modtime; - item->_inode = base._inode; - item->_instruction = CSYNC_INSTRUCTION_RENAME; - item->_direction = SyncFileItem::Down; - item->_renameTarget = path._target; - item->_file = originalPath; - item->_originalFile = originalPath; - path._original = originalPath; - path._local = originalPath; - done = true; - - qCInfo(lcDisco) << "Rename detected (down) " << item->_file << " -> " << item->_renameTarget; - // FIXME! check that the server version of origialPath is gone! + auto postProcessRename = [this, item, base, originalPath](PathTuple &path) { + _discoveryData->_renamedItems.insert(originalPath); + item->_modtime = base._modtime; + item->_inode = base._inode; + item->_instruction = CSYNC_INSTRUCTION_RENAME; + item->_direction = SyncFileItem::Down; + item->_renameTarget = path._target; + item->_file = originalPath; + item->_originalFile = originalPath; + path._original = originalPath; + path._local = originalPath; + qCInfo(lcDisco) << "Rename detected (down) " << item->_file << " -> " << item->_renameTarget; + }; + + if (wasDeletedOnServer) { + postProcessRename(path); + done = true; + } else { + // we need to make a request to the server to know that the original file is deleted on the server + _pendingAsyncJobs++; + auto job = new PropfindJob(_discoveryData->_account, originalPath, this); + auto considerNew = [=] { + // The original file still exist, consider it is new. + postProcessNew(); + qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->isDirectory(); + if (item->isDirectory()) { + auto job = new ProcessDirectoryJob(item, recurseQueryServer, ParentDontExist, _discoveryData, this); + job->_currentFolder = path; + connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); + connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); + _queuedJobs.push_back(job); + } else { + emit itemDiscovered(item); + } + + _pendingAsyncJobs--; + progress(); + }; + connect(job, &PropfindJob::result, this, considerNew); + connect(job, &PropfindJob::finishedWithError, this, [=](QNetworkReply *reply) mutable { + if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() != 404) { + qDebug() << reply->request().url() << reply->error() << reply->errorString(); + qFatal("TODO: Handle error"); + } + if (_discoveryData->_renamedItems.contains(originalPath)) { + // Somehow another item claimed this original path, consider as if it existed + considerNew(); + return; + } + + // In case the deleted item was discovered in parallel + auto it = _discoveryData->_deletedItem.find(originalPath); + if (it != _discoveryData->_deletedItem.end()) { + ASSERT((*it)->_instruction == CSYNC_INSTRUCTION_REMOVE); + (*it)->_instruction = CSYNC_INSTRUCTION_NONE; + } + delete _discoveryData->_queuedDeletedDirectories.take(originalPath); + + // Normal use case: this is a rename. + postProcessRename(path); + + qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->isDirectory(); + + if (item->isDirectory()) { + auto job = new ProcessDirectoryJob(item, recurseQueryServer, NormalQuery, _discoveryData, this); + job->_currentFolder = path; + connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); + connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); + _queuedJobs.push_back(job); + } else { + emit itemDiscovered(item); + } + _pendingAsyncJobs--; + progress(); + }); + job->start(); + done = true; // Ideally, if the origin still exist on the server, we should continue searching... but that'd be difficult + item = nullptr; + } }; if (!_discoveryData->_statedb->getFileRecordsByFileId(serverEntry.fileId, renameCandidateProcessing)) { qFatal("TODO: Handle DB ERROR"); } - } - - if (item->_instruction == CSYNC_INSTRUCTION_NEW && item->isDirectory()) { - if (_discoveryData->checkSelectiveSyncNewFolder(path._server, serverEntry.remotePerm)) { - return; + if (!item) { + return; // We wend async } } - // Turn new remote files into virtual files if the option is enabled. - if (item->_instruction == CSYNC_INSTRUCTION_NEW - && _discoveryData->_syncOptions._newFilesAreVirtual - && item->_type == ItemTypeFile) { - item->_type = ItemTypeVirtualFile; - item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix); + if (item->_instruction == CSYNC_INSTRUCTION_NEW) { + postProcessNew(); } + } else if (dbEntry._etag != serverEntry.etag) { item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; @@ -737,6 +814,10 @@ void ProcessDirectoryJob::subJobFinished() void ProcessDirectoryJob::progress() { + int maxRunning = 3; // FIXME + if (_pendingAsyncJobs + _runningJobs.size() > maxRunning) + return; + if (!_queuedJobs.empty()) { auto f = _queuedJobs.front(); _queuedJobs.pop_front(); @@ -744,7 +825,7 @@ void ProcessDirectoryJob::progress() f->start(); return; } - if (_runningJobs.empty()) { + if (_runningJobs.empty() && _pendingAsyncJobs == 0) { if (_dirItem) { if (_childModified && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { // re-create directory that has modified contents diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 14e9ac1bb..10647690f 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -162,8 +162,8 @@ private: QVector _localEntries; bool _hasServerEntries = false; bool _hasLocalEntries = false; + int _pendingAsyncJobs = 0; QPointer _serverJob; - //QScopedPointer _localJob; std::deque _queuedJobs; QVector _runningJobs; SyncFileItemPtr _dirItem;