From: Olivier Goffart Date: Wed, 11 Jul 2018 14:55:00 +0000 (+0200) Subject: New discovery algo: Remote move X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~559 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=f9a9be59e6b8a3d834a34aa7127ab5ef2721f477;p=nextcloud-desktop.git New discovery algo: Remote move TestSyncMove::testRemoteChangeInMovedFolder --- diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index dc383f05a..c144cee91 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -340,100 +340,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f } goto out; - } else { - qCInfo(lcUpdate, "Checking for rename based on fileid %s", fs->file_id.constData()); - - /* Remote Replica Rename check */ - fs->instruction = CSYNC_INSTRUCTION_NEW; - - bool done = false; - auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) { - if (done) - return; - if (!base.isValid()) - return; - - if (base._type == ItemTypeVirtualFileDownload) { - // Remote rename of a virtual file we have locally scheduled - // for download. We just consider this NEW but mark it for download. - fs->type = ItemTypeVirtualFileDownload; - done = true; - return; - } - - // Some things prohibit rename detection entirely. - // Since we don't do the same checks again in reconcile, we can't - // just skip the candidate, but have to give up completely. - if (base._type != fs->type - && base._type != ItemTypeVirtualFile) { - qCWarning(lcUpdate, "file types different, not a rename"); - done = true; - return; - } - if (fs->type != ItemTypeDirectory && base._etag != fs->etag) { - /* File with different etag, don't do a rename, but download the file again */ - qCWarning(lcUpdate, "file etag different, not a rename"); - done = true; - return; - } - - // Now we know there is a sane rename candidate. - - // Rename of a virtual file - if (base._type == ItemTypeVirtualFile && fs->type == ItemTypeFile) { - fs->type = ItemTypeVirtualFile; - fs->path.append(ctx->virtual_file_suffix); - } - - // Record directory renames - if (fs->type == ItemTypeDirectory) { - // If the same folder was already renamed by a different entry, - // skip to the next candidate - if (ctx->renames.folder_renamed_to.count(base._path) > 0) { - qCWarning(lcUpdate, "folder already has a rename entry, skipping"); - return; - } - csync_rename_record(ctx, base._path, fs->path); - } - - /* A remote rename can also mean Encryption Mangled Name. - * if we find one of those in the database, we ignore it. - */ - if (!base._e2eMangledName.isEmpty()) { - qCWarning(lcUpdate, "Encrypted file can not rename"); - done = true; - return; - } - - qCInfo(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); - - fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; - done = true; - }; - - if (!ctx->statedb->getFileRecordsByFileId(fs->file_id, renameCandidateProcessing)) { - ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; - return -1; - } - - if (fs->instruction == CSYNC_INSTRUCTION_NEW - && fs->type == ItemTypeDirectory - && ctx->current == REMOTE_REPLICA - && ctx->callbacks.checkSelectiveSyncNewFolderHook) { - if (ctx->callbacks.checkSelectiveSyncNewFolderHook(ctx->callbacks.update_callback_userdata, fs->path, fs->remotePerm)) { - return 1; - } - } - - // Turn new remote files into virtual files if the option is enabled. - if (ctx->new_files_are_virtual - && fs->instruction == CSYNC_INSTRUCTION_NEW - && fs->type == ItemTypeFile) { - fs->type = ItemTypeVirtualFile; - fs->path.append(ctx->virtual_file_suffix); - } - - goto out; + } else { /*... PORTED ... */ } } diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 8cb2adfe8..dcb2bd900 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -63,7 +63,7 @@ DiscoverServerJob::DiscoverServerJob(const AccountPtr &account, const QString &p void ProcessDirectoryJob::start() { if (_queryServer == NormalQuery) { - _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder, this); + _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder._server, this); connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) { if (results) { _serverEntries = *results; @@ -80,9 +80,6 @@ void ProcessDirectoryJob::start() _hasServerEntries = true; } - if (!_currentFolder.isEmpty() && _currentFolder.back() != '/') - _currentFolder += '/'; - if (_queryLocal == NormalQuery) { /*QDirIterator dirIt(_propagator->_localDir + _currentFolder); while (dirIt.hasNext()) { @@ -91,9 +88,9 @@ void ProcessDirectoryJob::start() i.name = dirIt.fileName(); }*/ - auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder).toUtf8()); + auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder._local).toUtf8()); if (!dh) { - qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder).toUtf8(); + qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder._local); qFatal("TODO: ERROR HANDLING"); // should be the same as in csync_update; } @@ -106,7 +103,7 @@ void ProcessDirectoryJob::start() if (state.invalidChars > 0 || state.remainingChars > 0) { _childIgnored = true; auto item = SyncFileItemPtr::create(); - item->_file = _currentFolder + i.name; + item->_file = _currentFolder._target + i.name; item->_instruction = CSYNC_INSTRUCTION_IGNORE; item->_status = SyncFileItem::NormalError; item->_errorString = tr("Filename encoding is not valid"); @@ -151,8 +148,7 @@ void ProcessDirectoryJob::process() if (_queryServer == ParentNotChanged) { // fetch all the name from the DB - auto pathU8 = _currentFolder.toUtf8(); - pathU8.chop(1); + auto pathU8 = _currentFolder._original.toUtf8(); // FIXME cache, and do that better (a query that do not get stuff recursively) if (!_discoveryData->_statedb->getFilesBelowPath(pathU8, [&](const SyncJournalFileRecord &rec) { if (rec._path.indexOf("/", pathU8.size() + 1) > 0) @@ -163,9 +159,8 @@ void ProcessDirectoryJob::process() } } - for (const auto &f : entriesNames) { - QString path = _currentFolder + f; + auto path = _currentFolder.addName(f); auto localEntry = localEntriesHash.value(f); auto serverEntry = serverEntriesHash.value(f); @@ -174,18 +169,18 @@ void ProcessDirectoryJob::process() // local stat function. // Recall file shall not be ignored (#4420) bool isHidden = localEntry.isHidden || (f[0] == '.' && f != QLatin1String(".sys.admin#recall#")); - if (handleExcluded(path, localEntry.isDirectory || serverEntry.isDirectory, isHidden)) + if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden)) continue; SyncJournalFileRecord record; - if (!_discoveryData->_statedb->getFileRecord(path, &record)) { + if (!_discoveryData->_statedb->getFileRecord(path._original, &record)) { qFatal("TODO: DB ERROR HANDLING"); } - if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path)) { + if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._server)) { processBlacklisted(path, localEntry, record); continue; } - processFile(path, localEntry, serverEntry, record); + processFile(std::move(path), localEntry, serverEntry, record); } progress(); @@ -297,7 +292,7 @@ bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory, } #endif break; - case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE: + case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE: // FIXME! item->_errorString = tr("The filename cannot be encoded on your file system."); break; } @@ -307,20 +302,24 @@ bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory, return true; } -void ProcessDirectoryJob::processFile(const QString &path, +void ProcessDirectoryJob::processFile(PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry) { - qCInfo(lcDisco).nospace() << "Processing " << path + qCInfo(lcDisco).nospace() << "Processing " << path._original << " | valid: " << dbEntry.isValid() << "/" << localEntry.isValid() << "/" << serverEntry.isValid() << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size << " | etag: " << dbEntry._etag << "//" << serverEntry.etag << " | checksum: " << dbEntry._checksumHeader << "//" << serverEntry.checksumHeader; - auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); - item->_file = path; + if (_discoveryData->_renamedItems.contains(path._original)) { + qCDebug(lcDisco) << "Ignoring renamed"; + return; // Ignore this. + } + auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); + item->_file = path._target; auto recurseQueryServer = _queryServer; if (_queryServer == NormalQuery && serverEntry.isValid()) { item->_checksumHeader = serverEntry.checksumHeader; @@ -330,12 +329,123 @@ void ProcessDirectoryJob::processFile(const QString &path, item->_etag = serverEntry.etag; item->_previousSize = localEntry.size; item->_previousModtime = localEntry.modtime; - if (!dbEntry.isValid()) { + if (!dbEntry.isValid()) { // New file? item->_instruction = CSYNC_INSTRUCTION_NEW; - // TODO! rename; item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; item->_size = serverEntry.size; + + if (!localEntry.isValid()) { + // Check for renames (if there is a file with the same file id) + bool done = false; + auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) { + if (done) + return; + if (!base.isValid()) + return; + + if (base._type == ItemTypeVirtualFileDownload) { + // Remote rename of a virtual file we have locally scheduled + // for download. We just consider this NEW but mark it for download. + item->_type = ItemTypeVirtualFileDownload; + done = true; + return; + } + + // Some things prohibit rename detection entirely. + // Since we don't do the same checks again in reconcile, we can't + // just skip the candidate, but have to give up completely. + if (base._type != item->_type && base._type != ItemTypeVirtualFile) { + qCDebug(lcDisco, "file types different, not a rename"); + done = true; + return; + } + if (!serverEntry.isDirectory && base._etag != serverEntry.etag) { + /* File with different etag, don't do a rename, but download the file again */ + qCInfo(lcDisco, "file etag different, not a rename"); + done = true; + return; + } + + // Now we know there is a sane rename candidate. + QString originalPath = QString::fromUtf8(base._path); + + // Rename of a virtual file + if (base._type == ItemTypeVirtualFile && item->_type == ItemTypeFile) { + item->_type = ItemTypeVirtualFile; + item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix); + qFatal("FIXME rename virtual file"); // Need to be tested, i'm not sure about it now + } + + if (_discoveryData->_renamedItems.contains(originalPath)) { + qCInfo(lcDisco, "folder already has a rename entry, skipping"); + return; + } + + /* A remote rename can also mean Encryption Mangled Name. + * if we find one of those in the database, we ignore it. + */ + else if (!base._e2eMangledName.isEmpty()) { + qCWarning(lcDisco, "Encrypted file can not rename"); + done = true; + return; + } + + else if (item->_type == ItemTypeFile) { + csync_file_stat_t buf; + if (csync_vio_local_stat((_discoveryData->_localDir + path._local).toUtf8(), &buf)) { + qFatal("FIXME! ERROR HANDLING"); + return; + } + if (buf.modtime != base._modtime || buf.size != base._fileSize) { + // File has changed locally, not a rename. + return; + } + } + + + auto it = _discoveryData->_deletedItem.find(originalPath); + if (it != _discoveryData->_deletedItem.end()) { + if ((*it)->_instruction != CSYNC_INSTRUCTION_REMOVE) + return; + (*it)->_instruction = CSYNC_INSTRUCTION_NONE; + } + delete _discoveryData->_queuedDeletedDirectories.take(originalPath); + _discoveryData->_renamedItems.insert(originalPath); + + item->_modtime = base._modtime; + item->_inode = base._inode; + item->_instruction = CSYNC_INSTRUCTION_RENAME; + item->_direction = SyncFileItem::Down; + item->_renameTarget = path._target; + item->_file = originalPath; + item->_originalFile = originalPath; + path._original = originalPath; + path._local = originalPath; + done = true; + + qCInfo(lcDisco) << "Rename detected (down) " << item->_file << " -> " << item->_renameTarget; + + // FIXME! chech that the server version of origialPath is gone! + }; + if (!_discoveryData->_statedb->getFileRecordsByFileId(serverEntry.fileId, renameCandidateProcessing)) { + qFatal("TODO: Handle DB ERROR"); + } + } + + if (item->_instruction == CSYNC_INSTRUCTION_NEW && item->isDirectory()) { + if (_discoveryData->checkSelectiveSyncNewFolder(path._server, serverEntry.remotePerm)) { + return; + } + } + + // Turn new remote files into virtual files if the option is enabled. + if (item->_instruction == CSYNC_INSTRUCTION_NEW + && _discoveryData->_syncOptions._newFilesAreVirtual + && item->_type == ItemTypeFile) { + item->_type = ItemTypeVirtualFile; + item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix); + } } else if (dbEntry._etag != serverEntry.etag) { item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; @@ -352,7 +462,7 @@ void ProcessDirectoryJob::processFile(const QString &path, recurseQueryServer = ParentNotChanged; } } - bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC; + bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_RENAME; _childModified |= serverModified; if (localEntry.isValid()) { item->_inode = localEntry.inode; @@ -382,7 +492,7 @@ void ProcessDirectoryJob::processFile(const QString &path, // Do we have an UploadInfo for this? // Maybe the Upload was completed, but the connection was broken just before // we recieved the etag (Issue #5106) - auto up = _discoveryData->_statedb->getUploadInfo(path); + auto up = _discoveryData->_statedb->getUploadInfo(path._original); if (up._valid && up._contentChecksum == remoteChecksumHeader) { // Solve the conflict into an upload, or nothing item->_instruction = up._modtime == localEntry.modtime ? CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC; @@ -391,8 +501,8 @@ void ProcessDirectoryJob::processFile(const QString &path, // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because // we must not store the size/modtime from the file system) OCC::SyncJournalFileRecord rec; - if (_discoveryData->_statedb->getFileRecord(path, &rec)) { - rec._path = path.toUtf8(); + if (_discoveryData->_statedb->getFileRecord(path._original, &rec)) { + rec._path = path._original.toUtf8(); rec._etag = serverEntry.etag; rec._fileId = serverEntry.fileId; rec._modtime = serverEntry.modtime; @@ -442,22 +552,26 @@ void ProcessDirectoryJob::processFile(const QString &path, // Checksum comparison at this stage is only enabled for .eml files, // check #4754 #4755 - bool isEmlFile = path.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive); + bool isEmlFile = path._original.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive); if (isEmlFile && dbEntry._fileSize == localEntry.size && !dbEntry._checksumHeader.isEmpty()) { QByteArray type = parseChecksumHeaderType(dbEntry._checksumHeader); if (!type.isEmpty()) { // TODO: compute async? - QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path, type); + QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path._local, type); if (!checksum.isEmpty()) { item->_checksumHeader = makeChecksumHeader(type, checksum); if (item->_checksumHeader == dbEntry._checksumHeader) { - qCDebug(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path; + qCInfo(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path._local; item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; } } } } } + } else if (_queryServer != ParentNotChanged && !serverEntry.isValid()) { + // Not locally, not on the server. The entry is stale! + qCInfo(lcDisco) << "Stale DB entry"; + return; } else if (!serverModified) { if (!dbEntry._serverHasIgnoredFiles) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; @@ -474,24 +588,34 @@ void ProcessDirectoryJob::processFile(const QString &path, if (recurseQueryServer != ParentNotChanged && !serverEntry.isValid()) recurseQueryServer = ParentDontExist; - auto job = new ProcessDirectoryJob(item, recurseQueryServer, localEntry.isValid() ? NormalQuery : ParentDontExist, + auto job = new ProcessDirectoryJob(item, recurseQueryServer, + localEntry.isValid() || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist, _discoveryData, this); - connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); - connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); - _queuedJobs.push_back(job); + job->_currentFolder = path; + if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) { + job->setParent(_discoveryData); + _discoveryData->_queuedDeletedDirectories[path._original] = job; + } else { + connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); + connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); + _queuedJobs.push_back(job); + } } else { + if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) { + _discoveryData->_deletedItem[path._original] = item; + } emit itemDiscovered(item); } } -void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::LocalInfo &localEntry, +void ProcessDirectoryJob::processBlacklisted(const PathTuple &path, const OCC::LocalInfo &localEntry, const SyncJournalFileRecord &dbEntry) { if (!localEntry.isValid()) return; auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); - item->_file = path; + item->_file = path._target; item->_inode = localEntry.inode; if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; @@ -507,6 +631,7 @@ void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::Loc if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) { auto job = new ProcessDirectoryJob(item, InBlackList, NormalQuery, _discoveryData, this); + job->_currentFolder = path; connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); _queuedJobs.push_back(job); diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 8c7b729eb..7494c6c51 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -115,24 +115,43 @@ public: ParentNotChanged, InBlackList }; explicit ProcessDirectoryJob(const SyncFileItemPtr &dirItem, QueryMode queryServer, QueryMode queryLocal, - const QSharedPointer &data, QObject *parent) + DiscoveryPhase *data, QObject *parent) : QObject(parent) , _dirItem(dirItem) , _queryServer(queryServer) , _queryLocal(queryLocal) , _discoveryData(data) - , _currentFolder(dirItem ? dirItem->_file : QString()) + { } void start(); void abort(); private: + struct PathTuple + { + QString _original; // Path as in the DB + QString _target; // Path that will be the result after the sync + QString _server; // Path on the server + QString _local; // Path locally + PathTuple addName(const QString &name) const + { + PathTuple result; + result._original = _original.isEmpty() ? name : _original + QLatin1Char('/') + name; + auto buildString = [&](const QString &other) { + return other == _original ? result._original : other.isEmpty() ? name : other + QLatin1Char('/') + name; + }; + result._target = buildString(_target); + result._server = buildString(_server); + result._local = buildString(_local); + return result; + } + }; void process(); // return true if the file is excluded bool handleExcluded(const QString &path, bool isDirectory, bool isHidden); - void processFile(const QString &, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &); - void processBlacklisted(const QString &path, const LocalInfo &, const SyncJournalFileRecord &); + void processFile(PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &dbEntry); + void processBlacklisted(const PathTuple &, const LocalInfo &, const SyncJournalFileRecord &dbEntry); void subJobFinished(); void progress(); @@ -147,8 +166,9 @@ private: SyncFileItemPtr _dirItem; QueryMode _queryServer; QueryMode _queryLocal; - QSharedPointer _discoveryData; - QString _currentFolder; + DiscoveryPhase *_discoveryData; + + PathTuple _currentFolder; bool _childModified = false; // the directory contains modified item what would prevent deletion bool _childIgnored = false; // The directory contains ignored item that would prevent deletion diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index 0058e846e..5b86848e9 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -19,11 +19,13 @@ #include #include #include +#include #include "networkjobs.h" #include #include #include #include "syncoptions.h" +#include "syncfileitem.h" class ExcludedFiles; @@ -115,6 +117,10 @@ public: bool isInSelectiveSyncBlackList(const QString &path) const; bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp); + QMap _deletedItem; + QMap> _queuedDeletedDirectories; + QSet _renamedItems; + signals: void finished(int result); void folderDiscovered(bool local, QString folderUrl); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 0b26a3ae4..2f819aae5 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -828,15 +828,15 @@ void SyncEngine::slotStartDiscovery() _progressInfo->_status = ProgressInfo::Discovery; emit transmissionProgress(*_progressInfo); - auto ddata = QSharedPointer::create(); - ddata->_account = _account; - ddata->_excludes = _excludedFiles.data(); - ddata->_statedb = _journal; - ddata->_localDir = _localPath; - ddata->_remoteFolder = _remotePath; - ddata->_syncOptions = _syncOptions; - ddata->_selectiveSyncBlackList = selectiveSyncBlackList; - ddata->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); + _discoveryPhase.reset(new DiscoveryPhase); + _discoveryPhase->_account = _account; + _discoveryPhase->_excludes = _excludedFiles.data(); + _discoveryPhase->_statedb = _journal; + _discoveryPhase->_localDir = _localPath; + _discoveryPhase->_remoteFolder = _remotePath; + _discoveryPhase->_syncOptions = _syncOptions; + _discoveryPhase->_selectiveSyncBlackList = selectiveSyncBlackList; + _discoveryPhase->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); if (!ok) { qCWarning(lcEngine) << "Unable to read selective sync list, aborting."; csyncError(tr("Unable to read from the sync journal.")); @@ -855,66 +855,80 @@ void SyncEngine::slotStartDiscovery() // version check doesn't make sense for custom servers. invalidFilenamePattern = "[\\\\:?*\"<>|]"; } - ddata->_invalidFilenamePattern = invalidFilenamePattern; - ddata->_ignoreHiddenFiles = ignoreHiddenFiles(); + _discoveryPhase->_invalidFilenamePattern = invalidFilenamePattern; + _discoveryPhase->_ignoreHiddenFiles = ignoreHiddenFiles(); + + connect(_discoveryPhase.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered); + connect(_discoveryPhase.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder); - connect(ddata.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered); - connect(ddata.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder); _discoveryJob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery, - ddata, this); - connect(_discoveryJob.data(), &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); }); - connect(_discoveryJob.data(), &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) { - if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) { - // For directories, metadata-only updates will be done after all their files are propagated. - - // Update the database now already: New remote fileid or Etag or RemotePerm - // Or for files that were detected as "resolved conflict". - // Or a local inode/mtime change - - // In case of "resolved conflict": there should have been a conflict because they - // both were new, or both had their local mtime or remote etag modified, but the - // size and mtime is the same on the server. This typically happens when the - // database is removed. Nothing will be done for those files, but we still need - // to update the database. - - // This metadata update *could* be a propagation job of its own, but since it's - // quick to do and we don't want to create a potentially large number of - // mini-jobs later on, we just update metadata right now. - - if (item->_direction == SyncFileItem::Down) { - QString filePath = _localPath + item->_file; - - // If the 'W' remote permission changed, update the local filesystem - SyncJournalFileRecord prev; - if (_journal->getFileRecord(item->_file, &prev) - && prev.isValid() - && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) { - const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite); - FileSystem::setFileReadOnlyWeak(filePath, isReadOnly); - } + _discoveryPhase.data(), this); + // FIXME! this sucks + auto runQueuedJob = [this](ProcessDirectoryJob *job, const auto &runQueuedJob) -> void { + connect(job, &ProcessDirectoryJob::finished, this, [this, runQueuedJob] { + sender()->deleteLater(); + if (!_discoveryPhase->_queuedDeletedDirectories.isEmpty()) { + auto job = qobject_cast(_discoveryPhase->_queuedDeletedDirectories.take(_discoveryPhase->_queuedDeletedDirectories.firstKey()).data()); + ASSERT(job); + runQueuedJob(job, runQueuedJob); + } else { + slotDiscoveryJobFinished(0); + } + }); + connect(job, &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) { + if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) { + // For directories, metadata-only updates will be done after all their files are propagated. + + // Update the database now already: New remote fileid or Etag or RemotePerm + // Or for files that were detected as "resolved conflict". + // Or a local inode/mtime change + + // In case of "resolved conflict": there should have been a conflict because they + // both were new, or both had their local mtime or remote etag modified, but the + // size and mtime is the same on the server. This typically happens when the + // database is removed. Nothing will be done for those files, but we still need + // to update the database. + + // This metadata update *could* be a propagation job of its own, but since it's + // quick to do and we don't want to create a potentially large number of + // mini-jobs later on, we just update metadata right now. + + if (item->_direction == SyncFileItem::Down) { + QString filePath = _localPath + item->_file; + + // If the 'W' remote permission changed, update the local filesystem + SyncJournalFileRecord prev; + if (_journal->getFileRecord(item->_file, &prev) + && prev.isValid() + && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) { + const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite); + FileSystem::setFileReadOnlyWeak(filePath, isReadOnly); + } - _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath)); + _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath)); - // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example - emit itemCompleted(item); - } else { - // The local tree is walked first and doesn't have all the info from the server. - // Update only outdated data from the disk. - // FIXME! I think this is no longer the case so a setFileRecordMetadata should work - _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode); + // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example + emit itemCompleted(item); + } else { + // The local tree is walked first and doesn't have all the info from the server. + // Update only outdated data from the disk. + // FIXME! I think this is no longer the case so a setFileRecordMetadata should work + _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode); + } + _hasNoneFiles = true; + return; + } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) { + _hasNoneFiles = true; + return; } - _hasNoneFiles = true; - return; - } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) { - _hasNoneFiles = true; - return; - } - _syncItems.append(item); - slotNewItem(item); - }); - _discoveryJob->start(); + _syncItems.append(item); + slotNewItem(item); + }); + job->start(); + }; + runQueuedJob(_discoveryJob.data(), runQueuedJob); /* * FIXME diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 7416d5d18..97a9f12e4 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -243,6 +243,7 @@ private: QString _remoteRootEtag; SyncJournalDb *_journal; QPointer _discoveryJob; + QScopedPointer _discoveryPhase; QSharedPointer _propagator; // After a sync, only the syncdb entries whose filenames appear in this