From: Olivier Goffart Date: Wed, 18 Jul 2018 11:44:41 +0000 (+0200) Subject: New Discovery Algo: Support the DatabaseAndFilesystem mode for local discovery X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~546 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=f4a51678123bec30b4177aa305ca124ccc7757d8;p=nextcloud-desktop.git New Discovery Algo: Support the DatabaseAndFilesystem mode for local discovery --- diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 20aa18000..3ee2fa6e3 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -83,6 +83,13 @@ void ProcessDirectoryJob::start() _hasServerEntries = true; } + if (_queryLocal == NormalQuery) { + if (!_discoveryData->_shouldDiscoverLocaly(_currentFolder._local) + && (_currentFolder._local == _currentFolder._original || !_discoveryData->_shouldDiscoverLocaly(_currentFolder._original))) { + _queryLocal = ParentNotChanged; + } + } + if (_queryLocal == NormalQuery) { /*QDirIterator dirIt(_propagator->_localDir + _currentFolder); while (dirIt.hasNext()) { @@ -187,14 +194,14 @@ void ProcessDirectoryJob::process() } _localEntries.clear(); - if (_queryServer == ParentNotChanged) { + if (_queryServer == ParentNotChanged || _queryLocal == ParentNotChanged) { // fetch all the name from the DB auto pathU8 = _currentFolder._original.toUtf8(); // FIXME cache, and do that better (a query that do not get stuff recursively) if (!_discoveryData->_statedb->getFilesBelowPath(pathU8, [&](const SyncJournalFileRecord &rec) { if (rec._path.indexOf("/", pathU8.size() + 1) > 0) return; - auto name = QString::fromUtf8(rec._path.mid(pathU8.size() + 1)); + auto name = pathU8.isEmpty() ? rec._path : QString::fromUtf8(rec._path.mid(pathU8.size() + 1)); if (rec._type == ItemTypeVirtualFile || rec._type == ItemTypeVirtualFileDownload) { name.chop(_discoveryData->_syncOptions._virtualFileSuffix.size()); } @@ -205,15 +212,21 @@ void ProcessDirectoryJob::process() } } + for (const auto &f : entriesNames) { auto localEntry = localEntriesHash.value(f); auto serverEntry = serverEntriesHash.value(f); + SyncJournalFileRecord record = dbEntriesHash.value(f); PathTuple path; if ((localEntry.isValid() && localEntry.isVirtualFile)) { Q_ASSERT(localEntry.name.endsWith(_discoveryData->_syncOptions._virtualFileSuffix)); path = _currentFolder.addName(localEntry.name); path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size()); + } else if (_queryLocal == ParentNotChanged && record.isValid() && record._type == ItemTypeVirtualFile) { + QString name = f + _discoveryData->_syncOptions._virtualFileSuffix; + path = _currentFolder.addName(name); + path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size()); } else { path = _currentFolder.addName(f); } @@ -226,8 +239,7 @@ void ProcessDirectoryJob::process() if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden)) continue; - SyncJournalFileRecord record = dbEntriesHash[f]; - if (_queryServer != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) { + if (_queryServer != ParentNotChanged && _queryLocal != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) { qFatal("TODO: DB ERROR HANDLING"); } if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._original)) { @@ -358,8 +370,9 @@ void ProcessDirectoryJob::processFile(PathTuple path, const SyncJournalFileRecord &dbEntry) { const char *hasServer = serverEntry.isValid() ? "true" : _queryServer == ParentNotChanged ? "db" : "false"; + const char *hasLocal = localEntry.isValid() ? "true" : _queryLocal == ParentNotChanged ? "db" : "false"; qCInfo(lcDisco).nospace() << "Processing " << path._original - << " | valid: " << dbEntry.isValid() << "/" << localEntry.isValid() << "/" << hasServer + << " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size << " | etag: " << dbEntry._etag << "//" << serverEntry.etag @@ -597,7 +610,7 @@ void ProcessDirectoryJob::processFile(PathTuple path, item->_size = serverEntry.size; if (serverEntry.isDirectory && dbEntry._type == ItemTypeDirectory) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; - } else if (!localEntry.isValid()) { + } else if (!localEntry.isValid() && _queryLocal != ParentNotChanged) { // Deleted locally, changed on server item->_instruction = CSYNC_INSTRUCTION_NEW; } else { @@ -859,6 +872,12 @@ void ProcessDirectoryJob::processFile(PathTuple path, } } } + } else if (_queryLocal == ParentNotChanged && dbEntry.isValid()) { + if (_queryServer != ParentNotChanged && !serverEntry.isValid()) { + // Not modified locally (ParentNotChanged), bit not on the server: Removed on the server. + item->_instruction = CSYNC_INSTRUCTION_REMOVE; + item->_direction = SyncFileItem::Down; + } } else if (_queryServer != ParentNotChanged && !serverEntry.isValid()) { // Not locally, not on the server. The entry is stale! qCInfo(lcDisco) << "Stale DB entry"; @@ -887,12 +906,15 @@ void ProcessDirectoryJob::processFile(PathTuple path, qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->_type; - if (item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory) { - if (item->_instruction == CSYNC_INSTRUCTION_SYNC) { - item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; - } + if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_SYNC) + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + + bool recurse = item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory; + if (_queryLocal != NormalQuery && _queryServer != NormalQuery) + recurse = false; + if (recurse) { auto job = new ProcessDirectoryJob(item, recurseQueryServer, - localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist, + _queryLocal == ParentNotChanged ? ParentNotChanged : localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist, _discoveryData, this); job->_currentFolder = path; if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) { diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index 4b09f6fc6..7cc31de3f 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -113,6 +113,7 @@ public: ExcludedFiles *_excludes; QString _invalidFilenamePattern; // FIXME: maybe move in ExcludedFiles bool _ignoreHiddenFiles = false; + std::function _shouldDiscoverLocaly; bool isInSelectiveSyncBlackList(const QString &path) const; bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp); diff --git a/src/libsync/localdiscoverytracker.cpp b/src/libsync/localdiscoverytracker.cpp index 0e69c69ad..a55c47724 100644 --- a/src/libsync/localdiscoverytracker.cpp +++ b/src/libsync/localdiscoverytracker.cpp @@ -26,7 +26,7 @@ LocalDiscoveryTracker::LocalDiscoveryTracker() { } -void LocalDiscoveryTracker::addTouchedPath(const QByteArray &relativePath) +void LocalDiscoveryTracker::addTouchedPath(const QString &relativePath) { qCDebug(lcLocalDiscoveryTracker) << "inserted touched" << relativePath; _localDiscoveryPaths.insert(relativePath); @@ -42,7 +42,7 @@ void LocalDiscoveryTracker::startSyncFullDiscovery() void LocalDiscoveryTracker::startSyncPartialDiscovery() { if (lcLocalDiscoveryTracker().isDebugEnabled()) { - QByteArrayList paths; + QStringList paths; for (auto &path : _localDiscoveryPaths) paths.append(path); qCDebug(lcLocalDiscoveryTracker) << "partial discovery with paths: " << paths; @@ -52,7 +52,7 @@ void LocalDiscoveryTracker::startSyncPartialDiscovery() _localDiscoveryPaths.clear(); } -const std::set &LocalDiscoveryTracker::localDiscoveryPaths() const +const std::set &LocalDiscoveryTracker::localDiscoveryPaths() const { return _localDiscoveryPaths; } diff --git a/src/libsync/localdiscoverytracker.h b/src/libsync/localdiscoverytracker.h index 913365913..1893e9402 100644 --- a/src/libsync/localdiscoverytracker.h +++ b/src/libsync/localdiscoverytracker.h @@ -57,7 +57,7 @@ public: * This should be a full relative file path, example: * foo/bar/file.txt */ - void addTouchedPath(const QByteArray &relativePath); + void addTouchedPath(const QString &relativePath); /** Call when a sync run starts that rediscovers all local files */ void startSyncFullDiscovery(); @@ -66,7 +66,7 @@ public: void startSyncPartialDiscovery(); /** Access list of files that shall be locally rediscovered. */ - const std::set &localDiscoveryPaths() const; + const std::set &localDiscoveryPaths() const; public slots: /** @@ -87,7 +87,7 @@ private: * Mostly a collection of files the filewatchers have reported as touched. * Also includes files that have had errors in the last sync run. */ - std::set _localDiscoveryPaths; + std::set _localDiscoveryPaths; /** * The paths that the current sync run used for local discovery. @@ -95,7 +95,7 @@ private: * For failing syncs, this list will be merged into _localDiscoveryPaths * again when the sync is done to make sure everything is retried. */ - std::set _previousLocalDiscoveryPaths; + std::set _previousLocalDiscoveryPaths; }; } // namespace OCC diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index aa1ea5bd5..68a98fea2 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -763,9 +763,6 @@ void SyncEngine::startSync() _csync_ctx->read_remote_from_db = true; _lastLocalDiscoveryStyle = _localDiscoveryStyle; - _csync_ctx->should_discover_locally_fn = [this](const QByteArray &path) { - return shouldDiscoverLocally(path); - }; _csync_ctx->new_files_are_virtual = _syncOptions._newFilesAreVirtual; _csync_ctx->virtual_file_suffix = _syncOptions._virtualFileSuffix.toUtf8(); @@ -875,6 +872,7 @@ void SyncEngine::slotStartDiscovery() _discoveryPhase->_syncOptions = _syncOptions; _discoveryPhase->_selectiveSyncBlackList = selectiveSyncBlackList; _discoveryPhase->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); + _discoveryPhase->_shouldDiscoverLocaly = [this](const QString &s) { return shouldDiscoverLocally(s); }; if (!ok) { qCWarning(lcEngine) << "Unable to read selective sync list, aborting."; csyncError(tr("Unable to read from the sync journal.")); @@ -1522,13 +1520,13 @@ AccountPtr SyncEngine::account() const return _account; } -void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set paths) +void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set paths) { _localDiscoveryStyle = style; _localDiscoveryPaths = std::move(paths); } -bool SyncEngine::shouldDiscoverLocally(const QByteArray &path) const +bool SyncEngine::shouldDiscoverLocally(const QString &path) const { if (_localDiscoveryStyle == LocalDiscoveryStyle::FilesystemOnly) return true; diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 175bdc6f7..c724a4416 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -113,7 +113,7 @@ public: * revert afterwards. Use _lastLocalDiscoveryStyle to discover the last * sync's style. */ - void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set paths = {}); + void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set paths = {}); /** * Returns whether the given folder-relative path should be locally discovered @@ -122,7 +122,7 @@ public: * Example: If path is 'foo/bar' and style is DatabaseAndFilesystem and dirs contains * 'foo/bar/touched_file', then the result will be true. */ - bool shouldDiscoverLocally(const QByteArray &path) const; + bool shouldDiscoverLocally(const QString &path) const; /** Access the last sync run's local discovery style */ LocalDiscoveryStyle lastLocalDiscoveryStyle() const { return _lastLocalDiscoveryStyle; } @@ -314,7 +314,7 @@ private: /** The kind of local discovery the last sync run used */ LocalDiscoveryStyle _lastLocalDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly; LocalDiscoveryStyle _localDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly; - std::set _localDiscoveryPaths; + std::set _localDiscoveryPaths; }; }