New Discovery Algo: Support the DatabaseAndFilesystem mode for local discovery

diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp
index 20aa18000a8e109155a89d11b48515f2c80d7652..3ee2fa6e3849c3f3ed70567ee5cfb6d124095ff8 100644 (file)
--- a/src/libsync/discovery.cpp
+++ b/src/libsync/discovery.cpp
@@ -83,6 +83,13 @@ void ProcessDirectoryJob::start()
         _hasServerEntries = true;
     }
 
+    if (_queryLocal == NormalQuery) {
+        if (!_discoveryData->_shouldDiscoverLocaly(_currentFolder._local)
+            && (_currentFolder._local == _currentFolder._original || !_discoveryData->_shouldDiscoverLocaly(_currentFolder._original))) {
+            _queryLocal = ParentNotChanged;
+        }
+    }
+
     if (_queryLocal == NormalQuery) {
         /*QDirIterator dirIt(_propagator->_localDir + _currentFolder);
         while (dirIt.hasNext()) {
@@ -187,14 +194,14 @@ void ProcessDirectoryJob::process()
     }
     _localEntries.clear();
 
-    if (_queryServer == ParentNotChanged) {
+    if (_queryServer == ParentNotChanged || _queryLocal == ParentNotChanged) {
         // fetch all the name from the DB
         auto pathU8 = _currentFolder._original.toUtf8();
         // FIXME cache, and do that better (a query that do not get stuff recursively)
         if (!_discoveryData->_statedb->getFilesBelowPath(pathU8, [&](const SyncJournalFileRecord &rec) {
                 if (rec._path.indexOf("/", pathU8.size() + 1) > 0)
                     return;
-                auto name = QString::fromUtf8(rec._path.mid(pathU8.size() + 1));
+                auto name = pathU8.isEmpty() ? rec._path : QString::fromUtf8(rec._path.mid(pathU8.size() + 1));
                 if (rec._type == ItemTypeVirtualFile || rec._type == ItemTypeVirtualFileDownload) {
                     name.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
                 }
@@ -205,15 +212,21 @@ void ProcessDirectoryJob::process()
         }
     }
 
+
     for (const auto &f : entriesNames) {
         auto localEntry = localEntriesHash.value(f);
         auto serverEntry = serverEntriesHash.value(f);
+        SyncJournalFileRecord record = dbEntriesHash.value(f);
         PathTuple path;
 
         if ((localEntry.isValid() && localEntry.isVirtualFile)) {
             Q_ASSERT(localEntry.name.endsWith(_discoveryData->_syncOptions._virtualFileSuffix));
             path = _currentFolder.addName(localEntry.name);
             path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
+        } else if (_queryLocal == ParentNotChanged && record.isValid() && record._type == ItemTypeVirtualFile) {
+            QString name = f + _discoveryData->_syncOptions._virtualFileSuffix;
+            path = _currentFolder.addName(name);
+            path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
         } else {
             path = _currentFolder.addName(f);
         }
@@ -226,8 +239,7 @@ void ProcessDirectoryJob::process()
         if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden))
             continue;
 
-        SyncJournalFileRecord record = dbEntriesHash[f];
-        if (_queryServer != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) {
+        if (_queryServer != ParentNotChanged && _queryLocal != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) {
             qFatal("TODO: DB ERROR HANDLING");
         }
         if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._original)) {
@@ -358,8 +370,9 @@ void ProcessDirectoryJob::processFile(PathTuple path,
     const SyncJournalFileRecord &dbEntry)
 {
     const char *hasServer = serverEntry.isValid() ? "true" : _queryServer == ParentNotChanged ? "db" : "false";
+    const char *hasLocal = localEntry.isValid() ? "true" : _queryLocal == ParentNotChanged ? "db" : "false";
     qCInfo(lcDisco).nospace() << "Processing " << path._original
-                              << " | valid: " << dbEntry.isValid() << "/" << localEntry.isValid() << "/" << hasServer
+                              << " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer
                               << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime
                               << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size
                               << " | etag: " << dbEntry._etag << "//" << serverEntry.etag
@@ -597,7 +610,7 @@ void ProcessDirectoryJob::processFile(PathTuple path,
             item->_size = serverEntry.size;
             if (serverEntry.isDirectory && dbEntry._type == ItemTypeDirectory) {
                 item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
-            } else if (!localEntry.isValid()) {
+            } else if (!localEntry.isValid() && _queryLocal != ParentNotChanged) {
                 // Deleted locally, changed on server
                 item->_instruction = CSYNC_INSTRUCTION_NEW;
             } else {
@@ -859,6 +872,12 @@ void ProcessDirectoryJob::processFile(PathTuple path,
                 }
             }
         }
+    } else if (_queryLocal == ParentNotChanged && dbEntry.isValid()) {
+        if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
+            // Not modified locally (ParentNotChanged), bit not on the server:  Removed on the server.
+            item->_instruction = CSYNC_INSTRUCTION_REMOVE;
+            item->_direction = SyncFileItem::Down;
+        }
     } else if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
         // Not locally, not on the server. The entry is stale!
         qCInfo(lcDisco) << "Stale DB entry";
@@ -887,12 +906,15 @@ void ProcessDirectoryJob::processFile(PathTuple path,
 
     qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->_type;
 
-    if (item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory) {
-        if (item->_instruction == CSYNC_INSTRUCTION_SYNC) {
-            item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
-        }
+    if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_SYNC)
+        item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
+
+    bool recurse = item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory;
+    if (_queryLocal != NormalQuery && _queryServer != NormalQuery)
+        recurse = false;
+    if (recurse) {
         auto job = new ProcessDirectoryJob(item, recurseQueryServer,
-            localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
+            _queryLocal == ParentNotChanged ? ParentNotChanged : localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
             _discoveryData, this);
         job->_currentFolder = path;
         if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) {

diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h
index 4b09f6fc602d307ae5fe91c2623c804dd00b7dba..7cc31de3ff972a9e2fc35d8a029191d316911b72 100644 (file)
--- a/src/libsync/discoveryphase.h
+++ b/src/libsync/discoveryphase.h
@@ -113,6 +113,7 @@ public:
     ExcludedFiles *_excludes;
     QString _invalidFilenamePattern; // FIXME: maybe move in ExcludedFiles
     bool _ignoreHiddenFiles = false;
+    std::function<bool(const QString &)> _shouldDiscoverLocaly;
 
     bool isInSelectiveSyncBlackList(const QString &path) const;
     bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp);

diff --git a/src/libsync/localdiscoverytracker.cpp b/src/libsync/localdiscoverytracker.cpp
index 0e69c69ad745962bdc7d9da12a57a94b00213e89..a55c4772440ef17bfd69f1ea950c8ea679416de0 100644 (file)
--- a/src/libsync/localdiscoverytracker.cpp
+++ b/src/libsync/localdiscoverytracker.cpp
@@ -26,7 +26,7 @@ LocalDiscoveryTracker::LocalDiscoveryTracker()
 {
 }
 
-void LocalDiscoveryTracker::addTouchedPath(const QByteArray &relativePath)
+void LocalDiscoveryTracker::addTouchedPath(const QString &relativePath)
 {
     qCDebug(lcLocalDiscoveryTracker) << "inserted touched" << relativePath;
     _localDiscoveryPaths.insert(relativePath);
@@ -42,7 +42,7 @@ void LocalDiscoveryTracker::startSyncFullDiscovery()
 void LocalDiscoveryTracker::startSyncPartialDiscovery()
 {
     if (lcLocalDiscoveryTracker().isDebugEnabled()) {
-        QByteArrayList paths;
+        QStringList paths;
         for (auto &path : _localDiscoveryPaths)
             paths.append(path);
         qCDebug(lcLocalDiscoveryTracker) << "partial discovery with paths: " << paths;
@@ -52,7 +52,7 @@ void LocalDiscoveryTracker::startSyncPartialDiscovery()
     _localDiscoveryPaths.clear();
 }
 
-const std::set<QByteArray> &LocalDiscoveryTracker::localDiscoveryPaths() const
+const std::set<QString> &LocalDiscoveryTracker::localDiscoveryPaths() const
 {
     return _localDiscoveryPaths;
 }

diff --git a/src/libsync/localdiscoverytracker.h b/src/libsync/localdiscoverytracker.h
index 9133659131fd3452078a517a01d8fc5917e619bd..1893e9402dd037e055336805488793f796467a60 100644 (file)
--- a/src/libsync/localdiscoverytracker.h
+++ b/src/libsync/localdiscoverytracker.h
@@ -57,7 +57,7 @@ public:
      * This should be a full relative file path, example:
      *   foo/bar/file.txt
      */
-    void addTouchedPath(const QByteArray &relativePath);
+    void addTouchedPath(const QString &relativePath);
 
     /** Call when a sync run starts that rediscovers all local files */
     void startSyncFullDiscovery();
@@ -66,7 +66,7 @@ public:
     void startSyncPartialDiscovery();
 
     /** Access list of files that shall be locally rediscovered. */
-    const std::set<QByteArray> &localDiscoveryPaths() const;
+    const std::set<QString> &localDiscoveryPaths() const;
 
 public slots:
     /**
@@ -87,7 +87,7 @@ private:
      * Mostly a collection of files the filewatchers have reported as touched.
      * Also includes files that have had errors in the last sync run.
      */
-    std::set<QByteArray> _localDiscoveryPaths;
+    std::set<QString> _localDiscoveryPaths;
 
     /**
      * The paths that the current sync run used for local discovery.
@@ -95,7 +95,7 @@ private:
      * For failing syncs, this list will be merged into _localDiscoveryPaths
      * again when the sync is done to make sure everything is retried.
      */
-    std::set<QByteArray> _previousLocalDiscoveryPaths;
+    std::set<QString> _previousLocalDiscoveryPaths;
 };
 
 } // namespace OCC

diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp
index aa1ea5bd5f3c82d4b472d854335d3ac90a9c0518..68a98fea2facd033714e9b4708516c0751230619 100644 (file)
--- a/src/libsync/syncengine.cpp
+++ b/src/libsync/syncengine.cpp
@@ -763,9 +763,6 @@ void SyncEngine::startSync()
     _csync_ctx->read_remote_from_db = true;
 
     _lastLocalDiscoveryStyle = _localDiscoveryStyle;
-    _csync_ctx->should_discover_locally_fn = [this](const QByteArray &path) {
-        return shouldDiscoverLocally(path);
-    };
 
     _csync_ctx->new_files_are_virtual = _syncOptions._newFilesAreVirtual;
     _csync_ctx->virtual_file_suffix = _syncOptions._virtualFileSuffix.toUtf8();
@@ -875,6 +872,7 @@ void SyncEngine::slotStartDiscovery()
     _discoveryPhase->_syncOptions = _syncOptions;
     _discoveryPhase->_selectiveSyncBlackList = selectiveSyncBlackList;
     _discoveryPhase->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
+    _discoveryPhase->_shouldDiscoverLocaly = [this](const QString &s) { return shouldDiscoverLocally(s); };
     if (!ok) {
         qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
         csyncError(tr("Unable to read from the sync journal."));
@@ -1522,13 +1520,13 @@ AccountPtr SyncEngine::account() const
     return _account;
 }
 
-void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QByteArray> paths)
+void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QString> paths)
 {
     _localDiscoveryStyle = style;
     _localDiscoveryPaths = std::move(paths);
 }
 
-bool SyncEngine::shouldDiscoverLocally(const QByteArray &path) const
+bool SyncEngine::shouldDiscoverLocally(const QString &path) const
 {
     if (_localDiscoveryStyle == LocalDiscoveryStyle::FilesystemOnly)
         return true;

diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h
index 175bdc6f7f768945e16a3e832a8b440670b80f9f..c724a4416e72c8519e0200151d5292853fc1458d 100644 (file)
--- a/src/libsync/syncengine.h
+++ b/src/libsync/syncengine.h
@@ -113,7 +113,7 @@ public:
      * revert afterwards. Use _lastLocalDiscoveryStyle to discover the last
      * sync's style.
      */
-    void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QByteArray> paths = {});
+    void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QString> paths = {});
 
     /**
      * Returns whether the given folder-relative path should be locally discovered
@@ -122,7 +122,7 @@ public:
      * Example: If path is 'foo/bar' and style is DatabaseAndFilesystem and dirs contains
      *     'foo/bar/touched_file', then the result will be true.
      */
-    bool shouldDiscoverLocally(const QByteArray &path) const;
+    bool shouldDiscoverLocally(const QString &path) const;
 
     /** Access the last sync run's local discovery style */
     LocalDiscoveryStyle lastLocalDiscoveryStyle() const { return _lastLocalDiscoveryStyle; }
@@ -314,7 +314,7 @@ private:
     /** The kind of local discovery the last sync run used */
     LocalDiscoveryStyle _lastLocalDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly;
     LocalDiscoveryStyle _localDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly;
-    std::set<QByteArray> _localDiscoveryPaths;
+    std::set<QString> _localDiscoveryPaths;
 };
 }