New discovery algo: Remote move
authorOlivier Goffart <ogoffart@woboq.com>
Wed, 11 Jul 2018 14:55:00 +0000 (16:55 +0200)
committerKevin Ottens <kevin.ottens@nextcloud.com>
Tue, 15 Dec 2020 09:57:57 +0000 (10:57 +0100)
TestSyncMove::testRemoteChangeInMovedFolder

src/csync/csync_update.cpp
src/libsync/discovery.cpp
src/libsync/discovery.h
src/libsync/discoveryphase.h
src/libsync/syncengine.cpp
src/libsync/syncengine.h

index dc383f05a78be7f9e71c4d3bc18939404275c08a..c144cee91c4e81d6213b31a9f454de305f2050fb 100644 (file)
@@ -340,100 +340,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f
           }
           goto out;
 
-      } else {
-          qCInfo(lcUpdate, "Checking for rename based on fileid %s", fs->file_id.constData());
-
-          /* Remote Replica Rename check */
-          fs->instruction = CSYNC_INSTRUCTION_NEW;
-
-          bool done = false;
-          auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) {
-              if (done)
-                  return;
-              if (!base.isValid())
-                  return;
-
-              if (base._type == ItemTypeVirtualFileDownload) {
-                  // Remote rename of a virtual file we have locally scheduled
-                  // for download. We just consider this NEW but mark it for download.
-                  fs->type = ItemTypeVirtualFileDownload;
-                  done = true;
-                  return;
-              }
-
-              // Some things prohibit rename detection entirely.
-              // Since we don't do the same checks again in reconcile, we can't
-              // just skip the candidate, but have to give up completely.
-              if (base._type != fs->type
-                  && base._type != ItemTypeVirtualFile) {
-                  qCWarning(lcUpdate, "file types different, not a rename");
-                  done = true;
-                  return;
-              }
-              if (fs->type != ItemTypeDirectory && base._etag != fs->etag) {
-                  /* File with different etag, don't do a rename, but download the file again */
-                  qCWarning(lcUpdate, "file etag different, not a rename");
-                  done = true;
-                  return;
-              }
-
-              // Now we know there is a sane rename candidate.
-
-              // Rename of a virtual file
-              if (base._type == ItemTypeVirtualFile && fs->type == ItemTypeFile) {
-                  fs->type = ItemTypeVirtualFile;
-                  fs->path.append(ctx->virtual_file_suffix);
-              }
-
-              // Record directory renames
-              if (fs->type == ItemTypeDirectory) {
-                  // If the same folder was already renamed by a different entry,
-                  // skip to the next candidate
-                  if (ctx->renames.folder_renamed_to.count(base._path) > 0) {
-                      qCWarning(lcUpdate, "folder already has a rename entry, skipping");
-                      return;
-                  }
-                  csync_rename_record(ctx, base._path, fs->path);
-              }
-
-              /* A remote rename can also mean Encryption Mangled Name.
-               * if we find one of those in the database, we ignore it.
-               */
-              if (!base._e2eMangledName.isEmpty()) {
-                  qCWarning(lcUpdate, "Encrypted file can not rename");
-                  done = true;
-                  return;
-              }
-
-              qCInfo(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData());
-
-              fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME;
-              done = true;
-          };
-
-          if (!ctx->statedb->getFileRecordsByFileId(fs->file_id, renameCandidateProcessing)) {
-              ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
-              return -1;
-          }
-
-          if (fs->instruction == CSYNC_INSTRUCTION_NEW
-              && fs->type == ItemTypeDirectory
-              && ctx->current == REMOTE_REPLICA
-              && ctx->callbacks.checkSelectiveSyncNewFolderHook) {
-              if (ctx->callbacks.checkSelectiveSyncNewFolderHook(ctx->callbacks.update_callback_userdata, fs->path, fs->remotePerm)) {
-                  return 1;
-              }
-          }
-
-          // Turn new remote files into virtual files if the option is enabled.
-          if (ctx->new_files_are_virtual
-              && fs->instruction == CSYNC_INSTRUCTION_NEW
-              && fs->type == ItemTypeFile) {
-              fs->type = ItemTypeVirtualFile;
-              fs->path.append(ctx->virtual_file_suffix);
-          }
-
-          goto out;
+      } else { /*... PORTED ... */
       }
   }
 
index 8cb2adfe864c3328ae95ebcf94b7df387cac8020..dcb2bd9006bf8fab74b1a9f45d87e414192d1196 100644 (file)
@@ -63,7 +63,7 @@ DiscoverServerJob::DiscoverServerJob(const AccountPtr &account, const QString &p
 void ProcessDirectoryJob::start()
 {
     if (_queryServer == NormalQuery) {
-        _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder, this);
+        _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder._server, this);
         connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) {
             if (results) {
                 _serverEntries = *results;
@@ -80,9 +80,6 @@ void ProcessDirectoryJob::start()
         _hasServerEntries = true;
     }
 
-    if (!_currentFolder.isEmpty() && _currentFolder.back() != '/')
-        _currentFolder += '/';
-
     if (_queryLocal == NormalQuery) {
         /*QDirIterator dirIt(_propagator->_localDir + _currentFolder);
         while (dirIt.hasNext()) {
@@ -91,9 +88,9 @@ void ProcessDirectoryJob::start()
             i.name = dirIt.fileName();
 
         }*/
-        auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder).toUtf8());
+        auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder._local).toUtf8());
         if (!dh) {
-            qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder).toUtf8();
+            qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder._local);
             qFatal("TODO: ERROR HANDLING");
             // should be the same as in csync_update;
         }
@@ -106,7 +103,7 @@ void ProcessDirectoryJob::start()
             if (state.invalidChars > 0 || state.remainingChars > 0) {
                 _childIgnored = true;
                 auto item = SyncFileItemPtr::create();
-                item->_file = _currentFolder + i.name;
+                item->_file = _currentFolder._target + i.name;
                 item->_instruction = CSYNC_INSTRUCTION_IGNORE;
                 item->_status = SyncFileItem::NormalError;
                 item->_errorString = tr("Filename encoding is not valid");
@@ -151,8 +148,7 @@ void ProcessDirectoryJob::process()
 
     if (_queryServer == ParentNotChanged) {
         // fetch all the name from the DB
-        auto pathU8 = _currentFolder.toUtf8();
-        pathU8.chop(1);
+        auto pathU8 = _currentFolder._original.toUtf8();
         // FIXME cache, and do that better (a query that do not get stuff recursively)
         if (!_discoveryData->_statedb->getFilesBelowPath(pathU8, [&](const SyncJournalFileRecord &rec) {
                 if (rec._path.indexOf("/", pathU8.size() + 1) > 0)
@@ -163,9 +159,8 @@ void ProcessDirectoryJob::process()
         }
     }
 
-
     for (const auto &f : entriesNames) {
-        QString path = _currentFolder + f;
+        auto path = _currentFolder.addName(f);
         auto localEntry = localEntriesHash.value(f);
         auto serverEntry = serverEntriesHash.value(f);
 
@@ -174,18 +169,18 @@ void ProcessDirectoryJob::process()
         // local stat function.
         // Recall file shall not be ignored (#4420)
         bool isHidden = localEntry.isHidden || (f[0] == '.' && f != QLatin1String(".sys.admin#recall#"));
-        if (handleExcluded(path, localEntry.isDirectory || serverEntry.isDirectory, isHidden))
+        if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden))
             continue;
 
         SyncJournalFileRecord record;
-        if (!_discoveryData->_statedb->getFileRecord(path, &record)) {
+        if (!_discoveryData->_statedb->getFileRecord(path._original, &record)) {
             qFatal("TODO: DB ERROR HANDLING");
         }
-        if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path)) {
+        if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._server)) {
             processBlacklisted(path, localEntry, record);
             continue;
         }
-        processFile(path, localEntry, serverEntry, record);
+        processFile(std::move(path), localEntry, serverEntry, record);
     }
 
     progress();
@@ -297,7 +292,7 @@ bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory,
         }
 #endif
         break;
-    case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE:
+    case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE: // FIXME!
         item->_errorString = tr("The filename cannot be encoded on your file system.");
         break;
     }
@@ -307,20 +302,24 @@ bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory,
     return true;
 }
 
-void ProcessDirectoryJob::processFile(const QString &path,
+void ProcessDirectoryJob::processFile(PathTuple path,
     const LocalInfo &localEntry, const RemoteInfo &serverEntry,
     const SyncJournalFileRecord &dbEntry)
 {
-    qCInfo(lcDisco).nospace() << "Processing " << path
+    qCInfo(lcDisco).nospace() << "Processing " << path._original
                               << " | valid: " << dbEntry.isValid() << "/" << localEntry.isValid() << "/" << serverEntry.isValid()
                               << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime
                               << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size
                               << " | etag: " << dbEntry._etag << "//" << serverEntry.etag
                               << " | checksum: " << dbEntry._checksumHeader << "//" << serverEntry.checksumHeader;
 
-    auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry);
-    item->_file = path;
+    if (_discoveryData->_renamedItems.contains(path._original)) {
+        qCDebug(lcDisco) << "Ignoring renamed";
+        return; // Ignore this.
+    }
 
+    auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry);
+    item->_file = path._target;
     auto recurseQueryServer = _queryServer;
     if (_queryServer == NormalQuery && serverEntry.isValid()) {
         item->_checksumHeader = serverEntry.checksumHeader;
@@ -330,12 +329,123 @@ void ProcessDirectoryJob::processFile(const QString &path,
         item->_etag = serverEntry.etag;
         item->_previousSize = localEntry.size;
         item->_previousModtime = localEntry.modtime;
-        if (!dbEntry.isValid()) {
+        if (!dbEntry.isValid()) { // New file?
             item->_instruction = CSYNC_INSTRUCTION_NEW;
-            // TODO! rename;
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
             item->_size = serverEntry.size;
+
+            if (!localEntry.isValid()) {
+                // Check for renames (if there is a file with the same file id)
+                bool done = false;
+                auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) {
+                    if (done)
+                        return;
+                    if (!base.isValid())
+                        return;
+
+                    if (base._type == ItemTypeVirtualFileDownload) {
+                        // Remote rename of a virtual file we have locally scheduled
+                        // for download. We just consider this NEW but mark it for download.
+                        item->_type = ItemTypeVirtualFileDownload;
+                        done = true;
+                        return;
+                    }
+
+                    // Some things prohibit rename detection entirely.
+                    // Since we don't do the same checks again in reconcile, we can't
+                    // just skip the candidate, but have to give up completely.
+                    if (base._type != item->_type && base._type != ItemTypeVirtualFile) {
+                        qCDebug(lcDisco, "file types different, not a rename");
+                        done = true;
+                        return;
+                    }
+                    if (!serverEntry.isDirectory && base._etag != serverEntry.etag) {
+                        /* File with different etag, don't do a rename, but download the file again */
+                        qCInfo(lcDisco, "file etag different, not a rename");
+                        done = true;
+                        return;
+                    }
+
+                    // Now we know there is a sane rename candidate.
+                    QString originalPath = QString::fromUtf8(base._path);
+
+                    // Rename of a virtual file
+                    if (base._type == ItemTypeVirtualFile && item->_type == ItemTypeFile) {
+                        item->_type = ItemTypeVirtualFile;
+                        item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix);
+                        qFatal("FIXME rename virtual file"); // Need to be tested, i'm not sure about it now
+                    }
+
+                    if (_discoveryData->_renamedItems.contains(originalPath)) {
+                        qCInfo(lcDisco, "folder already has a rename entry, skipping");
+                        return;
+                    }
+
+                    /* A remote rename can also mean Encryption Mangled Name.
+                     * if we find one of those in the database, we ignore it.
+                     */
+                    else if (!base._e2eMangledName.isEmpty()) {
+                        qCWarning(lcDisco, "Encrypted file can not rename");
+                        done = true;
+                        return;
+                    }
+
+                    else if (item->_type == ItemTypeFile) {
+                        csync_file_stat_t buf;
+                        if (csync_vio_local_stat((_discoveryData->_localDir + path._local).toUtf8(), &buf)) {
+                            qFatal("FIXME! ERROR HANDLING");
+                            return;
+                        }
+                        if (buf.modtime != base._modtime || buf.size != base._fileSize) {
+                            // File has changed locally, not a rename.
+                            return;
+                        }
+                    }
+
+
+                    auto it = _discoveryData->_deletedItem.find(originalPath);
+                    if (it != _discoveryData->_deletedItem.end()) {
+                        if ((*it)->_instruction != CSYNC_INSTRUCTION_REMOVE)
+                            return;
+                        (*it)->_instruction = CSYNC_INSTRUCTION_NONE;
+                    }
+                    delete _discoveryData->_queuedDeletedDirectories.take(originalPath);
+                    _discoveryData->_renamedItems.insert(originalPath);
+
+                    item->_modtime = base._modtime;
+                    item->_inode = base._inode;
+                    item->_instruction = CSYNC_INSTRUCTION_RENAME;
+                    item->_direction = SyncFileItem::Down;
+                    item->_renameTarget = path._target;
+                    item->_file = originalPath;
+                    item->_originalFile = originalPath;
+                    path._original = originalPath;
+                    path._local = originalPath;
+                    done = true;
+
+                    qCInfo(lcDisco) << "Rename detected (down) " << item->_file << " -> " << item->_renameTarget;
+
+                    // FIXME!  chech that the server version of origialPath is gone!
+                };
+                if (!_discoveryData->_statedb->getFileRecordsByFileId(serverEntry.fileId, renameCandidateProcessing)) {
+                    qFatal("TODO: Handle DB ERROR");
+                }
+            }
+
+            if (item->_instruction == CSYNC_INSTRUCTION_NEW && item->isDirectory()) {
+                if (_discoveryData->checkSelectiveSyncNewFolder(path._server, serverEntry.remotePerm)) {
+                    return;
+                }
+            }
+
+            // Turn new remote files into virtual files if the option is enabled.
+            if (item->_instruction == CSYNC_INSTRUCTION_NEW
+                && _discoveryData->_syncOptions._newFilesAreVirtual
+                && item->_type == ItemTypeFile) {
+                item->_type = ItemTypeVirtualFile;
+                item->_file.append(_discoveryData->_syncOptions._virtualFileSuffix);
+            }
         } else if (dbEntry._etag != serverEntry.etag) {
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
@@ -352,7 +462,7 @@ void ProcessDirectoryJob::processFile(const QString &path,
             recurseQueryServer = ParentNotChanged;
         }
     }
-    bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC;
+    bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_RENAME;
     _childModified |= serverModified;
     if (localEntry.isValid()) {
         item->_inode = localEntry.inode;
@@ -382,7 +492,7 @@ void ProcessDirectoryJob::processFile(const QString &path,
                     // Do we have an UploadInfo for this?
                     // Maybe the Upload was completed, but the connection was broken just before
                     // we recieved the etag (Issue #5106)
-                    auto up = _discoveryData->_statedb->getUploadInfo(path);
+                    auto up = _discoveryData->_statedb->getUploadInfo(path._original);
                     if (up._valid && up._contentChecksum == remoteChecksumHeader) {
                         // Solve the conflict into an upload, or nothing
                         item->_instruction = up._modtime == localEntry.modtime ? CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC;
@@ -391,8 +501,8 @@ void ProcessDirectoryJob::processFile(const QString &path,
                         // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because
                         // we must not store the size/modtime from the file system)
                         OCC::SyncJournalFileRecord rec;
-                        if (_discoveryData->_statedb->getFileRecord(path, &rec)) {
-                            rec._path = path.toUtf8();
+                        if (_discoveryData->_statedb->getFileRecord(path._original, &rec)) {
+                            rec._path = path._original.toUtf8();
                             rec._etag = serverEntry.etag;
                             rec._fileId = serverEntry.fileId;
                             rec._modtime = serverEntry.modtime;
@@ -442,22 +552,26 @@ void ProcessDirectoryJob::processFile(const QString &path,
 
             // Checksum comparison at this stage is only enabled for .eml files,
             // check #4754 #4755
-            bool isEmlFile = path.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive);
+            bool isEmlFile = path._original.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive);
             if (isEmlFile && dbEntry._fileSize == localEntry.size && !dbEntry._checksumHeader.isEmpty()) {
                 QByteArray type = parseChecksumHeaderType(dbEntry._checksumHeader);
                 if (!type.isEmpty()) {
                     // TODO: compute async?
-                    QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path, type);
+                    QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path._local, type);
                     if (!checksum.isEmpty()) {
                         item->_checksumHeader = makeChecksumHeader(type, checksum);
                         if (item->_checksumHeader == dbEntry._checksumHeader) {
-                            qCDebug(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path;
+                            qCInfo(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path._local;
                             item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
                         }
                     }
                 }
             }
         }
+    } else if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
+        // Not locally, not on the server. The entry is stale!
+        qCInfo(lcDisco) << "Stale DB entry";
+        return;
     } else if (!serverModified) {
         if (!dbEntry._serverHasIgnoredFiles) {
             item->_instruction = CSYNC_INSTRUCTION_REMOVE;
@@ -474,24 +588,34 @@ void ProcessDirectoryJob::processFile(const QString &path,
 
         if (recurseQueryServer != ParentNotChanged && !serverEntry.isValid())
             recurseQueryServer = ParentDontExist;
-        auto job = new ProcessDirectoryJob(item, recurseQueryServer, localEntry.isValid() ? NormalQuery : ParentDontExist,
+        auto job = new ProcessDirectoryJob(item, recurseQueryServer,
+            localEntry.isValid() || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
             _discoveryData, this);
-        connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered);
-        connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
-        _queuedJobs.push_back(job);
+        job->_currentFolder = path;
+        if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) {
+            job->setParent(_discoveryData);
+            _discoveryData->_queuedDeletedDirectories[path._original] = job;
+        } else {
+            connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered);
+            connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
+            _queuedJobs.push_back(job);
+        }
     } else {
+        if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) {
+            _discoveryData->_deletedItem[path._original] = item;
+        }
         emit itemDiscovered(item);
     }
 }
 
-void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::LocalInfo &localEntry,
+void ProcessDirectoryJob::processBlacklisted(const PathTuple &path, const OCC::LocalInfo &localEntry,
     const SyncJournalFileRecord &dbEntry)
 {
     if (!localEntry.isValid())
         return;
 
     auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry);
-    item->_file = path;
+    item->_file = path._target;
     item->_inode = localEntry.inode;
     if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) {
         item->_instruction = CSYNC_INSTRUCTION_REMOVE;
@@ -507,6 +631,7 @@ void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::Loc
 
     if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) {
         auto job = new ProcessDirectoryJob(item, InBlackList, NormalQuery, _discoveryData, this);
+        job->_currentFolder = path;
         connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered);
         connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
         _queuedJobs.push_back(job);
index 8c7b729ebd586ac4195378ae1740c895666c807a..7494c6c5180e67abdcc806e0c9f18fce3777d11d 100644 (file)
@@ -115,24 +115,43 @@ public:
         ParentNotChanged,
         InBlackList };
     explicit ProcessDirectoryJob(const SyncFileItemPtr &dirItem, QueryMode queryServer, QueryMode queryLocal,
-        const QSharedPointer<const DiscoveryPhase> &data, QObject *parent)
+        DiscoveryPhase *data, QObject *parent)
         : QObject(parent)
         , _dirItem(dirItem)
         , _queryServer(queryServer)
         , _queryLocal(queryLocal)
         , _discoveryData(data)
-        , _currentFolder(dirItem ? dirItem->_file : QString())
+
     {
     }
     void start();
     void abort();
 
 private:
+    struct PathTuple
+    {
+        QString _original; // Path as in the DB
+        QString _target; // Path that will be the result after the sync
+        QString _server; // Path on the server
+        QString _local; // Path locally
+        PathTuple addName(const QString &name) const
+        {
+            PathTuple result;
+            result._original = _original.isEmpty() ? name : _original + QLatin1Char('/') + name;
+            auto buildString = [&](const QString &other) {
+                return other == _original ? result._original : other.isEmpty() ? name : other + QLatin1Char('/') + name;
+            };
+            result._target = buildString(_target);
+            result._server = buildString(_server);
+            result._local = buildString(_local);
+            return result;
+        }
+    };
     void process();
     // return true if the file is excluded
     bool handleExcluded(const QString &path, bool isDirectory, bool isHidden);
-    void processFile(const QString &, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &);
-    void processBlacklisted(const QString &path, const LocalInfo &, const SyncJournalFileRecord &);
+    void processFile(PathTuple, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &dbEntry);
+    void processBlacklisted(const PathTuple &, const LocalInfo &, const SyncJournalFileRecord &dbEntry);
     void subJobFinished();
     void progress();
 
@@ -147,8 +166,9 @@ private:
     SyncFileItemPtr _dirItem;
     QueryMode _queryServer;
     QueryMode _queryLocal;
-    QSharedPointer<const DiscoveryPhase> _discoveryData;
-    QString _currentFolder;
+    DiscoveryPhase *_discoveryData;
+
+    PathTuple _currentFolder;
     bool _childModified = false; // the directory contains modified item what would prevent deletion
     bool _childIgnored = false; // The directory contains ignored item that would prevent deletion
 
index 0058e846e6cee7ad235d576404fa12db15dc5df4..5b86848e946dfdbfe3ffe70661a0a85a8062283a 100644 (file)
 #include <QStringList>
 #include <csync.h>
 #include <QMap>
+#include <QSet>
 #include "networkjobs.h"
 #include <QMutex>
 #include <QWaitCondition>
 #include <deque>
 #include "syncoptions.h"
+#include "syncfileitem.h"
 
 class ExcludedFiles;
 
@@ -115,6 +117,10 @@ public:
     bool isInSelectiveSyncBlackList(const QString &path) const;
     bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp);
 
+    QMap<QString, SyncFileItemPtr> _deletedItem;
+    QMap<QString, QPointer<QObject>> _queuedDeletedDirectories;
+    QSet<QString> _renamedItems;
+
 signals:
     void finished(int result);
     void folderDiscovered(bool local, QString folderUrl);
index 0b26a3ae4ac44c0498d7900e6300e078577a1fc7..2f819aae529e72e37bdc21c547b250c113edac79 100644 (file)
@@ -828,15 +828,15 @@ void SyncEngine::slotStartDiscovery()
     _progressInfo->_status = ProgressInfo::Discovery;
     emit transmissionProgress(*_progressInfo);
 
-    auto ddata = QSharedPointer<DiscoveryPhase>::create();
-    ddata->_account = _account;
-    ddata->_excludes = _excludedFiles.data();
-    ddata->_statedb = _journal;
-    ddata->_localDir = _localPath;
-    ddata->_remoteFolder = _remotePath;
-    ddata->_syncOptions = _syncOptions;
-    ddata->_selectiveSyncBlackList = selectiveSyncBlackList;
-    ddata->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
+    _discoveryPhase.reset(new DiscoveryPhase);
+    _discoveryPhase->_account = _account;
+    _discoveryPhase->_excludes = _excludedFiles.data();
+    _discoveryPhase->_statedb = _journal;
+    _discoveryPhase->_localDir = _localPath;
+    _discoveryPhase->_remoteFolder = _remotePath;
+    _discoveryPhase->_syncOptions = _syncOptions;
+    _discoveryPhase->_selectiveSyncBlackList = selectiveSyncBlackList;
+    _discoveryPhase->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
     if (!ok) {
         qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
         csyncError(tr("Unable to read from the sync journal."));
@@ -855,66 +855,80 @@ void SyncEngine::slotStartDiscovery()
         // version check doesn't make sense for custom servers.
         invalidFilenamePattern = "[\\\\:?*\"<>|]";
     }
-    ddata->_invalidFilenamePattern = invalidFilenamePattern;
-    ddata->_ignoreHiddenFiles = ignoreHiddenFiles();
+    _discoveryPhase->_invalidFilenamePattern = invalidFilenamePattern;
+    _discoveryPhase->_ignoreHiddenFiles = ignoreHiddenFiles();
+
+    connect(_discoveryPhase.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered);
+    connect(_discoveryPhase.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder);
 
-    connect(ddata.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered);
-    connect(ddata.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder);
 
     _discoveryJob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery,
-        ddata, this);
-    connect(_discoveryJob.data(), &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); });
-    connect(_discoveryJob.data(), &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) {
-        if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) {
-            // For directories, metadata-only updates will be done after all their files are propagated.
-
-            // Update the database now already:  New remote fileid or Etag or RemotePerm
-            // Or for files that were detected as "resolved conflict".
-            // Or a local inode/mtime change
-
-            // In case of "resolved conflict": there should have been a conflict because they
-            // both were new, or both had their local mtime or remote etag modified, but the
-            // size and mtime is the same on the server.  This typically happens when the
-            // database is removed. Nothing will be done for those files, but we still need
-            // to update the database.
-
-            // This metadata update *could* be a propagation job of its own, but since it's
-            // quick to do and we don't want to create a potentially large number of
-            // mini-jobs later on, we just update metadata right now.
-
-            if (item->_direction == SyncFileItem::Down) {
-                QString filePath = _localPath + item->_file;
-
-                // If the 'W' remote permission changed, update the local filesystem
-                SyncJournalFileRecord prev;
-                if (_journal->getFileRecord(item->_file, &prev)
-                    && prev.isValid()
-                    && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) {
-                    const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite);
-                    FileSystem::setFileReadOnlyWeak(filePath, isReadOnly);
-                }
+        _discoveryPhase.data(), this);
+    // FIXME! this sucks
+    auto runQueuedJob = [this](ProcessDirectoryJob *job, const auto &runQueuedJob) -> void {
+        connect(job, &ProcessDirectoryJob::finished, this, [this, runQueuedJob] {
+            sender()->deleteLater();
+            if (!_discoveryPhase->_queuedDeletedDirectories.isEmpty()) {
+                auto job = qobject_cast<ProcessDirectoryJob *>(_discoveryPhase->_queuedDeletedDirectories.take(_discoveryPhase->_queuedDeletedDirectories.firstKey()).data());
+                ASSERT(job);
+                runQueuedJob(job, runQueuedJob);
+            } else {
+                slotDiscoveryJobFinished(0);
+            }
+        });
+        connect(job, &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) {
+            if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) {
+                // For directories, metadata-only updates will be done after all their files are propagated.
+
+                // Update the database now already:  New remote fileid or Etag or RemotePerm
+                // Or for files that were detected as "resolved conflict".
+                // Or a local inode/mtime change
+
+                // In case of "resolved conflict": there should have been a conflict because they
+                // both were new, or both had their local mtime or remote etag modified, but the
+                // size and mtime is the same on the server.  This typically happens when the
+                // database is removed. Nothing will be done for those files, but we still need
+                // to update the database.
+
+                // This metadata update *could* be a propagation job of its own, but since it's
+                // quick to do and we don't want to create a potentially large number of
+                // mini-jobs later on, we just update metadata right now.
+
+                if (item->_direction == SyncFileItem::Down) {
+                    QString filePath = _localPath + item->_file;
+
+                    // If the 'W' remote permission changed, update the local filesystem
+                    SyncJournalFileRecord prev;
+                    if (_journal->getFileRecord(item->_file, &prev)
+                        && prev.isValid()
+                        && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) {
+                        const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite);
+                        FileSystem::setFileReadOnlyWeak(filePath, isReadOnly);
+                    }
 
-                _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath));
+                    _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath));
 
-                // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example
-                emit itemCompleted(item);
-            } else {
-                // The local tree is walked first and doesn't have all the info from the server.
-                // Update only outdated data from the disk.
-                // FIXME!  I think this is no longer the case so a setFileRecordMetadata should work
-                _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode);
+                    // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example
+                    emit itemCompleted(item);
+                } else {
+                    // The local tree is walked first and doesn't have all the info from the server.
+                    // Update only outdated data from the disk.
+                    // FIXME!  I think this is no longer the case so a setFileRecordMetadata should work
+                    _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode);
+                }
+                _hasNoneFiles = true;
+                return;
+            } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) {
+                _hasNoneFiles = true;
+                return;
             }
-            _hasNoneFiles = true;
-            return;
-        } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) {
-            _hasNoneFiles = true;
-            return;
-        }
 
-        _syncItems.append(item);
-        slotNewItem(item);
-    });
-    _discoveryJob->start();
+            _syncItems.append(item);
+            slotNewItem(item);
+        });
+        job->start();
+    };
+    runQueuedJob(_discoveryJob.data(), runQueuedJob);
 
     /*
      * FIXME
index 7416d5d18531b6fa3bd369964f7b635686ffeee6..97a9f12e4c5ba6191fa0f2a4bcd6d1b575281472 100644 (file)
@@ -243,6 +243,7 @@ private:
     QString _remoteRootEtag;
     SyncJournalDb *_journal;
     QPointer<ProcessDirectoryJob> _discoveryJob;
+    QScopedPointer<DiscoveryPhase> _discoveryPhase;
     QSharedPointer<OwncloudPropagator> _propagator;
 
     // After a sync, only the syncdb entries whose filenames appear in this