New discovery algo
authorOlivier Goffart <ogoffart@woboq.com>
Tue, 10 Jul 2018 12:50:32 +0000 (14:50 +0200)
committerKevin Ottens <kevin.ottens@nextcloud.com>
Tue, 15 Dec 2020 09:57:57 +0000 (10:57 +0100)
Make TestSyncEngine::testSelectiveSyncBug pass

src/csync/csync_reconcile.cpp
src/csync/csync_update.cpp
src/libsync/discovery.cpp
src/libsync/discovery.h
src/libsync/discoveryphase.cpp
src/libsync/discoveryphase.h
src/libsync/syncengine.cpp
src/libsync/syncengine.h

index 54a4ac2712b06a34338e129814743623da4ad952..4891c506ccd6b5f2a8b03c7e352562b59cd0e11a 100644 (file)
@@ -344,7 +344,6 @@ static void _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx)
             break;
         }
     } else {
-        bool is_conflict = true;
         /*
      * file found on the other replica
      */
@@ -368,75 +367,7 @@ static void _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx)
             /* file on other replica is changed or new */
             case CSYNC_INSTRUCTION_NEW:
             case CSYNC_INSTRUCTION_EVAL:
-                if (other->type == ItemTypeDirectory &&
-                        cur->type == ItemTypeDirectory) {
-                    // Folders of the same path are always considered equals
-                    is_conflict = false;
-                } else {
-                    // If the size or mtime is different, it's definitely a conflict.
-                    is_conflict = ((other->size != cur->size) || (other->modtime != cur->modtime));
-
-                    // It could be a conflict even if size and mtime match!
-                    //
-                    // In older client versions we always treated these cases as a
-                    // non-conflict. This behavior is preserved in case the server
-                    // doesn't provide a content checksum.
-                    //
-                    // When it does have one, however, we do create a job, but the job
-                    // will compare hashes and avoid the download if possible.
-                    QByteArray remoteChecksumHeader =
-                        (ctx->current == REMOTE_REPLICA ? cur->checksumHeader : other->checksumHeader);
-                    if (!remoteChecksumHeader.isEmpty()) {
-                        is_conflict = true;
-
-                        // Do we have an UploadInfo for this?
-                        // Maybe the Upload was completed, but the connection was broken just before
-                        // we recieved the etag (Issue #5106)
-                        auto up = ctx->statedb->getUploadInfo(cur->path);
-                        if (up._valid && up._contentChecksum == remoteChecksumHeader) {
-                            // Solve the conflict into an upload, or nothing
-                            auto remoteNode = ctx->current == REMOTE_REPLICA ? cur : other;
-                            auto localNode = ctx->current == REMOTE_REPLICA ? other : cur;
-                            remoteNode->instruction = CSYNC_INSTRUCTION_NONE;
-                            localNode->instruction = up._modtime == localNode->modtime && up._size == localNode->size ?
-                                CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC;
-
-                            // Update the etag and other server metadata in the journal already
-                            // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because
-                            // we must not store the size/modtime from the file system)
-                            OCC::SyncJournalFileRecord rec;
-                            if (ctx->statedb->getFileRecord(remoteNode->path, &rec)) {
-                                rec._path = remoteNode->path;
-                                rec._etag = remoteNode->etag;
-                                rec._fileId = remoteNode->file_id;
-                                rec._modtime = remoteNode->modtime;
-                                rec._type = remoteNode->type;
-                                rec._fileSize = remoteNode->size;
-                                rec._remotePerm = remoteNode->remotePerm;
-                                rec._checksumHeader = remoteNode->checksumHeader;
-                                ctx->statedb->setFileRecordMetadata(rec);
-                            }
-                            break;
-                        }
-                    }
-
-                    // SO: If there is no checksum, we can have !is_conflict here
-                    // even though the files have different content! This is an
-                    // intentional tradeoff. Downloading and comparing files would
-                    // be technically correct in this situation but leads to too
-                    // much waste.
-                    // In particular this kind of NEW/NEW situation with identical
-                    // sizes and mtimes pops up when the local database is lost for
-                    // whatever reason.
-                }
-                if (ctx->current == REMOTE_REPLICA) {
-                    // If the files are considered equal, only update the DB with the etag from remote
-                    cur->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA;
-                    other->instruction = CSYNC_INSTRUCTION_NONE;
-                } else {
-                    cur->instruction = CSYNC_INSTRUCTION_NONE;
-                    other->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA;
-                }
+                // PORTED
 
                 break;
                 /* file on the other replica has not been modified */
index 9b96ad454fb2cc934a566930bf779b8985a82a7f..6edfa0b46d3f4431c7fbb6e5eb584295f0868dec 100644 (file)
@@ -794,6 +794,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
         }
     }
 
+#if 0
     // Now process to have a relative path to the sync root for the local replica, or to the data root on the remote.
     dirent->path = fullpath;
     if (ctx->current == LOCAL_REPLICA) {
@@ -819,6 +820,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
       goto error;
     }
 
+PORTED
     if (recurse && rc == 0
         && (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) {
       rc = csync_ftw(ctx, fullpath, fn, depth - 1);
@@ -849,6 +851,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
 
     ctx->current_fs = previous_fs;
     ctx->remote.read_from_db = read_from_db;
+#endif
   }
 
   csync_vio_closedir(ctx, dh);
index 11cd65115abe2a4e30785568266729b72acfc38f..3bb961f50df8953d7298c9e7d89d73a16ae511d7 100644 (file)
 #include "discovery.h"
 #include "common/syncjournaldb.h"
 #include "syncfileitem.h"
-#include "owncloudpropagator.h" // FIXME! remove;
 #include <QDebug>
 #include <algorithm>
 #include <set>
 #include <QDirIterator>
 #include "vio/csync_vio_local.h"
 #include "common/checksums.h"
+#include "csync_exclude.h"
 
 namespace OCC {
 
@@ -58,11 +58,10 @@ DiscoverServerJob::DiscoverServerJob(const AccountPtr &account, const QString &p
         });
 }
 
-
 void ProcessDirectoryJob::start()
 {
     if (_queryServer == NormalQuery) {
-        _serverJob = new DiscoverServerJob(_propagator->account(), _propagator->_remoteFolder + _currentFolder, this);
+        _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder, this);
         connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) {
             if (results) {
                 _serverEntries = *results;
@@ -90,9 +89,9 @@ void ProcessDirectoryJob::start()
             i.name = dirIt.fileName();
 
         }*/
-        auto dh = csync_vio_local_opendir((_propagator->_localDir + _currentFolder).toUtf8());
+        auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder).toUtf8());
         if (!dh) {
-            qDebug() << "COULD NOT OPEN" << (_propagator->_localDir + _currentFolder).toUtf8();
+            qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder).toUtf8();
             qFatal("TODO: ERROR HANDLING");
             // should be the same as in csync_update;
         }
@@ -139,8 +138,12 @@ void ProcessDirectoryJob::process()
             continue;
 
         SyncJournalFileRecord record;
-        if (!_propagator->_journal->getFileRecord(path, &record)) {
-            qFatal("TODO: ERROR HANDLING");
+        if (!_discoveryData->_statedb->getFileRecord(path, &record)) {
+            qFatal("TODO: DB ERROR HANDLING");
+        }
+        if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path)) {
+            processBlacklisted(path, localEntriesHash.value(f), record);
+            continue;
         }
         processFile(path, localEntriesHash.value(f), serverEntriesHash.value(f), record);
     }
@@ -151,7 +154,7 @@ void ProcessDirectoryJob::process()
 bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory)
 {
     // FIXME! call directly, without char* conversion
-    auto excluded = _excludes->csyncTraversalMatchFun()(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile);
+    auto excluded = _discoveryData->_excludes->csyncTraversalMatchFun()(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile);
     if (excluded == CSYNC_NOT_EXCLUDED /* FIXME && item->_type != ItemTypeSoftLink */) {
         return false;
     } else if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED || excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) {
@@ -246,7 +249,6 @@ void ProcessDirectoryJob::processFile(const QString &path,
         item->_fileId = serverEntry.fileId;
         item->_remotePerm = serverEntry.remotePerm;
         item->_type = serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile;
-        item->_size = serverEntry.size;
         item->_previousSize = localEntry.size;
         item->_previousModtime = localEntry.modtime;
         if (!dbEntry.isValid()) {
@@ -254,10 +256,16 @@ void ProcessDirectoryJob::processFile(const QString &path,
             // TODO! rename;
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
+            item->_size = serverEntry.size;
         } else if (dbEntry._etag != serverEntry.etag) {
-            item->_instruction = CSYNC_INSTRUCTION_SYNC;
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
+            item->_size = serverEntry.size;
+            if (serverEntry.isDirectory && dbEntry._type == ItemTypeDirectory) {
+                item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
+            } else {
+                item->_instruction = CSYNC_INSTRUCTION_SYNC;
+            }
         } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId) {
             item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
             item->_direction = SyncFileItem::Down;
@@ -269,16 +277,70 @@ void ProcessDirectoryJob::processFile(const QString &path,
     _childModified |= serverModified;
     if (localEntry.isValid()) {
         item->_inode = localEntry.inode;
-        if (dbEntry.isValid() && dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) {
+        if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) {
             if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
                 item->_instruction = CSYNC_INSTRUCTION_REMOVE;
-                item->_direction = SyncFileItem::Down; // Does not matter
+                item->_direction = SyncFileItem::Down;
             } else if (!serverModified && dbEntry._inode != localEntry.inode) {
                 item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
                 item->_direction = SyncFileItem::Down; // Does not matter
             }
         } else if (serverModified) {
-            item->_instruction = CSYNC_INSTRUCTION_CONFLICT;
+            if (serverEntry.isDirectory && localEntry.isDirectory) {
+                // Folders of the same path are always considered equals
+                item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
+            } else {
+                // It could be a conflict even if size and mtime match!
+                //
+                // In older client versions we always treated these cases as a
+                // non-conflict. This behavior is preserved in case the server
+                // doesn't provide a content checksum.
+                //
+                // When it does have one, however, we do create a job, but the job
+                // will compare hashes and avoid the download if possible.
+                QByteArray remoteChecksumHeader = serverEntry.checksumHeader;
+                if (!remoteChecksumHeader.isEmpty()) {
+                    // Do we have an UploadInfo for this?
+                    // Maybe the Upload was completed, but the connection was broken just before
+                    // we recieved the etag (Issue #5106)
+                    auto up = _discoveryData->_statedb->getUploadInfo(path);
+                    if (up._valid && up._contentChecksum == remoteChecksumHeader) {
+                        // Solve the conflict into an upload, or nothing
+                        item->_instruction = up._modtime == localEntry.modtime ? CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC;
+
+                        // Update the etag and other server metadata in the journal already
+                        // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because
+                        // we must not store the size/modtime from the file system)
+                        OCC::SyncJournalFileRecord rec;
+                        if (_discoveryData->_statedb->getFileRecord(path, &rec)) {
+                            rec._path = path.toUtf8();
+                            rec._etag = serverEntry.etag;
+                            rec._fileId = serverEntry.fileId;
+                            rec._modtime = serverEntry.modtime;
+                            rec._type = item->_type;
+                            rec._fileSize = serverEntry.size;
+                            rec._remotePerm = serverEntry.remotePerm;
+                            rec._checksumHeader = serverEntry.checksumHeader;
+                            _discoveryData->_statedb->setFileRecordMetadata(rec);
+                        }
+                    } else {
+                        item->_instruction = CSYNC_INSTRUCTION_CONFLICT;
+                    }
+                } else {
+                    // If the size or mtime is different, it's definitely a conflict.
+                    bool isConflict = (serverEntry.size != localEntry.size) || (serverEntry.modtime != localEntry.modtime);
+
+                    // SO: If there is no checksum, we can have !is_conflict here
+                    // even though the files have different content! This is an
+                    // intentional tradeoff. Downloading and comparing files would
+                    // be technically correct in this situation but leads to too
+                    // much waste.
+                    // In particular this kind of NEW/NEW situation with identical
+                    // sizes and mtimes pops up when the local database is lost for
+                    // whatever reason.
+                    item->_instruction = isConflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA;
+                }
+            }
         } else if (!dbEntry.isValid()) {
             item->_instruction = CSYNC_INSTRUCTION_NEW;
             item->_direction = SyncFileItem::Up;
@@ -305,7 +367,7 @@ void ProcessDirectoryJob::processFile(const QString &path,
                 QByteArray type = parseChecksumHeaderType(dbEntry._checksumHeader);
                 if (!type.isEmpty()) {
                     // TODO: compute async?
-                    QByteArray checksum = ComputeChecksum::computeNow(_propagator->_localDir + path, type);
+                    QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path, type);
                     if (!checksum.isEmpty()) {
                         item->_checksumHeader = makeChecksumHeader(type, checksum);
                         if (item->_checksumHeader == dbEntry._checksumHeader) {
@@ -317,26 +379,63 @@ void ProcessDirectoryJob::processFile(const QString &path,
             }
         }
     } else if (!serverModified) {
-        item->_instruction = CSYNC_INSTRUCTION_REMOVE;
-        item->_direction = SyncFileItem::Up;
+        if (!dbEntry._serverHasIgnoredFiles) {
+            item->_instruction = CSYNC_INSTRUCTION_REMOVE;
+            item->_direction = SyncFileItem::Up;
+        }
     }
 
     qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->isDirectory();
 
     if (item->isDirectory()) {
+        if (item->_instruction == CSYNC_INSTRUCTION_SYNC) {
+            item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
+        }
+
         if (recurseQueryServer != ParentNotChanged && !serverEntry.isValid())
             recurseQueryServer = ParentDontExist;
         auto job = new ProcessDirectoryJob(item, recurseQueryServer, localEntry.isValid() ? NormalQuery : ParentDontExist,
-            _propagator, _excludes, this);
+            _discoveryData, this);
         connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered);
         connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
         _queuedJobs.push_back(job);
     } else {
-        if (item->_instruction != CSYNC_INSTRUCTION_NONE)
-            emit itemDiscovered(item);
+        emit itemDiscovered(item);
     }
 }
 
+void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::LocalInfo &localEntry,
+    const SyncJournalFileRecord &dbEntry)
+{
+    if (!localEntry.isValid())
+        return;
+
+    auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry);
+    item->_file = path;
+    item->_inode = localEntry.inode;
+    if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) {
+        item->_instruction = CSYNC_INSTRUCTION_REMOVE;
+        item->_direction = SyncFileItem::Down;
+    } else {
+        item->_instruction = CSYNC_INSTRUCTION_IGNORE;
+        item->_status = SyncFileItem::FileIgnored;
+        item->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist");
+        _childIgnored = true;
+    }
+
+    qCInfo(lcDisco) << "Discovered (blacklisted) " << item->_file << item->_instruction << item->_direction << item->isDirectory();
+
+    if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) {
+        auto job = new ProcessDirectoryJob(item, InBlackList, NormalQuery, _discoveryData, this);
+        connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered);
+        connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished);
+        _queuedJobs.push_back(job);
+    } else {
+        emit itemDiscovered(item);
+    }
+}
+
+
 void ProcessDirectoryJob::subJobFinished()
 {
     auto job = qobject_cast<ProcessDirectoryJob *>(sender());
@@ -364,7 +463,22 @@ void ProcessDirectoryJob::progress()
         return;
     }
     if (_runningJobs.empty()) {
+        if (_dirItem) {
+            if (_childModified && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) {
+                // re-create directory that has modified contents
+                _dirItem->_instruction = CSYNC_INSTRUCTION_NEW;
+            }
+            if (_childIgnored && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) {
+                // Do not remove a directory that has ignored files
+                _dirItem->_instruction = CSYNC_INSTRUCTION_NONE;
+            }
+        }
         emit finished();
     }
 }
+void ProcessDirectoryJob::abort()
+{
+    // This should delete all the sub jobs, and so abort everything
+    deleteLater();
+}
 }
index a8c56bbf718c803ecdbe758ef297f7f558efedac..3ba55155f5c617094103a3ccc7003073d71c8144 100644 (file)
@@ -23,7 +23,6 @@ class ExcludedFiles;
 
 namespace OCC {
 class SyncJournalDb;
-class OwncloudPropagator;
 
 enum ErrorTag { Error };
 
@@ -70,7 +69,6 @@ public:
     }
 };
 
-
 struct RemoteInfo
 {
     QString name;
@@ -113,25 +111,27 @@ class ProcessDirectoryJob : public QObject
 public:
     enum QueryMode { NormalQuery,
         ParentDontExist,
-        ParentNotChanged };
+        ParentNotChanged,
+        InBlackList };
     explicit ProcessDirectoryJob(const SyncFileItemPtr &dirItem, QueryMode queryServer, QueryMode queryLocal,
-        OwncloudPropagator *propagator, ExcludedFiles *excludes, QObject *parent)
+        const QSharedPointer<const DiscoveryPhase> &data, QObject *parent)
         : QObject(parent)
         , _dirItem(dirItem)
         , _queryServer(queryServer)
         , _queryLocal(queryLocal)
-        , _propagator(propagator)
-        , _excludes(excludes)
+        , _discoveryData(data)
         , _currentFolder(dirItem ? dirItem->_file : QString())
     {
     }
     void start();
+    void abort();
 
 private:
     void process();
     // return true if the file is excluded
     bool handleExcluded(const QString &path, bool isDirectory);
     void processFile(const QString &, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &);
+    void processBlacklisted(const QString &path, const LocalInfo &, const SyncJournalFileRecord &);
     void subJobFinished();
     void progress();
 
@@ -146,11 +146,10 @@ private:
     SyncFileItemPtr _dirItem;
     QueryMode _queryServer;
     QueryMode _queryLocal;
-    OwncloudPropagator *_propagator; // FIXME: remove this. We need that for the account and local/remote path only.
-    ExcludedFiles *_excludes; // FIXME: Move also in the replacement of the propagator
+    QSharedPointer<const DiscoveryPhase> _discoveryData;
     QString _currentFolder;
-    bool _childModified = false;
-    bool _childIgnored = false;
+    bool _childModified = false; // the directory contains modified item what would prevent deletion
+    bool _childIgnored = false; // The directory contains ignored item that would prevent deletion
 
 signals:
     void itemDiscovered(const SyncFileItemPtr &item);
index 29c70b63732e8e67f856314a12461deb472255ce..5c65841ea3e66b9eff6a92b6f115fbf4c54f4ce7 100644 (file)
@@ -60,7 +60,7 @@ static bool findPathInList(const QStringList &list, const QString &path)
     return pathSlash.startsWith(*it);
 }
 
-bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const
+bool DiscoveryPhase::isInSelectiveSyncBlackList(const QString &path) const
 {
     if (_selectiveSyncBlackList.isEmpty()) {
         // If there is no black list, everything is allowed
@@ -68,10 +68,11 @@ bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const
     }
 
     // Block if it is in the black list
-    if (findPathInList(_selectiveSyncBlackList, QString::fromUtf8(path))) {
+    if (findPathInList(_selectiveSyncBlackList, path)) {
         return true;
     }
 
+    /** FIXME
     // Also try to adjust the path if there was renames
     if (csync_rename_count(_csync_ctx)) {
         QByteArray adjusted = csync_rename_adjust_parent_path_source(_csync_ctx, path);
@@ -79,16 +80,12 @@ bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const
             return findPathInList(_selectiveSyncBlackList, QString::fromUtf8(adjusted));
         }
     }
+    */
 
     return false;
 }
 
-int DiscoveryJob::isInSelectiveSyncBlackListCallback(void *data, const QByteArray &path)
-{
-    return static_cast<DiscoveryJob *>(data)->isInSelectiveSyncBlackList(path);
-}
-
-bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm)
+bool DiscoveryPhase::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm)
 {
     if (_syncOptions._confirmExternalStorage
         && remotePerm.hasPermission(RemotePermissions::IsMounted)) {
@@ -121,11 +118,12 @@ bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermis
     // Go in the main thread to do a PROPFIND to know the size of this folder
     qint64 result = -1;
 
+    /* FIXME TOTO
     {
         QMutexLocker locker(&_vioMutex);
         emit doGetSizeSignal(path, &result);
         _vioWaitCondition.wait(&_vioMutex);
-    }
+    }*/
 
     if (result >= limit) {
         // we tell the UI there is a new folder
@@ -146,12 +144,7 @@ bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermis
     }
 }
 
-int DiscoveryJob::checkSelectiveSyncNewFolderCallback(void *data, const QByteArray &path, RemotePermissions remotePerm)
-{
-    return static_cast<DiscoveryJob *>(data)->checkSelectiveSyncNewFolder(QString::fromUtf8(path), remotePerm);
-}
-
-
+/* FIXME  (used to be called every time we were doing a propfind)
 void DiscoveryJob::update_job_update_callback(bool local,
     const char *dirUrl,
     void *userdata)
@@ -173,7 +166,7 @@ void DiscoveryJob::update_job_update_callback(bool local,
             emit updateJob->folderDiscovered(local, path);
         }
     }
-}
+}*/
 
 // Only use for error cases! It will always set an error errno
 int get_errno_from_http_errcode(int err, const QString &reason)
@@ -470,98 +463,7 @@ void DiscoverySingleDirectoryJob::lsJobFinishedWithErrorSlot(QNetworkReply *r)
     deleteLater();
 }
 
-void DiscoveryMainThread::setupHooks(DiscoveryJob *discoveryJob, const QString &pathPrefix)
-{
-    _discoveryJob = discoveryJob;
-    _pathPrefix = pathPrefix;
-
-    connect(discoveryJob, &DiscoveryJob::doOpendirSignal,
-        this, &DiscoveryMainThread::doOpendirSlot,
-        Qt::QueuedConnection);
-    connect(discoveryJob, &DiscoveryJob::doGetSizeSignal,
-        this, &DiscoveryMainThread::doGetSizeSlot,
-        Qt::QueuedConnection);
-}
-
-// Coming from owncloud_opendir -> DiscoveryJob::vio_opendir_hook -> doOpendirSignal
-void DiscoveryMainThread::doOpendirSlot(const QString &subPath, DiscoveryDirectoryResult *r)
-{
-    QString fullPath = _pathPrefix;
-    if (!_pathPrefix.endsWith('/')) {
-        fullPath += '/';
-    }
-    fullPath += subPath;
-    // remove trailing slash
-    while (fullPath.endsWith('/')) {
-        fullPath.chop(1);
-    }
-
-    _discoveryJob->update_job_update_callback(/*local=*/false, subPath.toUtf8(), _discoveryJob);
-
-    // Result gets written in there
-    _currentDiscoveryDirectoryResult = r;
-    _currentDiscoveryDirectoryResult->path = fullPath;
-
-    // Schedule the DiscoverySingleDirectoryJob
-    _singleDirJob = new DiscoverySingleDirectoryJob(_account, fullPath, this);
-    QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithResult,
-        this, &DiscoveryMainThread::singleDirectoryJobResultSlot);
-    QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithError,
-        this, &DiscoveryMainThread::singleDirectoryJobFinishedWithErrorSlot);
-    QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::firstDirectoryPermissions,
-        this, &DiscoveryMainThread::singleDirectoryJobFirstDirectoryPermissionsSlot);
-    QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::etagConcatenation,
-        this, &DiscoveryMainThread::etagConcatenation);
-    QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::etag,
-        this, &DiscoveryMainThread::etag);
-
-    if (!_firstFolderProcessed) {
-        _singleDirJob->setIsRootPath();
-    }
-
-    _singleDirJob->start();
-}
-
-
-void DiscoveryMainThread::singleDirectoryJobResultSlot()
-{
-    if (!_currentDiscoveryDirectoryResult) {
-        return; // possibly aborted
-    }
-
-    _currentDiscoveryDirectoryResult->list = _singleDirJob->takeResults();
-    _currentDiscoveryDirectoryResult->code = 0;
-
-    qCDebug(lcDiscovery) << "Have" << _currentDiscoveryDirectoryResult->list.size() << "results for " << _currentDiscoveryDirectoryResult->path;
-
-    _currentDiscoveryDirectoryResult = nullptr; // the sync thread owns it now
-
-    if (!_firstFolderProcessed) {
-        _firstFolderProcessed = true;
-        _dataFingerprint = _singleDirJob->_dataFingerprint;
-    }
-
-    _discoveryJob->_vioMutex.lock();
-    _discoveryJob->_vioWaitCondition.wakeAll();
-    _discoveryJob->_vioMutex.unlock();
-}
-
-void DiscoveryMainThread::singleDirectoryJobFinishedWithErrorSlot(int csyncErrnoCode, const QString &msg)
-{
-    if (!_currentDiscoveryDirectoryResult) {
-        return; // possibly aborted
-    }
-    qCDebug(lcDiscovery) << csyncErrnoCode << msg;
-
-    _currentDiscoveryDirectoryResult->code = csyncErrnoCode;
-    _currentDiscoveryDirectoryResult->msg = msg;
-    _currentDiscoveryDirectoryResult = nullptr; // the sync thread owns it now
-
-    _discoveryJob->_vioMutex.lock();
-    _discoveryJob->_vioWaitCondition.wakeAll();
-    _discoveryJob->_vioMutex.unlock();
-}
-
+/*
 void DiscoveryMainThread::singleDirectoryJobFirstDirectoryPermissionsSlot(RemotePermissions p)
 {
     // Should be thread safe since the sync thread is blocked
@@ -622,115 +524,5 @@ void DiscoveryMainThread::slotGetSizeResult(const QVariantMap &map)
     _discoveryJob->_vioWaitCondition.wakeAll();
 }
 
-
-// called from SyncEngine
-void DiscoveryMainThread::abort()
-{
-    if (_singleDirJob) {
-        disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithError, this, nullptr);
-        disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::firstDirectoryPermissions, this, nullptr);
-        disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithResult, this, nullptr);
-        _singleDirJob->abort();
-    }
-    if (_currentDiscoveryDirectoryResult) {
-        if (_discoveryJob->_vioMutex.tryLock()) {
-            _currentDiscoveryDirectoryResult->msg = tr("Aborted by the user"); // Actually also created somewhere else by sync engine
-            _currentDiscoveryDirectoryResult->code = EIO;
-            _currentDiscoveryDirectoryResult = nullptr;
-            _discoveryJob->_vioWaitCondition.wakeAll();
-            _discoveryJob->_vioMutex.unlock();
-        }
-    }
-    if (_currentGetSizeResult) {
-        _currentGetSizeResult = nullptr;
-        QMutexLocker locker(&_discoveryJob->_vioMutex);
-        _discoveryJob->_vioWaitCondition.wakeAll();
-    }
-}
-
-csync_vio_handle_t *DiscoveryJob::remote_vio_opendir_hook(const char *url,
-    void *userdata)
-{
-    auto *discoveryJob = static_cast<DiscoveryJob *>(userdata);
-    if (discoveryJob) {
-        qCDebug(lcDiscovery) << discoveryJob << url << "Calling into main thread...";
-
-        QScopedPointer<DiscoveryDirectoryResult> directoryResult(new DiscoveryDirectoryResult());
-        directoryResult->code = EIO;
-
-        discoveryJob->_vioMutex.lock();
-        const QString qurl = QString::fromUtf8(url);
-        emit discoveryJob->doOpendirSignal(qurl, directoryResult.data());
-        discoveryJob->_vioWaitCondition.wait(&discoveryJob->_vioMutex, ULONG_MAX); // FIXME timeout?
-        discoveryJob->_vioMutex.unlock();
-
-        qCDebug(lcDiscovery) << discoveryJob << url << "...Returned from main thread";
-
-        // Upon awakening from the _vioWaitCondition, iterator should be a valid iterator.
-        if (directoryResult->code != 0) {
-            qCDebug(lcDiscovery) << directoryResult->code << "when opening" << url << "msg=" << directoryResult->msg;
-            errno = directoryResult->code;
-            // save the error string to the context
-            discoveryJob->_csync_ctx->error_string = directoryResult->msg;
-            return nullptr;
-        }
-
-        return directoryResult.take();
-    }
-    return nullptr;
-}
-
-
-std::unique_ptr<csync_file_stat_t> DiscoveryJob::remote_vio_readdir_hook(csync_vio_handle_t *dhandle,
-    void *userdata)
-{
-    auto *discoveryJob = static_cast<DiscoveryJob *>(userdata);
-    if (discoveryJob) {
-        auto *directoryResult = static_cast<DiscoveryDirectoryResult *>(dhandle);
-        if (!directoryResult->list.empty()) {
-            auto file_stat = std::move(directoryResult->list.front());
-            directoryResult->list.pop_front();
-            return file_stat;
-        }
-    }
-    return nullptr;
-}
-
-void DiscoveryJob::remote_vio_closedir_hook(csync_vio_handle_t *dhandle, void *userdata)
-{
-    auto *discoveryJob = static_cast<DiscoveryJob *>(userdata);
-    if (discoveryJob) {
-        auto *directoryResult = static_cast<DiscoveryDirectoryResult *>(dhandle);
-        QString path = directoryResult->path;
-        qCDebug(lcDiscovery) << discoveryJob << path;
-        // just deletes the struct and the iterator, the data itself is owned by the SyncEngine/DiscoveryMainThread
-        delete directoryResult;
-    }
-}
-
-void DiscoveryJob::start()
-{
-    _selectiveSyncBlackList.sort();
-    _selectiveSyncWhiteList.sort();
-    _csync_ctx->callbacks.update_callback_userdata = this;
-    _csync_ctx->callbacks.update_callback = update_job_update_callback;
-    _csync_ctx->callbacks.checkSelectiveSyncBlackListHook = isInSelectiveSyncBlackListCallback;
-    _csync_ctx->callbacks.checkSelectiveSyncNewFolderHook = checkSelectiveSyncNewFolderCallback;
-
-    _csync_ctx->callbacks.remote_opendir_hook = remote_vio_opendir_hook;
-    _csync_ctx->callbacks.remote_readdir_hook = remote_vio_readdir_hook;
-    _csync_ctx->callbacks.remote_closedir_hook = remote_vio_closedir_hook;
-    _csync_ctx->callbacks.vio_userdata = this;
-
-    _lastUpdateProgressCallbackCall.invalidate();
-    int ret = csync_update(_csync_ctx);
-
-    _csync_ctx->callbacks.checkSelectiveSyncNewFolderHook = nullptr;
-    _csync_ctx->callbacks.checkSelectiveSyncBlackListHook = nullptr;
-    _csync_ctx->callbacks.update_callback = nullptr;
-    _csync_ctx->callbacks.update_callback_userdata = nullptr;
-
-    emit finished(ret);
-    deleteLater();
-}
+*/
 }
index cb027edc7fc45d4528e4a4c82648fd0181c8f119..a0b387104fd1fc0a4d62fe8388794854029342f6 100644 (file)
 #include <deque>
 #include "syncoptions.h"
 
+class ExcludedFiles;
+
 namespace OCC {
 
 class Account;
+class SyncJournalDb;
 
 /**
  * The Discovery Phase was once called "update" phase in csync terms.
@@ -94,111 +97,26 @@ public:
     QByteArray _dataFingerprint;
 };
 
-// Lives in main thread. Deleted by the SyncEngine
-class DiscoveryJob;
-class DiscoveryMainThread : public QObject
+class DiscoveryPhase : public QObject
 {
     Q_OBJECT
-
-    QPointer<DiscoveryJob> _discoveryJob;
-    QPointer<DiscoverySingleDirectoryJob> _singleDirJob;
-    QString _pathPrefix; // remote path
-    AccountPtr _account;
-    DiscoveryDirectoryResult *_currentDiscoveryDirectoryResult;
-    qint64 *_currentGetSizeResult;
-    bool _firstFolderProcessed;
-
 public:
-    DiscoveryMainThread(AccountPtr account)
-        : QObject()
-        , _account(account)
-        , _currentDiscoveryDirectoryResult(nullptr)
-        , _currentGetSizeResult(nullptr)
-        , _firstFolderProcessed(false)
-    {
-    }
-    void abort();
-
-    QByteArray _dataFingerprint;
-
-
-public slots:
-    // From DiscoveryJob:
-    void doOpendirSlot(const QString &url, DiscoveryDirectoryResult *);
-    void doGetSizeSlot(const QString &path, qint64 *result);
-
-    // From Job:
-    void singleDirectoryJobResultSlot();
-    void singleDirectoryJobFinishedWithErrorSlot(int csyncErrnoCode, const QString &msg);
-    void singleDirectoryJobFirstDirectoryPermissionsSlot(RemotePermissions);
-
-    void slotGetSizeFinishedWithError();
-    void slotGetSizeResult(const QVariantMap &);
-signals:
-    void etag(const QString &);
-    void etagConcatenation(const QString &);
-
-public:
-    void setupHooks(DiscoveryJob *discoveryJob, const QString &pathPrefix);
-};
-
-/**
- * @brief The DiscoveryJob class
- *
- * Lives in the other thread, deletes itself in !start()
- *
- * @ingroup libsync
- */
-class DiscoveryJob : public QObject
-{
-    Q_OBJECT
-    friend class DiscoveryMainThread;
-    CSYNC *_csync_ctx;
-    QElapsedTimer _lastUpdateProgressCallbackCall;
+    QString _localDir; // absolute path to the local directory. ends with '/'
+    QString _remoteFolder; // remote folder, ends with '/'
+    SyncJournalDb *_statedb;
+    AccountPtr _account;
+    SyncOptions _syncOptions;
+    QStringList _selectiveSyncBlackList;
+    QStringList _selectiveSyncWhiteList;
+    ExcludedFiles *_excludes;
 
-    /**
-     * return true if the given path should be ignored,
-     * false if the path should be synced
-     */
-    bool isInSelectiveSyncBlackList(const QByteArray &path) const;
-    static int isInSelectiveSyncBlackListCallback(void *, const QByteArray &);
+    bool isInSelectiveSyncBlackList(const QString &path) const;
     bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp);
-    static int checkSelectiveSyncNewFolderCallback(void *data, const QByteArray &path, RemotePermissions rm);
-
-    // Just for progress
-    static void update_job_update_callback(bool local,
-        const char *dirname,
-        void *userdata);
-
-    // For using QNAM to get the directory listings
-    static csync_vio_handle_t *remote_vio_opendir_hook(const char *url,
-        void *userdata);
-    static std::unique_ptr<csync_file_stat_t> remote_vio_readdir_hook(csync_vio_handle_t *dhandle,
-        void *userdata);
-    static void remote_vio_closedir_hook(csync_vio_handle_t *dhandle,
-        void *userdata);
-    QMutex _vioMutex;
-    QWaitCondition _vioWaitCondition;
 
-
-public:
-    explicit DiscoveryJob(CSYNC *ctx, QObject *parent = nullptr)
-        : QObject(parent)
-        , _csync_ctx(ctx)
-    {
-    }
-    QStringList _selectiveSyncBlackList;
-    QStringList _selectiveSyncWhiteList;
-    SyncOptions _syncOptions;
-    Q_INVOKABLE void start();
 signals:
     void finished(int result);
     void folderDiscovered(bool local, QString folderUrl);
 
-    // After the discovery job has been woken up again (_vioWaitCondition)
-    void doOpendirSignal(QString url, DiscoveryDirectoryResult *);
-    void doGetSizeSignal(const QString &path, qint64 *result);
-
     // A new folder was discovered and was not synced because of the confirmation feature
     void newBigFolder(const QString &folder, bool isExternal);
 };
index 5d5a9133e0d24284ee6dd7e3c4f78dad174036d8..07fbb652ec5c909172b430d34fdd5682b2b8f6be 100644 (file)
@@ -481,8 +481,8 @@ int SyncEngine::treewalkFile(csync_file_stat_t * /*file*/, csync_file_stat_t * /
     }
 
 
-#if 0
-PORTED
+
+
     switch (file->error_status) {
 
     case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK:
@@ -511,7 +511,7 @@ PORTED
         break;
 
     }
-#endif
+
 
     if (item->_instruction == CSYNC_INSTRUCTION_IGNORE && utf8DecodeError) {
         item->_status = SyncFileItem::NormalError;
@@ -550,60 +550,9 @@ PORTED
     }
     case CSYNC_INSTRUCTION_UPDATE_METADATA:
         dir = SyncFileItem::None;
-        // For directories, metadata-only updates will be done after all their files are propagated.
-        if (!isDirectory) {
-
-            // Update the database now already:  New remote fileid or Etag or RemotePerm
-            // Or for files that were detected as "resolved conflict".
-            // Or a local inode/mtime change
-
-            // In case of "resolved conflict": there should have been a conflict because they
-            // both were new, or both had their local mtime or remote etag modified, but the
-            // size and mtime is the same on the server.  This typically happens when the
-            // database is removed. Nothing will be done for those files, but we still need
-            // to update the database.
-
-            // This metadata update *could* be a propagation job of its own, but since it's
-            // quick to do and we don't want to create a potentially large number of
-            // mini-jobs later on, we just update metadata right now.
-
-            if (remote) {
-                QString filePath = _localPath + item->_file;
-
-                if (other && other->type != ItemTypeVirtualFile && other->type != ItemTypeVirtualFileDownload) {
-                    // Even if the mtime is different on the server, we always want to keep the mtime from
-                    // the file system in the DB, this is to avoid spurious upload on the next sync
-                    item->_modtime = other->modtime;
-                    // same for the size
-                    item->_size = other->size;
-                }
-
-                // If the 'W' remote permission changed, update the local filesystem
-                SyncJournalFileRecord prev;
-                if (_journal->getFileRecord(item->_file, &prev)
-                    && prev.isValid()
-                    && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) {
-                    const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite);
-                    FileSystem::setFileReadOnlyWeak(filePath, isReadOnly);
-                }
-
-                _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath));
-
-                // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example
-                emit itemCompleted(item);
-            } else {
-                // The local tree is walked first and doesn't have all the info from the server.
-                // Update only outdated data from the disk.
-                _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode);
-            }
 
-            if (!other || other->instruction == CSYNC_INSTRUCTION_NONE || other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA) {
-                _hasNoneFiles = true;
-            }
+        ... ported ...
 
-            // Technically we're done with this item.
-            return re;
-        }
         break;
     case CSYNC_INSTRUCTION_RENAME:
         dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
@@ -900,74 +849,94 @@ void SyncEngine::slotStartDiscovery()
     emit transmissionProgress(*_progressInfo);
 
     qCInfo(lcEngine) << "#### Discovery start ####################################################";
+    qCInfo(lcEngine) << "Server" << account()->serverVersion()
+                     << (account()->isHttp2Supported() ? "Using HTTP/2" : "");
     _progressInfo->_status = ProgressInfo::Discovery;
     emit transmissionProgress(*_progressInfo);
 
-    _propagator = QSharedPointer<OwncloudPropagator>(
-        new OwncloudPropagator(_account, _localPath, _remotePath, _journal));
-    _propagator->setSyncOptions(_syncOptions);
-    connect(_propagator.data(), &OwncloudPropagator::itemCompleted,
-        this, &SyncEngine::slotItemCompleted);
-    connect(_propagator.data(), &OwncloudPropagator::progress,
-        this, &SyncEngine::slotProgress);
-    connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection);
-    connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile);
-    connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile);
-    connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage);
-    connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage);
-    connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem);
+    auto ddata = QSharedPointer<DiscoveryPhase>::create();
+    ddata->_account = _account;
+    ddata->_excludes = _excludedFiles.data();
+    ddata->_statedb = _journal;
+    ddata->_selectiveSyncBlackList = selectiveSyncBlackList;
+    ddata->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
+    ddata->_localDir = _localPath;
+    ddata->_remoteFolder = _remotePath;
+    ddata->_syncOptions = _syncOptions;
+    if (!ok) {
+        qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
+        csyncError(tr("Unable to read from the sync journal."));
+        finalize(false);
+        return;
+    }
+
+    connect(ddata.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered);
+    connect(ddata.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder);
 
+    _discoveryJob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery,
+        ddata, this);
+    connect(_discoveryJob.data(), &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); });
+    connect(_discoveryJob.data(), &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) {
+        if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) {
+            // For directories, metadata-only updates will be done after all their files are propagated.
+
+            // Update the database now already:  New remote fileid or Etag or RemotePerm
+            // Or for files that were detected as "resolved conflict".
+            // Or a local inode/mtime change
+
+            // In case of "resolved conflict": there should have been a conflict because they
+            // both were new, or both had their local mtime or remote etag modified, but the
+            // size and mtime is the same on the server.  This typically happens when the
+            // database is removed. Nothing will be done for those files, but we still need
+            // to update the database.
+
+            // This metadata update *could* be a propagation job of its own, but since it's
+            // quick to do and we don't want to create a potentially large number of
+            // mini-jobs later on, we just update metadata right now.
+
+            if (item->_direction == SyncFileItem::Down) {
+                QString filePath = _localPath + item->_file;
+
+                // If the 'W' remote permission changed, update the local filesystem
+                SyncJournalFileRecord prev;
+                if (_journal->getFileRecord(item->_file, &prev)
+                    && prev.isValid()
+                    && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) {
+                    const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite);
+                    FileSystem::setFileReadOnlyWeak(filePath, isReadOnly);
+                }
+
+                _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath));
+
+                // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example
+                emit itemCompleted(item);
+            } else {
+                // The local tree is walked first and doesn't have all the info from the server.
+                // Update only outdated data from the disk.
+                // FIXME!  I think this is no longer the case so a setFileRecordMetadata should work
+                _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode);
+            }
+            _hasNoneFiles = true;
+            return;
+        } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) {
+            _hasNoneFiles = true;
+            return;
+        }
 
-    auto djob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery,
-        _propagator.data(), _excludedFiles.data(), this);
-    connect(djob, &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); });
-    connect(djob, &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) {
         _syncItems.append(item);
         slotNewItem(item);
     });
-    djob->start();
+    _discoveryJob->start();
 
     /*
-    _discoveryMainThread = new DiscoveryMainThread(account());
-    _discoveryMainThread->setParent(this);
-    connect(this, &SyncEngine::finished, _discoveryMainThread.data(), &QObject::deleteLater);
-    qCInfo(lcEngine) << "Server" << account()->serverVersion()
-                     << (account()->isHttp2Supported() ? "Using HTTP/2" : "");
+     * FIXME
     if (account()->rootEtagChangesNotOnlySubFolderEtags()) {
         connect(_discoveryMainThread.data(), &DiscoveryMainThread::etag, this, &SyncEngine::slotRootEtagReceived);
     } else {
         connect(_discoveryMainThread.data(), &DiscoveryMainThread::etagConcatenation, this, &SyncEngine::slotRootEtagReceived);
     }
 
-
-    auto *discoveryJob = new Disco(_csync_ctx.data());
-    discoveryJob->_selectiveSyncBlackList = selectiveSyncBlackList;
-    discoveryJob->_selectiveSyncWhiteList =
-        _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
-    if (!ok) {
-        delete discoveryJob;
-        qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
-        csyncError(tr("Unable to read from the sync journal."));
-        finalize(false);
-        return;
-    }
-
-    discoveryJob->_syncOptions = _syncOptions;
-
-    connect(discoveryJob, &DiscoveryJob::finished, this, &SyncEngine::slotDiscoveryJobFinished);
-    connect(discoveryJob, &DiscoveryJob::folderDiscovered,
-        this, &SyncEngine::slotFolderDiscovered);
-
-    connect(discoveryJob, &DiscoveryJob::newBigFolder,
-        this, &SyncEngine::newBigFolder);
-
-
-    // This is used for the DiscoveryJob to be able to request the main thread/
-    // to read in directory contents.
-    _discoveryMainThread->setupHooks(discoveryJob, _remotePath);
-
-    // Starts the update in a seperate thread
-    QMetaObject::invokeMethod(discoveryJob, "start", Qt::QueuedConnection);*/
+    */
 }
 
 void SyncEngine::slotFolderDiscovered(bool local, const QString &folder)
@@ -1157,6 +1126,19 @@ void SyncEngine::slotDiscoveryJobFinished(int /*discoveryResult*/)
     // do a database commit
     _journal->commit("post treewalk");
 
+    _propagator = QSharedPointer<OwncloudPropagator>(
+        new OwncloudPropagator(_account, _localPath, _remotePath, _journal));
+    _propagator->setSyncOptions(_syncOptions);
+    connect(_propagator.data(), &OwncloudPropagator::itemCompleted,
+        this, &SyncEngine::slotItemCompleted);
+    connect(_propagator.data(), &OwncloudPropagator::progress,
+        this, &SyncEngine::slotProgress);
+    connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection);
+    connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile);
+    connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile);
+    connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage);
+    connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage);
+    connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem);
 
     // apply the network limits to the propagator
     setNetworkLimits(_uploadLimit, _downloadLimit);
@@ -1310,57 +1292,6 @@ void SyncEngine::checkForPermission(SyncFileItemVector &syncItems)
         // Do not propagate anything in the server if it is in the selective sync blacklist
         const QString path = (*it)->destination() + QLatin1Char('/');
 
-        // if reading the selective sync list from db failed, lets ignore all rather than nothing.
-        if (!selectiveListOk || std::binary_search(selectiveSyncBlackList.constBegin(), selectiveSyncBlackList.constEnd(),
-                                    path)) {
-            (*it)->_instruction = CSYNC_INSTRUCTION_IGNORE;
-            (*it)->_status = SyncFileItem::FileIgnored;
-            (*it)->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist");
-
-            if ((*it)->isDirectory()) {
-                auto it_base = it;
-                for (SyncFileItemVector::iterator it_next = it + 1; it_next != syncItems.end() && (*it_next)->_file.startsWith(path); ++it_next) {
-                    it = it_next;
-                    // We want to ignore almost all instructions for items inside selective-sync excluded folders.
-                    //The exception are DOWN/REMOVE actions that remove local files and folders that are
-                    //guaranteed to be up-to-date with their server copies.
-                    if ((*it)->_direction == SyncFileItem::Down && (*it)->_instruction == CSYNC_INSTRUCTION_REMOVE) {
-                        // We need to keep the "delete" items. So we need to un-ignore parent directories
-                        QString parentDir = (*it)->_file;
-                        do {
-                            parentDir = QFileInfo(parentDir).path();
-                            if (parentDir.isEmpty() || !parentDir.startsWith((*it_base)->destination())) {
-                                break;
-                            }
-                            // Find the parent directory in the syncItems vector. Since the vector
-                            // is sorted we can use a lower_bound, but we need a fake
-                            // SyncFileItemPtr needle to compare against
-                            if (!needle)
-                                needle = SyncFileItemPtr::create();
-                            needle->_file = parentDir;
-                            auto parent_it = std::lower_bound(it_base, it, needle);
-                            if (parent_it == syncItems.end() || (*parent_it)->destination() != parentDir) {
-                                break;
-                            }
-                            ASSERT((*parent_it)->isDirectory());
-                            if ((*parent_it)->_instruction != CSYNC_INSTRUCTION_IGNORE) {
-                                break; // already changed
-                            }
-                            (*parent_it)->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
-                            (*parent_it)->_status = SyncFileItem::NoStatus;
-                            (*parent_it)->_errorString.clear();
-
-                        } while (true);
-                        continue;
-                    }
-                    (*it)->_instruction = CSYNC_INSTRUCTION_IGNORE;
-                    (*it)->_status = SyncFileItem::FileIgnored;
-                    (*it)->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist");
-                }
-            }
-            continue;
-        }
-
         switch ((*it)->_instruction) {
         case CSYNC_INSTRUCTION_TYPE_CHANGE:
         case CSYNC_INSTRUCTION_NEW: {
@@ -1707,8 +1638,8 @@ void SyncEngine::abort()
     csync_request_abort(_csync_ctx.data());
 
     // Aborts the discovery phase job
-    if (_discoveryMainThread) {
-        _discoveryMainThread->abort();
+    if (_discoveryJob) {
+        _discoveryJob->abort();
     }
     // For the propagator
     if (_propagator) {
index a95a5e9bc26b756d749dc792180e0c1a692fc41b..7416d5d18531b6fa3bd369964f7b635686ffeee6 100644 (file)
@@ -47,6 +47,7 @@ namespace OCC {
 class SyncJournalFileRecord;
 class SyncJournalDb;
 class OwncloudPropagator;
+class ProcessDirectoryJob;
 
 enum AnotherSyncNeeded {
     NoFollowUpSync,
@@ -241,7 +242,7 @@ private:
     QString _remotePath;
     QString _remoteRootEtag;
     SyncJournalDb *_journal;
-    QPointer<DiscoveryMainThread> _discoveryMainThread;
+    QPointer<ProcessDirectoryJob> _discoveryJob;
     QSharedPointer<OwncloudPropagator> _propagator;
 
     // After a sync, only the syncdb entries whose filenames appear in this