From e934f6b27b5eaa8d6e1b030d1c751a069117e660 Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Tue, 10 Jul 2018 14:50:32 +0200 Subject: [PATCH] New discovery algo Make TestSyncEngine::testSelectiveSyncBug pass --- src/csync/csync_reconcile.cpp | 71 +--------- src/csync/csync_update.cpp | 3 + src/libsync/discovery.cpp | 152 +++++++++++++++++--- src/libsync/discovery.h | 19 ++- src/libsync/discoveryphase.cpp | 230 ++---------------------------- src/libsync/discoveryphase.h | 108 ++------------ src/libsync/syncengine.cpp | 251 ++++++++++++--------------------- src/libsync/syncengine.h | 3 +- 8 files changed, 263 insertions(+), 574 deletions(-) diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index 54a4ac271..4891c506c 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -344,7 +344,6 @@ static void _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) break; } } else { - bool is_conflict = true; /* * file found on the other replica */ @@ -368,75 +367,7 @@ static void _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) /* file on other replica is changed or new */ case CSYNC_INSTRUCTION_NEW: case CSYNC_INSTRUCTION_EVAL: - if (other->type == ItemTypeDirectory && - cur->type == ItemTypeDirectory) { - // Folders of the same path are always considered equals - is_conflict = false; - } else { - // If the size or mtime is different, it's definitely a conflict. - is_conflict = ((other->size != cur->size) || (other->modtime != cur->modtime)); - - // It could be a conflict even if size and mtime match! - // - // In older client versions we always treated these cases as a - // non-conflict. This behavior is preserved in case the server - // doesn't provide a content checksum. - // - // When it does have one, however, we do create a job, but the job - // will compare hashes and avoid the download if possible. - QByteArray remoteChecksumHeader = - (ctx->current == REMOTE_REPLICA ? cur->checksumHeader : other->checksumHeader); - if (!remoteChecksumHeader.isEmpty()) { - is_conflict = true; - - // Do we have an UploadInfo for this? - // Maybe the Upload was completed, but the connection was broken just before - // we recieved the etag (Issue #5106) - auto up = ctx->statedb->getUploadInfo(cur->path); - if (up._valid && up._contentChecksum == remoteChecksumHeader) { - // Solve the conflict into an upload, or nothing - auto remoteNode = ctx->current == REMOTE_REPLICA ? cur : other; - auto localNode = ctx->current == REMOTE_REPLICA ? other : cur; - remoteNode->instruction = CSYNC_INSTRUCTION_NONE; - localNode->instruction = up._modtime == localNode->modtime && up._size == localNode->size ? - CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC; - - // Update the etag and other server metadata in the journal already - // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because - // we must not store the size/modtime from the file system) - OCC::SyncJournalFileRecord rec; - if (ctx->statedb->getFileRecord(remoteNode->path, &rec)) { - rec._path = remoteNode->path; - rec._etag = remoteNode->etag; - rec._fileId = remoteNode->file_id; - rec._modtime = remoteNode->modtime; - rec._type = remoteNode->type; - rec._fileSize = remoteNode->size; - rec._remotePerm = remoteNode->remotePerm; - rec._checksumHeader = remoteNode->checksumHeader; - ctx->statedb->setFileRecordMetadata(rec); - } - break; - } - } - - // SO: If there is no checksum, we can have !is_conflict here - // even though the files have different content! This is an - // intentional tradeoff. Downloading and comparing files would - // be technically correct in this situation but leads to too - // much waste. - // In particular this kind of NEW/NEW situation with identical - // sizes and mtimes pops up when the local database is lost for - // whatever reason. - } - if (ctx->current == REMOTE_REPLICA) { - // If the files are considered equal, only update the DB with the etag from remote - cur->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; - other->instruction = CSYNC_INSTRUCTION_NONE; - } else { - cur->instruction = CSYNC_INSTRUCTION_NONE; - other->instruction = is_conflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; - } + // PORTED break; /* file on the other replica has not been modified */ diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 9b96ad454..6edfa0b46 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -794,6 +794,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, } } +#if 0 // Now process to have a relative path to the sync root for the local replica, or to the data root on the remote. dirent->path = fullpath; if (ctx->current == LOCAL_REPLICA) { @@ -819,6 +820,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, goto error; } +PORTED if (recurse && rc == 0 && (!ctx->current_fs || ctx->current_fs->instruction != CSYNC_INSTRUCTION_IGNORE)) { rc = csync_ftw(ctx, fullpath, fn, depth - 1); @@ -849,6 +851,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, ctx->current_fs = previous_fs; ctx->remote.read_from_db = read_from_db; +#endif } csync_vio_closedir(ctx, dh); diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 11cd65115..3bb961f50 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -15,13 +15,13 @@ #include "discovery.h" #include "common/syncjournaldb.h" #include "syncfileitem.h" -#include "owncloudpropagator.h" // FIXME! remove; #include #include #include #include #include "vio/csync_vio_local.h" #include "common/checksums.h" +#include "csync_exclude.h" namespace OCC { @@ -58,11 +58,10 @@ DiscoverServerJob::DiscoverServerJob(const AccountPtr &account, const QString &p }); } - void ProcessDirectoryJob::start() { if (_queryServer == NormalQuery) { - _serverJob = new DiscoverServerJob(_propagator->account(), _propagator->_remoteFolder + _currentFolder, this); + _serverJob = new DiscoverServerJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder, this); connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) { if (results) { _serverEntries = *results; @@ -90,9 +89,9 @@ void ProcessDirectoryJob::start() i.name = dirIt.fileName(); }*/ - auto dh = csync_vio_local_opendir((_propagator->_localDir + _currentFolder).toUtf8()); + auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder).toUtf8()); if (!dh) { - qDebug() << "COULD NOT OPEN" << (_propagator->_localDir + _currentFolder).toUtf8(); + qDebug() << "COULD NOT OPEN" << (_discoveryData->_localDir + _currentFolder).toUtf8(); qFatal("TODO: ERROR HANDLING"); // should be the same as in csync_update; } @@ -139,8 +138,12 @@ void ProcessDirectoryJob::process() continue; SyncJournalFileRecord record; - if (!_propagator->_journal->getFileRecord(path, &record)) { - qFatal("TODO: ERROR HANDLING"); + if (!_discoveryData->_statedb->getFileRecord(path, &record)) { + qFatal("TODO: DB ERROR HANDLING"); + } + if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path)) { + processBlacklisted(path, localEntriesHash.value(f), record); + continue; } processFile(path, localEntriesHash.value(f), serverEntriesHash.value(f), record); } @@ -151,7 +154,7 @@ void ProcessDirectoryJob::process() bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory) { // FIXME! call directly, without char* conversion - auto excluded = _excludes->csyncTraversalMatchFun()(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile); + auto excluded = _discoveryData->_excludes->csyncTraversalMatchFun()(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile); if (excluded == CSYNC_NOT_EXCLUDED /* FIXME && item->_type != ItemTypeSoftLink */) { return false; } else if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED || excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) { @@ -246,7 +249,6 @@ void ProcessDirectoryJob::processFile(const QString &path, item->_fileId = serverEntry.fileId; item->_remotePerm = serverEntry.remotePerm; item->_type = serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; - item->_size = serverEntry.size; item->_previousSize = localEntry.size; item->_previousModtime = localEntry.modtime; if (!dbEntry.isValid()) { @@ -254,10 +256,16 @@ void ProcessDirectoryJob::processFile(const QString &path, // TODO! rename; item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; + item->_size = serverEntry.size; } else if (dbEntry._etag != serverEntry.etag) { - item->_instruction = CSYNC_INSTRUCTION_SYNC; item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; + item->_size = serverEntry.size; + if (serverEntry.isDirectory && dbEntry._type == ItemTypeDirectory) { + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + } else { + item->_instruction = CSYNC_INSTRUCTION_SYNC; + } } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; item->_direction = SyncFileItem::Down; @@ -269,16 +277,70 @@ void ProcessDirectoryJob::processFile(const QString &path, _childModified |= serverModified; if (localEntry.isValid()) { item->_inode = localEntry.inode; - if (dbEntry.isValid() && dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) { + if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) { if (_queryServer != ParentNotChanged && !serverEntry.isValid()) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; - item->_direction = SyncFileItem::Down; // Does not matter + item->_direction = SyncFileItem::Down; } else if (!serverModified && dbEntry._inode != localEntry.inode) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; item->_direction = SyncFileItem::Down; // Does not matter } } else if (serverModified) { - item->_instruction = CSYNC_INSTRUCTION_CONFLICT; + if (serverEntry.isDirectory && localEntry.isDirectory) { + // Folders of the same path are always considered equals + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + } else { + // It could be a conflict even if size and mtime match! + // + // In older client versions we always treated these cases as a + // non-conflict. This behavior is preserved in case the server + // doesn't provide a content checksum. + // + // When it does have one, however, we do create a job, but the job + // will compare hashes and avoid the download if possible. + QByteArray remoteChecksumHeader = serverEntry.checksumHeader; + if (!remoteChecksumHeader.isEmpty()) { + // Do we have an UploadInfo for this? + // Maybe the Upload was completed, but the connection was broken just before + // we recieved the etag (Issue #5106) + auto up = _discoveryData->_statedb->getUploadInfo(path); + if (up._valid && up._contentChecksum == remoteChecksumHeader) { + // Solve the conflict into an upload, or nothing + item->_instruction = up._modtime == localEntry.modtime ? CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC; + + // Update the etag and other server metadata in the journal already + // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because + // we must not store the size/modtime from the file system) + OCC::SyncJournalFileRecord rec; + if (_discoveryData->_statedb->getFileRecord(path, &rec)) { + rec._path = path.toUtf8(); + rec._etag = serverEntry.etag; + rec._fileId = serverEntry.fileId; + rec._modtime = serverEntry.modtime; + rec._type = item->_type; + rec._fileSize = serverEntry.size; + rec._remotePerm = serverEntry.remotePerm; + rec._checksumHeader = serverEntry.checksumHeader; + _discoveryData->_statedb->setFileRecordMetadata(rec); + } + } else { + item->_instruction = CSYNC_INSTRUCTION_CONFLICT; + } + } else { + // If the size or mtime is different, it's definitely a conflict. + bool isConflict = (serverEntry.size != localEntry.size) || (serverEntry.modtime != localEntry.modtime); + + // SO: If there is no checksum, we can have !is_conflict here + // even though the files have different content! This is an + // intentional tradeoff. Downloading and comparing files would + // be technically correct in this situation but leads to too + // much waste. + // In particular this kind of NEW/NEW situation with identical + // sizes and mtimes pops up when the local database is lost for + // whatever reason. + item->_instruction = isConflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; + } + } } else if (!dbEntry.isValid()) { item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Up; @@ -305,7 +367,7 @@ void ProcessDirectoryJob::processFile(const QString &path, QByteArray type = parseChecksumHeaderType(dbEntry._checksumHeader); if (!type.isEmpty()) { // TODO: compute async? - QByteArray checksum = ComputeChecksum::computeNow(_propagator->_localDir + path, type); + QByteArray checksum = ComputeChecksum::computeNow(_discoveryData->_localDir + path, type); if (!checksum.isEmpty()) { item->_checksumHeader = makeChecksumHeader(type, checksum); if (item->_checksumHeader == dbEntry._checksumHeader) { @@ -317,26 +379,63 @@ void ProcessDirectoryJob::processFile(const QString &path, } } } else if (!serverModified) { - item->_instruction = CSYNC_INSTRUCTION_REMOVE; - item->_direction = SyncFileItem::Up; + if (!dbEntry._serverHasIgnoredFiles) { + item->_instruction = CSYNC_INSTRUCTION_REMOVE; + item->_direction = SyncFileItem::Up; + } } qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->isDirectory(); if (item->isDirectory()) { + if (item->_instruction == CSYNC_INSTRUCTION_SYNC) { + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + } + if (recurseQueryServer != ParentNotChanged && !serverEntry.isValid()) recurseQueryServer = ParentDontExist; auto job = new ProcessDirectoryJob(item, recurseQueryServer, localEntry.isValid() ? NormalQuery : ParentDontExist, - _propagator, _excludes, this); + _discoveryData, this); connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); _queuedJobs.push_back(job); } else { - if (item->_instruction != CSYNC_INSTRUCTION_NONE) - emit itemDiscovered(item); + emit itemDiscovered(item); } } +void ProcessDirectoryJob::processBlacklisted(const QString &path, const OCC::LocalInfo &localEntry, + const SyncJournalFileRecord &dbEntry) +{ + if (!localEntry.isValid()) + return; + + auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); + item->_file = path; + item->_inode = localEntry.inode; + if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry._type == ItemTypeDirectory))) { + item->_instruction = CSYNC_INSTRUCTION_REMOVE; + item->_direction = SyncFileItem::Down; + } else { + item->_instruction = CSYNC_INSTRUCTION_IGNORE; + item->_status = SyncFileItem::FileIgnored; + item->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist"); + _childIgnored = true; + } + + qCInfo(lcDisco) << "Discovered (blacklisted) " << item->_file << item->_instruction << item->_direction << item->isDirectory(); + + if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) { + auto job = new ProcessDirectoryJob(item, InBlackList, NormalQuery, _discoveryData, this); + connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); + connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); + _queuedJobs.push_back(job); + } else { + emit itemDiscovered(item); + } +} + + void ProcessDirectoryJob::subJobFinished() { auto job = qobject_cast(sender()); @@ -364,7 +463,22 @@ void ProcessDirectoryJob::progress() return; } if (_runningJobs.empty()) { + if (_dirItem) { + if (_childModified && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { + // re-create directory that has modified contents + _dirItem->_instruction = CSYNC_INSTRUCTION_NEW; + } + if (_childIgnored && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { + // Do not remove a directory that has ignored files + _dirItem->_instruction = CSYNC_INSTRUCTION_NONE; + } + } emit finished(); } } +void ProcessDirectoryJob::abort() +{ + // This should delete all the sub jobs, and so abort everything + deleteLater(); +} } diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index a8c56bbf7..3ba55155f 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -23,7 +23,6 @@ class ExcludedFiles; namespace OCC { class SyncJournalDb; -class OwncloudPropagator; enum ErrorTag { Error }; @@ -70,7 +69,6 @@ public: } }; - struct RemoteInfo { QString name; @@ -113,25 +111,27 @@ class ProcessDirectoryJob : public QObject public: enum QueryMode { NormalQuery, ParentDontExist, - ParentNotChanged }; + ParentNotChanged, + InBlackList }; explicit ProcessDirectoryJob(const SyncFileItemPtr &dirItem, QueryMode queryServer, QueryMode queryLocal, - OwncloudPropagator *propagator, ExcludedFiles *excludes, QObject *parent) + const QSharedPointer &data, QObject *parent) : QObject(parent) , _dirItem(dirItem) , _queryServer(queryServer) , _queryLocal(queryLocal) - , _propagator(propagator) - , _excludes(excludes) + , _discoveryData(data) , _currentFolder(dirItem ? dirItem->_file : QString()) { } void start(); + void abort(); private: void process(); // return true if the file is excluded bool handleExcluded(const QString &path, bool isDirectory); void processFile(const QString &, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &); + void processBlacklisted(const QString &path, const LocalInfo &, const SyncJournalFileRecord &); void subJobFinished(); void progress(); @@ -146,11 +146,10 @@ private: SyncFileItemPtr _dirItem; QueryMode _queryServer; QueryMode _queryLocal; - OwncloudPropagator *_propagator; // FIXME: remove this. We need that for the account and local/remote path only. - ExcludedFiles *_excludes; // FIXME: Move also in the replacement of the propagator + QSharedPointer _discoveryData; QString _currentFolder; - bool _childModified = false; - bool _childIgnored = false; + bool _childModified = false; // the directory contains modified item what would prevent deletion + bool _childIgnored = false; // The directory contains ignored item that would prevent deletion signals: void itemDiscovered(const SyncFileItemPtr &item); diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 29c70b637..5c65841ea 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -60,7 +60,7 @@ static bool findPathInList(const QStringList &list, const QString &path) return pathSlash.startsWith(*it); } -bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const +bool DiscoveryPhase::isInSelectiveSyncBlackList(const QString &path) const { if (_selectiveSyncBlackList.isEmpty()) { // If there is no black list, everything is allowed @@ -68,10 +68,11 @@ bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const } // Block if it is in the black list - if (findPathInList(_selectiveSyncBlackList, QString::fromUtf8(path))) { + if (findPathInList(_selectiveSyncBlackList, path)) { return true; } + /** FIXME // Also try to adjust the path if there was renames if (csync_rename_count(_csync_ctx)) { QByteArray adjusted = csync_rename_adjust_parent_path_source(_csync_ctx, path); @@ -79,16 +80,12 @@ bool DiscoveryJob::isInSelectiveSyncBlackList(const QByteArray &path) const return findPathInList(_selectiveSyncBlackList, QString::fromUtf8(adjusted)); } } + */ return false; } -int DiscoveryJob::isInSelectiveSyncBlackListCallback(void *data, const QByteArray &path) -{ - return static_cast(data)->isInSelectiveSyncBlackList(path); -} - -bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm) +bool DiscoveryPhase::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm) { if (_syncOptions._confirmExternalStorage && remotePerm.hasPermission(RemotePermissions::IsMounted)) { @@ -121,11 +118,12 @@ bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermis // Go in the main thread to do a PROPFIND to know the size of this folder qint64 result = -1; + /* FIXME TOTO { QMutexLocker locker(&_vioMutex); emit doGetSizeSignal(path, &result); _vioWaitCondition.wait(&_vioMutex); - } + }*/ if (result >= limit) { // we tell the UI there is a new folder @@ -146,12 +144,7 @@ bool DiscoveryJob::checkSelectiveSyncNewFolder(const QString &path, RemotePermis } } -int DiscoveryJob::checkSelectiveSyncNewFolderCallback(void *data, const QByteArray &path, RemotePermissions remotePerm) -{ - return static_cast(data)->checkSelectiveSyncNewFolder(QString::fromUtf8(path), remotePerm); -} - - +/* FIXME (used to be called every time we were doing a propfind) void DiscoveryJob::update_job_update_callback(bool local, const char *dirUrl, void *userdata) @@ -173,7 +166,7 @@ void DiscoveryJob::update_job_update_callback(bool local, emit updateJob->folderDiscovered(local, path); } } -} +}*/ // Only use for error cases! It will always set an error errno int get_errno_from_http_errcode(int err, const QString &reason) @@ -470,98 +463,7 @@ void DiscoverySingleDirectoryJob::lsJobFinishedWithErrorSlot(QNetworkReply *r) deleteLater(); } -void DiscoveryMainThread::setupHooks(DiscoveryJob *discoveryJob, const QString &pathPrefix) -{ - _discoveryJob = discoveryJob; - _pathPrefix = pathPrefix; - - connect(discoveryJob, &DiscoveryJob::doOpendirSignal, - this, &DiscoveryMainThread::doOpendirSlot, - Qt::QueuedConnection); - connect(discoveryJob, &DiscoveryJob::doGetSizeSignal, - this, &DiscoveryMainThread::doGetSizeSlot, - Qt::QueuedConnection); -} - -// Coming from owncloud_opendir -> DiscoveryJob::vio_opendir_hook -> doOpendirSignal -void DiscoveryMainThread::doOpendirSlot(const QString &subPath, DiscoveryDirectoryResult *r) -{ - QString fullPath = _pathPrefix; - if (!_pathPrefix.endsWith('/')) { - fullPath += '/'; - } - fullPath += subPath; - // remove trailing slash - while (fullPath.endsWith('/')) { - fullPath.chop(1); - } - - _discoveryJob->update_job_update_callback(/*local=*/false, subPath.toUtf8(), _discoveryJob); - - // Result gets written in there - _currentDiscoveryDirectoryResult = r; - _currentDiscoveryDirectoryResult->path = fullPath; - - // Schedule the DiscoverySingleDirectoryJob - _singleDirJob = new DiscoverySingleDirectoryJob(_account, fullPath, this); - QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithResult, - this, &DiscoveryMainThread::singleDirectoryJobResultSlot); - QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithError, - this, &DiscoveryMainThread::singleDirectoryJobFinishedWithErrorSlot); - QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::firstDirectoryPermissions, - this, &DiscoveryMainThread::singleDirectoryJobFirstDirectoryPermissionsSlot); - QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::etagConcatenation, - this, &DiscoveryMainThread::etagConcatenation); - QObject::connect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::etag, - this, &DiscoveryMainThread::etag); - - if (!_firstFolderProcessed) { - _singleDirJob->setIsRootPath(); - } - - _singleDirJob->start(); -} - - -void DiscoveryMainThread::singleDirectoryJobResultSlot() -{ - if (!_currentDiscoveryDirectoryResult) { - return; // possibly aborted - } - - _currentDiscoveryDirectoryResult->list = _singleDirJob->takeResults(); - _currentDiscoveryDirectoryResult->code = 0; - - qCDebug(lcDiscovery) << "Have" << _currentDiscoveryDirectoryResult->list.size() << "results for " << _currentDiscoveryDirectoryResult->path; - - _currentDiscoveryDirectoryResult = nullptr; // the sync thread owns it now - - if (!_firstFolderProcessed) { - _firstFolderProcessed = true; - _dataFingerprint = _singleDirJob->_dataFingerprint; - } - - _discoveryJob->_vioMutex.lock(); - _discoveryJob->_vioWaitCondition.wakeAll(); - _discoveryJob->_vioMutex.unlock(); -} - -void DiscoveryMainThread::singleDirectoryJobFinishedWithErrorSlot(int csyncErrnoCode, const QString &msg) -{ - if (!_currentDiscoveryDirectoryResult) { - return; // possibly aborted - } - qCDebug(lcDiscovery) << csyncErrnoCode << msg; - - _currentDiscoveryDirectoryResult->code = csyncErrnoCode; - _currentDiscoveryDirectoryResult->msg = msg; - _currentDiscoveryDirectoryResult = nullptr; // the sync thread owns it now - - _discoveryJob->_vioMutex.lock(); - _discoveryJob->_vioWaitCondition.wakeAll(); - _discoveryJob->_vioMutex.unlock(); -} - +/* void DiscoveryMainThread::singleDirectoryJobFirstDirectoryPermissionsSlot(RemotePermissions p) { // Should be thread safe since the sync thread is blocked @@ -622,115 +524,5 @@ void DiscoveryMainThread::slotGetSizeResult(const QVariantMap &map) _discoveryJob->_vioWaitCondition.wakeAll(); } - -// called from SyncEngine -void DiscoveryMainThread::abort() -{ - if (_singleDirJob) { - disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithError, this, nullptr); - disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::firstDirectoryPermissions, this, nullptr); - disconnect(_singleDirJob.data(), &DiscoverySingleDirectoryJob::finishedWithResult, this, nullptr); - _singleDirJob->abort(); - } - if (_currentDiscoveryDirectoryResult) { - if (_discoveryJob->_vioMutex.tryLock()) { - _currentDiscoveryDirectoryResult->msg = tr("Aborted by the user"); // Actually also created somewhere else by sync engine - _currentDiscoveryDirectoryResult->code = EIO; - _currentDiscoveryDirectoryResult = nullptr; - _discoveryJob->_vioWaitCondition.wakeAll(); - _discoveryJob->_vioMutex.unlock(); - } - } - if (_currentGetSizeResult) { - _currentGetSizeResult = nullptr; - QMutexLocker locker(&_discoveryJob->_vioMutex); - _discoveryJob->_vioWaitCondition.wakeAll(); - } -} - -csync_vio_handle_t *DiscoveryJob::remote_vio_opendir_hook(const char *url, - void *userdata) -{ - auto *discoveryJob = static_cast(userdata); - if (discoveryJob) { - qCDebug(lcDiscovery) << discoveryJob << url << "Calling into main thread..."; - - QScopedPointer directoryResult(new DiscoveryDirectoryResult()); - directoryResult->code = EIO; - - discoveryJob->_vioMutex.lock(); - const QString qurl = QString::fromUtf8(url); - emit discoveryJob->doOpendirSignal(qurl, directoryResult.data()); - discoveryJob->_vioWaitCondition.wait(&discoveryJob->_vioMutex, ULONG_MAX); // FIXME timeout? - discoveryJob->_vioMutex.unlock(); - - qCDebug(lcDiscovery) << discoveryJob << url << "...Returned from main thread"; - - // Upon awakening from the _vioWaitCondition, iterator should be a valid iterator. - if (directoryResult->code != 0) { - qCDebug(lcDiscovery) << directoryResult->code << "when opening" << url << "msg=" << directoryResult->msg; - errno = directoryResult->code; - // save the error string to the context - discoveryJob->_csync_ctx->error_string = directoryResult->msg; - return nullptr; - } - - return directoryResult.take(); - } - return nullptr; -} - - -std::unique_ptr DiscoveryJob::remote_vio_readdir_hook(csync_vio_handle_t *dhandle, - void *userdata) -{ - auto *discoveryJob = static_cast(userdata); - if (discoveryJob) { - auto *directoryResult = static_cast(dhandle); - if (!directoryResult->list.empty()) { - auto file_stat = std::move(directoryResult->list.front()); - directoryResult->list.pop_front(); - return file_stat; - } - } - return nullptr; -} - -void DiscoveryJob::remote_vio_closedir_hook(csync_vio_handle_t *dhandle, void *userdata) -{ - auto *discoveryJob = static_cast(userdata); - if (discoveryJob) { - auto *directoryResult = static_cast(dhandle); - QString path = directoryResult->path; - qCDebug(lcDiscovery) << discoveryJob << path; - // just deletes the struct and the iterator, the data itself is owned by the SyncEngine/DiscoveryMainThread - delete directoryResult; - } -} - -void DiscoveryJob::start() -{ - _selectiveSyncBlackList.sort(); - _selectiveSyncWhiteList.sort(); - _csync_ctx->callbacks.update_callback_userdata = this; - _csync_ctx->callbacks.update_callback = update_job_update_callback; - _csync_ctx->callbacks.checkSelectiveSyncBlackListHook = isInSelectiveSyncBlackListCallback; - _csync_ctx->callbacks.checkSelectiveSyncNewFolderHook = checkSelectiveSyncNewFolderCallback; - - _csync_ctx->callbacks.remote_opendir_hook = remote_vio_opendir_hook; - _csync_ctx->callbacks.remote_readdir_hook = remote_vio_readdir_hook; - _csync_ctx->callbacks.remote_closedir_hook = remote_vio_closedir_hook; - _csync_ctx->callbacks.vio_userdata = this; - - _lastUpdateProgressCallbackCall.invalidate(); - int ret = csync_update(_csync_ctx); - - _csync_ctx->callbacks.checkSelectiveSyncNewFolderHook = nullptr; - _csync_ctx->callbacks.checkSelectiveSyncBlackListHook = nullptr; - _csync_ctx->callbacks.update_callback = nullptr; - _csync_ctx->callbacks.update_callback_userdata = nullptr; - - emit finished(ret); - deleteLater(); -} +*/ } diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index cb027edc7..a0b387104 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -25,9 +25,12 @@ #include #include "syncoptions.h" +class ExcludedFiles; + namespace OCC { class Account; +class SyncJournalDb; /** * The Discovery Phase was once called "update" phase in csync terms. @@ -94,111 +97,26 @@ public: QByteArray _dataFingerprint; }; -// Lives in main thread. Deleted by the SyncEngine -class DiscoveryJob; -class DiscoveryMainThread : public QObject +class DiscoveryPhase : public QObject { Q_OBJECT - - QPointer _discoveryJob; - QPointer _singleDirJob; - QString _pathPrefix; // remote path - AccountPtr _account; - DiscoveryDirectoryResult *_currentDiscoveryDirectoryResult; - qint64 *_currentGetSizeResult; - bool _firstFolderProcessed; - public: - DiscoveryMainThread(AccountPtr account) - : QObject() - , _account(account) - , _currentDiscoveryDirectoryResult(nullptr) - , _currentGetSizeResult(nullptr) - , _firstFolderProcessed(false) - { - } - void abort(); - - QByteArray _dataFingerprint; - - -public slots: - // From DiscoveryJob: - void doOpendirSlot(const QString &url, DiscoveryDirectoryResult *); - void doGetSizeSlot(const QString &path, qint64 *result); - - // From Job: - void singleDirectoryJobResultSlot(); - void singleDirectoryJobFinishedWithErrorSlot(int csyncErrnoCode, const QString &msg); - void singleDirectoryJobFirstDirectoryPermissionsSlot(RemotePermissions); - - void slotGetSizeFinishedWithError(); - void slotGetSizeResult(const QVariantMap &); -signals: - void etag(const QString &); - void etagConcatenation(const QString &); - -public: - void setupHooks(DiscoveryJob *discoveryJob, const QString &pathPrefix); -}; - -/** - * @brief The DiscoveryJob class - * - * Lives in the other thread, deletes itself in !start() - * - * @ingroup libsync - */ -class DiscoveryJob : public QObject -{ - Q_OBJECT - friend class DiscoveryMainThread; - CSYNC *_csync_ctx; - QElapsedTimer _lastUpdateProgressCallbackCall; + QString _localDir; // absolute path to the local directory. ends with '/' + QString _remoteFolder; // remote folder, ends with '/' + SyncJournalDb *_statedb; + AccountPtr _account; + SyncOptions _syncOptions; + QStringList _selectiveSyncBlackList; + QStringList _selectiveSyncWhiteList; + ExcludedFiles *_excludes; - /** - * return true if the given path should be ignored, - * false if the path should be synced - */ - bool isInSelectiveSyncBlackList(const QByteArray &path) const; - static int isInSelectiveSyncBlackListCallback(void *, const QByteArray &); + bool isInSelectiveSyncBlackList(const QString &path) const; bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp); - static int checkSelectiveSyncNewFolderCallback(void *data, const QByteArray &path, RemotePermissions rm); - - // Just for progress - static void update_job_update_callback(bool local, - const char *dirname, - void *userdata); - - // For using QNAM to get the directory listings - static csync_vio_handle_t *remote_vio_opendir_hook(const char *url, - void *userdata); - static std::unique_ptr remote_vio_readdir_hook(csync_vio_handle_t *dhandle, - void *userdata); - static void remote_vio_closedir_hook(csync_vio_handle_t *dhandle, - void *userdata); - QMutex _vioMutex; - QWaitCondition _vioWaitCondition; - -public: - explicit DiscoveryJob(CSYNC *ctx, QObject *parent = nullptr) - : QObject(parent) - , _csync_ctx(ctx) - { - } - QStringList _selectiveSyncBlackList; - QStringList _selectiveSyncWhiteList; - SyncOptions _syncOptions; - Q_INVOKABLE void start(); signals: void finished(int result); void folderDiscovered(bool local, QString folderUrl); - // After the discovery job has been woken up again (_vioWaitCondition) - void doOpendirSignal(QString url, DiscoveryDirectoryResult *); - void doGetSizeSignal(const QString &path, qint64 *result); - // A new folder was discovered and was not synced because of the confirmation feature void newBigFolder(const QString &folder, bool isExternal); }; diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 5d5a9133e..07fbb652e 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -481,8 +481,8 @@ int SyncEngine::treewalkFile(csync_file_stat_t * /*file*/, csync_file_stat_t * / } -#if 0 -PORTED + + switch (file->error_status) { case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK: @@ -511,7 +511,7 @@ PORTED break; } -#endif + if (item->_instruction == CSYNC_INSTRUCTION_IGNORE && utf8DecodeError) { item->_status = SyncFileItem::NormalError; @@ -550,60 +550,9 @@ PORTED } case CSYNC_INSTRUCTION_UPDATE_METADATA: dir = SyncFileItem::None; - // For directories, metadata-only updates will be done after all their files are propagated. - if (!isDirectory) { - - // Update the database now already: New remote fileid or Etag or RemotePerm - // Or for files that were detected as "resolved conflict". - // Or a local inode/mtime change - - // In case of "resolved conflict": there should have been a conflict because they - // both were new, or both had their local mtime or remote etag modified, but the - // size and mtime is the same on the server. This typically happens when the - // database is removed. Nothing will be done for those files, but we still need - // to update the database. - - // This metadata update *could* be a propagation job of its own, but since it's - // quick to do and we don't want to create a potentially large number of - // mini-jobs later on, we just update metadata right now. - - if (remote) { - QString filePath = _localPath + item->_file; - - if (other && other->type != ItemTypeVirtualFile && other->type != ItemTypeVirtualFileDownload) { - // Even if the mtime is different on the server, we always want to keep the mtime from - // the file system in the DB, this is to avoid spurious upload on the next sync - item->_modtime = other->modtime; - // same for the size - item->_size = other->size; - } - - // If the 'W' remote permission changed, update the local filesystem - SyncJournalFileRecord prev; - if (_journal->getFileRecord(item->_file, &prev) - && prev.isValid() - && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) { - const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite); - FileSystem::setFileReadOnlyWeak(filePath, isReadOnly); - } - - _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath)); - - // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example - emit itemCompleted(item); - } else { - // The local tree is walked first and doesn't have all the info from the server. - // Update only outdated data from the disk. - _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode); - } - if (!other || other->instruction == CSYNC_INSTRUCTION_NONE || other->instruction == CSYNC_INSTRUCTION_UPDATE_METADATA) { - _hasNoneFiles = true; - } + ... ported ... - // Technically we're done with this item. - return re; - } break; case CSYNC_INSTRUCTION_RENAME: dir = !remote ? SyncFileItem::Down : SyncFileItem::Up; @@ -900,74 +849,94 @@ void SyncEngine::slotStartDiscovery() emit transmissionProgress(*_progressInfo); qCInfo(lcEngine) << "#### Discovery start ####################################################"; + qCInfo(lcEngine) << "Server" << account()->serverVersion() + << (account()->isHttp2Supported() ? "Using HTTP/2" : ""); _progressInfo->_status = ProgressInfo::Discovery; emit transmissionProgress(*_progressInfo); - _propagator = QSharedPointer( - new OwncloudPropagator(_account, _localPath, _remotePath, _journal)); - _propagator->setSyncOptions(_syncOptions); - connect(_propagator.data(), &OwncloudPropagator::itemCompleted, - this, &SyncEngine::slotItemCompleted); - connect(_propagator.data(), &OwncloudPropagator::progress, - this, &SyncEngine::slotProgress); - connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection); - connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); - connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); - connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); - connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); - connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); + auto ddata = QSharedPointer::create(); + ddata->_account = _account; + ddata->_excludes = _excludedFiles.data(); + ddata->_statedb = _journal; + ddata->_selectiveSyncBlackList = selectiveSyncBlackList; + ddata->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); + ddata->_localDir = _localPath; + ddata->_remoteFolder = _remotePath; + ddata->_syncOptions = _syncOptions; + if (!ok) { + qCWarning(lcEngine) << "Unable to read selective sync list, aborting."; + csyncError(tr("Unable to read from the sync journal.")); + finalize(false); + return; + } + + connect(ddata.data(), &DiscoveryPhase::folderDiscovered, this, &SyncEngine::slotFolderDiscovered); + connect(ddata.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder); + _discoveryJob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery, + ddata, this); + connect(_discoveryJob.data(), &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); }); + connect(_discoveryJob.data(), &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) { + if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) { + // For directories, metadata-only updates will be done after all their files are propagated. + + // Update the database now already: New remote fileid or Etag or RemotePerm + // Or for files that were detected as "resolved conflict". + // Or a local inode/mtime change + + // In case of "resolved conflict": there should have been a conflict because they + // both were new, or both had their local mtime or remote etag modified, but the + // size and mtime is the same on the server. This typically happens when the + // database is removed. Nothing will be done for those files, but we still need + // to update the database. + + // This metadata update *could* be a propagation job of its own, but since it's + // quick to do and we don't want to create a potentially large number of + // mini-jobs later on, we just update metadata right now. + + if (item->_direction == SyncFileItem::Down) { + QString filePath = _localPath + item->_file; + + // If the 'W' remote permission changed, update the local filesystem + SyncJournalFileRecord prev; + if (_journal->getFileRecord(item->_file, &prev) + && prev.isValid() + && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) { + const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite); + FileSystem::setFileReadOnlyWeak(filePath, isReadOnly); + } + + _journal->setFileRecordMetadata(item->toSyncJournalFileRecordWithInode(filePath)); + + // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example + emit itemCompleted(item); + } else { + // The local tree is walked first and doesn't have all the info from the server. + // Update only outdated data from the disk. + // FIXME! I think this is no longer the case so a setFileRecordMetadata should work + _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode); + } + _hasNoneFiles = true; + return; + } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) { + _hasNoneFiles = true; + return; + } - auto djob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery, - _propagator.data(), _excludedFiles.data(), this); - connect(djob, &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); }); - connect(djob, &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) { _syncItems.append(item); slotNewItem(item); }); - djob->start(); + _discoveryJob->start(); /* - _discoveryMainThread = new DiscoveryMainThread(account()); - _discoveryMainThread->setParent(this); - connect(this, &SyncEngine::finished, _discoveryMainThread.data(), &QObject::deleteLater); - qCInfo(lcEngine) << "Server" << account()->serverVersion() - << (account()->isHttp2Supported() ? "Using HTTP/2" : ""); + * FIXME if (account()->rootEtagChangesNotOnlySubFolderEtags()) { connect(_discoveryMainThread.data(), &DiscoveryMainThread::etag, this, &SyncEngine::slotRootEtagReceived); } else { connect(_discoveryMainThread.data(), &DiscoveryMainThread::etagConcatenation, this, &SyncEngine::slotRootEtagReceived); } - - auto *discoveryJob = new Disco(_csync_ctx.data()); - discoveryJob->_selectiveSyncBlackList = selectiveSyncBlackList; - discoveryJob->_selectiveSyncWhiteList = - _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); - if (!ok) { - delete discoveryJob; - qCWarning(lcEngine) << "Unable to read selective sync list, aborting."; - csyncError(tr("Unable to read from the sync journal.")); - finalize(false); - return; - } - - discoveryJob->_syncOptions = _syncOptions; - - connect(discoveryJob, &DiscoveryJob::finished, this, &SyncEngine::slotDiscoveryJobFinished); - connect(discoveryJob, &DiscoveryJob::folderDiscovered, - this, &SyncEngine::slotFolderDiscovered); - - connect(discoveryJob, &DiscoveryJob::newBigFolder, - this, &SyncEngine::newBigFolder); - - - // This is used for the DiscoveryJob to be able to request the main thread/ - // to read in directory contents. - _discoveryMainThread->setupHooks(discoveryJob, _remotePath); - - // Starts the update in a seperate thread - QMetaObject::invokeMethod(discoveryJob, "start", Qt::QueuedConnection);*/ + */ } void SyncEngine::slotFolderDiscovered(bool local, const QString &folder) @@ -1157,6 +1126,19 @@ void SyncEngine::slotDiscoveryJobFinished(int /*discoveryResult*/) // do a database commit _journal->commit("post treewalk"); + _propagator = QSharedPointer( + new OwncloudPropagator(_account, _localPath, _remotePath, _journal)); + _propagator->setSyncOptions(_syncOptions); + connect(_propagator.data(), &OwncloudPropagator::itemCompleted, + this, &SyncEngine::slotItemCompleted); + connect(_propagator.data(), &OwncloudPropagator::progress, + this, &SyncEngine::slotProgress); + connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection); + connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); + connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); + connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); + connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); + connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); // apply the network limits to the propagator setNetworkLimits(_uploadLimit, _downloadLimit); @@ -1310,57 +1292,6 @@ void SyncEngine::checkForPermission(SyncFileItemVector &syncItems) // Do not propagate anything in the server if it is in the selective sync blacklist const QString path = (*it)->destination() + QLatin1Char('/'); - // if reading the selective sync list from db failed, lets ignore all rather than nothing. - if (!selectiveListOk || std::binary_search(selectiveSyncBlackList.constBegin(), selectiveSyncBlackList.constEnd(), - path)) { - (*it)->_instruction = CSYNC_INSTRUCTION_IGNORE; - (*it)->_status = SyncFileItem::FileIgnored; - (*it)->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist"); - - if ((*it)->isDirectory()) { - auto it_base = it; - for (SyncFileItemVector::iterator it_next = it + 1; it_next != syncItems.end() && (*it_next)->_file.startsWith(path); ++it_next) { - it = it_next; - // We want to ignore almost all instructions for items inside selective-sync excluded folders. - //The exception are DOWN/REMOVE actions that remove local files and folders that are - //guaranteed to be up-to-date with their server copies. - if ((*it)->_direction == SyncFileItem::Down && (*it)->_instruction == CSYNC_INSTRUCTION_REMOVE) { - // We need to keep the "delete" items. So we need to un-ignore parent directories - QString parentDir = (*it)->_file; - do { - parentDir = QFileInfo(parentDir).path(); - if (parentDir.isEmpty() || !parentDir.startsWith((*it_base)->destination())) { - break; - } - // Find the parent directory in the syncItems vector. Since the vector - // is sorted we can use a lower_bound, but we need a fake - // SyncFileItemPtr needle to compare against - if (!needle) - needle = SyncFileItemPtr::create(); - needle->_file = parentDir; - auto parent_it = std::lower_bound(it_base, it, needle); - if (parent_it == syncItems.end() || (*parent_it)->destination() != parentDir) { - break; - } - ASSERT((*parent_it)->isDirectory()); - if ((*parent_it)->_instruction != CSYNC_INSTRUCTION_IGNORE) { - break; // already changed - } - (*parent_it)->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; - (*parent_it)->_status = SyncFileItem::NoStatus; - (*parent_it)->_errorString.clear(); - - } while (true); - continue; - } - (*it)->_instruction = CSYNC_INSTRUCTION_IGNORE; - (*it)->_status = SyncFileItem::FileIgnored; - (*it)->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist"); - } - } - continue; - } - switch ((*it)->_instruction) { case CSYNC_INSTRUCTION_TYPE_CHANGE: case CSYNC_INSTRUCTION_NEW: { @@ -1707,8 +1638,8 @@ void SyncEngine::abort() csync_request_abort(_csync_ctx.data()); // Aborts the discovery phase job - if (_discoveryMainThread) { - _discoveryMainThread->abort(); + if (_discoveryJob) { + _discoveryJob->abort(); } // For the propagator if (_propagator) { diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index a95a5e9bc..7416d5d18 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -47,6 +47,7 @@ namespace OCC { class SyncJournalFileRecord; class SyncJournalDb; class OwncloudPropagator; +class ProcessDirectoryJob; enum AnotherSyncNeeded { NoFollowUpSync, @@ -241,7 +242,7 @@ private: QString _remotePath; QString _remoteRootEtag; SyncJournalDb *_journal; - QPointer _discoveryMainThread; + QPointer _discoveryJob; QSharedPointer _propagator; // After a sync, only the syncdb entries whose filenames appear in this -- 2.30.2