From: Olivier Goffart Date: Tue, 10 Jul 2018 08:21:45 +0000 (+0200) Subject: New discovery algorithm: Initial work. X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~567 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=ec681ab2a5ef239c1a3a0ac4f18a7dc45561acad;p=nextcloud-desktop.git New discovery algorithm: Initial work. SyncEngineTest testFileDownload is passing --- diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 7799eb01f..b665249e9 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -170,6 +170,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f } } +#if 0 // PORTED if (excluded > CSYNC_NOT_EXCLUDED || fs->type == ItemTypeSoftLink) { fs->instruction = CSYNC_INSTRUCTION_IGNORE; if (ctx->current_fs) { @@ -178,6 +179,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f goto out; } +#endif /* Update detection: Check if a database entry exists. * If not, the file is either new or has been renamed. To see if it is @@ -471,6 +473,8 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f out: +#if 0 +PORTED /* Set the ignored error string. */ if (fs->instruction == CSYNC_INSTRUCTION_IGNORE) { if( fs->type == ItemTypeSoftLink ) { @@ -501,6 +505,7 @@ out: && fs->type != ItemTypeDirectory) { fs->child_modified = true; } +#endif // If conflict files are uploaded, they won't be marked as IGNORE / CSYNC_FILE_EXCLUDE_CONFLICT // but we still want them marked! diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index ee08c83ac..cf744fe1e 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -22,6 +22,7 @@ set(libsync_SRCS capabilities.cpp clientproxy.cpp cookiejar.cpp + discovery.cpp discoveryphase.cpp encryptfolderjob.cpp filesystem.cpp diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp new file mode 100644 index 000000000..bb3d2b845 --- /dev/null +++ b/src/libsync/discovery.cpp @@ -0,0 +1,335 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "discovery.h" +#include "common/syncjournaldb.h" +#include "syncfileitem.h" +#include "owncloudpropagator.h" // FIXME! remove; +#include +#include +#include +#include +#include "vio/csync_vio_local.h" + +namespace OCC { + +static RemoteInfo remoteInfoFromCSync(const csync_file_stat_t &x) +{ + RemoteInfo ri; + ri.name = QFileInfo(QString::fromUtf8(x.path)).fileName(); + ri.etag = x.etag; + ri.fileId = x.file_id; + ri.checksumHeader = x.checksumHeader; + ri.modtime = x.modtime; + ri.size = x.size; + ri.isDirectory = x.type == ItemTypeDirectory; + ri.remotePerm = x.remotePerm; + return ri; +} + +DiscoverServerJob::DiscoverServerJob(const AccountPtr &account, const QString &path, QObject *parent) + : DiscoverySingleDirectoryJob(account, path, parent) +{ + connect(this, &DiscoverySingleDirectoryJob::finishedWithResult, this, [this] { + auto csync_results = takeResults(); + QVector results; + std::transform(csync_results.begin(), csync_results.end(), std::back_inserter(results), + [](const auto &x) { return remoteInfoFromCSync(*x); }); + emit this->finished(results); + }); + + connect(this, &DiscoverySingleDirectoryJob::finishedWithError, this, + [this](int, const QString &msg) { + emit this->finished({ Error, msg }); + }); +} + + +void ProcessDirectoryJob::start() +{ + if (_queryServer == NormalQuery) { + _serverJob = new DiscoverServerJob(_propagator->account(), _propagator->_remoteFolder + _currentFolder, this); + connect(_serverJob.data(), &DiscoverServerJob::finished, this, [this](const auto &results) { + if (results) { + _serverEntries = *results; + _hasServerEntries = true; + if (_hasLocalEntries) + process(); + } else { + qWarning() << results.errorMessage(); + qFatal("TODO: ERROR HANDLING"); + } + }); + _serverJob->start(); + } else { + _hasServerEntries = true; + } + + if (!_currentFolder.isEmpty() && _currentFolder.back() != '/') + _currentFolder += '/'; + + if (_queryLocal == NormalQuery) { + /*QDirIterator dirIt(_propagator->_localDir + _currentFolder); + while (dirIt.hasNext()) { + auto x = dirIt.next(); + LocalInfo i; + i.name = dirIt.fileName(); + + }*/ + auto dh = csync_vio_local_opendir((_propagator->_localDir + _currentFolder).toUtf8()); + if (!dh) { + qDebug() << "COULD NOT OPEN" << (_propagator->_localDir + _currentFolder).toUtf8(); + qFatal("TODO: ERROR HANDLING"); + // should be the same as in csync_update; + } + while (auto dirent = csync_vio_local_readdir(dh)) { + LocalInfo i; + i.name = QString::fromUtf8(dirent->path); // FIXME! conversion errors + i.modtime = dirent->modtime; + i.size = dirent->size; + i.inode = dirent->inode; + i.isDirectory = dirent->type == ItemTypeDirectory; + if (dirent->type != ItemTypeDirectory && dirent->type != ItemTypeFile) + qFatal("FIXME: NEED TO CARE ABOUT THE OTHER STUFF "); + _localEntries.push_back(i); + } + csync_vio_local_closedir(dh); + } + _hasLocalEntries = true; + if (_hasServerEntries) + process(); +} + +void ProcessDirectoryJob::process() +{ + ASSERT(_hasLocalEntries && _hasServerEntries); + + QString localDir; + + std::set entriesNames; // sorted + QHash serverEntriesHash; + QHash localEntriesHash; + for (auto &e : _serverEntries) { + entriesNames.insert(e.name); + serverEntriesHash[e.name] = std::move(e); + } + _serverEntries.clear(); + for (auto &e : _localEntries) { + entriesNames.insert(e.name); + localEntriesHash[e.name] = std::move(e); + } + _localEntries.clear(); + for (const auto &f : entriesNames) { + QString path = _currentFolder + f; + if (handleExcluded(path, (localEntriesHash.value(f).isDirectory || serverEntriesHash.value(f).isDirectory))) + continue; + + SyncJournalFileRecord record; + if (!_propagator->_journal->getFileRecord(path, &record)) { + qFatal("TODO: ERROR HANDLING"); + } + processFile(path, localEntriesHash.value(f), serverEntriesHash.value(f), record); + } + + progress(); +} + +bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory) +{ + // FIXME! call directly, without char* conversion + auto excluded = _excludes->csyncTraversalMatchFun()(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile); + if (excluded == CSYNC_NOT_EXCLUDED /* FIXME && item->_type != ItemTypeSoftLink */) { + return false; + } + + auto item = SyncFileItemPtr::create(); + item->_file = path; + item->_instruction = CSYNC_INSTRUCTION_IGNORE; + + /*if( fs->type == ItemTypeSoftLink ) { + fs->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */ + switch (excluded) { + case CSYNC_FILE_EXCLUDE_LIST: + item->_errorString = tr("File is listed on the ignore list."); + break; + case CSYNC_FILE_EXCLUDE_INVALID_CHAR: + if (item->_file.endsWith('.')) { + item->_errorString = tr("File names ending with a period are not supported on this file system."); + } else { + char invalid = '\0'; + foreach (char x, QByteArray("\\:?*\"<>|")) { + if (item->_file.contains(x)) { + invalid = x; + break; + } + } + if (invalid) { + item->_errorString = tr("File names containing the character '%1' are not supported on this file system.") + .arg(QLatin1Char(invalid)); + } else { + item->_errorString = tr("The file name is a reserved name on this file system."); + } + } + break; + case CSYNC_FILE_EXCLUDE_TRAILING_SPACE: + item->_errorString = tr("Filename contains trailing spaces."); + break; + case CSYNC_FILE_EXCLUDE_LONG_FILENAME: + item->_errorString = tr("Filename is too long."); + break; + case CSYNC_FILE_EXCLUDE_HIDDEN: + item->_errorString = tr("File/Folder is ignored because it's hidden."); + break; + case CSYNC_FILE_EXCLUDE_STAT_FAILED: + item->_errorString = tr("Stat failed."); + break; + case CSYNC_FILE_EXCLUDE_CONFLICT: + qFatal("TODO: conflicts"); +#if 0 + item->_status = SyncFileItem::Conflict; + if (_propagator->account()->capabilities().uploadConflictFiles()) { + // For uploaded conflict files, files with no action performed on them should + // be displayed: but we mustn't overwrite the instruction if something happens + // to the file! + if (remote && item->_instruction == CSYNC_INSTRUCTION_NONE) { + item->_errorString = tr("Unresolved conflict."); + item->_instruction = CSYNC_INSTRUCTION_IGNORE; + } + } else { + item->_errorString = tr("Conflict: Server version downloaded, local copy renamed and not uploaded."); + } +#endif + break; + case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE: + item->_errorString = tr("The filename cannot be encoded on your file system."); + break; + } + + _childIgnored = true; + emit itemDiscovered(item); + return true; +} + +void ProcessDirectoryJob::processFile(const QString &path, + const LocalInfo &localEntry, const RemoteInfo &serverEntry, + const SyncJournalFileRecord &dbEntry) +{ + auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); + item->_file = path; + + auto recurseQueryServer = _queryServer; + if (_queryServer == NormalQuery && serverEntry.isValid()) { + item->_checksumHeader = serverEntry.checksumHeader; + item->_fileId = serverEntry.fileId; + item->_remotePerm = serverEntry.remotePerm; + item->_type = serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; + item->_size = serverEntry.size; + item->_previousSize = localEntry.size; + item->_previousModtime = localEntry.modtime; + if (!dbEntry.isValid()) { + item->_instruction = CSYNC_INSTRUCTION_NEW; + // TODO! rename; + item->_direction = SyncFileItem::Down; + item->_modtime = serverEntry.modtime; + } else if (dbEntry._etag != serverEntry.etag) { + item->_instruction = CSYNC_INSTRUCTION_SYNC; + item->_direction = SyncFileItem::Down; + item->_modtime = serverEntry.modtime; + } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId) { + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + item->_direction = SyncFileItem::Down; + } else { + recurseQueryServer = ParentNotChanged; + } + } + bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC; + _childModified |= serverModified; + if (localEntry.isValid()) { + item->_inode = localEntry.inode; + if (dbEntry.isValid() && dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) { + if (_queryServer != ParentNotChanged && !serverEntry.isValid()) { + item->_instruction = CSYNC_INSTRUCTION_REMOVE; + item->_direction = SyncFileItem::Down; // Does not matter + } else if (!serverModified && dbEntry._inode != localEntry.inode) { + item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + item->_direction = SyncFileItem::Down; // Does not matter + } + } else if (serverModified) { + item->_instruction = CSYNC_INSTRUCTION_CONFLICT; + } else if (!dbEntry.isValid()) { + item->_instruction = CSYNC_INSTRUCTION_NEW; + item->_direction = SyncFileItem::Up; + // TODO! rename; + item->_size = localEntry.size; + item->_modtime = localEntry.modtime; + item->_type = serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; + _childModified = true; + } else { + item->_instruction = CSYNC_INSTRUCTION_SYNC; + item->_direction = SyncFileItem::Up; + item->_size = localEntry.size; + item->_modtime = localEntry.modtime; + item->_previousSize = serverEntry.size; + item->_previousModtime = serverEntry.modtime; + _childModified = true; + } + } else if (!serverModified) { + item->_instruction = CSYNC_INSTRUCTION_REMOVE; + item->_direction = SyncFileItem::Up; + } + + qDebug() << "Discovered" << item->_file << item->_instruction << item->_direction; + + if (item->isDirectory()) { + auto job = new ProcessDirectoryJob(item, recurseQueryServer, localEntry.isValid() ? NormalQuery : ParentDontExist, + _propagator, _excludes, this); + connect(job, &ProcessDirectoryJob::itemDiscovered, this, &ProcessDirectoryJob::itemDiscovered); + connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); + _queuedJobs.push_back(job); + } else { + emit itemDiscovered(item); + } +} + +void ProcessDirectoryJob::subJobFinished() +{ + auto job = qobject_cast(sender()); + ASSERT(job); + + _childIgnored |= job->_childIgnored; + _childModified |= job->_childModified; + + if (job->_dirItem) + emit itemDiscovered(job->_dirItem); + + int count = _runningJobs.removeAll(job); + ASSERT(count == 1); + job->deleteLater(); + progress(); +} + +void ProcessDirectoryJob::progress() +{ + if (!_queuedJobs.empty()) { + auto f = _queuedJobs.front(); + _queuedJobs.pop_front(); + _runningJobs.push_back(f); + f->start(); + return; + } + if (_runningJobs.empty()) { + emit finished(); + } +} +} diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h new file mode 100644 index 000000000..a8c56bbf7 --- /dev/null +++ b/src/libsync/discovery.h @@ -0,0 +1,159 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#pragma once + +#include +#include "discoveryphase.h" +#include "syncfileitem.h" +#include "common/asserts.h" + +class ExcludedFiles; + +namespace OCC { +class SyncJournalDb; +class OwncloudPropagator; + +enum ErrorTag { Error }; + +template +class Result +{ + union { + T _result; + QString _errorString; + }; + bool _isError; + +public: + Result(T value) + : _result(std::move(value)) + , _isError(false){}; + Result(ErrorTag, QString str) + : _errorString(std::move(str)) + , _isError(true) + { + } + ~Result() + { + if (_isError) + _errorString.~QString(); + else + _result.~T(); + } + explicit operator bool() const { return !_isError; } + const T &operator*() const & + { + ASSERT(!_isError); + return _result; + } + T operator*() && + { + ASSERT(!_isError); + return std::move(_result); + } + QString errorMessage() const + { + ASSERT(_isError); + return _errorString; + } +}; + + +struct RemoteInfo +{ + QString name; + QByteArray etag; + QByteArray fileId; + QByteArray checksumHeader; + OCC::RemotePermissions remotePerm; + time_t modtime = 0; + int64_t size = 0; + bool isDirectory = false; + bool isValid() const { return !name.isNull(); } +}; + +struct LocalInfo +{ + QString name; + time_t modtime = 0; + int64_t size = 0; + uint64_t inode = 0; + bool isDirectory = false; + bool isValid() const { return !name.isNull(); } +}; + +/** + * Do the propfind on the server. + * TODO: merge with DiscoverySingleDirectoryJob + */ +class DiscoverServerJob : public DiscoverySingleDirectoryJob +{ + Q_OBJECT +public: + explicit DiscoverServerJob(const AccountPtr &account, const QString &path, QObject *parent = 0); +signals: + void finished(const Result> &result); +}; + +class ProcessDirectoryJob : public QObject +{ + Q_OBJECT +public: + enum QueryMode { NormalQuery, + ParentDontExist, + ParentNotChanged }; + explicit ProcessDirectoryJob(const SyncFileItemPtr &dirItem, QueryMode queryServer, QueryMode queryLocal, + OwncloudPropagator *propagator, ExcludedFiles *excludes, QObject *parent) + : QObject(parent) + , _dirItem(dirItem) + , _queryServer(queryServer) + , _queryLocal(queryLocal) + , _propagator(propagator) + , _excludes(excludes) + , _currentFolder(dirItem ? dirItem->_file : QString()) + { + } + void start(); + +private: + void process(); + // return true if the file is excluded + bool handleExcluded(const QString &path, bool isDirectory); + void processFile(const QString &, const LocalInfo &, const RemoteInfo &, const SyncJournalFileRecord &); + void subJobFinished(); + void progress(); + + QVector _serverEntries; + QVector _localEntries; + bool _hasServerEntries = false; + bool _hasLocalEntries = false; + QPointer _serverJob; + //QScopedPointer _localJob; + std::deque _queuedJobs; + QVector _runningJobs; + SyncFileItemPtr _dirItem; + QueryMode _queryServer; + QueryMode _queryLocal; + OwncloudPropagator *_propagator; // FIXME: remove this. We need that for the account and local/remote path only. + ExcludedFiles *_excludes; // FIXME: Move also in the replacement of the propagator + QString _currentFolder; + bool _childModified = false; + bool _childIgnored = false; + +signals: + void itemDiscovered(const SyncFileItemPtr &item); + void finished(); +}; +} diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 5797aeb01..5d5a9133e 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -27,7 +27,7 @@ #include "propagatedownload.h" #include "common/asserts.h" #include "configfile.h" - +#include "discovery.h" #ifdef Q_OS_WIN #include @@ -99,15 +99,11 @@ SyncEngine::SyncEngine(AccountPtr account, const QString &localPath, _clearTouchedFilesTimer.setSingleShot(true); _clearTouchedFilesTimer.setInterval(30 * 1000); connect(&_clearTouchedFilesTimer, &QTimer::timeout, this, &SyncEngine::slotClearTouchedFiles); - - _thread.setObjectName("SyncEngine_Thread"); } SyncEngine::~SyncEngine() { abort(); - _thread.quit(); - _thread.wait(); _excludedFiles.reset(); } @@ -380,8 +376,9 @@ void SyncEngine::conflictRecordMaintenance() * * See doc/dev/sync-algorithm.md for an overview. */ -int SyncEngine::treewalkFile(csync_file_stat_t *file, csync_file_stat_t *other, bool remote) +int SyncEngine::treewalkFile(csync_file_stat_t * /*file*/, csync_file_stat_t * /*other*/, bool /*remote*/) { +#if 0 // FIXME adapt if (!file) return -1; @@ -483,67 +480,19 @@ int SyncEngine::treewalkFile(csync_file_stat_t *file, csync_file_stat_t *other, _seenFiles.insert(renameTarget); } + +#if 0 +PORTED switch (file->error_status) { - case CSYNC_STATUS_OK: - break; + case CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK: item->_errorString = tr("Symbolic links are not supported in syncing."); break; - case CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST: - item->_errorString = tr("File is listed on the ignore list."); - break; - case CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS: - if (item->_file.endsWith('.')) { - item->_errorString = tr("File names ending with a period are not supported on this file system."); - } else { - char invalid = '\0'; - foreach (char x, QByteArray("\\:?*\"<>|")) { - if (item->_file.contains(x)) { - invalid = x; - break; - } - } - if (invalid) { - item->_errorString = tr("File names containing the character '%1' are not supported on this file system.") - .arg(QLatin1Char(invalid)); - } else { - item->_errorString = tr("The file name is a reserved name on this file system."); - } - } - break; - case CSYNC_STATUS_INDIVIDUAL_TRAILING_SPACE: - item->_errorString = tr("Filename contains trailing spaces."); - break; - case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME: - item->_errorString = tr("Filename is too long."); - break; - case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN: - item->_errorString = tr("File/Folder is ignored because it's hidden."); - break; - case CSYNC_STATUS_INDIVIDUAL_TOO_DEEP: + + case CSYNC_STATUS_INDIVIDUAL_TOO_DEEP: item->_errorString = tr("Folder hierarchy is too deep"); break; - case CSYNC_STATUS_INDIVIDUAL_CANNOT_ENCODE: - item->_errorString = tr("The filename cannot be encoded on your file system."); - break; - case CSYNC_STATUS_INDIVIDUAL_IS_CONFLICT_FILE: - item->_status = SyncFileItem::Conflict; - if (account()->capabilities().uploadConflictFiles()) { - // For uploaded conflict files, files with no action performed on them should - // be displayed: but we mustn't overwrite the instruction if something happens - // to the file! - if (remote && item->_instruction == CSYNC_INSTRUCTION_NONE) { - item->_errorString = tr("Unresolved conflict."); - item->_instruction = CSYNC_INSTRUCTION_IGNORE; - } - } else { - item->_errorString = tr("Conflict: Server version downloaded, local copy renamed and not uploaded."); - } - break; - case CSYNC_STATUS_INDIVIDUAL_STAT_FAILED: - item->_errorString = tr("Stat failed."); - break; - case CSYNC_STATUS_SERVICE_UNAVAILABLE: + case CSYNC_STATUS_SERVICE_UNAVAILABLE: item->_errorString = QLatin1String("Server temporarily unavailable."); break; case CSYNC_STATUS_STORAGE_UNAVAILABLE: @@ -560,10 +509,9 @@ int SyncEngine::treewalkFile(csync_file_stat_t *file, csync_file_stat_t *other, item->_errorString = QLatin1String("Directory not accessible on client, permission denied."); item->_status = SyncFileItem::SoftError; break; - default: - ASSERT(false, "Non handled error-status"); - /* No error string */ + } +#endif if (item->_instruction == CSYNC_INSTRUCTION_IGNORE && utf8DecodeError) { item->_status = SyncFileItem::NormalError; @@ -710,14 +658,13 @@ int SyncEngine::treewalkFile(csync_file_stat_t *file, csync_file_stat_t *other, _needsUpdate = true; - if (other) { - item->_previousModtime = other->modtime; - item->_previousSize = other->size; - } + slotNewItem(item); _syncItemMap.insert(key, item); return re; +#endif + return 0; } void SyncEngine::handleSyncError(CSYNC *ctx, const char *state) @@ -817,7 +764,7 @@ void SyncEngine::startSync() qCWarning(lcEngine) << "Could not determine free space available at" << _localPath; } - _syncItemMap.clear(); + _syncItems.clear(); _needsUpdate = false; csync_resume(_csync_ctx.data()); @@ -956,11 +903,31 @@ void SyncEngine::slotStartDiscovery() _progressInfo->_status = ProgressInfo::Discovery; emit transmissionProgress(*_progressInfo); - // Usually the discovery runs in the background: We want to avoid - // stealing too much time from other processes that the user might - // be interacting with at the time. - _thread.start(QThread::LowPriority); + _propagator = QSharedPointer( + new OwncloudPropagator(_account, _localPath, _remotePath, _journal)); + _propagator->setSyncOptions(_syncOptions); + connect(_propagator.data(), &OwncloudPropagator::itemCompleted, + this, &SyncEngine::slotItemCompleted); + connect(_propagator.data(), &OwncloudPropagator::progress, + this, &SyncEngine::slotProgress); + connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection); + connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); + connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); + connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); + connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); + connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); + + + auto djob = new ProcessDirectoryJob(SyncFileItemPtr(), ProcessDirectoryJob::NormalQuery, ProcessDirectoryJob::NormalQuery, + _propagator.data(), _excludedFiles.data(), this); + connect(djob, &ProcessDirectoryJob::finished, this, [this] { slotDiscoveryJobFinished(0); sender()->deleteLater(); }); + connect(djob, &ProcessDirectoryJob::itemDiscovered, this, [this](const auto &item) { + _syncItems.append(item); + slotNewItem(item); + }); + djob->start(); + /* _discoveryMainThread = new DiscoveryMainThread(account()); _discoveryMainThread->setParent(this); connect(this, &SyncEngine::finished, _discoveryMainThread.data(), &QObject::deleteLater); @@ -972,7 +939,8 @@ void SyncEngine::slotStartDiscovery() connect(_discoveryMainThread.data(), &DiscoveryMainThread::etagConcatenation, this, &SyncEngine::slotRootEtagReceived); } - auto *discoveryJob = new DiscoveryJob(_csync_ctx.data()); + + auto *discoveryJob = new Disco(_csync_ctx.data()); discoveryJob->_selectiveSyncBlackList = selectiveSyncBlackList; discoveryJob->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok); @@ -985,7 +953,7 @@ void SyncEngine::slotStartDiscovery() } discoveryJob->_syncOptions = _syncOptions; - discoveryJob->moveToThread(&_thread); + connect(discoveryJob, &DiscoveryJob::finished, this, &SyncEngine::slotDiscoveryJobFinished); connect(discoveryJob, &DiscoveryJob::folderDiscovered, this, &SyncEngine::slotFolderDiscovered); @@ -999,7 +967,7 @@ void SyncEngine::slotStartDiscovery() _discoveryMainThread->setupHooks(discoveryJob, _remotePath); // Starts the update in a seperate thread - QMetaObject::invokeMethod(discoveryJob, "start", Qt::QueuedConnection); + QMetaObject::invokeMethod(discoveryJob, "start", Qt::QueuedConnection);*/ } void SyncEngine::slotFolderDiscovered(bool local, const QString &folder) @@ -1031,8 +999,8 @@ void SyncEngine::slotNewItem(const SyncFileItemPtr &item) _progressInfo->adjustTotalsForFile(*item); } -void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) -{ +void SyncEngine::slotDiscoveryJobFinished(int /*discoveryResult*/) +{ /* if (discoveryResult < 0) { handleSyncError(_csync_ctx.data(), "csync_update"); return; @@ -1155,19 +1123,19 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) restoreOldFiles(syncItems); } } - +*/ // Sort items per destination - std::sort(syncItems.begin(), syncItems.end()); + std::sort(_syncItems.begin(), _syncItems.end()); // make sure everything is allowed - checkForPermission(syncItems); + // TODO checkForPermission(_syncItems); // Re-init the csync context to free memory _csync_ctx->reinitialize(); _localDiscoveryPaths.clear(); // To announce the beginning of the sync - emit aboutToPropagate(syncItems); + emit aboutToPropagate(_syncItems); // it's important to do this before ProgressInfo::start(), to announce start of new sync _progressInfo->_status = ProgressInfo::Propagation; @@ -1189,33 +1157,21 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) // do a database commit _journal->commit("post treewalk"); - _propagator = QSharedPointer( - new OwncloudPropagator(_account, _localPath, _remotePath, _journal)); - _propagator->setSyncOptions(_syncOptions); - connect(_propagator.data(), &OwncloudPropagator::itemCompleted, - this, &SyncEngine::slotItemCompleted); - connect(_propagator.data(), &OwncloudPropagator::progress, - this, &SyncEngine::slotProgress); - connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotFinished, Qt::QueuedConnection); - connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile); - connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile); - connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage); - connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage); - connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem); // apply the network limits to the propagator setNetworkLimits(_uploadLimit, _downloadLimit); - deleteStaleDownloadInfos(syncItems); - deleteStaleUploadInfos(syncItems); - deleteStaleErrorBlacklistEntries(syncItems); + deleteStaleDownloadInfos(_syncItems); + deleteStaleUploadInfos(_syncItems); + deleteStaleErrorBlacklistEntries(_syncItems); _journal->commit("post stale entry removal"); // Emit the started signal only after the propagator has been set up. if (_needsUpdate) emit(started()); - _propagator->start(syncItems); + _propagator->start(_syncItems); + _syncItems.clear(); qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Post-Reconcile Finished")) << "ms"; } @@ -1263,6 +1219,8 @@ void SyncEngine::slotFinished(bool success) _anotherSyncNeeded = ImmediateFollowUp; } +#if 0 + FIXME if (success) { _journal->setDataFingerprint(_discoveryMainThread->_dataFingerprint); } @@ -1270,6 +1228,7 @@ void SyncEngine::slotFinished(bool success) if (!_journal->postSyncCleanup(_seenFiles, _temporarilyUnavailablePaths)) { qCDebug(lcEngine) << "Cleaning of synced "; } +#endif conflictRecordMaintenance(); @@ -1287,9 +1246,6 @@ void SyncEngine::slotFinished(bool success) void SyncEngine::finalize(bool success) { - _thread.quit(); - _thread.wait(); - _csync_ctx->reinitialize(); _journal->close(); diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index df9138e0d..a95a5e9bc 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -231,7 +231,7 @@ private: static bool s_anySyncRunning; //true when one sync is running somewhere (for debugging) // Must only be acessed during update and reconcile - QMap _syncItemMap; + QVector _syncItems; AccountPtr _account; QScopedPointer _csync_ctx; @@ -258,7 +258,6 @@ private: // while the remote says storage not available. QSet _temporarilyUnavailablePaths; - QThread _thread; QScopedPointer _progressInfo; diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 04b60ad6e..fab9d72f1 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -55,7 +55,7 @@ SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QStri SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRecord &rec) { - SyncFileItemPtr item(new SyncFileItem); + auto item = SyncFileItemPtr::create(); item->_file = QString::fromUtf8(rec._path); item->_inode = rec._inode; item->_modtime = rec._modtime; diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index f98ea8c0a..04765dfa0 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -35,12 +35,14 @@ using SyncFileItemPtr = QSharedPointer; */ class SyncFileItem { + Q_GADGET public: enum Direction { None = 0, Up, Down }; + Q_ENUM(Direction) enum Status { // stored in 4 bits NoStatus, @@ -82,6 +84,7 @@ public: */ BlacklistedError }; + Q_ENUM(Status) SyncJournalFileRecord toSyncJournalFileRecordWithInode(const QString &localFileName);