From b673ab996ef0e588765ffd1e2d07f094397e2375 Mon Sep 17 00:00:00 2001 From: alex-z Date: Fri, 4 Nov 2022 17:38:59 +0100 Subject: [PATCH] Edit locally job will only sync a specific file by modifying the state of the Sync Engine. All other running syncs will get terminated. Syncing is now optimized to run faster and it checks if the file has changed on the server before syncing it. Signed-off-by: alex-z --- src/gui/editlocallyjob.cpp | 305 +++++++++++++++++++++++++---- src/gui/editlocallyjob.h | 25 ++- src/gui/editlocallymanager.cpp | 3 + src/gui/editlocallymanager.h | 2 - src/gui/folder.cpp | 15 +- src/libsync/CMakeLists.txt | 1 + src/libsync/account.cpp | 20 +- src/libsync/account.h | 3 + src/libsync/discovery.cpp | 17 ++ src/libsync/discovery.h | 80 ++++---- src/libsync/discoveryphase.cpp | 1 + src/libsync/discoveryphase.h | 2 + src/libsync/helpers.cpp | 25 +++ src/libsync/helpers.h | 25 +++ src/libsync/networkjobs.cpp | 20 +- src/libsync/networkjobs.h | 3 - src/libsync/owncloudpropagator_p.h | 1 + src/libsync/syncengine.cpp | 64 +++++- src/libsync/syncengine.h | 13 ++ src/libsync/syncfileitem.cpp | 71 +++++++ src/libsync/syncfileitem.h | 4 + 21 files changed, 592 insertions(+), 108 deletions(-) create mode 100644 src/libsync/helpers.cpp create mode 100644 src/libsync/helpers.h diff --git a/src/gui/editlocallyjob.cpp b/src/gui/editlocallyjob.cpp index 63f9515f8..93ca47628 100644 --- a/src/gui/editlocallyjob.cpp +++ b/src/gui/editlocallyjob.cpp @@ -118,52 +118,199 @@ void EditLocallyJob::remoteTokenCheckResultReceived(const int statusCode) return; } - proceedWithSetup(); + findAfolderAndConstructPaths(); } void EditLocallyJob::proceedWithSetup() { if (!_tokenVerified) { qCWarning(lcEditLocallyJob) << "Could not proceed with setup as token is not verified."; + showError(tr("Could not validate the request to open a file from server."), tr("Please try again.")); return; } - const auto foundFiles = FolderMan::instance()->findFileInLocalFolders(_relPath, _accountState->account()); - - if (foundFiles.isEmpty()) { - if (isRelPathExcluded(_relPath)) { - showError(tr("Could not find a file for local editing. Make sure it is not excluded via selective sync."), _relPath); - } else { - showError(tr("Could not find a file for local editing. Make sure its path is valid and it is synced locally."), _relPath); - } + const auto relPathSplit = _relPath.split(QLatin1Char('/')); + if (relPathSplit.isEmpty()) { + showError(tr("Could not find a file for local editing. Make sure its path is valid and it is synced locally."), _relPath); return; } - _localFilePath = foundFiles.first(); - _folderForFile = FolderMan::instance()->folderForPath(_localFilePath); + _fileName = relPathSplit.last(); + + _folderForFile = findFolderForFile(_relPath, _userId); if (!_folderForFile) { - showError(tr("Could not find a folder to sync."), _relPath); + showError(tr("Could not find a file for local editing. Make sure it is not excluded via selective sync."), _relPath); return; } - const auto relPathSplit = _relPath.split(QLatin1Char('/')); - if (relPathSplit.isEmpty()) { + if (_relPathParent != QStringLiteral("/") && (!_fileParentItem || _fileParentItem->isEmpty())) { showError(tr("Could not find a file for local editing. Make sure its path is valid and it is synced locally."), _relPath); return; } - _fileName = relPathSplit.last(); + _localFilePath = _folderForFile->path() + _relativePathToRemoteRoot; Systray::instance()->destroyEditFileLocallyLoadingDialog(); Q_EMIT setupFinished(); } +void EditLocallyJob::findAfolderAndConstructPaths() +{ + _folderForFile = findFolderForFile(_relPath, _userId); + + if (!_folderForFile) { + showError(tr("Could not find a file for local editing. Make sure it is not excluded via selective sync."), _relPath); + return; + } + + _relativePathToRemoteRoot = getRelativePathToRemoteRootForFile(); + + if (_relativePathToRemoteRoot.isEmpty()) { + qCWarning(lcEditLocallyJob) << "_relativePathToRemoteRoot is empty for" << _relPath; + showError(tr("Could not find a file for local editing. Make sure it is not excluded via selective sync."), _relPath); + return; + } + + _relPathParent = getRelativePathParent(); + + if (_relPathParent.isEmpty()) { + showError(tr("Could not find a file for local editing. Make sure it is not excluded via selective sync."), _relPath); + return; + } + + if (_relPathParent == QStringLiteral("/")) { + proceedWithSetup(); + return; + } + + fetchRemoteFileParentInfo(); +} + QString EditLocallyJob::prefixSlashToPath(const QString &path) { return path.startsWith('/') ? path : QChar::fromLatin1('/') + path; } +void EditLocallyJob::fetchRemoteFileParentInfo() +{ + Q_ASSERT(_relPathParent != QStringLiteral("/")); + + if (_relPathParent == QStringLiteral("/")) { + qCWarning(lcEditLocallyJob) << "LsColJob must only be used for nested folders."; + return; + } + + const auto job = new LsColJob(_accountState->account(), QDir::cleanPath(_folderForFile->remotePathTrailingSlash() + _relPathParent), this); + const QList props{QByteArrayLiteral("resourcetype"), + QByteArrayLiteral("getlastmodified"), + QByteArrayLiteral("getetag"), + QByteArrayLiteral("http://owncloud.org/ns:size"), + QByteArrayLiteral("http://owncloud.org/ns:id"), + QByteArrayLiteral("http://owncloud.org/ns:permissions"), + QByteArrayLiteral("http://owncloud.org/ns:checksums")}; + + job->setProperties(props); + connect(job, &LsColJob::directoryListingIterated, this, &EditLocallyJob::slotDirectoryListingIterated); + connect(job, &LsColJob::finishedWithoutError, this, &EditLocallyJob::proceedWithSetup); + connect(job, &LsColJob::finishedWithError, this, &EditLocallyJob::slotLsColJobFinishedWithError); + job->start(); +} + +bool EditLocallyJob::checkIfFileParentSyncIsNeeded() +{ + if (_relPathParent == QLatin1String("/")) { + return true; + } + + Q_ASSERT(_fileParentItem && !_fileParentItem->isEmpty()); + + if (!_fileParentItem || _fileParentItem->isEmpty()) { + return true; + } + + SyncJournalFileRecord rec; + if (!_folderForFile->journalDb()->getFileRecord(_fileParentItem->_file, &rec) || !rec.isValid()) { + // we don't have this folder locally, so let's sync it + _fileParentItem->_direction = SyncFileItem::Down; + _fileParentItem->_instruction = CSYNC_INSTRUCTION_NEW; + } else if (rec._etag != _fileParentItem->_etag && rec._modtime != _fileParentItem->_modtime) { + // we just need to update metadata as the folder is already present locally + _fileParentItem->_direction = rec._modtime < _fileParentItem->_modtime ? SyncFileItem::Down : SyncFileItem::Up; + _fileParentItem->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + } else { + _fileParentItem->_direction = SyncFileItem::Down; + _fileParentItem->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; + SyncJournalFileRecord recFile; + if (_folderForFile->journalDb()->getFileRecord(_relativePathToRemoteRoot, &recFile) && recFile.isValid()) { + return false; + } + } + return true; +} + +void EditLocallyJob::startSyncBeforeOpening() +{ + eraseBlacklistRecordForItem(); + if (!checkIfFileParentSyncIsNeeded()) { + openFile(); + return; + } + + // connect to a SyncEngine::itemDiscovered so we can complete the job as soon as the file in question is discovered + QObject::connect(&_folderForFile->syncEngine(), &SyncEngine::itemDiscovered, this, &EditLocallyJob::slotItemDiscovered); + _folderForFile->syncEngine().setSingleItemDiscoveryOptions({_relPathParent == QStringLiteral("/") ? QString{} : _relPathParent, _relativePathToRemoteRoot, _fileParentItem}); + FolderMan::instance()->forceSyncForFolder(_folderForFile); +} + +void EditLocallyJob::eraseBlacklistRecordForItem() +{ + if (!_folderForFile || !_fileParentItem) { + qCWarning(lcEditLocallyJob) << "_folderForFile or _fileParentItem is invalid!"; + return; + } + Q_ASSERT(!_folderForFile->isSyncRunning()); + if (_folderForFile->isSyncRunning()) { + qCWarning(lcEditLocallyJob) << "_folderForFile is syncing"; + return; + } + if (_folderForFile->journalDb()->errorBlacklistEntry(_fileParentItem->_file).isValid()) { + _folderForFile->journalDb()->wipeErrorBlacklistEntry(_fileParentItem->_file); + } +} + +const QString EditLocallyJob::getRelativePathToRemoteRootForFile() const +{ + Q_ASSERT(_folderForFile); + if (!_folderForFile) { + return {}; + } + + if (_folderForFile->remotePathTrailingSlash().size() == 1) { + return _relPath; + } else { + const auto remoteFolderPathWithTrailingSlash = _folderForFile->remotePathTrailingSlash(); + const auto remoteFolderPathWithoutLeadingSlash = + remoteFolderPathWithTrailingSlash.startsWith(QLatin1Char('/')) ? remoteFolderPathWithTrailingSlash.mid(1) : remoteFolderPathWithTrailingSlash; + + return _relPath.startsWith(remoteFolderPathWithoutLeadingSlash) ? _relPath.mid(remoteFolderPathWithoutLeadingSlash.size()) : _relPath; + } +} + +const QString EditLocallyJob::getRelativePathParent() const +{ + Q_ASSERT(!_relativePathToRemoteRoot.isEmpty()); + if (_relativePathToRemoteRoot.isEmpty()) { + return {}; + } + auto relativePathToRemoteRootSplit = _relativePathToRemoteRoot.split(QLatin1Char('/')); + if (relativePathToRemoteRootSplit.size() > 1) { + relativePathToRemoteRootSplit.removeLast(); + return relativePathToRemoteRootSplit.join(QLatin1Char('/')); + } + return QStringLiteral("/"); +} + bool EditLocallyJob::isTokenValid(const QString &token) { if (token.isEmpty()) { @@ -201,24 +348,49 @@ bool EditLocallyJob::isRelPathValid(const QString &relPath) return true; } -bool EditLocallyJob::isRelPathExcluded(const QString &relPath) +OCC::Folder *EditLocallyJob::findFolderForFile(const QString &relPath, const QString &userId) { if (relPath.isEmpty()) { - return false; + return nullptr; } const auto folderMap = FolderMan::instance()->map(); + + const auto relPathSplit = relPath.split(QLatin1Char('/')); + + // a file is on the first level of remote root, so, we just need a proper folder that points to a remote root + if (relPathSplit.size() == 1) { + const auto foundIt = std::find_if(std::begin(folderMap), std::end(folderMap), [&userId](const OCC::Folder *folder) { + return folder->remotePath() == QStringLiteral("/") && folder->accountState()->account()->userIdAtHostWithPort() == userId; + }); + + return foundIt != std::end(folderMap) ? foundIt.value() : nullptr; + } + + const auto relPathWithSlash = relPath.startsWith(QStringLiteral("/")) ? relPath : QStringLiteral("/") + relPath; + for (const auto &folder : folderMap) { - bool result = false; + // make sure we properly handle folders with non-root(nested) remote paths + if ((folder->remotePath() != QStringLiteral("/") && !relPathWithSlash.startsWith(folder->remotePath())) + || folder->accountState()->account()->userIdAtHostWithPort() != userId) { + continue; + } + auto result = false; const auto excludedThroughSelectiveSync = folder->journalDb()->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &result); + auto isExcluded = false; for (const auto &excludedPath : excludedThroughSelectiveSync) { if (relPath.startsWith(excludedPath)) { - return true; + isExcluded = true; + break; } } + if (isExcluded) { + continue; + } + return folder; } - return false; + return nullptr; } void EditLocallyJob::showError(const QString &message, const QString &informativeText) @@ -271,32 +443,95 @@ void EditLocallyJob::startEditLocally() Systray::instance()->createEditFileLocallyLoadingDialog(_fileName); - _folderForFile->startSync(); - const auto syncFinishedConnection = connect(_folderForFile, &Folder::syncFinished, - this, &EditLocallyJob::folderSyncFinished); + if (_folderForFile->isSyncRunning()) { + // in case sync is already running - terminate it and start a new one + _syncTerminatedConnection = connect(_folderForFile, &Folder::syncFinished, this, [this]() { + disconnect(_syncTerminatedConnection); + _syncTerminatedConnection = {}; + startSyncBeforeOpening(); + }); + _folderForFile->slotTerminateSync(); - EditLocallyManager::instance()->folderSyncFinishedConnections.insert(_localFilePath, - syncFinishedConnection); + return; + } + startSyncBeforeOpening(); } -void EditLocallyJob::folderSyncFinished(const OCC::SyncResult &result) +void EditLocallyJob::slotItemCompleted(const OCC::SyncFileItemPtr &item) { - Q_UNUSED(result) - disconnectSyncFinished(); - openFile(); + Q_ASSERT(item && !item->isEmpty()); + if (!item || item->isEmpty()) { + qCWarning(lcEditLocallyJob) << "invalid item"; + } + if (item->_file == _relativePathToRemoteRoot) { + disconnect(&_folderForFile->syncEngine(), &SyncEngine::itemCompleted, this, &EditLocallyJob::slotItemCompleted); + disconnect(&_folderForFile->syncEngine(), &SyncEngine::itemDiscovered, this, &EditLocallyJob::slotItemDiscovered); + openFile(); + } } -void EditLocallyJob::disconnectSyncFinished() const +void EditLocallyJob::slotLsColJobFinishedWithError(QNetworkReply *reply) { - if(_localFilePath.isEmpty()) { + const auto contentType = reply->header(QNetworkRequest::ContentTypeHeader).toString(); + const auto invalidContentType = !contentType.contains(QStringLiteral("application/xml; charset=utf-8")) + && !contentType.contains(QStringLiteral("application/xml; charset=\"utf-8\"")) && !contentType.contains(QStringLiteral("text/xml; charset=utf-8")) + && !contentType.contains(QStringLiteral("text/xml; charset=\"utf-8\"")); + const auto httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + qCWarning(lcEditLocallyJob) << "LSCOL job error" << reply->errorString() << httpCode << reply->error(); + + const auto message = reply->error() == QNetworkReply::NoError && invalidContentType + ? tr("Server error: PROPFIND reply is not XML formatted!") : reply->errorString(); + qCWarning(lcEditLocallyJob) << "Could not proceed with setup as file PROPFIND job has failed." << httpCode << message; + showError(tr("Could not find a remote file info for local editing. Make sure its path is valid."), _relPath); +} + +void EditLocallyJob::slotDirectoryListingIterated(const QString &name, const QMap &properties) +{ + Q_ASSERT(_relPathParent != QStringLiteral("/")); + + if (_relPathParent == QStringLiteral("/")) { + qCWarning(lcEditLocallyJob) << "LsColJob must only be used for nested folders."; + return; + } + + const auto job = qobject_cast(sender()); + Q_ASSERT(job); + if (!job) { + qCWarning(lcEditLocallyJob) << "Must call slotDirectoryListingIterated from a signal."; return; } - const auto manager = EditLocallyManager::instance(); + if (name.endsWith(_relPathParent)) { + // let's remove remote dav path and remote root from the beginning of the name + const auto nameWithoutDavPath = name.mid(_accountState->account()->davPath().size()); + + const auto remoteFolderPathWithTrailingSlash = _folderForFile->remotePathTrailingSlash(); + const auto remoteFolderPathWithoutLeadingSlash = remoteFolderPathWithTrailingSlash.startsWith(QLatin1Char('/')) + ? remoteFolderPathWithTrailingSlash.mid(1) : remoteFolderPathWithTrailingSlash; - if (const auto existingConnection = manager->folderSyncFinishedConnections.value(_localFilePath)) { - disconnect(existingConnection); - manager->folderSyncFinishedConnections.remove(_localFilePath); + const auto cleanName = nameWithoutDavPath.startsWith(remoteFolderPathWithoutLeadingSlash) + ? nameWithoutDavPath.mid(remoteFolderPathWithoutLeadingSlash.size()) : nameWithoutDavPath; + disconnect(job, &LsColJob::directoryListingIterated, this, &EditLocallyJob::slotDirectoryListingIterated); + _fileParentItem = SyncFileItem::fromProperties(cleanName, properties); + } +} + +void EditLocallyJob::slotItemDiscovered(const OCC::SyncFileItemPtr &item) +{ + Q_ASSERT(item && !item->isEmpty()); + if (!item || item->isEmpty()) { + qCWarning(lcEditLocallyJob) << "invalid item"; + } + if (item->_file == _relativePathToRemoteRoot) { + disconnect(&_folderForFile->syncEngine(), &SyncEngine::itemDiscovered, this, &EditLocallyJob::slotItemDiscovered); + if (item->_instruction == CSYNC_INSTRUCTION_NONE) { + // return early if the file is already in sync + slotItemCompleted(item); + return; + } + // or connect to the SyncEngine::itemCompleted and wait till the file gets sycned + QObject::connect(&_folderForFile->syncEngine(), &SyncEngine::itemCompleted, this, &EditLocallyJob::slotItemCompleted); } } diff --git a/src/gui/editlocallyjob.h b/src/gui/editlocallyjob.h index 258382a80..a2293bffc 100644 --- a/src/gui/editlocallyjob.h +++ b/src/gui/editlocallyjob.h @@ -17,6 +17,7 @@ #include #include "accountstate.h" +#include "syncfileitem.h" namespace OCC { @@ -38,7 +39,7 @@ public: [[nodiscard]] static bool isTokenValid(const QString &token); [[nodiscard]] static bool isRelPathValid(const QString &relPath); - [[nodiscard]] static bool isRelPathExcluded(const QString &relPath); + [[nodiscard]] static OCC::Folder *findFolderForFile(const QString &relPath, const QString &userId); [[nodiscard]] static QString prefixSlashToPath(const QString &path); signals: @@ -51,31 +52,47 @@ public slots: void startEditLocally(); private slots: + void fetchRemoteFileParentInfo(); + void startSyncBeforeOpening(); + void eraseBlacklistRecordForItem(); + void startTokenRemoteCheck(); void proceedWithSetup(); + void findAfolderAndConstructPaths(); void showError(const QString &message, const QString &informativeText); void showErrorNotification(const QString &message, const QString &informativeText) const; void showErrorMessageBox(const QString &message, const QString &informativeText) const; void remoteTokenCheckResultReceived(const int statusCode); - void folderSyncFinished(const OCC::SyncResult &result); + void slotItemDiscovered(const OCC::SyncFileItemPtr &item); + void slotItemCompleted(const OCC::SyncFileItemPtr &item); + + void slotLsColJobFinishedWithError(QNetworkReply *reply); + void slotDirectoryListingIterated(const QString &name, const QMap &properties); - void disconnectSyncFinished() const; void openFile(); private: + [[nodiscard]] bool checkIfFileParentSyncIsNeeded(); // returns true if sync will be needed, false otherwise + [[nodiscard]] const QString getRelativePathToRemoteRootForFile() const; // returns either '/' or a (relative path - Folder::remotePath()) for folders pointing to a non-root remote path e.g. '/subfolder' instead of '/' + [[nodiscard]] const QString getRelativePathParent() const; + bool _tokenVerified = false; AccountStatePtr _accountState; QString _userId; - QString _relPath; + QString _relPath; // full remote path for a file (as on the server) + QString _relativePathToRemoteRoot; // (relative path - Folder::remotePath()) for folders pointing to a non-root remote path e.g. '/subfolder' instead of '/' + QString _relPathParent; // a folder where the file resides ('/' if it is in the first level of a remote root, or e.g. a '/subfolder/a/b/c if it resides in a nested folder) QString _token; + SyncFileItemPtr _fileParentItem; QString _fileName; QString _localFilePath; Folder *_folderForFile = nullptr; std::unique_ptr _checkTokenJob; + QMetaObject::Connection _syncTerminatedConnection = {}; }; } diff --git a/src/gui/editlocallymanager.cpp b/src/gui/editlocallymanager.cpp index 567a4abbd..09776b31a 100644 --- a/src/gui/editlocallymanager.cpp +++ b/src/gui/editlocallymanager.cpp @@ -71,6 +71,9 @@ void EditLocallyManager::createJob(const QString &userId, const QString &relPath, const QString &token) { + if (_jobs.contains(token)) { + return; + } const EditLocallyJobPtr job(new EditLocallyJob(userId, relPath, token)); // We need to make sure the job sticks around until it is finished _jobs.insert(token, job); diff --git a/src/gui/editlocallymanager.h b/src/gui/editlocallymanager.h index cf42f1ed7..079ba2c83 100644 --- a/src/gui/editlocallymanager.h +++ b/src/gui/editlocallymanager.h @@ -28,8 +28,6 @@ class EditLocallyManager : public QObject public: [[nodiscard]] static EditLocallyManager *instance(); - QHash folderSyncFinishedConnections; - public slots: void editLocally(const QUrl &url); diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index a14d8b7c3..0c89e52a3 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -831,8 +831,11 @@ bool Folder::reloadExcludes() void Folder::startSync(const QStringList &pathList) { - Q_UNUSED(pathList) - + const auto singleItemDiscoveryOptions = _engine->singleItemDiscoveryOptions(); + Q_ASSERT(!singleItemDiscoveryOptions.discoveryDirItem || singleItemDiscoveryOptions.discoveryDirItem->isDirectory()); + if (singleItemDiscoveryOptions.discoveryDirItem && !singleItemDiscoveryOptions.discoveryDirItem->isDirectory()) { + qCCritical(lcFolder) << "startSync only accepts directory SyncFileItem, not a file."; + } if (isBusy()) { qCCritical(lcFolder) << "ERROR csync is still running and new sync requested."; return; @@ -868,7 +871,13 @@ void Folder::startSync(const QStringList &pathList) bool periodicFullLocalDiscoveryNow = fullLocalDiscoveryInterval.count() >= 0 // negative means we don't require periodic full runs && _timeSinceLastFullLocalDiscovery.hasExpired(fullLocalDiscoveryInterval.count()); - if (_folderWatcher && _folderWatcher->isReliable() + + if (!singleItemDiscoveryOptions.filePathRelative.isEmpty() + && singleItemDiscoveryOptions.discoveryDirItem && !singleItemDiscoveryOptions.discoveryDirItem->isEmpty()) { + qCInfo(lcFolder) << "Going to sync just one file"; + _engine->setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, {singleItemDiscoveryOptions.discoveryPath}); + _localDiscoveryTracker->startSyncPartialDiscovery(); + } else if (_folderWatcher && _folderWatcher->isReliable() && hasDoneFullLocalDiscovery && !periodicFullLocalDiscoveryNow) { qCInfo(lcFolder) << "Allowing local discovery to read from the database"; diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 4d1aacd7b..590dbc4c7 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -34,6 +34,7 @@ set(libsync_SRCS encryptfolderjob.cpp filesystem.h filesystem.cpp + helpers.cpp httplogger.h httplogger.cpp logger.h diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 4b0042927..a881745a5 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -64,7 +64,6 @@ constexpr int checksumRecalculateRequestServerVersionMinSupportedMajor = 24; } namespace OCC { - Q_LOGGING_CATEGORY(lcAccount, "nextcloud.sync.account", QtInfoMsg) const char app_password[] = "_app-password"; @@ -162,6 +161,25 @@ QString Account::displayName() const return dn; } +QString Account::userIdAtHostWithPort() const +{ + const auto credentialsUserSplit = credentials() ? credentials()->user().split(QLatin1Char('@')) : QStringList{}; + + if (credentialsUserSplit.isEmpty()) { + return {}; + } + + const auto userName = credentialsUserSplit.first(); + + QString dn = QStringLiteral("%1@%2").arg(userName, _url.host()); + const auto port = url().port(); + if (port > 0 && port != 80 && port != 443) { + dn.append(QLatin1Char(':')); + dn.append(QString::number(port)); + } + return dn; +} + QString Account::davDisplayName() const { return _displayName; diff --git a/src/libsync/account.h b/src/libsync/account.h index 667c41fb6..713f32886 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -115,6 +115,9 @@ public: /// The name of the account as shown in the toolbar [[nodiscard]] QString displayName() const; + /// User id in a form 'user@example.de, optionally port is added (if it is not 80 or 443) + [[nodiscard]] QString userIdAtHostWithPort() const; + /// The name of the account that is displayed as nicely as possible, /// e.g. the actual name of the user (John Doe). If this cannot be /// provided, defaults to davUser (e.g. johndoe) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 368fc84a3..aee314020 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -59,6 +59,17 @@ ProcessDirectoryJob::ProcessDirectoryJob(const PathTuple &path, const SyncFileIt computePinState(parent->_pinState); } +ProcessDirectoryJob::ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState, const PathTuple &path, const SyncFileItemPtr &dirItem, QueryMode queryLocal, qint64 lastSyncTimestamp, QObject *parent) + : QObject(parent) + , _dirItem(dirItem) + , _lastSyncTimestamp(lastSyncTimestamp) + , _queryLocal(queryLocal) + , _discoveryData(data) + , _currentFolder(path) +{ + computePinState(basePinState); +} + void ProcessDirectoryJob::start() { qCInfo(lcDisco) << "STARTING" << _currentFolder._server << _queryServer << _currentFolder._local << _queryLocal; @@ -162,6 +173,11 @@ void ProcessDirectoryJob::process() PathTuple path; path = _currentFolder.addName(e.nameOverride.isEmpty() ? f.first : e.nameOverride); + if (!_discoveryData->_listExclusiveFiles.isEmpty() && !_discoveryData->_listExclusiveFiles.contains(path._server)) { + qCInfo(lcDisco) << "Skipping a file:" << path._server << "as it is not listed in the _listExclusiveFiles"; + continue; + } + if (isVfsWithSuffix()) { // Without suffix vfs the paths would be good. But since the dbEntry and localEntry // can have different names from f.first when suffix vfs is on, make sure the @@ -213,6 +229,7 @@ void ProcessDirectoryJob::process() processFile(std::move(path), e.localEntry, e.serverEntry, e.dbEntry); } + _discoveryData->_listExclusiveFiles.clear(); QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); } diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 0667aaad2..1be9f5372 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -49,8 +49,45 @@ class ProcessDirectoryJob : public QObject { Q_OBJECT - struct PathTuple; public: + + /** Structure representing a path during discovery. A same path may have different value locally + * or on the server in case of renames. + * + * These strings never start or ends with slashes. They are all relative to the folder's root. + * Usually they are all the same and are even shared instance of the same QString. + * + * _server and _local paths will differ if there are renames, example: + * remote renamed A/ to B/ and local renamed A/X to A/Y then + * target: B/Y/file + * original: A/X/file + * local: A/Y/file + * server: B/X/file + */ + struct PathTuple { + QString _original; // Path as in the DB (before the sync) + QString _target; // Path that will be the result after the sync (and will be in the DB) + QString _server; // Path on the server (before the sync) + QString _local; // Path locally (before the sync) + static QString pathAppend(const QString &base, const QString &name) + { + return base.isEmpty() ? name : base + QLatin1Char('/') + name; + } + [[nodiscard]] PathTuple addName(const QString &name) const + { + PathTuple result; + result._original = pathAppend(_original, name); + auto buildString = [&](const QString &other) { + // Optimize by trying to keep all string implicitly shared if they are the same (common case) + return other == _original ? result._original : pathAppend(other, name); + }; + result._target = buildString(_target); + result._server = buildString(_server); + result._local = buildString(_local); + return result; + } + }; + enum QueryMode { NormalQuery, ParentDontExist, // Do not query this folder because it does not exist @@ -71,6 +108,9 @@ public: QueryMode queryLocal, QueryMode queryServer, qint64 lastSyncTimestamp, ProcessDirectoryJob *parent); + explicit ProcessDirectoryJob(DiscoveryPhase *data, PinState basePinState, const PathTuple &path, const SyncFileItemPtr &dirItem, + QueryMode queryLocal, qint64 lastSyncTimestamp, QObject *parent); + void start(); /** Start up to nbJobs, return the number of job started; emit finished() when done */ int processSubJobs(int nbJobs); @@ -96,44 +136,6 @@ private: LocalInfo localEntry; }; - /** Structure representing a path during discovery. A same path may have different value locally - * or on the server in case of renames. - * - * These strings never start or ends with slashes. They are all relative to the folder's root. - * Usually they are all the same and are even shared instance of the same QString. - * - * _server and _local paths will differ if there are renames, example: - * remote renamed A/ to B/ and local renamed A/X to A/Y then - * target: B/Y/file - * original: A/X/file - * local: A/Y/file - * server: B/X/file - */ - struct PathTuple - { - QString _original; // Path as in the DB (before the sync) - QString _target; // Path that will be the result after the sync (and will be in the DB) - QString _server; // Path on the server (before the sync) - QString _local; // Path locally (before the sync) - static QString pathAppend(const QString &base, const QString &name) - { - return base.isEmpty() ? name : base + QLatin1Char('/') + name; - } - [[nodiscard]] PathTuple addName(const QString &name) const - { - PathTuple result; - result._original = pathAppend(_original, name); - auto buildString = [&](const QString &other) { - // Optimize by trying to keep all string implicitly shared if they are the same (common case) - return other == _original ? result._original : pathAppend(other, name); - }; - result._target = buildString(_target); - result._server = buildString(_server); - result._local = buildString(_local); - return result; - } - }; - /** Iterate over entries inside the directory (non-recursively). * * Called once _serverEntries and _localEntries are filled diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index c2e06adab..760e9d593 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -14,6 +14,7 @@ #include "discoveryphase.h" #include "discovery.h" +#include "helpers.h" #include "account.h" #include "clientsideencryptionjobs.h" diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index 52471a6dc..b905d5d7f 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -294,6 +294,8 @@ public: QHash _filesNeedingScheduledSync; QVector _filesUnscheduleSync; + QStringList _listExclusiveFiles; + signals: void fatalError(const QString &errorString); void itemDiscovered(const OCC::SyncFileItemPtr &item); diff --git a/src/libsync/helpers.cpp b/src/libsync/helpers.cpp new file mode 100644 index 000000000..ff78ba447 --- /dev/null +++ b/src/libsync/helpers.cpp @@ -0,0 +1,25 @@ +#include "helpers.h" + +namespace OCC +{ +QByteArray parseEtag(const char *header) +{ + if (!header) { + return {}; + } + QByteArray result = header; + + // Weak E-Tags can appear when gzip compression is on, see #3946 + if (result.startsWith("W/")) { + result = result.mid(2); + } + + // https://github.com/owncloud/client/issues/1195 + result.replace("-gzip", ""); + + if (result.length() >= 2 && result.startsWith('"') && result.endsWith('"')) { + result = result.mid(1, result.length() - 2); + } + return result; +} +} // namespace OCC \ No newline at end of file diff --git a/src/libsync/helpers.h b/src/libsync/helpers.h new file mode 100644 index 000000000..fe3a76331 --- /dev/null +++ b/src/libsync/helpers.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#pragma once + +#include "owncloudlib.h" +#include + +namespace OCC +{ +/** Strips quotes and gzip annotations */ +OWNCLOUDSYNC_EXPORT QByteArray parseEtag(const char *header); + +} // namespace OCC \ No newline at end of file diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index a2a17567f..dd26a9550 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -38,6 +38,7 @@ #include "networkjobs.h" #include "account.h" +#include "helpers.h" #include "owncloudpropagator.h" #include "clientsideencryption.h" @@ -59,25 +60,6 @@ Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "nextcloud.sync.networkjob.determinea Q_LOGGING_CATEGORY(lcSimpleFileJob, "nextcloud.sync.networkjob.simplefilejob", QtInfoMsg) const int notModifiedStatusCode = 304; -QByteArray parseEtag(const char *header) -{ - if (!header) - return QByteArray(); - QByteArray arr = header; - - // Weak E-Tags can appear when gzip compression is on, see #3946 - if (arr.startsWith("W/")) - arr = arr.mid(2); - - // https://github.com/owncloud/client/issues/1195 - arr.replace("-gzip", ""); - - if (arr.length() >= 2 && arr.startsWith('"') && arr.endsWith('"')) { - arr = arr.mid(1, arr.length() - 2); - } - return arr; -} - RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) { diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 51924396e..fb9536043 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -30,9 +30,6 @@ class QJsonObject; namespace OCC { -/** Strips quotes and gzip annotations */ -OWNCLOUDSYNC_EXPORT QByteArray parseEtag(const char *header); - struct HttpError { int code; // HTTP error code diff --git a/src/libsync/owncloudpropagator_p.h b/src/libsync/owncloudpropagator_p.h index e203c57fe..b227d0720 100644 --- a/src/libsync/owncloudpropagator_p.h +++ b/src/libsync/owncloudpropagator_p.h @@ -16,6 +16,7 @@ #pragma once #include "owncloudpropagator.h" +#include "helpers.h" #include "syncfileitem.h" #include "networkjobs.h" #include "syncengine.h" diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index d1cbd438f..0e72b761d 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -320,6 +320,8 @@ void SyncEngine::conflictRecordMaintenance() void OCC::SyncEngine::slotItemDiscovered(const OCC::SyncFileItemPtr &item) { + emit itemDiscovered(item); + if (Utility::isConflictFile(item->_file)) _seenConflictFiles.insert(item->_file); if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) { @@ -633,8 +635,54 @@ void SyncEngine::startSync() connect(_discoveryPhase.data(), &DiscoveryPhase::silentlyExcluded, _syncFileStatusTracker.data(), &SyncFileStatusTracker::slotAddSilentlyExcluded); - auto discoveryJob = new ProcessDirectoryJob( - _discoveryPhase.data(), PinState::AlwaysLocal, _journal->keyValueStoreGetInt("last_sync", 0), _discoveryPhase.data()); + ProcessDirectoryJob *discoveryJob = nullptr; + + if (!singleItemDiscoveryOptions().filePathRelative.isEmpty()) { + _discoveryPhase->_listExclusiveFiles.clear(); + _discoveryPhase->_listExclusiveFiles.push_back(singleItemDiscoveryOptions().filePathRelative); + } + + if (!singleItemDiscoveryOptions().discoveryPath.isEmpty() && singleItemDiscoveryOptions().discoveryDirItem) { + ProcessDirectoryJob::PathTuple path = {}; + path._local = path._original = path._server = path._target = singleItemDiscoveryOptions().discoveryPath; + + SyncJournalFileRecord rec; + const auto localQueryMode = _journal->getFileRecord(singleItemDiscoveryOptions().discoveryDirItem->_file, &rec) && rec.isValid() + ? ProcessDirectoryJob::NormalQuery + : ProcessDirectoryJob::ParentDontExist; + + const auto pinState = [this, &rec]() { + if (!_syncOptions._vfs || _syncOptions._vfs->mode() == Vfs::Off) { + return PinState::AlwaysLocal; + } + if (!rec.isValid()) { + return PinState::OnlineOnly; + } + const auto pinStateInDb = _journal->internalPinStates().rawForPath(singleItemDiscoveryOptions().discoveryDirItem->_file.toUtf8()); + if (pinStateInDb) { + return *pinStateInDb; + } + return PinState::Unspecified; + }(); + + discoveryJob = new ProcessDirectoryJob( + _discoveryPhase.data(), + pinState, + path, + singleItemDiscoveryOptions().discoveryDirItem, + localQueryMode, + _journal->keyValueStoreGetInt("last_sync", 0), + _discoveryPhase.data() + ); + } else { + discoveryJob = new ProcessDirectoryJob( + _discoveryPhase.data(), + PinState::AlwaysLocal, + _journal->keyValueStoreGetInt("last_sync", 0), + _discoveryPhase.data() + ); + } + _discoveryPhase->startJob(discoveryJob); connect(discoveryJob, &ProcessDirectoryJob::etag, this, &SyncEngine::slotRootEtagReceived); connect(_discoveryPhase.data(), &DiscoveryPhase::addErrorToGui, this, &SyncEngine::addErrorToGui); @@ -874,6 +922,8 @@ void SyncEngine::slotPropagationFinished(bool success) void SyncEngine::finalize(bool success) { + setSingleItemDiscoveryOptions({}); + qCInfo(lcEngine) << "Sync run took " << _stopWatch.addLapTime(QLatin1String("Sync Finished")) << "ms"; _stopWatch.stop(); @@ -1003,6 +1053,16 @@ void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set paths = {}); + void setSingleItemDiscoveryOptions(const SingleItemDiscoveryOptions &singleItemDiscoveryOptions); + [[nodiscard]] const SyncEngine::SingleItemDiscoveryOptions &singleItemDiscoveryOptions() const; + void addAcceptedInvalidFileName(const QString& filePath); signals: @@ -157,6 +166,8 @@ signals: void transmissionProgress(const OCC::ProgressInfo &progress); + void itemDiscovered(const SyncFileItemPtr &); + /// We've produced a new sync error of a type. void syncError(const QString &message, OCC::ErrorCategory category = OCC::ErrorCategory::Normal); @@ -375,6 +386,8 @@ private: // A vector of all the (unique) scheduled sync timers QVector> _scheduledSyncTimers; + + SingleItemDiscoveryOptions _singleItemDiscoveryOptions; }; } diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 700b0d3e4..1727a8938 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -13,8 +13,10 @@ */ #include "syncfileitem.h" +#include "common/checksums.h" #include "common/syncjournalfilerecord.h" #include "common/utility.h" +#include "helpers.h" #include "filesystem.h" #include @@ -98,4 +100,73 @@ SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRec return item; } +SyncFileItemPtr SyncFileItem::fromProperties(const QString &filePath, const QMap &properties) +{ + SyncFileItemPtr item(new SyncFileItem); + item->_file = filePath; + item->_originalFile = filePath; + + const auto isDirectory = properties.value(QStringLiteral("resourcetype")).contains(QStringLiteral("collection")); + item->_type = isDirectory ? ItemTypeDirectory : ItemTypeFile; + + item->_size = isDirectory ? 0 : properties.value(QStringLiteral("size")).toInt(); + item->_fileId = properties.value(QStringLiteral("id")).toUtf8(); + + if (properties.contains(QStringLiteral("permissions"))) { + item->_remotePerm = RemotePermissions::fromServerString(properties.value("permissions")); + } + + if (!properties.value(QStringLiteral("share-types")).isEmpty()) { + item->_remotePerm.setPermission(RemotePermissions::IsShared); + } + + item->_isShared = item->_remotePerm.hasPermission(RemotePermissions::IsShared); + item->_lastShareStateFetchedTimestamp = QDateTime::currentMSecsSinceEpoch(); + + item->_isEncrypted = properties.value(QStringLiteral("is-encrypted")) == QStringLiteral("1"); + item->_locked = + properties.value(QStringLiteral("lock")) == QStringLiteral("1") ? SyncFileItem::LockStatus::LockedItem : SyncFileItem::LockStatus::UnlockedItem; + item->_lockOwnerDisplayName = properties.value(QStringLiteral("lock-owner-displayname")); + item->_lockOwnerId = properties.value(QStringLiteral("lock-owner")); + item->_lockEditorApp = properties.value(QStringLiteral("lock-owner-editor")); + + { + auto ok = false; + const auto intConvertedValue = properties.value(QStringLiteral("lock-owner-type")).toULongLong(&ok); + item->_lockOwnerType = ok ? static_cast(intConvertedValue) : SyncFileItem::LockOwnerType::UserLock; + } + + { + auto ok = false; + const auto intConvertedValue = properties.value(QStringLiteral("lock-time")).toULongLong(&ok); + item->_lockTime = ok ? intConvertedValue : 0; + } + + { + auto ok = false; + const auto intConvertedValue = properties.value(QStringLiteral("lock-timeout")).toULongLong(&ok); + item->_lockTimeout = ok ? intConvertedValue : 0; + } + + const auto date = QDateTime::fromString(properties.value(QStringLiteral("getlastmodified")), Qt::RFC2822Date); + Q_ASSERT(date.isValid()); + if (date.toSecsSinceEpoch() > 0) { + item->_modtime = date.toSecsSinceEpoch(); + } + + if (properties.contains(QStringLiteral("getetag"))) { + item->_etag = parseEtag(properties.value(QStringLiteral("getetag")).toUtf8()); + } + + if (properties.contains(QStringLiteral("checksums"))) { + item->_checksumHeader = findBestChecksum(properties.value("checksums").toUtf8()); + } + + // direction and instruction are decided later + item->_direction = SyncFileItem::None; + item->_instruction = CSYNC_INSTRUCTION_NONE; + + return item; +} + } diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index ed94c9eda..e517ee362 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -124,6 +124,10 @@ public: */ static SyncFileItemPtr fromSyncJournalFileRecord(const SyncJournalFileRecord &rec); + /** Creates a basic SyncFileItem from remote properties + */ + [[nodiscard]] static SyncFileItemPtr fromProperties(const QString &filePath, const QMap &properties); + SyncFileItem() : _type(ItemTypeSkip) -- 2.30.2