From: Markus Goetz Date: Mon, 9 Sep 2019 13:41:57 +0000 (+0200) Subject: Discovery: List local directories from thread #7456 #7439 X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2~21^2~468^2~183 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=26b5e3635139f738a1e6acc3053f253d2574f4b0;p=nextcloud-desktop.git Discovery: List local directories from thread #7456 #7439 --- diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index e1c6311cb..f1dfd903c 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -18,9 +18,11 @@ #include #include #include -#include #include #include "vio/csync_vio_local.h" +#include +#include +#include #include "common/checksums.h" #include "csync_exclude.h" #include "csync_util.h" @@ -50,14 +52,16 @@ void ProcessDirectoryJob::start() } if (_queryLocal == NormalQuery) { - if (!runLocalQuery() && serverJob) - serverJob->abort(); + _localJob = startAsyncLocalQuery(); + } else { + _localQueryDone = true; } - _localQueryDone = true; - // Process is being called when both local and server entries are fetched. - if (_serverQueryDone) + // FIXME: serverJob->abort() if local failed..? This used to be in code before + + if (_localQueryDone && _serverQueryDone) { process(); + } } void ProcessDirectoryJob::process() @@ -1426,72 +1430,59 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() return serverJob; } -bool ProcessDirectoryJob::runLocalQuery() +DiscoverySingleLocalDirectoryJob *ProcessDirectoryJob::startAsyncLocalQuery() { QString localPath = _discoveryData->_localDir + _currentFolder._local; - if (localPath.endsWith('/')) // Happens if _currentFolder._local.isEmpty() - localPath.chop(1); - auto dh = csync_vio_local_opendir(localPath); - if (!dh) { - qCInfo(lcDisco) << "Error while opening directory" << (localPath) << errno; - QString errorString = tr("Error while opening directory %1").arg(localPath); - if (errno == EACCES) { - errorString = tr("Directory not accessible on client, permission denied"); - if (_dirItem) { - _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE; - _dirItem->_errorString = errorString; - emit finished(); - return false; - } - } else if (errno == ENOENT) { - errorString = tr("Directory not found: %1").arg(localPath); - } else if (errno == ENOTDIR) { - // Not a directory.. - // Just consider it is empty - return true; - } - emit _discoveryData->fatalError(errorString); - return false; - } - errno = 0; - while (auto dirent = csync_vio_local_readdir(dh, _discoveryData->_syncOptions._vfs.data())) { - if (dirent->type == ItemTypeSkip) - continue; - LocalInfo i; - static QTextCodec *codec = QTextCodec::codecForName("UTF-8"); - ASSERT(codec); - QTextCodec::ConverterState state; - i.name = codec->toUnicode(dirent->path, dirent->path.size(), &state); - if (state.invalidChars > 0 || state.remainingChars > 0) { - _childIgnored = true; - auto item = SyncFileItemPtr::create(); - item->_file = _currentFolder._target + i.name; - item->_instruction = CSYNC_INSTRUCTION_IGNORE; - item->_status = SyncFileItem::NormalError; - item->_errorString = tr("Filename encoding is not valid"); - emit _discoveryData->itemDiscovered(item); - continue; + auto localJob = new DiscoverySingleLocalDirectoryJob(_discoveryData->_account, localPath, _discoveryData->_syncOptions._vfs.data(), this); + + _discoveryData->_currentlyActiveJobs++; + _pendingAsyncJobs++; + + connect(localJob, &DiscoverySingleLocalDirectoryJob::itemDiscovered, _discoveryData, &DiscoveryPhase::itemDiscovered); + + connect(localJob, &DiscoverySingleLocalDirectoryJob::childIgnored, this, [this](bool b) { + _childIgnored = b; + }); + + connect(localJob, &DiscoverySingleLocalDirectoryJob::finishedFatalError, this, [this](const QString &msg) { + _discoveryData->_currentlyActiveJobs--; + _pendingAsyncJobs--; + + emit _discoveryData->fatalError(msg); + }); + + connect(localJob, &DiscoverySingleLocalDirectoryJob::finishedNonFatalError, this, [this](const QString &msg) { + _discoveryData->_currentlyActiveJobs--; + _pendingAsyncJobs--; + + if (_dirItem) { + _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE; + _dirItem->_errorString = msg; + emit this->finished(); + } else { + // Fatal for the root job since it has no SyncFileItem + emit _discoveryData->fatalError(msg); } - i.modtime = dirent->modtime; - i.size = dirent->size; - i.inode = dirent->inode; - i.isDirectory = dirent->type == ItemTypeDirectory; - i.isHidden = dirent->is_hidden; - i.isSymLink = dirent->type == ItemTypeSoftLink; - i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload; - i.type = dirent->type; - _localNormalQueryEntries.push_back(i); - } - csync_vio_local_closedir(dh); - if (errno != 0) { - // Note: Windows vio converts any error into EACCES - qCWarning(lcDisco) << "readdir failed for file in " << _currentFolder._local << " - errno: " << errno; - emit _discoveryData->fatalError(tr("Error while reading directory %1").arg(localPath)); - return false; - } - return true; + }); + + connect(localJob, &DiscoverySingleLocalDirectoryJob::finished, this, [this](const auto &results) { + _discoveryData->_currentlyActiveJobs--; + _pendingAsyncJobs--; + + _localNormalQueryEntries = results; + _localQueryDone = true; + + if (_serverQueryDone) + this->process(); + }); + + QThreadPool *pool = QThreadPool::globalInstance(); + pool->start(localJob); + + return localJob; } + bool ProcessDirectoryJob::isVfsWithSuffix() const { return _discoveryData->_syncOptions._vfs->mode() == Vfs::WithSuffix; diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 96295c9db..e731562b2 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -202,11 +202,12 @@ private: */ DiscoverySingleDirectoryJob *startAsyncServerQuery(); - /** Discover the local directory now + /** Discover the local directory * * Fills _localNormalQueryEntries. */ - bool runLocalQuery(); + DiscoverySingleLocalDirectoryJob *startAsyncLocalQuery(); + /** Sets _pinState, the directory's pin state * @@ -242,6 +243,9 @@ private: RemotePermissions _rootPermissions; QPointer _serverJob; + QPointer _localJob; + + /** Number of currently running async jobs. * * These "async jobs" have nothing to do with the jobs for subdirectories diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 233cf16a4..f4ca7eaa3 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -20,10 +20,13 @@ #include "common/checksums.h" #include +#include "vio/csync_vio_local.h" #include #include +#include #include +#include #include @@ -214,6 +217,78 @@ void DiscoveryPhase::scheduleMoreJobs() } } +DiscoverySingleLocalDirectoryJob::DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent) + : QObject(parent), QRunnable(), _localPath(localPath), _account(account), _vfs(vfs) +{ + qRegisterMetaType >("QVector"); +} + +// Use as QRunnable +void DiscoverySingleLocalDirectoryJob::run() { + QString localPath = _localPath; + if (localPath.endsWith('/')) // Happens if _currentFolder._local.isEmpty() + localPath.chop(1); + + auto dh = csync_vio_local_opendir(localPath); + if (!dh) { + qCInfo(lcDiscovery) << "Error while opening directory" << (localPath) << errno; + QString errorString = tr("Error while opening directory %1").arg(localPath); + if (errno == EACCES) { + errorString = tr("Directory not accessible on client, permission denied"); + emit finishedNonFatalError(errorString); + return; + } else if (errno == ENOENT) { + errorString = tr("Directory not found: %1").arg(localPath); + } else if (errno == ENOTDIR) { + // Not a directory.. + // Just consider it is empty + return; + } + emit finishedFatalError(errorString); + return; + } + errno = 0; + QVector results; + while (auto dirent = csync_vio_local_readdir(dh, _vfs)) { + if (dirent->type == ItemTypeSkip) + continue; + LocalInfo i; + static QTextCodec *codec = QTextCodec::codecForName("UTF-8"); + ASSERT(codec); + QTextCodec::ConverterState state; + i.name = codec->toUnicode(dirent->path, dirent->path.size(), &state); + if (state.invalidChars > 0 || state.remainingChars > 0) { + emit childIgnored(true); + auto item = SyncFileItemPtr::create(); + //item->_file = _currentFolder._target + i.name; + // FIXME ^^ do we really need to use _target or is local fine? + item->_file = _localPath + i.name; + item->_instruction = CSYNC_INSTRUCTION_IGNORE; + item->_status = SyncFileItem::NormalError; + item->_errorString = tr("Filename encoding is not valid"); + emit itemDiscovered(item); + continue; + } + i.modtime = dirent->modtime; + i.size = dirent->size; + i.inode = dirent->inode; + i.isDirectory = dirent->type == ItemTypeDirectory; + i.isHidden = dirent->is_hidden; + i.isSymLink = dirent->type == ItemTypeSoftLink; + i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload; + i.type = dirent->type; + results.push_back(i); + } + csync_vio_local_closedir(dh); + if (errno != 0) { + // Note: Windows vio converts any error into EACCES + qCWarning(lcDiscovery) << "readdir failed for file in " << localPath << " - errno: " << errno; + emit finishedFatalError(tr("Error while reading directory %1").arg(localPath)); + return; + } + emit finished(results); +} + DiscoverySingleDirectoryJob::DiscoverySingleDirectoryJob(const AccountPtr &account, const QString &path, QObject *parent) : QObject(parent) , _subPath(path) diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index 8fbaafc7c..ba098c545 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -23,6 +23,7 @@ #include "networkjobs.h" #include #include +#include #include #include "syncoptions.h" #include "syncfileitem.h" @@ -76,6 +77,33 @@ struct LocalInfo bool isValid() const { return !name.isNull(); } }; +/** + * @brief Run list on a local directory and process the results for Discovery + * + * @ingroup libsync + */ +class DiscoverySingleLocalDirectoryJob : public QObject, public QRunnable +{ + Q_OBJECT +public: + explicit DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent = 0); + + void run() Q_DECL_OVERRIDE; +signals: + void finished(const QVector &result); + void finishedFatalError(const QString &errorString); + void finishedNonFatalError(const QString &errorString); + + void itemDiscovered(const SyncFileItemPtr &item); + void childIgnored(bool b); +private slots: +private: + QString _localPath; + AccountPtr _account; + OCC::Vfs* _vfs; +public: +}; + /** * @brief Run a PROPFIND on a directory and process the results for Discovery diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp index c3bc04991..eadf56ba2 100644 --- a/test/testsyncengine.cpp +++ b/test/testsyncengine.cpp @@ -711,6 +711,18 @@ private slots: } #endif + void testEmptyLocalButHasRemote() + { + FakeFolder fakeFolder{ FileInfo{} }; + fakeFolder.remoteModifier().mkdir("foo"); + + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + QVERIFY(fakeFolder.currentLocalState().find("foo")); + + } + // Check that server mtime is set on directories on initial propagation void testDirectoryInitialMtime() {