Discovery: List local directories from thread #7456 #7439

diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp
index e1c6311cb238dc95d88ff332ad6cda1b5b9bc808..f1dfd903ca001598a23c1714c0194910ef577a60 100644 (file)
--- a/src/libsync/discovery.cpp
+++ b/src/libsync/discovery.cpp
@@ -18,9 +18,11 @@
 #include <QDebug>
 #include <algorithm>
 #include <set>
-#include <QDirIterator>
 #include <QTextCodec>
 #include "vio/csync_vio_local.h"
+#include <QFileInfo>
+#include <QFile>
+#include <QThreadPool>
 #include "common/checksums.h"
 #include "csync_exclude.h"
 #include "csync_util.h"
@@ -50,14 +52,16 @@ void ProcessDirectoryJob::start()
     }
 
     if (_queryLocal == NormalQuery) {
-        if (!runLocalQuery() && serverJob)
-            serverJob->abort();
+        _localJob = startAsyncLocalQuery();
+    } else {
+        _localQueryDone = true;
     }
-    _localQueryDone = true;
 
-    // Process is being called when both local and server entries are fetched.
-    if (_serverQueryDone)
+    // FIXME: serverJob->abort() if local failed..? This used to be in code before
+
+    if (_localQueryDone && _serverQueryDone) {
         process();
+    }
 }
 
 void ProcessDirectoryJob::process()
@@ -1426,72 +1430,59 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery()
     return serverJob;
 }
 
-bool ProcessDirectoryJob::runLocalQuery()
+DiscoverySingleLocalDirectoryJob *ProcessDirectoryJob::startAsyncLocalQuery()
 {
     QString localPath = _discoveryData->_localDir + _currentFolder._local;
-    if (localPath.endsWith('/')) // Happens if _currentFolder._local.isEmpty()
-        localPath.chop(1);
-    auto dh = csync_vio_local_opendir(localPath);
-    if (!dh) {
-        qCInfo(lcDisco) << "Error while opening directory" << (localPath) << errno;
-        QString errorString = tr("Error while opening directory %1").arg(localPath);
-        if (errno == EACCES) {
-            errorString = tr("Directory not accessible on client, permission denied");
-            if (_dirItem) {
-                _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE;
-                _dirItem->_errorString = errorString;
-                emit finished();
-                return false;
-            }
-        } else if (errno == ENOENT) {
-            errorString = tr("Directory not found: %1").arg(localPath);
-        } else if (errno == ENOTDIR) {
-            // Not a directory..
-            // Just consider it is empty
-            return true;
-        }
-        emit _discoveryData->fatalError(errorString);
-        return false;
-    }
-    errno = 0;
-    while (auto dirent = csync_vio_local_readdir(dh, _discoveryData->_syncOptions._vfs.data())) {
-        if (dirent->type == ItemTypeSkip)
-            continue;
-        LocalInfo i;
-        static QTextCodec *codec = QTextCodec::codecForName("UTF-8");
-        ASSERT(codec);
-        QTextCodec::ConverterState state;
-        i.name = codec->toUnicode(dirent->path, dirent->path.size(), &state);
-        if (state.invalidChars > 0 || state.remainingChars > 0) {
-            _childIgnored = true;
-            auto item = SyncFileItemPtr::create();
-            item->_file = _currentFolder._target + i.name;
-            item->_instruction = CSYNC_INSTRUCTION_IGNORE;
-            item->_status = SyncFileItem::NormalError;
-            item->_errorString = tr("Filename encoding is not valid");
-            emit _discoveryData->itemDiscovered(item);
-            continue;
+    auto localJob = new DiscoverySingleLocalDirectoryJob(_discoveryData->_account, localPath, _discoveryData->_syncOptions._vfs.data(), this);
+
+    _discoveryData->_currentlyActiveJobs++;
+    _pendingAsyncJobs++;
+
+    connect(localJob, &DiscoverySingleLocalDirectoryJob::itemDiscovered, _discoveryData, &DiscoveryPhase::itemDiscovered);
+
+    connect(localJob, &DiscoverySingleLocalDirectoryJob::childIgnored, this, [this](bool b) {
+        _childIgnored = b;
+    });
+
+    connect(localJob, &DiscoverySingleLocalDirectoryJob::finishedFatalError, this, [this](const QString &msg) {
+        _discoveryData->_currentlyActiveJobs--;
+        _pendingAsyncJobs--;
+
+        emit _discoveryData->fatalError(msg);
+    });
+
+    connect(localJob, &DiscoverySingleLocalDirectoryJob::finishedNonFatalError, this, [this](const QString &msg) {
+        _discoveryData->_currentlyActiveJobs--;
+        _pendingAsyncJobs--;
+
+        if (_dirItem) {
+            _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE;
+            _dirItem->_errorString = msg;
+            emit this->finished();
+        } else {
+            // Fatal for the root job since it has no SyncFileItem
+            emit _discoveryData->fatalError(msg);
         }
-        i.modtime = dirent->modtime;
-        i.size = dirent->size;
-        i.inode = dirent->inode;
-        i.isDirectory = dirent->type == ItemTypeDirectory;
-        i.isHidden = dirent->is_hidden;
-        i.isSymLink = dirent->type == ItemTypeSoftLink;
-        i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload;
-        i.type = dirent->type;
-        _localNormalQueryEntries.push_back(i);
-    }
-    csync_vio_local_closedir(dh);
-    if (errno != 0) {
-        // Note: Windows vio converts any error into EACCES
-        qCWarning(lcDisco) << "readdir failed for file in " << _currentFolder._local << " - errno: " << errno;
-        emit _discoveryData->fatalError(tr("Error while reading directory %1").arg(localPath));
-        return false;
-    }
-    return true;
+    });
+
+    connect(localJob, &DiscoverySingleLocalDirectoryJob::finished, this, [this](const auto &results) {
+        _discoveryData->_currentlyActiveJobs--;
+        _pendingAsyncJobs--;
+
+        _localNormalQueryEntries = results;
+        _localQueryDone = true;
+
+        if (_serverQueryDone)
+            this->process();
+    });
+
+    QThreadPool *pool = QThreadPool::globalInstance();
+    pool->start(localJob);
+
+    return localJob;
 }
 
+
 bool ProcessDirectoryJob::isVfsWithSuffix() const
 {
     return _discoveryData->_syncOptions._vfs->mode() == Vfs::WithSuffix;

diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h
index 96295c9dbeb35c988a82b81668ca8ab1c7ad3603..e731562b21cb72f1fda968b1f0afa0df97d726c8 100644 (file)
--- a/src/libsync/discovery.h
+++ b/src/libsync/discovery.h
@@ -202,11 +202,12 @@ private:
      */
     DiscoverySingleDirectoryJob *startAsyncServerQuery();
 
-    /** Discover the local directory now
+    /** Discover the local directory
       *
       * Fills _localNormalQueryEntries.
       */
-    bool runLocalQuery();
+    DiscoverySingleLocalDirectoryJob *startAsyncLocalQuery();
+
 
     /** Sets _pinState, the directory's pin state
      *
@@ -242,6 +243,9 @@ private:
     RemotePermissions _rootPermissions;
     QPointer<DiscoverySingleDirectoryJob> _serverJob;
 
+    QPointer<DiscoverySingleLocalDirectoryJob> _localJob;
+
+
     /** Number of currently running async jobs.
      *
      * These "async jobs" have nothing to do with the jobs for subdirectories

diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp
index 233cf16a47214bf72e5fd2f97eec4a35f15ec381..f4ca7eaa3f0d90019f9322b4eedcfcf11127d187 100644 (file)
--- a/src/libsync/discoveryphase.cpp
+++ b/src/libsync/discoveryphase.cpp
@@ -20,10 +20,13 @@
 #include "common/checksums.h"
 
 #include <csync_exclude.h>
+#include "vio/csync_vio_local.h"
 
 #include <QLoggingCategory>
 #include <QUrl>
+#include <QFile>
 #include <QFileInfo>
+#include <QTextCodec>
 #include <cstring>
 
 
@@ -214,6 +217,78 @@ void DiscoveryPhase::scheduleMoreJobs()
     }
 }
 
+DiscoverySingleLocalDirectoryJob::DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent)
+ : QObject(parent), QRunnable(), _localPath(localPath), _account(account), _vfs(vfs)
+{
+    qRegisterMetaType<QVector<LocalInfo> >("QVector<LocalInfo>");
+}
+
+// Use as QRunnable
+void DiscoverySingleLocalDirectoryJob::run() {
+    QString localPath = _localPath;
+    if (localPath.endsWith('/')) // Happens if _currentFolder._local.isEmpty()
+        localPath.chop(1);
+
+    auto dh = csync_vio_local_opendir(localPath);
+    if (!dh) {
+        qCInfo(lcDiscovery) << "Error while opening directory" << (localPath) << errno;
+        QString errorString = tr("Error while opening directory %1").arg(localPath);
+        if (errno == EACCES) {
+            errorString = tr("Directory not accessible on client, permission denied");
+            emit finishedNonFatalError(errorString);
+            return;
+        } else if (errno == ENOENT) {
+            errorString = tr("Directory not found: %1").arg(localPath);
+        } else if (errno == ENOTDIR) {
+            // Not a directory..
+            // Just consider it is empty
+            return;
+        }
+        emit finishedFatalError(errorString);
+        return;
+    }
+    errno = 0;
+    QVector<LocalInfo> results;
+    while (auto dirent = csync_vio_local_readdir(dh, _vfs)) {
+        if (dirent->type == ItemTypeSkip)
+            continue;
+        LocalInfo i;
+        static QTextCodec *codec = QTextCodec::codecForName("UTF-8");
+        ASSERT(codec);
+        QTextCodec::ConverterState state;
+        i.name = codec->toUnicode(dirent->path, dirent->path.size(), &state);
+        if (state.invalidChars > 0 || state.remainingChars > 0) {
+            emit childIgnored(true);
+            auto item = SyncFileItemPtr::create();
+            //item->_file = _currentFolder._target + i.name;
+            // FIXME ^^ do we really need to use _target or is local fine?
+            item->_file = _localPath + i.name;
+            item->_instruction = CSYNC_INSTRUCTION_IGNORE;
+            item->_status = SyncFileItem::NormalError;
+            item->_errorString = tr("Filename encoding is not valid");
+            emit itemDiscovered(item);
+            continue;
+        }
+        i.modtime = dirent->modtime;
+        i.size = dirent->size;
+        i.inode = dirent->inode;
+        i.isDirectory = dirent->type == ItemTypeDirectory;
+        i.isHidden = dirent->is_hidden;
+        i.isSymLink = dirent->type == ItemTypeSoftLink;
+        i.isVirtualFile = dirent->type == ItemTypeVirtualFile || dirent->type == ItemTypeVirtualFileDownload;
+        i.type = dirent->type;
+        results.push_back(i);
+    }
+    csync_vio_local_closedir(dh);
+    if (errno != 0) {
+        // Note: Windows vio converts any error into EACCES
+        qCWarning(lcDiscovery) << "readdir failed for file in " << localPath << " - errno: " << errno;
+        emit finishedFatalError(tr("Error while reading directory %1").arg(localPath));
+        return;
+    }
+    emit finished(results);
+}
+
 DiscoverySingleDirectoryJob::DiscoverySingleDirectoryJob(const AccountPtr &account, const QString &path, QObject *parent)
     : QObject(parent)
     , _subPath(path)

diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h
index 8fbaafc7c9761791e6d7dd4021a3ccadea6fbc5b..ba098c5452fde5ed2f396c96dbeb63a5600ed5af 100644 (file)
--- a/src/libsync/discoveryphase.h
+++ b/src/libsync/discoveryphase.h
@@ -23,6 +23,7 @@
 #include "networkjobs.h"
 #include <QMutex>
 #include <QWaitCondition>
+#include <QRunnable>
 #include <deque>
 #include "syncoptions.h"
 #include "syncfileitem.h"
@@ -76,6 +77,33 @@ struct LocalInfo
     bool isValid() const { return !name.isNull(); }
 };
 
+/**
+ * @brief Run list on a local directory and process the results for Discovery
+ *
+ * @ingroup libsync
+ */
+class DiscoverySingleLocalDirectoryJob : public QObject, public QRunnable
+{
+    Q_OBJECT
+public:
+    explicit DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent = 0);
+
+    void run() Q_DECL_OVERRIDE;
+signals:
+    void finished(const QVector<LocalInfo> &result);
+    void finishedFatalError(const QString &errorString);
+    void finishedNonFatalError(const QString &errorString);
+
+    void itemDiscovered(const SyncFileItemPtr &item);
+    void childIgnored(bool b);
+private slots:
+private:
+    QString _localPath;
+    AccountPtr _account;
+    OCC::Vfs* _vfs;
+public:
+};
+
 
 /**
  * @brief Run a PROPFIND on a directory and process the results for Discovery

diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp
index c3bc049914f57275d12717d4a25d8821cd81689b..eadf56ba2d6c0e23835c16a909c0e7d69ae1fbab 100644 (file)
--- a/test/testsyncengine.cpp
+++ b/test/testsyncengine.cpp
@@ -711,6 +711,18 @@ private slots:
     }
 #endif
 
+    void testEmptyLocalButHasRemote()
+    {
+        FakeFolder fakeFolder{ FileInfo{} };
+        fakeFolder.remoteModifier().mkdir("foo");
+
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        QVERIFY(fakeFolder.currentLocalState().find("foo"));
+
+    }
+
     // Check that server mtime is set on directories on initial propagation
     void testDirectoryInitialMtime()
     {