SyncJournalDb: Reimplement the db_is_empty logic from csync

This reduces the initial sync local discovery time from
2.0 to 0.6 seconds in LargeSyncBench on my machine.

diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp
index d2b6fb2a78ab807c5413af9152793eb6d53dd618..d4c396e6b6f20d635bf2c86b1040827c4943266f 100644 (file)
--- a/src/common/syncjournaldb.cpp
+++ b/src/common/syncjournaldb.cpp
@@ -76,7 +76,9 @@ static QString defaultJournalMode(const QString &dbPath)
 SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent)
     : QObject(parent)
     , _dbFile(dbFilePath)
+    , _mutex(QMutex::Recursive)
     , _transaction(0)
+    , _metadataTableIsEmpty(false)
 {
     // Allow forcing the journal mode for debugging
     static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE"));
@@ -672,6 +674,10 @@ bool SyncJournalDb::checkConnect()
     // don't start a new transaction now
     commitInternal(QString("checkConnect End"), false);
 
+    // This avoid reading from the DB if we already know it is empty
+    // thereby speeding up the initial discovery significantly.
+    _metadataTableIsEmpty = (getFileRecordCount() == 0);
+
     // Hide 'em all!
     FileSystem::setFileHidden(databaseFilePath(), true);
     FileSystem::setFileHidden(databaseFilePath() + "-wal", true);
@@ -715,6 +721,7 @@ void SyncJournalDb::close()
 
     _db.close();
     _avoidReadFromDbOnNextSyncFilter.clear();
+    _metadataTableIsEmpty = false;
 }
 
 
@@ -973,6 +980,9 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)
             return false;
         }
 
+        // Can't be true anymore.
+        _metadataTableIsEmpty = false;
+
         return true;
     } else {
         qCWarning(lcDb) << "Failed to connect database.";
@@ -1020,6 +1030,9 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
     rec->_path.clear();
     Q_ASSERT(!rec->isValid());
 
+    if (_metadataTableIsEmpty)
+        return true; // no error, yet nothing found (rec->isValid() == false)
+
     if (!checkConnect())
         return false;
 
@@ -1028,7 +1041,6 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
         _getFileRecordQuery->bindValue(1, getPHash(filename));
 
         if (!_getFileRecordQuery->exec()) {
-            locker.unlock();
             close();
             return false;
         }
@@ -1040,9 +1052,7 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec
             if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE
                 QString err = _getFileRecordQuery->error();
                 qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err;
-                locker.unlock();
                 close();
-                locker.relock();
             }
         }
     }
@@ -1058,7 +1068,7 @@ bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *r
     rec->_path.clear();
     Q_ASSERT(!rec->isValid());
 
-    if (!inode)
+    if (!inode || _metadataTableIsEmpty)
         return true; // no error, yet nothing found (rec->isValid() == false)
 
     if (!checkConnect())
@@ -1087,7 +1097,7 @@ bool SyncJournalDb::getFileRecordByFileId(const QByteArray &fileId, SyncJournalF
     rec->_path.clear();
     Q_ASSERT(!rec->isValid());
 
-    if (fileId.isEmpty())
+    if (fileId.isEmpty() || _metadataTableIsEmpty)
         return true; // no error, yet nothing found (rec->isValid() == false)
 
     if (!checkConnect())
@@ -1111,6 +1121,9 @@ bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::functio
 {
     QMutexLocker locker(&_mutex);
 
+    if (_metadataTableIsEmpty)
+        return true; // no error, yet nothing found
+
     if (!checkConnect())
         return false;
 
@@ -1184,15 +1197,11 @@ int SyncJournalDb::getFileRecordCount()
 {
     QMutexLocker locker(&_mutex);
 
-    if (!checkConnect()) {
-        return -1;
-    }
-
     SqlQuery query(_db);
     query.prepare("SELECT COUNT(*) FROM metadata");
 
     if (!query.exec()) {
-        return 0;
+        return -1;
     }
 
     if (query.next()) {
@@ -1200,7 +1209,7 @@ int SyncJournalDb::getFileRecordCount()
         return count;
     }
 
-    return 0;
+    return -1;
 }
 
 bool SyncJournalDb::updateFileRecordChecksum(const QString &filename,
@@ -1775,7 +1784,6 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path)
 
     // We also need to remove the ETags so the update phase refreshes the directory paths
     // on the next sync
-    locker.unlock();
     avoidReadFromDbOnNextSync(path);
 }
 

diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h
index 10324aba7e45819e74df22e51cb2db8a95e9010c..cfdced9a349b9f11a10a20acae87c6846cf7f619 100644 (file)
--- a/src/common/syncjournaldb.h
+++ b/src/common/syncjournaldb.h
@@ -66,7 +66,6 @@ public:
     bool setFileRecordMetadata(const SyncJournalFileRecord &record);
 
     bool deleteFileRecord(const QString &filename, bool recursively = false);
-    int getFileRecordCount();
     bool updateFileRecordChecksum(const QString &filename,
         const QByteArray &contentChecksum,
         const QByteArray &contentChecksumType);
@@ -215,6 +214,7 @@ public:
     void clearFileTable();
 
 private:
+    int getFileRecordCount();
     bool updateDatabaseStructure();
     bool updateMetadataTableStructure();
     bool updateErrorBlacklistTableStructure();
@@ -237,6 +237,7 @@ private:
     QString _dbFile;
     QMutex _mutex; // Public functions are protected with the mutex.
     int _transaction;
+    bool _metadataTableIsEmpty;
 
     // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close
     QScopedPointer<SqlQuery> _getFileRecordQuery;

diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp
index d2838884ac06da304ea365b833824e58ba7ba465..4f0c1991622791bd511c83f8b7d8fcfe017da39c 100644 (file)
--- a/src/csync/csync.cpp
+++ b/src/csync/csync.cpp
@@ -310,7 +310,6 @@ int csync_s::reinitialize() {
 
   remote.read_from_db = 0;
   read_remote_from_db = true;
-  db_is_empty = false;
 
   local.files.clear();
   remote.files.clear();

diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h
index c898eeabeb11f0d47ee481ad98bd31d17eab84d1..60d12df91ef16c6977e48acf9f470a17c55e8206 100644 (file)
--- a/src/csync/csync_private.h
+++ b/src/csync/csync_private.h
@@ -142,12 +142,6 @@ struct OCSYNC_EXPORT csync_s {
    */
   bool read_remote_from_db = false;
 
-  /**
-   * If true, the DB is considered empty and all reads are skipped. (default is false)
-   * This is useful during the initial local discovery as it speeds it up significantly.
-   */
-  bool db_is_empty = false;
-
   bool ignore_hidden_files = true;
 
   csync_s(const char *localUri, OCC::SyncJournalDb *statedb);

diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp
index f329d42cddf74914068dd40bccc9a2922dc674d8..2c71ae7346a1a0e56e7902aa141b34f871f923eb 100644 (file)
--- a/src/libsync/syncengine.cpp
+++ b/src/libsync/syncengine.cpp
@@ -786,7 +786,6 @@ void SyncEngine::startSync()
 
     csync_resume(_csync_ctx.data());
 
-    int fileRecordCount = -1;
     if (!_journal->exists()) {
         qCInfo(lcEngine) << "New sync (no sync journal exists)";
     } else {
@@ -800,9 +799,8 @@ void SyncEngine::startSync()
     verStr.append(" on ").append(Utility::platformName());
     qCInfo(lcEngine) << verStr;
 
-    fileRecordCount = _journal->getFileRecordCount(); // this creates the DB if it does not exist yet
-
-    if (fileRecordCount == -1) {
+    // This creates the DB if it does not exist yet.
+    if (!_journal->isConnected()) {
         qCWarning(lcEngine) << "No way to create a sync journal!";
         csyncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder."));
         finalize(false);
@@ -812,10 +810,6 @@ void SyncEngine::startSync()
 
     _csync_ctx->read_remote_from_db = true;
 
-    // This tells csync to never read from the DB if it is empty
-    // thereby speeding up the initial discovery significantly.
-    _csync_ctx->db_is_empty = (fileRecordCount == 0);
-
     bool ok;
     auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok);
     if (ok) {