From: Jocelyn Turcotte Date: Thu, 14 Sep 2017 14:34:14 +0000 (+0200) Subject: SyncJournalDb: Reimplement the db_is_empty logic from csync X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~701^2~39 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=ecb4e96794340ba37914c770ef0e2dc4068eb5a8;p=nextcloud-desktop.git SyncJournalDb: Reimplement the db_is_empty logic from csync This reduces the initial sync local discovery time from 2.0 to 0.6 seconds in LargeSyncBench on my machine. --- diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index d2b6fb2a7..d4c396e6b 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -76,7 +76,9 @@ static QString defaultJournalMode(const QString &dbPath) SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent) : QObject(parent) , _dbFile(dbFilePath) + , _mutex(QMutex::Recursive) , _transaction(0) + , _metadataTableIsEmpty(false) { // Allow forcing the journal mode for debugging static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE")); @@ -672,6 +674,10 @@ bool SyncJournalDb::checkConnect() // don't start a new transaction now commitInternal(QString("checkConnect End"), false); + // This avoid reading from the DB if we already know it is empty + // thereby speeding up the initial discovery significantly. + _metadataTableIsEmpty = (getFileRecordCount() == 0); + // Hide 'em all! FileSystem::setFileHidden(databaseFilePath(), true); FileSystem::setFileHidden(databaseFilePath() + "-wal", true); @@ -715,6 +721,7 @@ void SyncJournalDb::close() _db.close(); _avoidReadFromDbOnNextSyncFilter.clear(); + _metadataTableIsEmpty = false; } @@ -973,6 +980,9 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) return false; } + // Can't be true anymore. + _metadataTableIsEmpty = false; + return true; } else { qCWarning(lcDb) << "Failed to connect database."; @@ -1020,6 +1030,9 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec rec->_path.clear(); Q_ASSERT(!rec->isValid()); + if (_metadataTableIsEmpty) + return true; // no error, yet nothing found (rec->isValid() == false) + if (!checkConnect()) return false; @@ -1028,7 +1041,6 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec _getFileRecordQuery->bindValue(1, getPHash(filename)); if (!_getFileRecordQuery->exec()) { - locker.unlock(); close(); return false; } @@ -1040,9 +1052,7 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE QString err = _getFileRecordQuery->error(); qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err; - locker.unlock(); close(); - locker.relock(); } } } @@ -1058,7 +1068,7 @@ bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *r rec->_path.clear(); Q_ASSERT(!rec->isValid()); - if (!inode) + if (!inode || _metadataTableIsEmpty) return true; // no error, yet nothing found (rec->isValid() == false) if (!checkConnect()) @@ -1087,7 +1097,7 @@ bool SyncJournalDb::getFileRecordByFileId(const QByteArray &fileId, SyncJournalF rec->_path.clear(); Q_ASSERT(!rec->isValid()); - if (fileId.isEmpty()) + if (fileId.isEmpty() || _metadataTableIsEmpty) return true; // no error, yet nothing found (rec->isValid() == false) if (!checkConnect()) @@ -1111,6 +1121,9 @@ bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::functio { QMutexLocker locker(&_mutex); + if (_metadataTableIsEmpty) + return true; // no error, yet nothing found + if (!checkConnect()) return false; @@ -1184,15 +1197,11 @@ int SyncJournalDb::getFileRecordCount() { QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return -1; - } - SqlQuery query(_db); query.prepare("SELECT COUNT(*) FROM metadata"); if (!query.exec()) { - return 0; + return -1; } if (query.next()) { @@ -1200,7 +1209,7 @@ int SyncJournalDb::getFileRecordCount() return count; } - return 0; + return -1; } bool SyncJournalDb::updateFileRecordChecksum(const QString &filename, @@ -1775,7 +1784,6 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path) // We also need to remove the ETags so the update phase refreshes the directory paths // on the next sync - locker.unlock(); avoidReadFromDbOnNextSync(path); } diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 10324aba7..cfdced9a3 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -66,7 +66,6 @@ public: bool setFileRecordMetadata(const SyncJournalFileRecord &record); bool deleteFileRecord(const QString &filename, bool recursively = false); - int getFileRecordCount(); bool updateFileRecordChecksum(const QString &filename, const QByteArray &contentChecksum, const QByteArray &contentChecksumType); @@ -215,6 +214,7 @@ public: void clearFileTable(); private: + int getFileRecordCount(); bool updateDatabaseStructure(); bool updateMetadataTableStructure(); bool updateErrorBlacklistTableStructure(); @@ -237,6 +237,7 @@ private: QString _dbFile; QMutex _mutex; // Public functions are protected with the mutex. int _transaction; + bool _metadataTableIsEmpty; // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close QScopedPointer _getFileRecordQuery; diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp index d2838884a..4f0c19916 100644 --- a/src/csync/csync.cpp +++ b/src/csync/csync.cpp @@ -310,7 +310,6 @@ int csync_s::reinitialize() { remote.read_from_db = 0; read_remote_from_db = true; - db_is_empty = false; local.files.clear(); remote.files.clear(); diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h index c898eeabe..60d12df91 100644 --- a/src/csync/csync_private.h +++ b/src/csync/csync_private.h @@ -142,12 +142,6 @@ struct OCSYNC_EXPORT csync_s { */ bool read_remote_from_db = false; - /** - * If true, the DB is considered empty and all reads are skipped. (default is false) - * This is useful during the initial local discovery as it speeds it up significantly. - */ - bool db_is_empty = false; - bool ignore_hidden_files = true; csync_s(const char *localUri, OCC::SyncJournalDb *statedb); diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index f329d42cd..2c71ae734 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -786,7 +786,6 @@ void SyncEngine::startSync() csync_resume(_csync_ctx.data()); - int fileRecordCount = -1; if (!_journal->exists()) { qCInfo(lcEngine) << "New sync (no sync journal exists)"; } else { @@ -800,9 +799,8 @@ void SyncEngine::startSync() verStr.append(" on ").append(Utility::platformName()); qCInfo(lcEngine) << verStr; - fileRecordCount = _journal->getFileRecordCount(); // this creates the DB if it does not exist yet - - if (fileRecordCount == -1) { + // This creates the DB if it does not exist yet. + if (!_journal->isConnected()) { qCWarning(lcEngine) << "No way to create a sync journal!"; csyncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder.")); finalize(false); @@ -812,10 +810,6 @@ void SyncEngine::startSync() _csync_ctx->read_remote_from_db = true; - // This tells csync to never read from the DB if it is empty - // thereby speeding up the initial discovery significantly. - _csync_ctx->db_is_empty = (fileRecordCount == 0); - bool ok; auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok); if (ok) {