From: Jocelyn Turcotte Date: Fri, 1 Sep 2017 16:11:43 +0000 (+0200) Subject: Move SyncJournalDB to src/common X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~701^2~81 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=81e32e1a086277bfc56abf1f818017704a8cb53d;p=nextcloud-desktop.git Move SyncJournalDB to src/common --- diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp index 60d54a1c4..d8d836ecf 100644 --- a/src/cmd/cmd.cpp +++ b/src/cmd/cmd.cpp @@ -31,7 +31,7 @@ #include "creds/httpcredentials.h" #include "simplesslerrorhandler.h" #include "syncengine.h" -#include "syncjournaldb.h" +#include "common/syncjournaldb.h" #include "config.h" #include "connectionvalidator.h" diff --git a/src/common/asserts.h b/src/common/asserts.h new file mode 100644 index 000000000..e96e57bb9 --- /dev/null +++ b/src/common/asserts.h @@ -0,0 +1,57 @@ +#ifndef OWNCLOUD_ASSERTS_H +#define OWNCLOUD_ASSERTS_H + +#include + +#if defined(QT_FORCE_ASSERTS) || !defined(QT_NO_DEBUG) +#define OC_ASSERT_MSG qFatal +#else +#define OC_ASSERT_MSG qCritical +#endif + +// For overloading macros by argument count +// See stackoverflow.com/questions/16683146/can-macros-be-overloaded-by-number-of-arguments +#define OC_ASSERT_CAT(A, B) A##B +#define OC_ASSERT_SELECT(NAME, NUM) OC_ASSERT_CAT(NAME##_, NUM) +#define OC_ASSERT_GET_COUNT(_1, _2, _3, COUNT, ...) COUNT +#define OC_ASSERT_VA_SIZE(...) OC_ASSERT_GET_COUNT(__VA_ARGS__, 3, 2, 1, 0) + +#define OC_ASSERT_OVERLOAD(NAME, ...) OC_ASSERT_SELECT(NAME, OC_ASSERT_VA_SIZE(__VA_ARGS__)) \ + (__VA_ARGS__) + +// Default assert: If the condition is false in debug builds, terminate. +// +// Prints a message on failure, even in release builds. +#define ASSERT(...) OC_ASSERT_OVERLOAD(ASSERT, __VA_ARGS__) +#define ASSERT_1(cond) \ + if (!(cond)) { \ + OC_ASSERT_MSG("ASSERT: \"%s\" in file %s, line %d", #cond, __FILE__, __LINE__); \ + } else { \ + } +#define ASSERT_2(cond, message) \ + if (!(cond)) { \ + OC_ASSERT_MSG("ASSERT: \"%s\" in file %s, line %d with message: %s", #cond, __FILE__, __LINE__, message); \ + } else { \ + } + +// Enforce condition to be true, even in release builds. +// +// Prints 'message' and aborts execution if 'cond' is false. +#define ENFORCE(...) OC_ASSERT_OVERLOAD(ENFORCE, __VA_ARGS__) +#define ENFORCE_1(cond) \ + if (!(cond)) { \ + qFatal("ENFORCE: \"%s\" in file %s, line %d", #cond, __FILE__, __LINE__); \ + } else { \ + } +#define ENFORCE_2(cond, message) \ + if (!(cond)) { \ + qFatal("ENFORCE: \"%s\" in file %s, line %d with message: %s", #cond, __FILE__, __LINE__, message); \ + } else { \ + } + +// An assert that is only present in debug builds: typically used for +// asserts that are too expensive for release mode. +// +// Q_ASSERT + +#endif diff --git a/src/common/c_jhash.h b/src/common/c_jhash.h new file mode 100644 index 000000000..699244a0d --- /dev/null +++ b/src/common/c_jhash.h @@ -0,0 +1,245 @@ +/* + * c_jhash.c Jenkins Hash + * + * Copyright (c) 1997 Bob Jenkins + * + * lookup8.c, by Bob Jenkins, January 4 1997, Public Domain. + * hash(), hash2(), hash3, and _c_mix() are externally useful functions. + * Routines to test the hash are included if SELF_TEST is defined. + * You can use this free for any purpose. It has no warranty. + * + * See http://burtleburtle.net/bob/hash/evahash.html + */ + +/** + * @file common/c_jhash.h + * + * @brief Interface of the cynapses jhash implementation + * + * @defgroup cynJHashInternals cynapses libc jhash function + * @ingroup cynLibraryAPI + * + * @{ + */ +#ifndef _C_JHASH_H +#define _C_JHASH_H + +#include + +#define c_hashsize(n) ((uint8_t) 1 << (n)) +#define c_hashmask(n) (xhashsize(n) - 1) + +/** + * _c_mix -- Mix 3 32-bit values reversibly. + * + * For every delta with one or two bit set, and the deltas of all three + * high bits or all three low bits, whether the original value of a,b,c + * is almost all zero or is uniformly distributed, + * If _c_mix() is run forward or backward, at least 32 bits in a,b,c + * have at least 1/4 probability of changing. + * If _c_mix() is run forward, every bit of c will change between 1/3 and + * 2/3 of the time. (Well, 22/100 and 78/100 for some 2-bit deltas.) + * _c_mix() was built out of 36 single-cycle latency instructions in a + * structure that could supported 2x parallelism, like so: + * a -= b; + * a -= c; x = (c>>13); + * b -= c; a ^= x; + * b -= a; x = (a<<8); + * c -= a; b ^= x; + * c -= b; x = (b>>13); + * ... + * + * Unfortunately, superscalar Pentiums and Sparcs can't take advantage + * of that parallelism. They've also turned some of those single-cycle + * latency instructions into multi-cycle latency instructions. Still, + * this is the fastest good hash I could find. There were about 2^^68 + * to choose from. I only looked at a billion or so. + */ +#define _c_mix(a,b,c) \ +{ \ + a -= b; a -= c; a ^= (c>>13); \ + b -= c; b -= a; b ^= (a<<8); \ + c -= a; c -= b; c ^= (b>>13); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<16); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>3); \ + b -= c; b -= a; b ^= (a<<10); \ + c -= a; c -= b; c ^= (b>>15); \ +} + +/** + * _c_mix64 -- Mix 3 64-bit values reversibly. + * + * _c_mix64() takes 48 machine instructions, but only 24 cycles on a superscalar + * machine (like Intel's new MMX architecture). It requires 4 64-bit + * registers for 4::2 parallelism. + * All 1-bit deltas, all 2-bit deltas, all deltas composed of top bits of + * (a,b,c), and all deltas of bottom bits were tested. All deltas were + * tested both on random keys and on keys that were nearly all zero. + * These deltas all cause every bit of c to change between 1/3 and 2/3 + * of the time (well, only 113/400 to 287/400 of the time for some + * 2-bit delta). These deltas all cause at least 80 bits to change + * among (a,b,c) when the _c_mix is run either forward or backward (yes it + * is reversible). + * This implies that a hash using _c_mix64 has no funnels. There may be + * characteristics with 3-bit deltas or bigger, I didn't test for + * those. + */ +#define _c_mix64(a,b,c) \ +{ \ + a -= b; a -= c; a ^= (c>>43); \ + b -= c; b -= a; b ^= (a<<9); \ + c -= a; c -= b; c ^= (b>>8); \ + a -= b; a -= c; a ^= (c>>38); \ + b -= c; b -= a; b ^= (a<<23); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>35); \ + b -= c; b -= a; b ^= (a<<49); \ + c -= a; c -= b; c ^= (b>>11); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<18); \ + c -= a; c -= b; c ^= (b>>22); \ +} + +/** + * @brief hash a variable-length key into a 32-bit value + * + * The best hash table sizes are powers of 2. There is no need to do + * mod a prime (mod is sooo slow!). If you need less than 32 bits, + * use a bitmask. For example, if you need only 10 bits, do + * h = (h & hashmask(10)); + * In which case, the hash table should have hashsize(10) elements. + * + * Use for hash table lookup, or anything where one collision in 2^32 is + * acceptable. Do NOT use for cryptographic purposes. + * + * @param k The key (the unaligned variable-length array of bytes). + * + * @param length The length of the key, counting by bytes. + * + * @param initval Initial value, can be any 4-byte value. + * + * @return Returns a 32-bit value. Every bit of the key affects every bit + * of the return value. Every 1-bit and 2-bit delta achieves + * avalanche. About 36+6len instructions. + */ +static inline uint32_t c_jhash(const uint8_t *k, uint32_t length, uint32_t initval) { + uint32_t a,b,c,len; + + /* Set up the internal state */ + len = length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + c = initval; /* the previous hash value */ + + while (len >= 12) { + a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24)); + b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24)); + c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24)); + _c_mix(a,b,c); + k += 12; len -= 12; + } + + /* handle the last 11 bytes */ + c += length; + /* all the case statements fall through */ + switch(len) { + case 11: c+=((uint32_t)k[10]<<24); + case 10: c+=((uint32_t)k[9]<<16); + case 9 : c+=((uint32_t)k[8]<<8); + /* the first byte of c is reserved for the length */ + case 8 : b+=((uint32_t)k[7]<<24); + case 7 : b+=((uint32_t)k[6]<<16); + case 6 : b+=((uint32_t)k[5]<<8); + case 5 : b+=k[4]; + case 4 : a+=((uint32_t)k[3]<<24); + case 3 : a+=((uint32_t)k[2]<<16); + case 2 : a+=((uint32_t)k[1]<<8); + case 1 : a+=k[0]; + /* case 0: nothing left to add */ + } + _c_mix(a,b,c); + + return c; +} + +/** + * @brief hash a variable-length key into a 64-bit value + * + * The best hash table sizes are powers of 2. There is no need to do + * mod a prime (mod is sooo slow!). If you need less than 64 bits, + * use a bitmask. For example, if you need only 10 bits, do + * h = (h & hashmask(10)); + * In which case, the hash table should have hashsize(10) elements. + * + * Use for hash table lookup, or anything where one collision in 2^^64 + * is acceptable. Do NOT use for cryptographic purposes. + * + * @param k The key (the unaligned variable-length array of bytes). + * @param length The length of the key, counting by bytes. + * @param intval Initial value, can be any 8-byte value. + * + * @return A 64-bit value. Every bit of the key affects every bit of + * the return value. No funnels. Every 1-bit and 2-bit delta + * achieves avalanche. About 41+5len instructions. + */ +static inline uint64_t c_jhash64(const uint8_t *k, uint64_t length, uint64_t intval) { + uint64_t a,b,c,len; + + /* Set up the internal state */ + len = length; + a = b = intval; /* the previous hash value */ + c = 0x9e3779b97f4a7c13LL; /* the golden ratio; an arbitrary value */ + + /* handle most of the key */ + while (len >= 24) + { + a += (k[0] +((uint64_t)k[ 1]<< 8)+((uint64_t)k[ 2]<<16)+((uint64_t)k[ 3]<<24) + +((uint64_t)k[4 ]<<32)+((uint64_t)k[ 5]<<40)+((uint64_t)k[ 6]<<48)+((uint64_t)k[ 7]<<56)); + b += (k[8] +((uint64_t)k[ 9]<< 8)+((uint64_t)k[10]<<16)+((uint64_t)k[11]<<24) + +((uint64_t)k[12]<<32)+((uint64_t)k[13]<<40)+((uint64_t)k[14]<<48)+((uint64_t)k[15]<<56)); + c += (k[16] +((uint64_t)k[17]<< 8)+((uint64_t)k[18]<<16)+((uint64_t)k[19]<<24) + +((uint64_t)k[20]<<32)+((uint64_t)k[21]<<40)+((uint64_t)k[22]<<48)+((uint64_t)k[23]<<56)); + _c_mix64(a,b,c); + k += 24; len -= 24; + } + + /* handle the last 23 bytes */ + c += length; + switch(len) { + case 23: c+=((uint64_t)k[22]<<56); + case 22: c+=((uint64_t)k[21]<<48); + case 21: c+=((uint64_t)k[20]<<40); + case 20: c+=((uint64_t)k[19]<<32); + case 19: c+=((uint64_t)k[18]<<24); + case 18: c+=((uint64_t)k[17]<<16); + case 17: c+=((uint64_t)k[16]<<8); + /* the first byte of c is reserved for the length */ + case 16: b+=((uint64_t)k[15]<<56); + case 15: b+=((uint64_t)k[14]<<48); + case 14: b+=((uint64_t)k[13]<<40); + case 13: b+=((uint64_t)k[12]<<32); + case 12: b+=((uint64_t)k[11]<<24); + case 11: b+=((uint64_t)k[10]<<16); + case 10: b+=((uint64_t)k[ 9]<<8); + case 9: b+=((uint64_t)k[ 8]); + case 8: a+=((uint64_t)k[ 7]<<56); + case 7: a+=((uint64_t)k[ 6]<<48); + case 6: a+=((uint64_t)k[ 5]<<40); + case 5: a+=((uint64_t)k[ 4]<<32); + case 4: a+=((uint64_t)k[ 3]<<24); + case 3: a+=((uint64_t)k[ 2]<<16); + case 2: a+=((uint64_t)k[ 1]<<8); + case 1: a+=((uint64_t)k[ 0]); + /* case 0: nothing left to add */ + } + _c_mix64(a,b,c); + + return c; +} + +/** + * }@ + */ +#endif /* _C_JHASH_H */ + diff --git a/src/common/checksums.cpp b/src/common/checksums.cpp new file mode 100644 index 000000000..6ffb57385 --- /dev/null +++ b/src/common/checksums.cpp @@ -0,0 +1,249 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "config.h" +#include "filesystembase.h" +#include "common/checksums.h" + +#include +#include + +/** \file checksums.cpp + * + * \brief Computing and validating file checksums + * + * Overview + * -------- + * + * Checksums are used in two distinct ways during synchronization: + * + * - to guard uploads and downloads against data corruption + * (transmission checksum) + * - to quickly check whether the content of a file has changed + * to avoid redundant uploads (content checksum) + * + * In principle both are independent and different checksumming + * algorithms can be used. To avoid redundant computations, it can + * make sense to use the same checksum algorithm though. + * + * Transmission Checksums + * ---------------------- + * + * The usage of transmission checksums is currently optional and needs + * to be explicitly enabled by adding 'transmissionChecksum=TYPE' to + * the '[General]' section of the config file. + * + * When enabled, the checksum will be calculated on upload and sent to + * the server in the OC-Checksum header with the format 'TYPE:CHECKSUM'. + * + * On download, the header with the same name is read and if the + * received data does not have the expected checksum, the download is + * rejected. + * + * Transmission checksums guard a specific sync action and are not stored + * in the database. + * + * Content Checksums + * ----------------- + * + * Sometimes the metadata of a local file changes while the content stays + * unchanged. Content checksums allow the sync client to avoid uploading + * the same data again by comparing the file's actual checksum to the + * checksum stored in the database. + * + * Content checksums are not sent to the server. + * + * Checksum Algorithms + * ------------------- + * + * - Adler32 (requires zlib) + * - MD5 + * - SHA1 + * + */ + +namespace OCC { + +Q_LOGGING_CATEGORY(lcChecksums, "sync.checksums", QtInfoMsg) + +QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum) +{ + if (checksumType.isEmpty() || checksum.isEmpty()) + return QByteArray(); + QByteArray header = checksumType; + header.append(':'); + header.append(checksum); + return header; +} + +bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum) +{ + if (header.isEmpty()) { + type->clear(); + checksum->clear(); + return true; + } + + const auto idx = header.indexOf(':'); + if (idx < 0) { + return false; + } + + *type = header.left(idx); + *checksum = header.mid(idx + 1); + return true; +} + + +QByteArray parseChecksumHeaderType(const QByteArray &header) +{ + const auto idx = header.indexOf(':'); + if (idx < 0) { + return QByteArray(); + } + return header.left(idx); +} + +bool uploadChecksumEnabled() +{ + static bool enabled = qgetenv("OWNCLOUD_DISABLE_CHECKSUM_UPLOAD").isEmpty(); + return enabled; +} + +QByteArray contentChecksumType() +{ + static QByteArray type = qgetenv("OWNCLOUD_CONTENT_CHECKSUM_TYPE"); + if (type.isNull()) { // can set to "" to disable checksumming + type = "SHA1"; + } + return type; +} + +ComputeChecksum::ComputeChecksum(QObject *parent) + : QObject(parent) +{ +} + +void ComputeChecksum::setChecksumType(const QByteArray &type) +{ + _checksumType = type; +} + +QByteArray ComputeChecksum::checksumType() const +{ + return _checksumType; +} + +void ComputeChecksum::start(const QString &filePath) +{ + // Calculate the checksum in a different thread first. + connect(&_watcher, SIGNAL(finished()), + this, SLOT(slotCalculationDone()), + Qt::UniqueConnection); + _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType())); +} + +QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray &checksumType) +{ + if (checksumType == checkSumMD5C) { + return FileSystem::calcMd5(filePath); + } else if (checksumType == checkSumSHA1C) { + return FileSystem::calcSha1(filePath); + } +#ifdef ZLIB_FOUND + else if (checksumType == checkSumAdlerC) { + return FileSystem::calcAdler32(filePath); + } +#endif + // for an unknown checksum or no checksum, we're done right now + if (!checksumType.isEmpty()) { + qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType; + } + return QByteArray(); +} + +void ComputeChecksum::slotCalculationDone() +{ + QByteArray checksum = _watcher.future().result(); + if (!checksum.isNull()) { + emit done(_checksumType, checksum); + } else { + emit done(QByteArray(), QByteArray()); + } +} + + +ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent) + : QObject(parent) +{ +} + +void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader) +{ + // If the incoming header is empty no validation can happen. Just continue. + if (checksumHeader.isEmpty()) { + emit validated(QByteArray(), QByteArray()); + return; + } + + if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) { + qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader; + emit validationFailed(tr("The checksum header is malformed.")); + return; + } + + auto calculator = new ComputeChecksum(this); + calculator->setChecksumType(_expectedChecksumType); + connect(calculator, SIGNAL(done(QByteArray, QByteArray)), + SLOT(slotChecksumCalculated(QByteArray, QByteArray))); + calculator->start(filePath); +} + +void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType, + const QByteArray &checksum) +{ + if (checksumType != _expectedChecksumType) { + emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType))); + return; + } + if (checksum != _expectedChecksum) { + emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed.")); + return; + } + emit validated(checksumType, checksum); +} + +CSyncChecksumHook::CSyncChecksumHook() +{ +} + +QByteArray CSyncChecksumHook::hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void * /*this_obj*/) +{ + QByteArray type = parseChecksumHeaderType(QByteArray(otherChecksumHeader)); + if (type.isEmpty()) + return NULL; + + QByteArray checksum = ComputeChecksum::computeNow(QString::fromUtf8(path), type); + if (checksum.isNull()) { + qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path; + return NULL; + } + + return makeChecksumHeader(type, checksum); +} + +} diff --git a/src/common/checksums.h b/src/common/checksums.h new file mode 100644 index 000000000..fa8bf3313 --- /dev/null +++ b/src/common/checksums.h @@ -0,0 +1,147 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#pragma once + +#include "ocsynclib.h" + +#include +#include +#include + +namespace OCC { + +/** + * Tags for checksum headers values. + * They are here for being shared between Upload- and Download Job + */ +static const char checkSumMD5C[] = "MD5"; +static const char checkSumSHA1C[] = "SHA1"; +static const char checkSumAdlerC[] = "Adler32"; + +class SyncJournalDb; + +/// Creates a checksum header from type and value. +OCSYNC_EXPORT QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum); + +/// Parses a checksum header +OCSYNC_EXPORT bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum); + +/// Convenience for getting the type from a checksum header, null if none +OCSYNC_EXPORT QByteArray parseChecksumHeaderType(const QByteArray &header); + +/// Checks OWNCLOUD_DISABLE_CHECKSUM_UPLOAD +OCSYNC_EXPORT bool uploadChecksumEnabled(); + +/// Checks OWNCLOUD_CONTENT_CHECKSUM_TYPE (default: SHA1) +OCSYNC_EXPORT QByteArray contentChecksumType(); + + +/** + * Computes the checksum of a file. + * \ingroup libsync + */ +class OCSYNC_EXPORT ComputeChecksum : public QObject +{ + Q_OBJECT +public: + explicit ComputeChecksum(QObject *parent = 0); + + /** + * Sets the checksum type to be used. The default is empty. + */ + void setChecksumType(const QByteArray &type); + + QByteArray checksumType() const; + + /** + * Computes the checksum for the given file path. + * + * done() is emitted when the calculation finishes. + */ + void start(const QString &filePath); + + /** + * Computes the checksum synchronously. + */ + static QByteArray computeNow(const QString &filePath, const QByteArray &checksumType); + +signals: + void done(const QByteArray &checksumType, const QByteArray &checksum); + +private slots: + void slotCalculationDone(); + +private: + QByteArray _checksumType; + + // watcher for the checksum calculation thread + QFutureWatcher _watcher; +}; + +/** + * Checks whether a file's checksum matches the expected value. + * @ingroup libsync + */ +class OCSYNC_EXPORT ValidateChecksumHeader : public QObject +{ + Q_OBJECT +public: + explicit ValidateChecksumHeader(QObject *parent = 0); + + /** + * Check a file's actual checksum against the provided checksumHeader + * + * If no checksum is there, or if a correct checksum is there, the signal validated() + * will be emitted. In case of any kind of error, the signal validationFailed() will + * be emitted. + */ + void start(const QString &filePath, const QByteArray &checksumHeader); + +signals: + void validated(const QByteArray &checksumType, const QByteArray &checksum); + void validationFailed(const QString &errMsg); + +private slots: + void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum); + +private: + QByteArray _expectedChecksumType; + QByteArray _expectedChecksum; +}; + +/** + * Hooks checksum computations into csync. + * @ingroup libsync + */ +class OCSYNC_EXPORT CSyncChecksumHook : public QObject +{ + Q_OBJECT +public: + explicit CSyncChecksumHook(); + + /** + * Returns the checksum value for \a path that is comparable to \a otherChecksum. + * + * Called from csync, where a instance of CSyncChecksumHook has + * to be set as userdata. + * The return value will be owned by csync. + */ + static QByteArray hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void *this_obj); +}; +} diff --git a/src/common/common.cmake b/src/common/common.cmake index ab1ca1e83..e448868c6 100644 --- a/src/common/common.cmake +++ b/src/common/common.cmake @@ -2,6 +2,10 @@ # Essentially they could be in the same directory but are separate to # help keep track of the different code licenses. set(common_SOURCES + ${CMAKE_CURRENT_LIST_DIR}/checksums.cpp ${CMAKE_CURRENT_LIST_DIR}/filesystembase.cpp + ${CMAKE_CURRENT_LIST_DIR}/ownsql.cpp + ${CMAKE_CURRENT_LIST_DIR}/syncjournaldb.cpp + ${CMAKE_CURRENT_LIST_DIR}/syncjournalfilerecord.cpp ${CMAKE_CURRENT_LIST_DIR}/utility.cpp ) diff --git a/src/common/ownsql.cpp b/src/common/ownsql.cpp new file mode 100644 index 000000000..2bda1ab09 --- /dev/null +++ b/src/common/ownsql.cpp @@ -0,0 +1,433 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include +#include + +#include "ownsql.h" +#include "common/utility.h" +#include "common/asserts.h" + +#define SQLITE_SLEEP_TIME_USEC 100000 +#define SQLITE_REPEAT_COUNT 20 + +#define SQLITE_DO(A) \ + if (1) { \ + _errId = (A); \ + if (_errId != SQLITE_OK) { \ + _error = QString::fromUtf8(sqlite3_errmsg(_db)); \ + } \ + } + +namespace OCC { + +Q_LOGGING_CATEGORY(lcSql, "sync.database.sql", QtInfoMsg) + +SqlDatabase::SqlDatabase() + : _db(0) + , _errId(0) +{ +} + +bool SqlDatabase::isOpen() +{ + return _db != 0; +} + +bool SqlDatabase::openHelper(const QString &filename, int sqliteFlags) +{ + if (isOpen()) { + return true; + } + + sqliteFlags |= SQLITE_OPEN_NOMUTEX; + + SQLITE_DO(sqlite3_open_v2(filename.toUtf8().constData(), &_db, sqliteFlags, 0)); + + if (_errId != SQLITE_OK) { + qCWarning(lcSql) << "Error:" << _error << "for" << filename; + if (_errId == SQLITE_CANTOPEN) { + qCWarning(lcSql) << "CANTOPEN extended errcode: " << sqlite3_extended_errcode(_db); +#if SQLITE_VERSION_NUMBER >= 3012000 + qCWarning(lcSql) << "CANTOPEN system errno: " << sqlite3_system_errno(_db); +#endif + } + close(); + return false; + } + + if (!_db) { + qCWarning(lcSql) << "Error: no database for" << filename; + return false; + } + + sqlite3_busy_timeout(_db, 5000); + + return true; +} + +bool SqlDatabase::checkDb() +{ + // quick_check can fail with a disk IO error when diskspace is low + SqlQuery quick_check(*this); + quick_check.prepare("PRAGMA quick_check;", /*allow_failure=*/true); + if (!quick_check.exec()) { + qCWarning(lcSql) << "Error running quick_check on database"; + return false; + } + + quick_check.next(); + QString result = quick_check.stringValue(0); + if (result != "ok") { + qCWarning(lcSql) << "quick_check returned failure:" << result; + return false; + } + + return true; +} + +bool SqlDatabase::openOrCreateReadWrite(const QString &filename) +{ + if (isOpen()) { + return true; + } + + if (!openHelper(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE)) { + return false; + } + + if (!checkDb()) { + // When disk space is low, checking the db may fail even though it's fine. + qint64 freeSpace = Utility::freeDiskSpace(QFileInfo(filename).dir().absolutePath()); + if (freeSpace != -1 && freeSpace < 1000000) { + qCWarning(lcSql) << "Consistency check failed, disk space is low, aborting" << freeSpace; + close(); + return false; + } + + qCCritical(lcSql) << "Consistency check failed, removing broken db" << filename; + close(); + QFile::remove(filename); + + return openHelper(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE); + } + + return true; +} + +bool SqlDatabase::openReadOnly(const QString &filename) +{ + if (isOpen()) { + return true; + } + + if (!openHelper(filename, SQLITE_OPEN_READONLY)) { + return false; + } + + if (!checkDb()) { + qCWarning(lcSql) << "Consistency check failed in readonly mode, giving up" << filename; + close(); + return false; + } + + return true; +} + +QString SqlDatabase::error() const +{ + const QString err(_error); + // _error.clear(); + return err; +} + +void SqlDatabase::close() +{ + if (_db) { + SQLITE_DO(sqlite3_close(_db)); + // Fatal because reopening an unclosed db might be problematic. + ENFORCE(_errId == SQLITE_OK, "Error when closing DB"); + _db = 0; + } +} + +bool SqlDatabase::transaction() +{ + if (!_db) { + return false; + } + SQLITE_DO(sqlite3_exec(_db, "BEGIN", 0, 0, 0)); + return _errId == SQLITE_OK; +} + +bool SqlDatabase::commit() +{ + if (!_db) { + return false; + } + SQLITE_DO(sqlite3_exec(_db, "COMMIT", 0, 0, 0)); + return _errId == SQLITE_OK; +} + +sqlite3 *SqlDatabase::sqliteDb() +{ + return _db; +} + +/* =========================================================================================== */ + +SqlQuery::SqlQuery(SqlDatabase &db) + : _db(db.sqliteDb()) + , _stmt(0) + , _errId(0) +{ +} + +SqlQuery::~SqlQuery() +{ + if (_stmt) { + finish(); + } +} + +SqlQuery::SqlQuery(const QString &sql, SqlDatabase &db) + : _db(db.sqliteDb()) + , _stmt(0) + , _errId(0) +{ + prepare(sql); +} + +int SqlQuery::prepare(const QString &sql, bool allow_failure) +{ + QString s(sql); + _sql = s.trimmed(); + if (_stmt) { + finish(); + } + if (!_sql.isEmpty()) { + int n = 0; + int rc; + do { + rc = sqlite3_prepare_v2(_db, _sql.toUtf8().constData(), -1, &_stmt, 0); + if ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED)) { + n++; + OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); + } + } while ((n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); + _errId = rc; + + if (_errId != SQLITE_OK) { + _error = QString::fromUtf8(sqlite3_errmsg(_db)); + qCWarning(lcSql) << "Sqlite prepare statement error:" << _error << "in" << _sql; + ENFORCE(allow_failure, "SQLITE Prepare error"); + } + } + return _errId; +} + +bool SqlQuery::isSelect() +{ + return (!_sql.isEmpty() && _sql.startsWith("SELECT", Qt::CaseInsensitive)); +} + +bool SqlQuery::isPragma() +{ + return (!_sql.isEmpty() && _sql.startsWith("PRAGMA", Qt::CaseInsensitive)); +} + +bool SqlQuery::exec() +{ + qCDebug(lcSql) << "SQL exec" << _sql; + + if (!_stmt) { + qCWarning(lcSql) << "Can't exec query, statement unprepared."; + return false; + } + + // Don't do anything for selects, that is how we use the lib :-| + if (!isSelect() && !isPragma()) { + int rc, n = 0; + do { + rc = sqlite3_step(_stmt); + if (rc == SQLITE_LOCKED) { + rc = sqlite3_reset(_stmt); /* This will also return SQLITE_LOCKED */ + n++; + OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); + } else if (rc == SQLITE_BUSY) { + OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); + n++; + } + } while ((n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); + _errId = rc; + + if (_errId != SQLITE_DONE && _errId != SQLITE_ROW) { + _error = QString::fromUtf8(sqlite3_errmsg(_db)); + qCWarning(lcSql) << "Sqlite exec statement error:" << _errId << _error << "in" << _sql; + if (_errId == SQLITE_IOERR) { + qCWarning(lcSql) << "IOERR extended errcode: " << sqlite3_extended_errcode(_db); +#if SQLITE_VERSION_NUMBER >= 3012000 + qCWarning(lcSql) << "IOERR system errno: " << sqlite3_system_errno(_db); +#endif + } + } else { + qCDebug(lcSql) << "Last exec affected" << numRowsAffected() << "rows."; + } + return (_errId == SQLITE_DONE); // either SQLITE_ROW or SQLITE_DONE + } + + return true; +} + +bool SqlQuery::next() +{ + SQLITE_DO(sqlite3_step(_stmt)); + return _errId == SQLITE_ROW; +} + +void SqlQuery::bindValue(int pos, const QVariant &value) +{ + qCDebug(lcSql) << "SQL bind" << pos << value; + + int res = -1; + if (!_stmt) { + ASSERT(false); + return; + } + + switch (value.type()) { + case QVariant::Int: + case QVariant::Bool: + res = sqlite3_bind_int(_stmt, pos, value.toInt()); + break; + case QVariant::Double: + res = sqlite3_bind_double(_stmt, pos, value.toDouble()); + break; + case QVariant::UInt: + case QVariant::LongLong: + res = sqlite3_bind_int64(_stmt, pos, value.toLongLong()); + break; + case QVariant::DateTime: { + const QDateTime dateTime = value.toDateTime(); + const QString str = dateTime.toString(QLatin1String("yyyy-MM-ddThh:mm:ss.zzz")); + res = sqlite3_bind_text16(_stmt, pos, str.utf16(), + str.size() * sizeof(ushort), SQLITE_TRANSIENT); + break; + } + case QVariant::Time: { + const QTime time = value.toTime(); + const QString str = time.toString(QLatin1String("hh:mm:ss.zzz")); + res = sqlite3_bind_text16(_stmt, pos, str.utf16(), + str.size() * sizeof(ushort), SQLITE_TRANSIENT); + break; + } + case QVariant::String: { + if (!value.toString().isNull()) { + // lifetime of string == lifetime of its qvariant + const QString *str = static_cast(value.constData()); + res = sqlite3_bind_text16(_stmt, pos, str->utf16(), + (str->size()) * sizeof(QChar), SQLITE_TRANSIENT); + } else { + res = sqlite3_bind_null(_stmt, pos); + } + break; + } + case QVariant::ByteArray: { + auto ba = value.toByteArray(); + res = sqlite3_bind_text(_stmt, pos, ba.constData(), ba.size(), SQLITE_TRANSIENT); + break; + } + default: { + QString str = value.toString(); + // SQLITE_TRANSIENT makes sure that sqlite buffers the data + res = sqlite3_bind_text16(_stmt, pos, str.utf16(), + (str.size()) * sizeof(QChar), SQLITE_TRANSIENT); + break; + } + } + if (res != SQLITE_OK) { + qCWarning(lcSql) << "ERROR binding SQL value:" << value << "error:" << res; + } + ASSERT(res == SQLITE_OK); +} + +bool SqlQuery::nullValue(int index) +{ + return sqlite3_column_type(_stmt, index) == SQLITE_NULL; +} + +QString SqlQuery::stringValue(int index) +{ + return QString::fromUtf16(static_cast(sqlite3_column_text16(_stmt, index))); +} + +int SqlQuery::intValue(int index) +{ + return sqlite3_column_int(_stmt, index); +} + +quint64 SqlQuery::int64Value(int index) +{ + return sqlite3_column_int64(_stmt, index); +} + +QByteArray SqlQuery::baValue(int index) +{ + return QByteArray(static_cast(sqlite3_column_blob(_stmt, index)), + sqlite3_column_bytes(_stmt, index)); +} + +QString SqlQuery::error() const +{ + return _error; +} + +int SqlQuery::errorId() const +{ + return _errId; +} + +QString SqlQuery::lastQuery() const +{ + return _sql; +} + +int SqlQuery::numRowsAffected() +{ + return sqlite3_changes(_db); +} + +void SqlQuery::finish() +{ + SQLITE_DO(sqlite3_finalize(_stmt)); + _stmt = 0; +} + +void SqlQuery::reset_and_clear_bindings() +{ + if (_stmt) { + SQLITE_DO(sqlite3_reset(_stmt)); + SQLITE_DO(sqlite3_clear_bindings(_stmt)); + } +} + +} // namespace OCC diff --git a/src/common/ownsql.h b/src/common/ownsql.h new file mode 100644 index 000000000..a7f3c2bf0 --- /dev/null +++ b/src/common/ownsql.h @@ -0,0 +1,104 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef OWNSQL_H +#define OWNSQL_H + +#include + +#include +#include + +#include "ocsynclib.h" + +namespace OCC { + +/** + * @brief The SqlDatabase class + * @ingroup libsync + */ +class OCSYNC_EXPORT SqlDatabase +{ + Q_DISABLE_COPY(SqlDatabase) +public: + explicit SqlDatabase(); + + bool isOpen(); + bool openOrCreateReadWrite(const QString &filename); + bool openReadOnly(const QString &filename); + bool transaction(); + bool commit(); + void close(); + QString error() const; + sqlite3 *sqliteDb(); + +private: + bool openHelper(const QString &filename, int sqliteFlags); + bool checkDb(); + + sqlite3 *_db; + QString _error; // last error string + int _errId; +}; + +/** + * @brief The SqlQuery class + * @ingroup libsync + */ +class OCSYNC_EXPORT SqlQuery +{ + Q_DISABLE_COPY(SqlQuery) +public: + explicit SqlQuery(SqlDatabase &db); + explicit SqlQuery(const QString &sql, SqlDatabase &db); + + ~SqlQuery(); + QString error() const; + int errorId() const; + + /// Checks whether the value at the given column index is NULL + bool nullValue(int index); + + + QString stringValue(int index); + int intValue(int index); + quint64 int64Value(int index); + QByteArray baValue(int index); + + bool isSelect(); + bool isPragma(); + bool exec(); + int prepare(const QString &sql, bool allow_failure = false); + bool next(); + void bindValue(int pos, const QVariant &value); + QString lastQuery() const; + int numRowsAffected(); + void reset_and_clear_bindings(); + void finish(); + +private: + sqlite3 *_db; + sqlite3_stmt *_stmt; + QString _error; + int _errId; + QString _sql; +}; + +} // namespace OCC + +#endif // OWNSQL_H diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp new file mode 100644 index 000000000..d1dae8f66 --- /dev/null +++ b/src/common/syncjournaldb.cpp @@ -0,0 +1,1882 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "common/syncjournaldb.h" +#include "version.h" +#include "filesystembase.h" +#include "common/asserts.h" +#include "common/checksums.h" + +#include "common/c_jhash.h" + +namespace OCC { + +Q_LOGGING_CATEGORY(lcDb, "sync.database", QtInfoMsg) + +static QString defaultJournalMode(const QString &dbPath) +{ +#ifdef Q_OS_WIN + // See #2693: Some exFAT file systems seem unable to cope with the + // WAL journaling mode. They work fine with DELETE. + QString fileSystem = FileSystem::fileSystemForPath(dbPath); + qCInfo(lcDb) << "Detected filesystem" << fileSystem << "for" << dbPath; + if (fileSystem.contains("FAT")) { + qCInfo(lcDb) << "Filesystem contains FAT - using DELETE journal mode"; + return "DELETE"; + } +#else + Q_UNUSED(dbPath) +#endif + return "WAL"; +} + +SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent) + : QObject(parent) + , _dbFile(dbFilePath) + , _transaction(0) +{ + // Allow forcing the journal mode for debugging + static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE")); + _journalMode = envJournalMode; + if (_journalMode.isEmpty()) { + _journalMode = defaultJournalMode(_dbFile); + } +} + +QString SyncJournalDb::makeDbName(const QString &localPath, + const QUrl &remoteUrl, + const QString &remotePath, + const QString &user) +{ + QString journalPath = QLatin1String("._sync_"); + + QString key = QString::fromUtf8("%1@%2:%3").arg(user, remoteUrl.toString(), remotePath); + + QByteArray ba = QCryptographicHash::hash(key.toUtf8(), QCryptographicHash::Md5); + journalPath.append(ba.left(6).toHex()); + journalPath.append(".db"); + + // If the journal doesn't exist and we can't create a file + // at that location, try again with a journal name that doesn't + // have the ._ prefix. + // + // The disadvantage of that filename is that it will only be ignored + // by client versions >2.3.2. + // + // See #5633: "._*" is often forbidden on samba shared folders. + + // If it exists already, the path is clearly usable + QFile file(QDir(localPath).filePath(journalPath)); + if (file.exists()) { + return journalPath; + } + + // Try to create a file there + if (file.open(QIODevice::ReadWrite)) { + // Ok, all good. + file.close(); + file.remove(); + return journalPath; + } + + // Can we create it if we drop the underscore? + QString alternateJournalPath = journalPath.mid(2).prepend("."); + QFile file2(QDir(localPath).filePath(alternateJournalPath)); + if (file2.open(QIODevice::ReadWrite)) { + // The alternative worked, use it + qCInfo(lcDb) << "Using alternate database path" << alternateJournalPath; + file2.close(); + file2.remove(); + return alternateJournalPath; + } + + // Neither worked, just keep the original and throw errors later + qCWarning(lcDb) << "Could not find a writable database path" << file.fileName(); + return journalPath; +} + +bool SyncJournalDb::maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath) +{ + const QString oldDbName = localPath + QLatin1String(".csync_journal.db"); + if (!FileSystem::fileExists(oldDbName)) { + return true; + } + const QString oldDbNameShm = oldDbName + "-shm"; + const QString oldDbNameWal = oldDbName + "-wal"; + + const QString newDbName = absoluteJournalPath; + const QString newDbNameShm = newDbName + "-shm"; + const QString newDbNameWal = newDbName + "-wal"; + + // Whenever there is an old db file, migrate it to the new db path. + // This is done to make switching from older versions to newer versions + // work correctly even if the user had previously used a new version + // and therefore already has an (outdated) new-style db file. + QString error; + + if (FileSystem::fileExists(newDbName)) { + if (!FileSystem::remove(newDbName, &error)) { + qCWarning(lcDb) << "Database migration: Could not remove db file" << newDbName + << "due to" << error; + return false; + } + } + if (FileSystem::fileExists(newDbNameWal)) { + if (!FileSystem::remove(newDbNameWal, &error)) { + qCWarning(lcDb) << "Database migration: Could not remove db WAL file" << newDbNameWal + << "due to" << error; + return false; + } + } + if (FileSystem::fileExists(newDbNameShm)) { + if (!FileSystem::remove(newDbNameShm, &error)) { + qCWarning(lcDb) << "Database migration: Could not remove db SHM file" << newDbNameShm + << "due to" << error; + return false; + } + } + + if (!FileSystem::rename(oldDbName, newDbName, &error)) { + qCWarning(lcDb) << "Database migration: could not rename " << oldDbName + << "to" << newDbName << ":" << error; + return false; + } + if (!FileSystem::rename(oldDbNameWal, newDbNameWal, &error)) { + qCWarning(lcDb) << "Database migration: could not rename " << oldDbNameWal + << "to" << newDbNameWal << ":" << error; + return false; + } + if (!FileSystem::rename(oldDbNameShm, newDbNameShm, &error)) { + qCWarning(lcDb) << "Database migration: could not rename " << oldDbNameShm + << "to" << newDbNameShm << ":" << error; + return false; + } + + qCInfo(lcDb) << "Journal successfully migrated from" << oldDbName << "to" << newDbName; + return true; +} + +bool SyncJournalDb::exists() +{ + QMutexLocker locker(&_mutex); + return (!_dbFile.isEmpty() && QFile::exists(_dbFile)); +} + +QString SyncJournalDb::databaseFilePath() const +{ + return _dbFile; +} + +// Note that this does not change the size of the -wal file, but it is supposed to make +// the normal .db faster since the changes from the wal will be incorporated into it. +// Then the next sync (and the SocketAPI) will have a faster access. +void SyncJournalDb::walCheckpoint() +{ + QElapsedTimer t; + t.start(); + SqlQuery pragma1(_db); + pragma1.prepare("PRAGMA wal_checkpoint(FULL);"); + if (pragma1.exec()) { + qCDebug(lcDb) << "took" << t.elapsed() << "msec"; + } +} + +void SyncJournalDb::startTransaction() +{ + if (_transaction == 0) { + if (!_db.transaction()) { + qCWarning(lcDb) << "ERROR starting transaction: " << _db.error(); + return; + } + _transaction = 1; + } else { + qCDebug(lcDb) << "Database Transaction is running, not starting another one!"; + } +} + +void SyncJournalDb::commitTransaction() +{ + if (_transaction == 1) { + if (!_db.commit()) { + qCWarning(lcDb) << "ERROR committing to the database: " << _db.error(); + return; + } + _transaction = 0; + } else { + qCDebug(lcDb) << "No database Transaction to commit"; + } +} + +bool SyncJournalDb::sqlFail(const QString &log, const SqlQuery &query) +{ + commitTransaction(); + qCWarning(lcDb) << "SQL Error" << log << query.error(); + ASSERT(false); + _db.close(); + return false; +} + +bool SyncJournalDb::checkConnect() +{ + if (_db.isOpen()) { + return true; + } + + if (_dbFile.isEmpty()) { + qCWarning(lcDb) << "Database filename" + _dbFile + " is empty"; + return false; + } + + // The database file is created by this call (SQLITE_OPEN_CREATE) + if (!_db.openOrCreateReadWrite(_dbFile)) { + QString error = _db.error(); + qCWarning(lcDb) << "Error opening the db: " << error; + return false; + } + + if (!QFile::exists(_dbFile)) { + qCWarning(lcDb) << "Database file" + _dbFile + " does not exist"; + return false; + } + + SqlQuery pragma1(_db); + pragma1.prepare("SELECT sqlite_version();"); + if (!pragma1.exec()) { + return sqlFail("SELECT sqlite_version()", pragma1); + } else { + pragma1.next(); + qCInfo(lcDb) << "sqlite3 version" << pragma1.stringValue(0); + } + + pragma1.prepare(QString("PRAGMA journal_mode=%1;").arg(_journalMode)); + if (!pragma1.exec()) { + return sqlFail("Set PRAGMA journal_mode", pragma1); + } else { + pragma1.next(); + qCInfo(lcDb) << "sqlite3 journal_mode=" << pragma1.stringValue(0); + } + + // For debugging purposes, allow temp_store to be set + static QString env_temp_store = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_TEMP_STORE")); + if (!env_temp_store.isEmpty()) { + pragma1.prepare(QString("PRAGMA temp_store = %1;").arg(env_temp_store)); + if (!pragma1.exec()) { + return sqlFail("Set PRAGMA temp_store", pragma1); + } + qCInfo(lcDb) << "sqlite3 with temp_store =" << env_temp_store; + } + + pragma1.prepare("PRAGMA synchronous = 1;"); + if (!pragma1.exec()) { + return sqlFail("Set PRAGMA synchronous", pragma1); + } + pragma1.prepare("PRAGMA case_sensitive_like = ON;"); + if (!pragma1.exec()) { + return sqlFail("Set PRAGMA case_sensitivity", pragma1); + } + + /* Because insert is so slow, we do everything in a transaction, and only need one call to commit */ + startTransaction(); + + SqlQuery createQuery(_db); + createQuery.prepare("CREATE TABLE IF NOT EXISTS metadata(" + "phash INTEGER(8)," + "pathlen INTEGER," + "path VARCHAR(4096)," + "inode INTEGER," + "uid INTEGER," + "gid INTEGER," + "mode INTEGER," + "modtime INTEGER(8)," + "type INTEGER," + "md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */ + // updateDatabaseStructure() will add + // fileid + // remotePerm + // filesize + // ignoredChildrenRemote + // contentChecksum + // contentChecksumTypeId + "PRIMARY KEY(phash)" + ");"); + + if (!createQuery.exec()) { + // In certain situations the io error can be avoided by switching + // to the DELETE journal mode, see #5723 + if (_journalMode != "DELETE" + && createQuery.errorId() == SQLITE_IOERR + && sqlite3_extended_errcode(_db.sqliteDb()) == SQLITE_IOERR_SHMMAP) { + qCWarning(lcDb) << "IO error SHMMAP on table creation, attempting with DELETE journal mode"; + + _journalMode = "DELETE"; + createQuery.finish(); + pragma1.finish(); + commitTransaction(); + _db.close(); + return checkConnect(); + } + + return sqlFail("Create table metadata", createQuery); + } + + createQuery.prepare("CREATE TABLE IF NOT EXISTS downloadinfo(" + "path VARCHAR(4096)," + "tmpfile VARCHAR(4096)," + "etag VARCHAR(32)," + "errorcount INTEGER," + "PRIMARY KEY(path)" + ");"); + + if (!createQuery.exec()) { + return sqlFail("Create table downloadinfo", createQuery); + } + + createQuery.prepare("CREATE TABLE IF NOT EXISTS uploadinfo(" + "path VARCHAR(4096)," + "chunk INTEGER," + "transferid INTEGER," + "errorcount INTEGER," + "size INTEGER(8)," + "modtime INTEGER(8)," + "PRIMARY KEY(path)" + ");"); + + if (!createQuery.exec()) { + return sqlFail("Create table uploadinfo", createQuery); + } + + // create the blacklist table. + createQuery.prepare("CREATE TABLE IF NOT EXISTS blacklist (" + "path VARCHAR(4096)," + "lastTryEtag VARCHAR[32]," + "lastTryModtime INTEGER[8]," + "retrycount INTEGER," + "errorstring VARCHAR[4096]," + "PRIMARY KEY(path)" + ");"); + + if (!createQuery.exec()) { + return sqlFail("Create table blacklist", createQuery); + } + + createQuery.prepare("CREATE TABLE IF NOT EXISTS poll(" + "path VARCHAR(4096)," + "modtime INTEGER(8)," + "pollpath VARCHAR(4096));"); + if (!createQuery.exec()) { + return sqlFail("Create table poll", createQuery); + } + + // create the selectivesync table. + createQuery.prepare("CREATE TABLE IF NOT EXISTS selectivesync (" + "path VARCHAR(4096)," + "type INTEGER" + ");"); + + if (!createQuery.exec()) { + return sqlFail("Create table selectivesync", createQuery); + } + + // create the checksumtype table. + createQuery.prepare("CREATE TABLE IF NOT EXISTS checksumtype(" + "id INTEGER PRIMARY KEY," + "name TEXT UNIQUE" + ");"); + if (!createQuery.exec()) { + return sqlFail("Create table version", createQuery); + } + + // create the checksumtype table. + createQuery.prepare("CREATE TABLE IF NOT EXISTS datafingerprint(" + "fingerprint TEXT UNIQUE" + ");"); + if (!createQuery.exec()) { + return sqlFail("Create table datafingerprint", createQuery); + } + + createQuery.prepare("CREATE TABLE IF NOT EXISTS version(" + "major INTEGER(8)," + "minor INTEGER(8)," + "patch INTEGER(8)," + "custom VARCHAR(256)" + ");"); + if (!createQuery.exec()) { + return sqlFail("Create table version", createQuery); + } + + bool forceRemoteDiscovery = false; + + SqlQuery versionQuery("SELECT major, minor, patch FROM version;", _db); + if (!versionQuery.next()) { + // If there was no entry in the table, it means we are likely upgrading from 1.5 + qCInfo(lcDb) << "possibleUpgradeFromMirall_1_5 detected!"; + forceRemoteDiscovery = true; + + createQuery.prepare("INSERT INTO version VALUES (?1, ?2, ?3, ?4);"); + createQuery.bindValue(1, MIRALL_VERSION_MAJOR); + createQuery.bindValue(2, MIRALL_VERSION_MINOR); + createQuery.bindValue(3, MIRALL_VERSION_PATCH); + createQuery.bindValue(4, MIRALL_VERSION_BUILD); + if (!createQuery.exec()) { + return sqlFail("Update version", createQuery); + } + + } else { + int major = versionQuery.intValue(0); + int minor = versionQuery.intValue(1); + int patch = versionQuery.intValue(2); + + if (major == 1 && minor == 8 && (patch == 0 || patch == 1)) { + qCInfo(lcDb) << "possibleUpgradeFromMirall_1_8_0_or_1 detected!"; + forceRemoteDiscovery = true; + } + + // There was a bug in versions <2.3.0 that could lead to stale + // local files and a remote discovery will fix them. + // See #5190 #5242. + if (major == 2 && minor < 3) { + qCInfo(lcDb) << "upgrade form client < 2.3.0 detected! forcing remote discovery"; + forceRemoteDiscovery = true; + } + + // Not comparing the BUILD id here, correct? + if (!(major == MIRALL_VERSION_MAJOR && minor == MIRALL_VERSION_MINOR && patch == MIRALL_VERSION_PATCH)) { + createQuery.prepare("UPDATE version SET major=?1, minor=?2, patch =?3, custom=?4 " + "WHERE major=?5 AND minor=?6 AND patch=?7;"); + createQuery.bindValue(1, MIRALL_VERSION_MAJOR); + createQuery.bindValue(2, MIRALL_VERSION_MINOR); + createQuery.bindValue(3, MIRALL_VERSION_PATCH); + createQuery.bindValue(4, MIRALL_VERSION_BUILD); + createQuery.bindValue(5, major); + createQuery.bindValue(6, minor); + createQuery.bindValue(7, patch); + if (!createQuery.exec()) { + return sqlFail("Update version", createQuery); + } + } + } + + commitInternal("checkConnect"); + + bool rc = updateDatabaseStructure(); + if (!rc) { + qCWarning(lcDb) << "Failed to update the database structure!"; + } + + /* + * If we are upgrading from a client version older than 1.5, + * we cannot read from the database because we need to fetch the files id and etags. + * + * If 1.8.0 caused missing data in the local tree, so we also don't read from DB + * to get back the files that were gone. + * In 1.8.1 we had a fix to re-get the data, but this one here is better + */ + if (forceRemoteDiscovery) { + forceRemoteDiscoveryNextSyncLocked(); + } + + _getFileRecordQuery.reset(new SqlQuery(_db)); + if (_getFileRecordQuery->prepare( + "SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize," + " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" + " FROM metadata" + " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" + " WHERE phash=?1")) { + return sqlFail("prepare _getFileRecordQuery", *_getFileRecordQuery); + } + + _setFileRecordQuery.reset(new SqlQuery(_db)); + if (_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " + "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);")) { + return sqlFail("prepare _setFileRecordQuery", *_setFileRecordQuery); + } + + _setFileRecordChecksumQuery.reset(new SqlQuery(_db)); + if (_setFileRecordChecksumQuery->prepare( + "UPDATE metadata" + " SET contentChecksum = ?2, contentChecksumTypeId = ?3" + " WHERE phash == ?1;")) { + return sqlFail("prepare _setFileRecordChecksumQuery", *_setFileRecordChecksumQuery); + } + + _setFileRecordLocalMetadataQuery.reset(new SqlQuery(_db)); + if (_setFileRecordLocalMetadataQuery->prepare( + "UPDATE metadata" + " SET inode=?2, modtime=?3, filesize=?4" + " WHERE phash == ?1;")) { + return sqlFail("prepare _setFileRecordLocalMetadataQuery", *_setFileRecordLocalMetadataQuery); + } + + _getDownloadInfoQuery.reset(new SqlQuery(_db)); + if (_getDownloadInfoQuery->prepare("SELECT tmpfile, etag, errorcount FROM " + "downloadinfo WHERE path=?1")) { + return sqlFail("prepare _getDownloadInfoQuery", *_getDownloadInfoQuery); + } + + _setDownloadInfoQuery.reset(new SqlQuery(_db)); + if (_setDownloadInfoQuery->prepare("INSERT OR REPLACE INTO downloadinfo " + "(path, tmpfile, etag, errorcount) " + "VALUES ( ?1 , ?2, ?3, ?4 )")) { + return sqlFail("prepare _setDownloadInfoQuery", *_setDownloadInfoQuery); + } + + _deleteDownloadInfoQuery.reset(new SqlQuery(_db)); + if (_deleteDownloadInfoQuery->prepare("DELETE FROM downloadinfo WHERE path=?1")) { + return sqlFail("prepare _deleteDownloadInfoQuery", *_deleteDownloadInfoQuery); + } + + _getUploadInfoQuery.reset(new SqlQuery(_db)); + if (_getUploadInfoQuery->prepare("SELECT chunk, transferid, errorcount, size, modtime FROM " + "uploadinfo WHERE path=?1")) { + return sqlFail("prepare _getUploadInfoQuery", *_getUploadInfoQuery); + } + + _setUploadInfoQuery.reset(new SqlQuery(_db)); + if (_setUploadInfoQuery->prepare("INSERT OR REPLACE INTO uploadinfo " + "(path, chunk, transferid, errorcount, size, modtime) " + "VALUES ( ?1 , ?2, ?3 , ?4 , ?5, ?6 )")) { + return sqlFail("prepare _setUploadInfoQuery", *_setUploadInfoQuery); + } + + _deleteUploadInfoQuery.reset(new SqlQuery(_db)); + if (_deleteUploadInfoQuery->prepare("DELETE FROM uploadinfo WHERE path=?1")) { + return sqlFail("prepare _deleteUploadInfoQuery", *_deleteUploadInfoQuery); + } + + + _deleteFileRecordPhash.reset(new SqlQuery(_db)); + if (_deleteFileRecordPhash->prepare("DELETE FROM metadata WHERE phash=?1")) { + return sqlFail("prepare _deleteFileRecordPhash", *_deleteFileRecordPhash); + } + + _deleteFileRecordRecursively.reset(new SqlQuery(_db)); + if (_deleteFileRecordRecursively->prepare("DELETE FROM metadata WHERE path LIKE(?||'/%')")) { + return sqlFail("prepare _deleteFileRecordRecursively", *_deleteFileRecordRecursively); + } + + QString sql("SELECT lastTryEtag, lastTryModtime, retrycount, errorstring, lastTryTime, ignoreDuration, renameTarget, errorCategory " + "FROM blacklist WHERE path=?1"); + if (Utility::fsCasePreserving()) { + // if the file system is case preserving we have to check the blacklist + // case insensitively + sql += QLatin1String(" COLLATE NOCASE"); + } + _getErrorBlacklistQuery.reset(new SqlQuery(_db)); + if (_getErrorBlacklistQuery->prepare(sql)) { + return sqlFail("prepare _getErrorBlacklistQuery", *_getErrorBlacklistQuery); + } + + _setErrorBlacklistQuery.reset(new SqlQuery(_db)); + if (_setErrorBlacklistQuery->prepare("INSERT OR REPLACE INTO blacklist " + "(path, lastTryEtag, lastTryModtime, retrycount, errorstring, lastTryTime, ignoreDuration, renameTarget, errorCategory) " + "VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)")) { + return sqlFail("prepare _setErrorBlacklistQuery", *_setErrorBlacklistQuery); + } + + _getSelectiveSyncListQuery.reset(new SqlQuery(_db)); + if (_getSelectiveSyncListQuery->prepare("SELECT path FROM selectivesync WHERE type=?1")) { + return sqlFail("prepare _getSelectiveSyncListQuery", *_getSelectiveSyncListQuery); + } + + _getChecksumTypeIdQuery.reset(new SqlQuery(_db)); + if (_getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1")) { + return sqlFail("prepare _getChecksumTypeIdQuery", *_getChecksumTypeIdQuery); + } + + _getChecksumTypeQuery.reset(new SqlQuery(_db)); + if (_getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1")) { + return sqlFail("prepare _getChecksumTypeQuery", *_getChecksumTypeQuery); + } + + _insertChecksumTypeQuery.reset(new SqlQuery(_db)); + if (_insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)")) { + return sqlFail("prepare _insertChecksumTypeQuery", *_insertChecksumTypeQuery); + } + + _getDataFingerprintQuery.reset(new SqlQuery(_db)); + if (_getDataFingerprintQuery->prepare("SELECT fingerprint FROM datafingerprint")) { + return sqlFail("prepare _getDataFingerprintQuery", *_getDataFingerprintQuery); + } + + _setDataFingerprintQuery1.reset(new SqlQuery(_db)); + if (_setDataFingerprintQuery1->prepare("DELETE FROM datafingerprint;")) { + return sqlFail("prepare _setDataFingerprintQuery1", *_setDataFingerprintQuery1); + } + _setDataFingerprintQuery2.reset(new SqlQuery(_db)); + if (_setDataFingerprintQuery2->prepare("INSERT INTO datafingerprint (fingerprint) VALUES (?1);")) { + return sqlFail("prepare _setDataFingerprintQuery2", *_setDataFingerprintQuery2); + } + + // don't start a new transaction now + commitInternal(QString("checkConnect End"), false); + + // Hide 'em all! + FileSystem::setFileHidden(databaseFilePath(), true); + FileSystem::setFileHidden(databaseFilePath() + "-wal", true); + FileSystem::setFileHidden(databaseFilePath() + "-shm", true); + FileSystem::setFileHidden(databaseFilePath() + "-journal", true); + + return rc; +} + +void SyncJournalDb::close() +{ + QMutexLocker locker(&_mutex); + qCInfo(lcDb) << "Closing DB" << _dbFile; + + commitTransaction(); + + _getFileRecordQuery.reset(0); + _setFileRecordQuery.reset(0); + _setFileRecordChecksumQuery.reset(0); + _setFileRecordLocalMetadataQuery.reset(0); + _getDownloadInfoQuery.reset(0); + _setDownloadInfoQuery.reset(0); + _deleteDownloadInfoQuery.reset(0); + _getUploadInfoQuery.reset(0); + _setUploadInfoQuery.reset(0); + _deleteUploadInfoQuery.reset(0); + _deleteFileRecordPhash.reset(0); + _deleteFileRecordRecursively.reset(0); + _getErrorBlacklistQuery.reset(0); + _setErrorBlacklistQuery.reset(0); + _getSelectiveSyncListQuery.reset(0); + _getChecksumTypeIdQuery.reset(0); + _getChecksumTypeQuery.reset(0); + _insertChecksumTypeQuery.reset(0); + _getDataFingerprintQuery.reset(0); + _setDataFingerprintQuery1.reset(0); + _setDataFingerprintQuery2.reset(0); + + _db.close(); + _avoidReadFromDbOnNextSyncFilter.clear(); +} + + +bool SyncJournalDb::updateDatabaseStructure() +{ + if (!updateMetadataTableStructure()) + return false; + if (!updateErrorBlacklistTableStructure()) + return false; + return true; +} + +bool SyncJournalDb::updateMetadataTableStructure() +{ + QStringList columns = tableColumns("metadata"); + bool re = true; + + // check if the file_id column is there and create it if not + if (!checkConnect()) { + return false; + } + + if (columns.indexOf(QLatin1String("fileid")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN fileid VARCHAR(128);"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: Add column fileid", query); + re = false; + } + + query.prepare("CREATE INDEX metadata_file_id ON metadata(fileid);"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: create index fileid", query); + re = false; + } + commitInternal("update database structure: add fileid col"); + } + if (columns.indexOf(QLatin1String("remotePerm")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN remotePerm VARCHAR(128);"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: add column remotePerm", query); + re = false; + } + commitInternal("update database structure (remotePerm)"); + } + if (columns.indexOf(QLatin1String("filesize")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN filesize BIGINT;"); + if (!query.exec()) { + sqlFail("updateDatabaseStructure: add column filesize", query); + re = false; + } + commitInternal("update database structure: add filesize col"); + } + + if (1) { + SqlQuery query(_db); + query.prepare("CREATE INDEX IF NOT EXISTS metadata_inode ON metadata(inode);"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: create index inode", query); + re = false; + } + commitInternal("update database structure: add inode index"); + } + + if (1) { + SqlQuery query(_db); + query.prepare("CREATE INDEX IF NOT EXISTS metadata_path ON metadata(path);"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: create index path", query); + re = false; + } + commitInternal("update database structure: add path index"); + } + + if (columns.indexOf(QLatin1String("ignoredChildrenRemote")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN ignoredChildrenRemote INT;"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: add ignoredChildrenRemote column", query); + re = false; + } + commitInternal("update database structure: add ignoredChildrenRemote col"); + } + + if (columns.indexOf(QLatin1String("contentChecksum")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: add contentChecksum column", query); + re = false; + } + commitInternal("update database structure: add contentChecksum col"); + } + if (columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query); + re = false; + } + commitInternal("update database structure: add contentChecksumTypeId col"); + } + + + return re; +} + +bool SyncJournalDb::updateErrorBlacklistTableStructure() +{ + QStringList columns = tableColumns("blacklist"); + bool re = true; + + // check if the file_id column is there and create it if not + if (!checkConnect()) { + return false; + } + + if (columns.indexOf(QLatin1String("lastTryTime")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE blacklist ADD COLUMN lastTryTime INTEGER(8);"); + if (!query.exec()) { + sqlFail("updateBlacklistTableStructure: Add lastTryTime fileid", query); + re = false; + } + query.prepare("ALTER TABLE blacklist ADD COLUMN ignoreDuration INTEGER(8);"); + if (!query.exec()) { + sqlFail("updateBlacklistTableStructure: Add ignoreDuration fileid", query); + re = false; + } + commitInternal("update database structure: add lastTryTime, ignoreDuration cols"); + } + if (columns.indexOf(QLatin1String("renameTarget")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE blacklist ADD COLUMN renameTarget VARCHAR(4096);"); + if (!query.exec()) { + sqlFail("updateBlacklistTableStructure: Add renameTarget", query); + re = false; + } + commitInternal("update database structure: add renameTarget col"); + } + + if (columns.indexOf(QLatin1String("errorCategory")) == -1) { + SqlQuery query(_db); + query.prepare("ALTER TABLE blacklist ADD COLUMN errorCategory INTEGER(8);"); + if (!query.exec()) { + sqlFail("updateBlacklistTableStructure: Add errorCategory", query); + re = false; + } + commitInternal("update database structure: add errorCategory col"); + } + + SqlQuery query(_db); + query.prepare("CREATE INDEX IF NOT EXISTS blacklist_index ON blacklist(path collate nocase);"); + if (!query.exec()) { + sqlFail("updateErrorBlacklistTableStructure: create index blacklit", query); + re = false; + } + + return re; +} + +QStringList SyncJournalDb::tableColumns(const QString &table) +{ + QStringList columns; + if (!table.isEmpty()) { + if (checkConnect()) { + QString q = QString("PRAGMA table_info('%1');").arg(table); + SqlQuery query(_db); + query.prepare(q); + + if (!query.exec()) { + return columns; + } + + while (query.next()) { + columns.append(query.stringValue(1)); + } + } + } + qCDebug(lcDb) << "Columns in the current journal: " << columns; + + return columns; +} + +qint64 SyncJournalDb::getPHash(const QString &file) +{ + QByteArray utf8File = file.toUtf8(); + int64_t h; + + if (file.isEmpty()) { + return -1; + } + + int len = utf8File.length(); + + h = c_jhash64((uint8_t *)utf8File.data(), len, 0); + return h; +} + +bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) +{ + SyncJournalFileRecord record = _record; + QMutexLocker locker(&_mutex); + + if (!_avoidReadFromDbOnNextSyncFilter.isEmpty()) { + // If we are a directory that should not be read from db next time, don't write the etag + QString prefix = record._path + "/"; + foreach (const QString &it, _avoidReadFromDbOnNextSyncFilter) { + if (it.startsWith(prefix)) { + qCInfo(lcDb) << "Filtered writing the etag of" << prefix << "because it is a prefix of" << it; + record._etag = "_invalid_"; + break; + } + } + } + + qCInfo(lcDb) << "Updating file record for path:" << record._path << "inode:" << record._inode + << "modtime:" << record._modtime << "type:" << record._type + << "etag:" << record._etag << "fileId:" << record._fileId << "remotePerm:" << record._remotePerm + << "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader; + + qlonglong phash = getPHash(record._path); + if (checkConnect()) { + QByteArray arr = record._path.toUtf8(); + int plen = arr.length(); + + QString etag(record._etag); + if (etag.isEmpty()) + etag = ""; + QString fileId(record._fileId); + if (fileId.isEmpty()) + fileId = ""; + QString remotePerm(record._remotePerm); + if (remotePerm.isEmpty()) + remotePerm = QString(); // have NULL in DB (vs empty) + QByteArray checksumType, checksum; + parseChecksumHeader(record._checksumHeader, &checksumType, &checksum); + int contentChecksumTypeId = mapChecksumType(checksumType); + _setFileRecordQuery->reset_and_clear_bindings(); + _setFileRecordQuery->bindValue(1, QString::number(phash)); + _setFileRecordQuery->bindValue(2, plen); + _setFileRecordQuery->bindValue(3, record._path); + _setFileRecordQuery->bindValue(4, record._inode); + _setFileRecordQuery->bindValue(5, 0); // uid Not used + _setFileRecordQuery->bindValue(6, 0); // gid Not used + _setFileRecordQuery->bindValue(7, 0); // mode Not used + _setFileRecordQuery->bindValue(8, QString::number(Utility::qDateTimeToTime_t(record._modtime))); + _setFileRecordQuery->bindValue(9, QString::number(record._type)); + _setFileRecordQuery->bindValue(10, etag); + _setFileRecordQuery->bindValue(11, fileId); + _setFileRecordQuery->bindValue(12, remotePerm); + _setFileRecordQuery->bindValue(13, record._fileSize); + _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1 : 0); + _setFileRecordQuery->bindValue(15, checksum); + _setFileRecordQuery->bindValue(16, contentChecksumTypeId); + + if (!_setFileRecordQuery->exec()) { + return false; + } + + _setFileRecordQuery->reset_and_clear_bindings(); + return true; + } else { + qCWarning(lcDb) << "Failed to connect database."; + return false; // checkConnect failed. + } +} + +bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively) +{ + QMutexLocker locker(&_mutex); + + if (checkConnect()) { + // if (!recursively) { + // always delete the actual file. + + qlonglong phash = getPHash(filename); + _deleteFileRecordPhash->reset_and_clear_bindings(); + _deleteFileRecordPhash->bindValue(1, QString::number(phash)); + + if (!_deleteFileRecordPhash->exec()) { + return false; + } + + _deleteFileRecordPhash->reset_and_clear_bindings(); + if (recursively) { + _deleteFileRecordRecursively->reset_and_clear_bindings(); + _deleteFileRecordRecursively->bindValue(1, filename); + if (!_deleteFileRecordRecursively->exec()) { + return false; + } + + _deleteFileRecordRecursively->reset_and_clear_bindings(); + } + return true; + } else { + qCWarning(lcDb) << "Failed to connect database."; + return false; // checkConnect failed. + } +} + + +SyncJournalFileRecord SyncJournalDb::getFileRecord(const QString &filename) +{ + QMutexLocker locker(&_mutex); + + qlonglong phash = getPHash(filename); + SyncJournalFileRecord rec; + + if (!filename.isEmpty() && checkConnect()) { + _getFileRecordQuery->reset_and_clear_bindings(); + _getFileRecordQuery->bindValue(1, QString::number(phash)); + + if (!_getFileRecordQuery->exec()) { + locker.unlock(); + close(); + return rec; + } + + if (_getFileRecordQuery->next()) { + rec._path = _getFileRecordQuery->stringValue(0); + rec._inode = _getFileRecordQuery->intValue(1); + //rec._uid = _getFileRecordQuery->value(2).toInt(&ok); Not Used + //rec._gid = _getFileRecordQuery->value(3).toInt(&ok); Not Used + //rec._mode = _getFileRecordQuery->intValue(4); + rec._modtime = Utility::qDateTimeFromTime_t(_getFileRecordQuery->int64Value(5)); + rec._type = _getFileRecordQuery->intValue(6); + rec._etag = _getFileRecordQuery->baValue(7); + rec._fileId = _getFileRecordQuery->baValue(8); + rec._remotePerm = _getFileRecordQuery->baValue(9); + rec._fileSize = _getFileRecordQuery->int64Value(10); + rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0); + rec._checksumHeader = _getFileRecordQuery->baValue(12); + _getFileRecordQuery->reset_and_clear_bindings(); + } else { + int errId = _getFileRecordQuery->errorId(); + if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE + QString err = _getFileRecordQuery->error(); + qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err; + locker.unlock(); + close(); + locker.relock(); + } + } + if (_getFileRecordQuery) { + _getFileRecordQuery->reset_and_clear_bindings(); + } + } + return rec; +} + +bool SyncJournalDb::postSyncCleanup(const QSet &filepathsToKeep, + const QSet &prefixesToKeep) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return false; + } + + SqlQuery query(_db); + query.prepare("SELECT phash, path FROM metadata order by path"); + + if (!query.exec()) { + return false; + } + + QStringList superfluousItems; + + while (query.next()) { + const QString file = query.stringValue(1); + bool keep = filepathsToKeep.contains(file); + if (!keep) { + foreach (const QString &prefix, prefixesToKeep) { + if (file.startsWith(prefix)) { + keep = true; + break; + } + } + } + if (!keep) { + superfluousItems.append(query.stringValue(0)); + } + } + + if (superfluousItems.count()) { + QString sql = "DELETE FROM metadata WHERE phash in (" + superfluousItems.join(",") + ")"; + qCInfo(lcDb) << "Sync Journal cleanup for" << superfluousItems; + SqlQuery delQuery(_db); + delQuery.prepare(sql); + if (!delQuery.exec()) { + return false; + } + } + + // Incorporate results back into main DB + walCheckpoint(); + + return true; +} + +int SyncJournalDb::getFileRecordCount() +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return -1; + } + + SqlQuery query(_db); + query.prepare("SELECT COUNT(*) FROM metadata"); + + if (!query.exec()) { + return 0; + } + + if (query.next()) { + int count = query.intValue(0); + return count; + } + + return 0; +} + +bool SyncJournalDb::updateFileRecordChecksum(const QString &filename, + const QByteArray &contentChecksum, + const QByteArray &contentChecksumType) +{ + QMutexLocker locker(&_mutex); + + qCInfo(lcDb) << "Updating file checksum" << filename << contentChecksum << contentChecksumType; + + qlonglong phash = getPHash(filename); + if (!checkConnect()) { + qCWarning(lcDb) << "Failed to connect database."; + return false; + } + + int checksumTypeId = mapChecksumType(contentChecksumType); + auto &query = _setFileRecordChecksumQuery; + + query->reset_and_clear_bindings(); + query->bindValue(1, QString::number(phash)); + query->bindValue(2, contentChecksum); + query->bindValue(3, checksumTypeId); + + if (!query->exec()) { + return false; + } + + query->reset_and_clear_bindings(); + return true; +} + +bool SyncJournalDb::updateLocalMetadata(const QString &filename, + qint64 modtime, quint64 size, quint64 inode) + +{ + QMutexLocker locker(&_mutex); + + qCInfo(lcDb) << "Updating local metadata for:" << filename << modtime << size << inode; + + qlonglong phash = getPHash(filename); + if (!checkConnect()) { + qCWarning(lcDb) << "Failed to connect database."; + return false; + } + + auto &query = _setFileRecordLocalMetadataQuery; + + query->reset_and_clear_bindings(); + query->bindValue(1, QString::number(phash)); + query->bindValue(2, inode); + query->bindValue(3, modtime); + query->bindValue(4, size); + + if (!query->exec()) { + return false; + } + + query->reset_and_clear_bindings(); + return true; +} + +bool SyncJournalDb::setFileRecordMetadata(const SyncJournalFileRecord &record) +{ + SyncJournalFileRecord existing = getFileRecord(record._path); + + // If there's no existing record, just insert the new one. + if (existing._path.isEmpty()) { + return setFileRecord(record); + } + + // Update the metadata on the existing record. + existing._inode = record._inode; + existing._modtime = record._modtime; + existing._type = record._type; + existing._etag = record._etag; + existing._fileId = record._fileId; + existing._remotePerm = record._remotePerm; + existing._fileSize = record._fileSize; + existing._serverHasIgnoredFiles = record._serverHasIgnoredFiles; + return setFileRecord(existing); +} + +static void toDownloadInfo(SqlQuery &query, SyncJournalDb::DownloadInfo *res) +{ + bool ok = true; + res->_tmpfile = query.stringValue(0); + res->_etag = query.baValue(1); + res->_errorCount = query.intValue(2); + res->_valid = ok; +} + +static bool deleteBatch(SqlQuery &query, const QStringList &entries, const QString &name) +{ + if (entries.isEmpty()) + return true; + + qCDebug(lcDb) << "Removing stale " << qPrintable(name) << " entries: " << entries.join(", "); + // FIXME: Was ported from execBatch, check if correct! + foreach (const QString &entry, entries) { + query.reset_and_clear_bindings(); + query.bindValue(1, entry); + if (!query.exec()) { + return false; + } + } + query.reset_and_clear_bindings(); // viel hilft viel ;-) + + return true; +} + +SyncJournalDb::DownloadInfo SyncJournalDb::getDownloadInfo(const QString &file) +{ + QMutexLocker locker(&_mutex); + + DownloadInfo res; + + if (checkConnect()) { + _getDownloadInfoQuery->reset_and_clear_bindings(); + _getDownloadInfoQuery->bindValue(1, file); + + if (!_getDownloadInfoQuery->exec()) { + return res; + } + + if (_getDownloadInfoQuery->next()) { + toDownloadInfo(*_getDownloadInfoQuery, &res); + } else { + res._valid = false; + } + _getDownloadInfoQuery->reset_and_clear_bindings(); + } + return res; +} + +void SyncJournalDb::setDownloadInfo(const QString &file, const SyncJournalDb::DownloadInfo &i) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return; + } + + if (i._valid) { + _setDownloadInfoQuery->reset_and_clear_bindings(); + _setDownloadInfoQuery->bindValue(1, file); + _setDownloadInfoQuery->bindValue(2, i._tmpfile); + _setDownloadInfoQuery->bindValue(3, i._etag); + _setDownloadInfoQuery->bindValue(4, i._errorCount); + + if (!_setDownloadInfoQuery->exec()) { + return; + } + + _setDownloadInfoQuery->reset_and_clear_bindings(); + + } else { + _deleteDownloadInfoQuery->reset_and_clear_bindings(); + _deleteDownloadInfoQuery->bindValue(1, file); + + if (!_deleteDownloadInfoQuery->exec()) { + return; + } + + _deleteDownloadInfoQuery->reset_and_clear_bindings(); + } +} + +QVector SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet &keep) +{ + QVector empty_result; + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return empty_result; + } + + SqlQuery query(_db); + // The selected values *must* match the ones expected by toDownloadInfo(). + query.prepare("SELECT tmpfile, etag, errorcount, path FROM downloadinfo"); + + if (!query.exec()) { + return empty_result; + } + + QStringList superfluousPaths; + QVector deleted_entries; + + while (query.next()) { + const QString file = query.stringValue(3); // path + if (!keep.contains(file)) { + superfluousPaths.append(file); + DownloadInfo info; + toDownloadInfo(query, &info); + deleted_entries.append(info); + } + } + + if (!deleteBatch(*_deleteDownloadInfoQuery, superfluousPaths, "downloadinfo")) + return empty_result; + + return deleted_entries; +} + +int SyncJournalDb::downloadInfoCount() +{ + int re = 0; + + QMutexLocker locker(&_mutex); + if (checkConnect()) { + SqlQuery query("SELECT count(*) FROM downloadinfo", _db); + + if (!query.exec()) { + sqlFail("Count number of downloadinfo entries failed", query); + } + if (query.next()) { + re = query.intValue(0); + } + } + return re; +} + +SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString &file) +{ + QMutexLocker locker(&_mutex); + + UploadInfo res; + + if (checkConnect()) { + _getUploadInfoQuery->reset_and_clear_bindings(); + _getUploadInfoQuery->bindValue(1, file); + + if (!_getUploadInfoQuery->exec()) { + return res; + } + + if (_getUploadInfoQuery->next()) { + bool ok = true; + res._chunk = _getUploadInfoQuery->intValue(0); + res._transferid = _getUploadInfoQuery->intValue(1); + res._errorCount = _getUploadInfoQuery->intValue(2); + res._size = _getUploadInfoQuery->int64Value(3); + res._modtime = Utility::qDateTimeFromTime_t(_getUploadInfoQuery->int64Value(4)); + res._valid = ok; + } + _getUploadInfoQuery->reset_and_clear_bindings(); + } + return res; +} + +void SyncJournalDb::setUploadInfo(const QString &file, const SyncJournalDb::UploadInfo &i) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return; + } + + if (i._valid) { + _setUploadInfoQuery->reset_and_clear_bindings(); + _setUploadInfoQuery->bindValue(1, file); + _setUploadInfoQuery->bindValue(2, i._chunk); + _setUploadInfoQuery->bindValue(3, i._transferid); + _setUploadInfoQuery->bindValue(4, i._errorCount); + _setUploadInfoQuery->bindValue(5, i._size); + _setUploadInfoQuery->bindValue(6, Utility::qDateTimeToTime_t(i._modtime)); + + if (!_setUploadInfoQuery->exec()) { + return; + } + + _setUploadInfoQuery->reset_and_clear_bindings(); + } else { + _deleteUploadInfoQuery->reset_and_clear_bindings(); + _deleteUploadInfoQuery->bindValue(1, file); + + if (!_deleteUploadInfoQuery->exec()) { + return; + } + + _deleteUploadInfoQuery->reset_and_clear_bindings(); + } +} + +QVector SyncJournalDb::deleteStaleUploadInfos(const QSet &keep) +{ + QMutexLocker locker(&_mutex); + QVector ids; + + if (!checkConnect()) { + return ids; + } + + SqlQuery query(_db); + query.prepare("SELECT path,transferid FROM uploadinfo"); + + if (!query.exec()) { + return ids; + } + + QStringList superfluousPaths; + + while (query.next()) { + const QString file = query.stringValue(0); + if (!keep.contains(file)) { + superfluousPaths.append(file); + ids.append(query.intValue(1)); + } + } + + deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo"); + return ids; +} + +SyncJournalErrorBlacklistRecord SyncJournalDb::errorBlacklistEntry(const QString &file) +{ + QMutexLocker locker(&_mutex); + SyncJournalErrorBlacklistRecord entry; + + if (file.isEmpty()) + return entry; + + // SELECT lastTryEtag, lastTryModtime, retrycount, errorstring + + if (checkConnect()) { + _getErrorBlacklistQuery->reset_and_clear_bindings(); + _getErrorBlacklistQuery->bindValue(1, file); + if (_getErrorBlacklistQuery->exec()) { + if (_getErrorBlacklistQuery->next()) { + entry._lastTryEtag = _getErrorBlacklistQuery->baValue(0); + entry._lastTryModtime = _getErrorBlacklistQuery->int64Value(1); + entry._retryCount = _getErrorBlacklistQuery->intValue(2); + entry._errorString = _getErrorBlacklistQuery->stringValue(3); + entry._lastTryTime = _getErrorBlacklistQuery->int64Value(4); + entry._ignoreDuration = _getErrorBlacklistQuery->int64Value(5); + entry._renameTarget = _getErrorBlacklistQuery->stringValue(6); + entry._errorCategory = static_cast( + _getErrorBlacklistQuery->intValue(7)); + entry._file = file; + } + _getErrorBlacklistQuery->reset_and_clear_bindings(); + } + } + + return entry; +} + +bool SyncJournalDb::deleteStaleErrorBlacklistEntries(const QSet &keep) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return false; + } + + SqlQuery query(_db); + query.prepare("SELECT path FROM blacklist"); + + if (!query.exec()) { + return false; + } + + QStringList superfluousPaths; + + while (query.next()) { + const QString file = query.stringValue(0); + if (!keep.contains(file)) { + superfluousPaths.append(file); + } + } + + SqlQuery delQuery(_db); + delQuery.prepare("DELETE FROM blacklist WHERE path = ?"); + return deleteBatch(delQuery, superfluousPaths, "blacklist"); +} + +int SyncJournalDb::errorBlackListEntryCount() +{ + int re = 0; + + QMutexLocker locker(&_mutex); + if (checkConnect()) { + SqlQuery query("SELECT count(*) FROM blacklist", _db); + + if (!query.exec()) { + sqlFail("Count number of blacklist entries failed", query); + } + if (query.next()) { + re = query.intValue(0); + } + } + return re; +} + +int SyncJournalDb::wipeErrorBlacklist() +{ + QMutexLocker locker(&_mutex); + if (checkConnect()) { + SqlQuery query(_db); + + query.prepare("DELETE FROM blacklist"); + + if (!query.exec()) { + sqlFail("Deletion of whole blacklist failed", query); + return -1; + } + return query.numRowsAffected(); + } + return -1; +} + +void SyncJournalDb::wipeErrorBlacklistEntry(const QString &file) +{ + if (file.isEmpty()) { + return; + } + + QMutexLocker locker(&_mutex); + if (checkConnect()) { + SqlQuery query(_db); + + query.prepare("DELETE FROM blacklist WHERE path=?1"); + query.bindValue(1, file); + if (!query.exec()) { + sqlFail("Deletion of blacklist item failed.", query); + } + } +} + +void SyncJournalDb::wipeErrorBlacklistCategory(SyncJournalErrorBlacklistRecord::Category category) +{ + QMutexLocker locker(&_mutex); + if (checkConnect()) { + SqlQuery query(_db); + + query.prepare("DELETE FROM blacklist WHERE errorCategory=?1"); + query.bindValue(1, category); + if (!query.exec()) { + sqlFail("Deletion of blacklist category failed.", query); + } + } +} + +void SyncJournalDb::setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item) +{ + QMutexLocker locker(&_mutex); + + qCInfo(lcDb) << "Setting blacklist entry for " << item._file << item._retryCount + << item._errorString << item._lastTryTime << item._ignoreDuration + << item._lastTryModtime << item._lastTryEtag << item._renameTarget + << item._errorCategory; + + if (!checkConnect()) { + return; + } + + _setErrorBlacklistQuery->bindValue(1, item._file); + _setErrorBlacklistQuery->bindValue(2, item._lastTryEtag); + _setErrorBlacklistQuery->bindValue(3, QString::number(item._lastTryModtime)); + _setErrorBlacklistQuery->bindValue(4, item._retryCount); + _setErrorBlacklistQuery->bindValue(5, item._errorString); + _setErrorBlacklistQuery->bindValue(6, QString::number(item._lastTryTime)); + _setErrorBlacklistQuery->bindValue(7, QString::number(item._ignoreDuration)); + _setErrorBlacklistQuery->bindValue(8, item._renameTarget); + _setErrorBlacklistQuery->bindValue(9, item._errorCategory); + _setErrorBlacklistQuery->exec(); + _setErrorBlacklistQuery->reset_and_clear_bindings(); +} + +QVector SyncJournalDb::getPollInfos() +{ + QMutexLocker locker(&_mutex); + + QVector res; + + if (!checkConnect()) + return res; + + SqlQuery query("SELECT path, modtime, pollpath FROM poll", _db); + + if (!query.exec()) { + return res; + } + + while (query.next()) { + PollInfo info; + info._file = query.stringValue(0); + info._modtime = query.int64Value(1); + info._url = query.stringValue(2); + res.append(info); + } + + query.finish(); + return res; +} + +void SyncJournalDb::setPollInfo(const SyncJournalDb::PollInfo &info) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return; + } + + if (info._url.isEmpty()) { + qCDebug(lcDb) << "Deleting Poll job" << info._file; + SqlQuery query("DELETE FROM poll WHERE path=?", _db); + query.bindValue(1, info._file); + query.exec(); + } else { + SqlQuery query("INSERT OR REPLACE INTO poll (path, modtime, pollpath) VALUES( ? , ? , ? )", _db); + query.bindValue(1, info._file); + query.bindValue(2, QString::number(info._modtime)); + query.bindValue(3, info._url); + query.exec(); + } +} + +QStringList SyncJournalDb::getSelectiveSyncList(SyncJournalDb::SelectiveSyncListType type, bool *ok) +{ + QStringList result; + ASSERT(ok); + + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + *ok = false; + return result; + } + + _getSelectiveSyncListQuery->reset_and_clear_bindings(); + _getSelectiveSyncListQuery->bindValue(1, int(type)); + if (!_getSelectiveSyncListQuery->exec()) { + *ok = false; + return result; + } + while (_getSelectiveSyncListQuery->next()) { + auto entry = _getSelectiveSyncListQuery->stringValue(0); + if (!entry.endsWith(QLatin1Char('/'))) { + entry.append(QLatin1Char('/')); + } + result.append(entry); + } + *ok = true; + + return result; +} + +void SyncJournalDb::setSelectiveSyncList(SyncJournalDb::SelectiveSyncListType type, const QStringList &list) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return; + } + + //first, delete all entries of this type + SqlQuery delQuery("DELETE FROM selectivesync WHERE type == ?1", _db); + delQuery.bindValue(1, int(type)); + if (!delQuery.exec()) { + qCWarning(lcDb) << "SQL error when deleting selective sync list" << list << delQuery.error(); + } + + SqlQuery insQuery("INSERT INTO selectivesync VALUES (?1, ?2)", _db); + foreach (const auto &path, list) { + insQuery.reset_and_clear_bindings(); + insQuery.bindValue(1, path); + insQuery.bindValue(2, int(type)); + if (!insQuery.exec()) { + qCWarning(lcDb) << "SQL error when inserting into selective sync" << type << path << delQuery.error(); + } + } +} + +void SyncJournalDb::avoidRenamesOnNextSync(const QString &path) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return; + } + + SqlQuery query(_db); + query.prepare("UPDATE metadata SET fileid = '', inode = '0' WHERE path == ?1 OR path LIKE(?2||'/%')"); + query.bindValue(1, path); + query.bindValue(2, path); + query.exec(); + + // We also need to remove the ETags so the update phase refreshes the directory paths + // on the next sync + locker.unlock(); + avoidReadFromDbOnNextSync(path); +} + +void SyncJournalDb::avoidReadFromDbOnNextSync(const QString &fileName) +{ + // Make sure that on the next sync, fileName is not read from the DB but uses the PROPFIND to + // get the info from the server + // We achieve that by clearing the etag of the parents directory recursively + + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return; + } + + SqlQuery query(_db); + // This query will match entries for which the path is a prefix of fileName + // Note: CSYNC_FTW_TYPE_DIR == 2 + query.prepare("UPDATE metadata SET md5='_invalid_' WHERE ?1 LIKE(path||'/%') AND type == 2;"); + query.bindValue(1, fileName); + query.exec(); + + // Prevent future overwrite of the etag for this sync + _avoidReadFromDbOnNextSyncFilter.append(fileName); +} + +void SyncJournalDb::forceRemoteDiscoveryNextSync() +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) { + return; + } + + forceRemoteDiscoveryNextSyncLocked(); +} + +void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked() +{ + qCInfo(lcDb) << "Forcing remote re-discovery by deleting folder Etags"; + SqlQuery deleteRemoteFolderEtagsQuery(_db); + deleteRemoteFolderEtagsQuery.prepare("UPDATE metadata SET md5='_invalid_' WHERE type=2;"); + deleteRemoteFolderEtagsQuery.exec(); +} + + +QByteArray SyncJournalDb::getChecksumType(int checksumTypeId) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return QByteArray(); + } + + // Retrieve the id + auto &query = *_getChecksumTypeQuery; + query.reset_and_clear_bindings(); + query.bindValue(1, checksumTypeId); + if (!query.exec()) { + return 0; + } + + if (!query.next()) { + qCWarning(lcDb) << "No checksum type mapping found for" << checksumTypeId; + return 0; + } + return query.baValue(0); +} + +int SyncJournalDb::mapChecksumType(const QByteArray &checksumType) +{ + if (checksumType.isEmpty()) { + return 0; + } + + // Ensure the checksum type is in the db + _insertChecksumTypeQuery->reset_and_clear_bindings(); + _insertChecksumTypeQuery->bindValue(1, checksumType); + if (!_insertChecksumTypeQuery->exec()) { + return 0; + } + + // Retrieve the id + _getChecksumTypeIdQuery->reset_and_clear_bindings(); + _getChecksumTypeIdQuery->bindValue(1, checksumType); + if (!_getChecksumTypeIdQuery->exec()) { + return 0; + } + + if (!_getChecksumTypeIdQuery->next()) { + qCWarning(lcDb) << "No checksum type mapping found for" << checksumType; + return 0; + } + return _getChecksumTypeIdQuery->intValue(0); +} + +QByteArray SyncJournalDb::dataFingerprint() +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return QByteArray(); + } + + _getDataFingerprintQuery->reset_and_clear_bindings(); + if (!_getDataFingerprintQuery->exec()) { + return QByteArray(); + } + + if (!_getDataFingerprintQuery->next()) { + return QByteArray(); + } + return _getDataFingerprintQuery->baValue(0); +} + +void SyncJournalDb::setDataFingerprint(const QByteArray &dataFingerprint) +{ + QMutexLocker locker(&_mutex); + if (!checkConnect()) { + return; + } + + _setDataFingerprintQuery1->reset_and_clear_bindings(); + _setDataFingerprintQuery1->exec(); + + _setDataFingerprintQuery2->reset_and_clear_bindings(); + _setDataFingerprintQuery2->bindValue(1, dataFingerprint); + _setDataFingerprintQuery2->exec(); +} + +void SyncJournalDb::clearFileTable() +{ + SqlQuery query(_db); + query.prepare("DELETE FROM metadata;"); + query.exec(); +} + +void SyncJournalDb::commit(const QString &context, bool startTrans) +{ + QMutexLocker lock(&_mutex); + commitInternal(context, startTrans); +} + +void SyncJournalDb::commitIfNeededAndStartNewTransaction(const QString &context) +{ + QMutexLocker lock(&_mutex); + if (_transaction == 1) { + commitInternal(context, true); + } else { + startTransaction(); + } +} + + +void SyncJournalDb::commitInternal(const QString &context, bool startTrans) +{ + qCDebug(lcDb) << "Transaction commit " << context << (startTrans ? "and starting new transaction" : ""); + commitTransaction(); + + if (startTrans) { + startTransaction(); + } +} + +SyncJournalDb::~SyncJournalDb() +{ + close(); +} + +bool SyncJournalDb::isConnected() +{ + QMutexLocker lock(&_mutex); + return checkConnect(); +} + +bool operator==(const SyncJournalDb::DownloadInfo &lhs, + const SyncJournalDb::DownloadInfo &rhs) +{ + return lhs._errorCount == rhs._errorCount + && lhs._etag == rhs._etag + && lhs._tmpfile == rhs._tmpfile + && lhs._valid == rhs._valid; +} + +bool operator==(const SyncJournalDb::UploadInfo &lhs, + const SyncJournalDb::UploadInfo &rhs) +{ + return lhs._errorCount == rhs._errorCount + && lhs._chunk == rhs._chunk + && lhs._modtime == rhs._modtime + && lhs._valid == rhs._valid + && lhs._size == rhs._size + && lhs._transferid == rhs._transferid; +} + +} // namespace OCC diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h new file mode 100644 index 000000000..0afefebcf --- /dev/null +++ b/src/common/syncjournaldb.h @@ -0,0 +1,280 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SYNCJOURNALDB_H +#define SYNCJOURNALDB_H + +#include +#include +#include +#include + +#include "common/utility.h" +#include "common/ownsql.h" +#include "common/syncjournalfilerecord.h" + +namespace OCC { +class SyncJournalFileRecord; + +/** + * @brief Class that handles the sync database + * + * This class is thread safe. All public functions lock the mutex. + * @ingroup libsync + */ +class OCSYNC_EXPORT SyncJournalDb : public QObject +{ + Q_OBJECT +public: + explicit SyncJournalDb(const QString &dbFilePath, QObject *parent = 0); + virtual ~SyncJournalDb(); + + /// Create a journal path for a specific configuration + static QString makeDbName(const QString &localPath, + const QUrl &remoteUrl, + const QString &remotePath, + const QString &user); + + /// Migrate a csync_journal to the new path, if necessary. Returns false on error + static bool maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath); + + // to verify that the record could be queried successfully check + // with SyncJournalFileRecord::isValid() + SyncJournalFileRecord getFileRecord(const QString &filename); + bool setFileRecord(const SyncJournalFileRecord &record); + + /// Like setFileRecord, but preserves checksums + bool setFileRecordMetadata(const SyncJournalFileRecord &record); + + bool deleteFileRecord(const QString &filename, bool recursively = false); + int getFileRecordCount(); + bool updateFileRecordChecksum(const QString &filename, + const QByteArray &contentChecksum, + const QByteArray &contentChecksumType); + bool updateLocalMetadata(const QString &filename, + qint64 modtime, quint64 size, quint64 inode); + bool exists(); + void walCheckpoint(); + + QString databaseFilePath() const; + + static qint64 getPHash(const QString &); + + void setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item); + void wipeErrorBlacklistEntry(const QString &file); + void wipeErrorBlacklistCategory(SyncJournalErrorBlacklistRecord::Category category); + int wipeErrorBlacklist(); + int errorBlackListEntryCount(); + + struct DownloadInfo + { + DownloadInfo() + : _errorCount(0) + , _valid(false) + { + } + QString _tmpfile; + QByteArray _etag; + int _errorCount; + bool _valid; + }; + struct UploadInfo + { + UploadInfo() + : _chunk(0) + , _transferid(0) + , _size(0) + , _errorCount(0) + , _valid(false) + { + } + int _chunk; + int _transferid; + quint64 _size; //currently unused + QDateTime _modtime; + int _errorCount; + bool _valid; + }; + + struct PollInfo + { + QString _file; + QString _url; + time_t _modtime; + }; + + DownloadInfo getDownloadInfo(const QString &file); + void setDownloadInfo(const QString &file, const DownloadInfo &i); + QVector getAndDeleteStaleDownloadInfos(const QSet &keep); + int downloadInfoCount(); + + UploadInfo getUploadInfo(const QString &file); + void setUploadInfo(const QString &file, const UploadInfo &i); + // Return the list of transfer ids that were removed. + QVector deleteStaleUploadInfos(const QSet &keep); + + SyncJournalErrorBlacklistRecord errorBlacklistEntry(const QString &); + bool deleteStaleErrorBlacklistEntries(const QSet &keep); + + void avoidRenamesOnNextSync(const QString &path); + void setPollInfo(const PollInfo &); + QVector getPollInfos(); + + enum SelectiveSyncListType { + /** The black list is the list of folders that are unselected in the selective sync dialog. + * For the sync engine, those folders are considered as if they were not there, so the local + * folders will be deleted */ + SelectiveSyncBlackList = 1, + /** When a shared folder has a size bigger than a configured size, it is by default not sync'ed + * Unless it is in the white list, in which case the folder is sync'ed and all its children. + * If a folder is both on the black and the white list, the black list wins */ + SelectiveSyncWhiteList = 2, + /** List of big sync folders that have not been confirmed by the user yet and that the UI + * should notify about */ + SelectiveSyncUndecidedList = 3 + }; + /* return the specified list from the database */ + QStringList getSelectiveSyncList(SelectiveSyncListType type, bool *ok); + /* Write the selective sync list (remove all other entries of that list */ + void setSelectiveSyncList(SelectiveSyncListType type, const QStringList &list); + + /** + * Make sure that on the next sync, fileName is not read from the DB but uses the PROPFIND to + * get the info from the server + * + * Specifically, this sets the md5 field of fileName and all its parents to _invalid_. + * That causes a metadata difference and a resulting discovery from the remote for the + * affected folders. + * + * Since folders in the selective sync list will not be rediscovered (csync_ftw, + * _csync_detect_update skip them), the _invalid_ marker will stay and it. And any + * child items in the db will be ignored when reading a remote tree from the database. + */ + void avoidReadFromDbOnNextSync(const QString &fileName); + + /** + * Ensures full remote discovery happens on the next sync. + * + * Equivalent to calling avoidReadFromDbOnNextSync() for all files. + */ + void forceRemoteDiscoveryNextSync(); + + bool postSyncCleanup(const QSet &filepathsToKeep, + const QSet &prefixesToKeep); + + /* Because sqlite transactions are really slow, we encapsulate everything in big transactions + * Commit will actually commit the transaction and create a new one. + */ + void commit(const QString &context, bool startTrans = true); + void commitIfNeededAndStartNewTransaction(const QString &context); + + void close(); + + /** + * return true if everything is correct + */ + bool isConnected(); + + /** + * Returns the checksum type for an id. + */ + QByteArray getChecksumType(int checksumTypeId); + + /** + * The data-fingerprint used to detect backup + */ + void setDataFingerprint(const QByteArray &dataFingerprint); + QByteArray dataFingerprint(); + + /** + * Delete any file entry. This will force the next sync to re-sync everything as if it was new, + * restoring everyfile on every remote. If a file is there both on the client and server side, + * it will be a conflict that will be automatically resolved if the file is the same. + */ + void clearFileTable(); + +private: + bool updateDatabaseStructure(); + bool updateMetadataTableStructure(); + bool updateErrorBlacklistTableStructure(); + bool sqlFail(const QString &log, const SqlQuery &query); + void commitInternal(const QString &context, bool startTrans = true); + void startTransaction(); + void commitTransaction(); + QStringList tableColumns(const QString &table); + bool checkConnect(); + + // Same as forceRemoteDiscoveryNextSync but without acquiring the lock + void forceRemoteDiscoveryNextSyncLocked(); + + // Returns the integer id of the checksum type + // + // Returns 0 on failure and for empty checksum types. + int mapChecksumType(const QByteArray &checksumType); + + SqlDatabase _db; + QString _dbFile; + QMutex _mutex; // Public functions are protected with the mutex. + int _transaction; + + // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close + QScopedPointer _getFileRecordQuery; + QScopedPointer _setFileRecordQuery; + QScopedPointer _setFileRecordChecksumQuery; + QScopedPointer _setFileRecordLocalMetadataQuery; + QScopedPointer _getDownloadInfoQuery; + QScopedPointer _setDownloadInfoQuery; + QScopedPointer _deleteDownloadInfoQuery; + QScopedPointer _getUploadInfoQuery; + QScopedPointer _setUploadInfoQuery; + QScopedPointer _deleteUploadInfoQuery; + QScopedPointer _deleteFileRecordPhash; + QScopedPointer _deleteFileRecordRecursively; + QScopedPointer _getErrorBlacklistQuery; + QScopedPointer _setErrorBlacklistQuery; + QScopedPointer _getSelectiveSyncListQuery; + QScopedPointer _getChecksumTypeIdQuery; + QScopedPointer _getChecksumTypeQuery; + QScopedPointer _insertChecksumTypeQuery; + QScopedPointer _getDataFingerprintQuery; + QScopedPointer _setDataFingerprintQuery1; + QScopedPointer _setDataFingerprintQuery2; + + /* This is the list of paths we called avoidReadFromDbOnNextSync on. + * It means that they should not be written to the DB in any case since doing + * that would write the etag and would void the purpose of avoidReadFromDbOnNextSync + */ + QList _avoidReadFromDbOnNextSyncFilter; + + /** The journal mode to use for the db. + * + * Typically WAL initially, but may be set to other modes via environment + * variable, for specific filesystems, or when WAL fails in a particular way. + */ + QString _journalMode; +}; + +bool OCSYNC_EXPORT +operator==(const SyncJournalDb::DownloadInfo &lhs, + const SyncJournalDb::DownloadInfo &rhs); +bool OCSYNC_EXPORT +operator==(const SyncJournalDb::UploadInfo &lhs, + const SyncJournalDb::UploadInfo &rhs); + +} // namespace OCC +#endif // SYNCJOURNALDB_H diff --git a/src/common/syncjournalfilerecord.cpp b/src/common/syncjournalfilerecord.cpp new file mode 100644 index 000000000..c6a77b997 --- /dev/null +++ b/src/common/syncjournalfilerecord.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "common/syncjournalfilerecord.h" +#include "common/utility.h" + +namespace OCC { + +SyncJournalFileRecord::SyncJournalFileRecord() + : _inode(0) + , _type(0) + , _fileSize(0) + , _serverHasIgnoredFiles(false) +{ +} + +QByteArray SyncJournalFileRecord::numericFileId() const +{ + // Use the id up until the first non-numeric character + for (int i = 0; i < _fileId.size(); ++i) { + if (_fileId[i] < '0' || _fileId[i] > '9') { + return _fileId.left(i); + } + } + return _fileId; +} + +bool SyncJournalErrorBlacklistRecord::isValid() const +{ + return !_file.isEmpty() + && (!_lastTryEtag.isEmpty() || _lastTryModtime != 0) + && _lastTryTime > 0; +} + +bool operator==(const SyncJournalFileRecord &lhs, + const SyncJournalFileRecord &rhs) +{ + return lhs._path == rhs._path + && lhs._inode == rhs._inode + && lhs._modtime.toTime_t() == rhs._modtime.toTime_t() + && lhs._type == rhs._type + && lhs._etag == rhs._etag + && lhs._fileId == rhs._fileId + && lhs._fileSize == rhs._fileSize + && lhs._remotePerm == rhs._remotePerm + && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles + && lhs._checksumHeader == rhs._checksumHeader; +} +} diff --git a/src/common/syncjournalfilerecord.h b/src/common/syncjournalfilerecord.h new file mode 100644 index 000000000..ca8430469 --- /dev/null +++ b/src/common/syncjournalfilerecord.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) by Klaas Freitag + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SYNCJOURNALFILERECORD_H +#define SYNCJOURNALFILERECORD_H + +#include +#include + +#include "ocsynclib.h" + +namespace OCC { + +class SyncFileItem; + +/** + * @brief The SyncJournalFileRecord class + * @ingroup libsync + */ +class OCSYNC_EXPORT SyncJournalFileRecord +{ +public: + SyncJournalFileRecord(); + + bool isValid() + { + return !_path.isEmpty(); + } + + /** Returns the numeric part of the full id in _fileId. + * + * On the server this is sometimes known as the internal file id. + * + * It is used in the construction of private links. + */ + QByteArray numericFileId() const; + + QString _path; + quint64 _inode; + QDateTime _modtime; + int _type; + QByteArray _etag; + QByteArray _fileId; + qint64 _fileSize; + QByteArray _remotePerm; + bool _serverHasIgnoredFiles; + QByteArray _checksumHeader; +}; + +bool OCSYNC_EXPORT +operator==(const SyncJournalFileRecord &lhs, + const SyncJournalFileRecord &rhs); + +class OCSYNC_EXPORT SyncJournalErrorBlacklistRecord +{ +public: + enum Category { + /// Normal errors have no special behavior + Normal = 0, + /// These get a special summary message + InsufficientRemoteStorage + }; + + SyncJournalErrorBlacklistRecord() + : _retryCount(0) + , _errorCategory(Category::Normal) + , _lastTryModtime(0) + , _lastTryTime(0) + , _ignoreDuration(0) + { + } + + /// The number of times the operation was unsuccessful so far. + int _retryCount; + + /// The last error string. + QString _errorString; + /// The error category. Sometimes used for special actions. + Category _errorCategory; + + time_t _lastTryModtime; + QByteArray _lastTryEtag; + + /// The last time the operation was attempted (in s since epoch). + time_t _lastTryTime; + + /// The number of seconds the file shall be ignored. + time_t _ignoreDuration; + + QString _file; + QString _renameTarget; + + bool isValid() const; +}; +} + +#endif // SYNCJOURNALFILERECORD_H diff --git a/src/csync/CMakeLists.txt b/src/csync/CMakeLists.txt index e265b95d9..acb1cc7c2 100644 --- a/src/csync/CMakeLists.txt +++ b/src/csync/CMakeLists.txt @@ -1,4 +1,5 @@ project(libcsync) +set(CMAKE_AUTOMOC TRUE) # global needed variables set(APPLICATION_NAME "ocsync") @@ -134,7 +135,7 @@ if(ZLIB_FOUND) endif(ZLIB_FOUND) find_package(Qt5Core REQUIRED) -qt5_use_modules(${CSYNC_LIBRARY} Core) +qt5_use_modules(${CSYNC_LIBRARY} Core Concurrent) # For src/common/utility_mac.cpp if (APPLE) diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp index ac096ae2b..6c36637b8 100644 --- a/src/csync/csync.cpp +++ b/src/csync/csync.cpp @@ -49,7 +49,7 @@ #include "csync_log.h" #include "csync_rename.h" -#include "c_jhash.h" +#include "common/c_jhash.h" csync_s::csync_s(const char *localUri, const char *db_file) { diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index ee7eee7a1..dd1cad68e 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -26,7 +26,7 @@ #include "csync_util.h" #include "csync_statedb.h" #include "csync_rename.h" -#include "c_jhash.h" +#include "common/c_jhash.h" #define CSYNC_LOG_CATEGORY_NAME "csync.reconciler" #include "csync_log.h" diff --git a/src/csync/csync_statedb.cpp b/src/csync/csync_statedb.cpp index 7f7ed0c29..89ac83fd7 100644 --- a/src/csync/csync_statedb.cpp +++ b/src/csync/csync_statedb.cpp @@ -41,7 +41,7 @@ #include "csync_exclude.h" #include "c_string.h" -#include "c_jhash.h" +#include "common/c_jhash.h" #include "c_utf8.h" #include "csync_time.h" diff --git a/src/csync/csync_util.cpp b/src/csync/csync_util.cpp index 4fc13a471..d90a2eacc 100644 --- a/src/csync/csync_util.cpp +++ b/src/csync/csync_util.cpp @@ -30,7 +30,7 @@ #include #include -#include "c_jhash.h" +#include "common/c_jhash.h" #include "csync_util.h" #include "vio/csync_vio.h" diff --git a/src/csync/std/c_jhash.h b/src/csync/std/c_jhash.h deleted file mode 100644 index 261a0a4a4..000000000 --- a/src/csync/std/c_jhash.h +++ /dev/null @@ -1,245 +0,0 @@ -/* - * c_jhash.c Jenkins Hash - * - * Copyright (c) 1997 Bob Jenkins - * - * lookup8.c, by Bob Jenkins, January 4 1997, Public Domain. - * hash(), hash2(), hash3, and _c_mix() are externally useful functions. - * Routines to test the hash are included if SELF_TEST is defined. - * You can use this free for any purpose. It has no warranty. - * - * See http://burtleburtle.net/bob/hash/evahash.html - */ - -/** - * @file c_jhash.h - * - * @brief Interface of the cynapses jhash implementation - * - * @defgroup cynJHashInternals cynapses libc jhash function - * @ingroup cynLibraryAPI - * - * @{ - */ -#ifndef _C_JHASH_H -#define _C_JHASH_H - -#include - -#define c_hashsize(n) ((uint8_t) 1 << (n)) -#define c_hashmask(n) (xhashsize(n) - 1) - -/** - * _c_mix -- Mix 3 32-bit values reversibly. - * - * For every delta with one or two bit set, and the deltas of all three - * high bits or all three low bits, whether the original value of a,b,c - * is almost all zero or is uniformly distributed, - * If _c_mix() is run forward or backward, at least 32 bits in a,b,c - * have at least 1/4 probability of changing. - * If _c_mix() is run forward, every bit of c will change between 1/3 and - * 2/3 of the time. (Well, 22/100 and 78/100 for some 2-bit deltas.) - * _c_mix() was built out of 36 single-cycle latency instructions in a - * structure that could supported 2x parallelism, like so: - * a -= b; - * a -= c; x = (c>>13); - * b -= c; a ^= x; - * b -= a; x = (a<<8); - * c -= a; b ^= x; - * c -= b; x = (b>>13); - * ... - * - * Unfortunately, superscalar Pentiums and Sparcs can't take advantage - * of that parallelism. They've also turned some of those single-cycle - * latency instructions into multi-cycle latency instructions. Still, - * this is the fastest good hash I could find. There were about 2^^68 - * to choose from. I only looked at a billion or so. - */ -#define _c_mix(a,b,c) \ -{ \ - a -= b; a -= c; a ^= (c>>13); \ - b -= c; b -= a; b ^= (a<<8); \ - c -= a; c -= b; c ^= (b>>13); \ - a -= b; a -= c; a ^= (c>>12); \ - b -= c; b -= a; b ^= (a<<16); \ - c -= a; c -= b; c ^= (b>>5); \ - a -= b; a -= c; a ^= (c>>3); \ - b -= c; b -= a; b ^= (a<<10); \ - c -= a; c -= b; c ^= (b>>15); \ -} - -/** - * _c_mix64 -- Mix 3 64-bit values reversibly. - * - * _c_mix64() takes 48 machine instructions, but only 24 cycles on a superscalar - * machine (like Intel's new MMX architecture). It requires 4 64-bit - * registers for 4::2 parallelism. - * All 1-bit deltas, all 2-bit deltas, all deltas composed of top bits of - * (a,b,c), and all deltas of bottom bits were tested. All deltas were - * tested both on random keys and on keys that were nearly all zero. - * These deltas all cause every bit of c to change between 1/3 and 2/3 - * of the time (well, only 113/400 to 287/400 of the time for some - * 2-bit delta). These deltas all cause at least 80 bits to change - * among (a,b,c) when the _c_mix is run either forward or backward (yes it - * is reversible). - * This implies that a hash using _c_mix64 has no funnels. There may be - * characteristics with 3-bit deltas or bigger, I didn't test for - * those. - */ -#define _c_mix64(a,b,c) \ -{ \ - a -= b; a -= c; a ^= (c>>43); \ - b -= c; b -= a; b ^= (a<<9); \ - c -= a; c -= b; c ^= (b>>8); \ - a -= b; a -= c; a ^= (c>>38); \ - b -= c; b -= a; b ^= (a<<23); \ - c -= a; c -= b; c ^= (b>>5); \ - a -= b; a -= c; a ^= (c>>35); \ - b -= c; b -= a; b ^= (a<<49); \ - c -= a; c -= b; c ^= (b>>11); \ - a -= b; a -= c; a ^= (c>>12); \ - b -= c; b -= a; b ^= (a<<18); \ - c -= a; c -= b; c ^= (b>>22); \ -} - -/** - * @brief hash a variable-length key into a 32-bit value - * - * The best hash table sizes are powers of 2. There is no need to do - * mod a prime (mod is sooo slow!). If you need less than 32 bits, - * use a bitmask. For example, if you need only 10 bits, do - * h = (h & hashmask(10)); - * In which case, the hash table should have hashsize(10) elements. - * - * Use for hash table lookup, or anything where one collision in 2^32 is - * acceptable. Do NOT use for cryptographic purposes. - * - * @param k The key (the unaligned variable-length array of bytes). - * - * @param length The length of the key, counting by bytes. - * - * @param initval Initial value, can be any 4-byte value. - * - * @return Returns a 32-bit value. Every bit of the key affects every bit - * of the return value. Every 1-bit and 2-bit delta achieves - * avalanche. About 36+6len instructions. - */ -static inline uint32_t c_jhash(const uint8_t *k, uint32_t length, uint32_t initval) { - uint32_t a,b,c,len; - - /* Set up the internal state */ - len = length; - a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ - c = initval; /* the previous hash value */ - - while (len >= 12) { - a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24)); - b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24)); - c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24)); - _c_mix(a,b,c); - k += 12; len -= 12; - } - - /* handle the last 11 bytes */ - c += length; - /* all the case statements fall through */ - switch(len) { - case 11: c+=((uint32_t)k[10]<<24); - case 10: c+=((uint32_t)k[9]<<16); - case 9 : c+=((uint32_t)k[8]<<8); - /* the first byte of c is reserved for the length */ - case 8 : b+=((uint32_t)k[7]<<24); - case 7 : b+=((uint32_t)k[6]<<16); - case 6 : b+=((uint32_t)k[5]<<8); - case 5 : b+=k[4]; - case 4 : a+=((uint32_t)k[3]<<24); - case 3 : a+=((uint32_t)k[2]<<16); - case 2 : a+=((uint32_t)k[1]<<8); - case 1 : a+=k[0]; - /* case 0: nothing left to add */ - } - _c_mix(a,b,c); - - return c; -} - -/** - * @brief hash a variable-length key into a 64-bit value - * - * The best hash table sizes are powers of 2. There is no need to do - * mod a prime (mod is sooo slow!). If you need less than 64 bits, - * use a bitmask. For example, if you need only 10 bits, do - * h = (h & hashmask(10)); - * In which case, the hash table should have hashsize(10) elements. - * - * Use for hash table lookup, or anything where one collision in 2^^64 - * is acceptable. Do NOT use for cryptographic purposes. - * - * @param k The key (the unaligned variable-length array of bytes). - * @param length The length of the key, counting by bytes. - * @param intval Initial value, can be any 8-byte value. - * - * @return A 64-bit value. Every bit of the key affects every bit of - * the return value. No funnels. Every 1-bit and 2-bit delta - * achieves avalanche. About 41+5len instructions. - */ -static inline uint64_t c_jhash64(const uint8_t *k, uint64_t length, uint64_t intval) { - uint64_t a,b,c,len; - - /* Set up the internal state */ - len = length; - a = b = intval; /* the previous hash value */ - c = 0x9e3779b97f4a7c13LL; /* the golden ratio; an arbitrary value */ - - /* handle most of the key */ - while (len >= 24) - { - a += (k[0] +((uint64_t)k[ 1]<< 8)+((uint64_t)k[ 2]<<16)+((uint64_t)k[ 3]<<24) - +((uint64_t)k[4 ]<<32)+((uint64_t)k[ 5]<<40)+((uint64_t)k[ 6]<<48)+((uint64_t)k[ 7]<<56)); - b += (k[8] +((uint64_t)k[ 9]<< 8)+((uint64_t)k[10]<<16)+((uint64_t)k[11]<<24) - +((uint64_t)k[12]<<32)+((uint64_t)k[13]<<40)+((uint64_t)k[14]<<48)+((uint64_t)k[15]<<56)); - c += (k[16] +((uint64_t)k[17]<< 8)+((uint64_t)k[18]<<16)+((uint64_t)k[19]<<24) - +((uint64_t)k[20]<<32)+((uint64_t)k[21]<<40)+((uint64_t)k[22]<<48)+((uint64_t)k[23]<<56)); - _c_mix64(a,b,c); - k += 24; len -= 24; - } - - /* handle the last 23 bytes */ - c += length; - switch(len) { - case 23: c+=((uint64_t)k[22]<<56); - case 22: c+=((uint64_t)k[21]<<48); - case 21: c+=((uint64_t)k[20]<<40); - case 20: c+=((uint64_t)k[19]<<32); - case 19: c+=((uint64_t)k[18]<<24); - case 18: c+=((uint64_t)k[17]<<16); - case 17: c+=((uint64_t)k[16]<<8); - /* the first byte of c is reserved for the length */ - case 16: b+=((uint64_t)k[15]<<56); - case 15: b+=((uint64_t)k[14]<<48); - case 14: b+=((uint64_t)k[13]<<40); - case 13: b+=((uint64_t)k[12]<<32); - case 12: b+=((uint64_t)k[11]<<24); - case 11: b+=((uint64_t)k[10]<<16); - case 10: b+=((uint64_t)k[ 9]<<8); - case 9: b+=((uint64_t)k[ 8]); - case 8: a+=((uint64_t)k[ 7]<<56); - case 7: a+=((uint64_t)k[ 6]<<48); - case 6: a+=((uint64_t)k[ 5]<<40); - case 5: a+=((uint64_t)k[ 4]<<32); - case 4: a+=((uint64_t)k[ 3]<<24); - case 3: a+=((uint64_t)k[ 2]<<16); - case 2: a+=((uint64_t)k[ 1]<<8); - case 1: a+=((uint64_t)k[ 0]); - /* case 0: nothing left to add */ - } - _c_mix64(a,b,c); - - return c; -} - -/** - * }@ - */ -#endif /* _C_JHASH_H */ - diff --git a/src/csync/vio/csync_vio.cpp b/src/csync/vio/csync_vio.cpp index 57eb46c39..dab04d3df 100644 --- a/src/csync/vio/csync_vio.cpp +++ b/src/csync/vio/csync_vio.cpp @@ -31,7 +31,7 @@ #include "vio/csync_vio.h" #include "vio/csync_vio_local.h" #include "csync_statedb.h" -#include "std/c_jhash.h" +#include "common/c_jhash.h" #define CSYNC_LOG_CATEGORY_NAME "csync.vio.main" diff --git a/src/gui/creds/httpcredentialsgui.cpp b/src/gui/creds/httpcredentialsgui.cpp index 98e4df907..bc533db86 100644 --- a/src/gui/creds/httpcredentialsgui.cpp +++ b/src/gui/creds/httpcredentialsgui.cpp @@ -24,7 +24,7 @@ #include "account.h" #include "networkjobs.h" #include -#include "asserts.h" +#include "common/asserts.h" using namespace QKeychain; diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 961e8754c..864c496bf 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -22,7 +22,7 @@ #include "logger.h" #include "configfile.h" #include "networkjobs.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "syncresult.h" #include "clientproxy.h" #include "syncengine.h" diff --git a/src/gui/folder.h b/src/gui/folder.h index 1c67c0722..7b503a6d6 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -19,7 +19,7 @@ #include "syncresult.h" #include "progressdispatcher.h" -#include "syncjournaldb.h" +#include "common/syncjournaldb.h" #include "clientproxy.h" #include "networkjobs.h" diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index bb56db2ef..475eb4f1a 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -23,7 +23,7 @@ #include "accountmanager.h" #include "filesystem.h" #include "lockwatcher.h" -#include "asserts.h" +#include "common/asserts.h" #include #ifdef Q_OS_MAC diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index 16e2ab0e0..a7b38dc43 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -15,7 +15,7 @@ #include "folderstatusmodel.h" #include "folderman.h" #include "accountstate.h" -#include "asserts.h" +#include "common/asserts.h" #include #include #include "folderstatusdelegate.h" diff --git a/src/gui/folderwizard.cpp b/src/gui/folderwizard.cpp index 26ce66dfd..283f34856 100644 --- a/src/gui/folderwizard.cpp +++ b/src/gui/folderwizard.cpp @@ -22,7 +22,7 @@ #include "accountstate.h" #include "creds/abstractcredentials.h" #include "wizard/owncloudwizard.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/gui/issueswidget.cpp b/src/gui/issueswidget.cpp index 19e69c014..40f2822b7 100644 --- a/src/gui/issueswidget.cpp +++ b/src/gui/issueswidget.cpp @@ -29,7 +29,7 @@ #include "accountstate.h" #include "account.h" #include "accountmanager.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "elidedlabel.h" #include "ui_issueswidget.h" diff --git a/src/gui/notificationwidget.cpp b/src/gui/notificationwidget.cpp index 0457857b8..bcada1075 100644 --- a/src/gui/notificationwidget.cpp +++ b/src/gui/notificationwidget.cpp @@ -15,7 +15,7 @@ #include "notificationwidget.h" #include "QProgressIndicator.h" #include "common/utility.h" -#include "asserts.h" +#include "common/asserts.h" #include diff --git a/src/gui/owncloudgui.cpp b/src/gui/owncloudgui.cpp index cb0cafbd8..e5a398c8e 100644 --- a/src/gui/owncloudgui.cpp +++ b/src/gui/owncloudgui.cpp @@ -32,7 +32,7 @@ #include "accountstate.h" #include "openfilemanager.h" #include "accountmanager.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "creds/abstractcredentials.h" #include diff --git a/src/gui/socketapi.cpp b/src/gui/socketapi.cpp index fe7ca5f4d..9261cef83 100644 --- a/src/gui/socketapi.cpp +++ b/src/gui/socketapi.cpp @@ -21,7 +21,7 @@ #include "folderman.h" #include "folder.h" #include "theme.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "syncengine.h" #include "syncfileitem.h" #include "filesystem.h" @@ -30,7 +30,7 @@ #include "accountstate.h" #include "account.h" #include "capabilities.h" -#include "asserts.h" +#include "common/asserts.h" #include "guiutility.h" #include diff --git a/src/gui/socketapi.h b/src/gui/socketapi.h index d4f6e7bf8..2b7b4fe9f 100644 --- a/src/gui/socketapi.h +++ b/src/gui/socketapi.h @@ -18,7 +18,7 @@ #include "syncfileitem.h" #include "syncfilestatus.h" -#include "ownsql.h" +// #include "ownsql.h" #if defined(Q_OS_MAC) #include "socketapisocket_mac.h" diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index c7ac8ef08..be5cb885e 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -54,12 +54,8 @@ set(libsync_SRCS syncfileitem.cpp syncfilestatus.cpp syncfilestatustracker.cpp - syncjournaldb.cpp - syncjournalfilerecord.cpp syncresult.cpp theme.cpp - ownsql.cpp - checksums.cpp excludedfiles.cpp creds/dummycredentials.cpp creds/abstractcredentials.cpp @@ -125,9 +121,9 @@ GENERATE_EXPORT_HEADER( ${synclib_NAME} ) if(TOKEN_AUTH_ONLY) - qt5_use_modules(${synclib_NAME} Network Concurrent) + qt5_use_modules(${synclib_NAME} Network) else() - qt5_use_modules(${synclib_NAME} Widgets Network Concurrent) + qt5_use_modules(${synclib_NAME} Widgets Network) endif() set_target_properties( ${synclib_NAME} PROPERTIES diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index d70e7ac9e..7f2fadb29 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -20,7 +20,7 @@ #include "creds/abstractcredentials.h" #include "capabilities.h" #include "theme.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/asserts.h b/src/libsync/asserts.h deleted file mode 100644 index e96e57bb9..000000000 --- a/src/libsync/asserts.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef OWNCLOUD_ASSERTS_H -#define OWNCLOUD_ASSERTS_H - -#include - -#if defined(QT_FORCE_ASSERTS) || !defined(QT_NO_DEBUG) -#define OC_ASSERT_MSG qFatal -#else -#define OC_ASSERT_MSG qCritical -#endif - -// For overloading macros by argument count -// See stackoverflow.com/questions/16683146/can-macros-be-overloaded-by-number-of-arguments -#define OC_ASSERT_CAT(A, B) A##B -#define OC_ASSERT_SELECT(NAME, NUM) OC_ASSERT_CAT(NAME##_, NUM) -#define OC_ASSERT_GET_COUNT(_1, _2, _3, COUNT, ...) COUNT -#define OC_ASSERT_VA_SIZE(...) OC_ASSERT_GET_COUNT(__VA_ARGS__, 3, 2, 1, 0) - -#define OC_ASSERT_OVERLOAD(NAME, ...) OC_ASSERT_SELECT(NAME, OC_ASSERT_VA_SIZE(__VA_ARGS__)) \ - (__VA_ARGS__) - -// Default assert: If the condition is false in debug builds, terminate. -// -// Prints a message on failure, even in release builds. -#define ASSERT(...) OC_ASSERT_OVERLOAD(ASSERT, __VA_ARGS__) -#define ASSERT_1(cond) \ - if (!(cond)) { \ - OC_ASSERT_MSG("ASSERT: \"%s\" in file %s, line %d", #cond, __FILE__, __LINE__); \ - } else { \ - } -#define ASSERT_2(cond, message) \ - if (!(cond)) { \ - OC_ASSERT_MSG("ASSERT: \"%s\" in file %s, line %d with message: %s", #cond, __FILE__, __LINE__, message); \ - } else { \ - } - -// Enforce condition to be true, even in release builds. -// -// Prints 'message' and aborts execution if 'cond' is false. -#define ENFORCE(...) OC_ASSERT_OVERLOAD(ENFORCE, __VA_ARGS__) -#define ENFORCE_1(cond) \ - if (!(cond)) { \ - qFatal("ENFORCE: \"%s\" in file %s, line %d", #cond, __FILE__, __LINE__); \ - } else { \ - } -#define ENFORCE_2(cond, message) \ - if (!(cond)) { \ - qFatal("ENFORCE: \"%s\" in file %s, line %d with message: %s", #cond, __FILE__, __LINE__, message); \ - } else { \ - } - -// An assert that is only present in debug builds: typically used for -// asserts that are too expensive for release mode. -// -// Q_ASSERT - -#endif diff --git a/src/libsync/checksums.cpp b/src/libsync/checksums.cpp deleted file mode 100644 index 448b5d477..000000000 --- a/src/libsync/checksums.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ -#include "config.h" -#include "filesystem.h" -#include "checksums.h" -#include "syncfileitem.h" -#include "propagatorjobs.h" -#include "account.h" - -#include -#include - -/** \file checksums.cpp - * - * \brief Computing and validating file checksums - * - * Overview - * -------- - * - * Checksums are used in two distinct ways during synchronization: - * - * - to guard uploads and downloads against data corruption - * (transmission checksum) - * - to quickly check whether the content of a file has changed - * to avoid redundant uploads (content checksum) - * - * In principle both are independent and different checksumming - * algorithms can be used. To avoid redundant computations, it can - * make sense to use the same checksum algorithm though. - * - * Transmission Checksums - * ---------------------- - * - * The usage of transmission checksums is currently optional and needs - * to be explicitly enabled by adding 'transmissionChecksum=TYPE' to - * the '[General]' section of the config file. - * - * When enabled, the checksum will be calculated on upload and sent to - * the server in the OC-Checksum header with the format 'TYPE:CHECKSUM'. - * - * On download, the header with the same name is read and if the - * received data does not have the expected checksum, the download is - * rejected. - * - * Transmission checksums guard a specific sync action and are not stored - * in the database. - * - * Content Checksums - * ----------------- - * - * Sometimes the metadata of a local file changes while the content stays - * unchanged. Content checksums allow the sync client to avoid uploading - * the same data again by comparing the file's actual checksum to the - * checksum stored in the database. - * - * Content checksums are not sent to the server. - * - * Checksum Algorithms - * ------------------- - * - * - Adler32 (requires zlib) - * - MD5 - * - SHA1 - * - */ - -namespace OCC { - -Q_LOGGING_CATEGORY(lcChecksums, "sync.checksums", QtInfoMsg) - -QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum) -{ - if (checksumType.isEmpty() || checksum.isEmpty()) - return QByteArray(); - QByteArray header = checksumType; - header.append(':'); - header.append(checksum); - return header; -} - -bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum) -{ - if (header.isEmpty()) { - type->clear(); - checksum->clear(); - return true; - } - - const auto idx = header.indexOf(':'); - if (idx < 0) { - return false; - } - - *type = header.left(idx); - *checksum = header.mid(idx + 1); - return true; -} - - -QByteArray parseChecksumHeaderType(const QByteArray &header) -{ - const auto idx = header.indexOf(':'); - if (idx < 0) { - return QByteArray(); - } - return header.left(idx); -} - -bool uploadChecksumEnabled() -{ - static bool enabled = qgetenv("OWNCLOUD_DISABLE_CHECKSUM_UPLOAD").isEmpty(); - return enabled; -} - -QByteArray contentChecksumType() -{ - static QByteArray type = qgetenv("OWNCLOUD_CONTENT_CHECKSUM_TYPE"); - if (type.isNull()) { // can set to "" to disable checksumming - type = "SHA1"; - } - return type; -} - -ComputeChecksum::ComputeChecksum(QObject *parent) - : QObject(parent) -{ -} - -void ComputeChecksum::setChecksumType(const QByteArray &type) -{ - _checksumType = type; -} - -QByteArray ComputeChecksum::checksumType() const -{ - return _checksumType; -} - -void ComputeChecksum::start(const QString &filePath) -{ - // Calculate the checksum in a different thread first. - connect(&_watcher, SIGNAL(finished()), - this, SLOT(slotCalculationDone()), - Qt::UniqueConnection); - _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType())); -} - -QByteArray ComputeChecksum::computeNow(const QString &filePath, const QByteArray &checksumType) -{ - if (checksumType == checkSumMD5C) { - return FileSystem::calcMd5(filePath); - } else if (checksumType == checkSumSHA1C) { - return FileSystem::calcSha1(filePath); - } -#ifdef ZLIB_FOUND - else if (checksumType == checkSumAdlerC) { - return FileSystem::calcAdler32(filePath); - } -#endif - // for an unknown checksum or no checksum, we're done right now - if (!checksumType.isEmpty()) { - qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType; - } - return QByteArray(); -} - -void ComputeChecksum::slotCalculationDone() -{ - QByteArray checksum = _watcher.future().result(); - if (!checksum.isNull()) { - emit done(_checksumType, checksum); - } else { - emit done(QByteArray(), QByteArray()); - } -} - - -ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent) - : QObject(parent) -{ -} - -void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &checksumHeader) -{ - // If the incoming header is empty no validation can happen. Just continue. - if (checksumHeader.isEmpty()) { - emit validated(QByteArray(), QByteArray()); - return; - } - - if (!parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum)) { - qCWarning(lcChecksums) << "Checksum header malformed:" << checksumHeader; - emit validationFailed(tr("The checksum header is malformed.")); - return; - } - - auto calculator = new ComputeChecksum(this); - calculator->setChecksumType(_expectedChecksumType); - connect(calculator, SIGNAL(done(QByteArray, QByteArray)), - SLOT(slotChecksumCalculated(QByteArray, QByteArray))); - calculator->start(filePath); -} - -void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray &checksumType, - const QByteArray &checksum) -{ - if (checksumType != _expectedChecksumType) { - emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(QString::fromLatin1(_expectedChecksumType))); - return; - } - if (checksum != _expectedChecksum) { - emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed.")); - return; - } - emit validated(checksumType, checksum); -} - -CSyncChecksumHook::CSyncChecksumHook() -{ -} - -QByteArray CSyncChecksumHook::hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void * /*this_obj*/) -{ - QByteArray type = parseChecksumHeaderType(QByteArray(otherChecksumHeader)); - if (type.isEmpty()) - return NULL; - - QByteArray checksum = ComputeChecksum::computeNow(QString::fromUtf8(path), type); - if (checksum.isNull()) { - qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path; - return NULL; - } - - return makeChecksumHeader(type, checksum); -} - -} diff --git a/src/libsync/checksums.h b/src/libsync/checksums.h deleted file mode 100644 index 3c31c8c03..000000000 --- a/src/libsync/checksums.h +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#pragma once - -#include "owncloudlib.h" -#include "accountfwd.h" - -#include -#include -#include - -namespace OCC { - -class SyncJournalDb; - -/// Creates a checksum header from type and value. -QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum); - -/// Parses a checksum header -bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum); - -/// Convenience for getting the type from a checksum header, null if none -QByteArray parseChecksumHeaderType(const QByteArray &header); - -/// Checks OWNCLOUD_DISABLE_CHECKSUM_UPLOAD -bool uploadChecksumEnabled(); - -/// Checks OWNCLOUD_CONTENT_CHECKSUM_TYPE (default: SHA1) -QByteArray contentChecksumType(); - - -/** - * Computes the checksum of a file. - * \ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT ComputeChecksum : public QObject -{ - Q_OBJECT -public: - explicit ComputeChecksum(QObject *parent = 0); - - /** - * Sets the checksum type to be used. The default is empty. - */ - void setChecksumType(const QByteArray &type); - - QByteArray checksumType() const; - - /** - * Computes the checksum for the given file path. - * - * done() is emitted when the calculation finishes. - */ - void start(const QString &filePath); - - /** - * Computes the checksum synchronously. - */ - static QByteArray computeNow(const QString &filePath, const QByteArray &checksumType); - -signals: - void done(const QByteArray &checksumType, const QByteArray &checksum); - -private slots: - void slotCalculationDone(); - -private: - QByteArray _checksumType; - - // watcher for the checksum calculation thread - QFutureWatcher _watcher; -}; - -/** - * Checks whether a file's checksum matches the expected value. - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT ValidateChecksumHeader : public QObject -{ - Q_OBJECT -public: - explicit ValidateChecksumHeader(QObject *parent = 0); - - /** - * Check a file's actual checksum against the provided checksumHeader - * - * If no checksum is there, or if a correct checksum is there, the signal validated() - * will be emitted. In case of any kind of error, the signal validationFailed() will - * be emitted. - */ - void start(const QString &filePath, const QByteArray &checksumHeader); - -signals: - void validated(const QByteArray &checksumType, const QByteArray &checksum); - void validationFailed(const QString &errMsg); - -private slots: - void slotChecksumCalculated(const QByteArray &checksumType, const QByteArray &checksum); - -private: - QByteArray _expectedChecksumType; - QByteArray _expectedChecksum; -}; - -/** - * Hooks checksum computations into csync. - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT CSyncChecksumHook : public QObject -{ - Q_OBJECT -public: - explicit CSyncChecksumHook(); - - /** - * Returns the checksum value for \a path that is comparable to \a otherChecksum. - * - * Called from csync, where a instance of CSyncChecksumHook has - * to be set as userdata. - * The return value will be owned by csync. - */ - static QByteArray hook(const QByteArray &path, const QByteArray &otherChecksumHeader, void *this_obj); -}; -} diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 0fa6b77ff..6fd3df6f0 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -17,7 +17,7 @@ #include "configfile.h" #include "theme.h" #include "common/utility.h" -#include "asserts.h" +#include "common/asserts.h" #include "creds/abstractcredentials.h" diff --git a/src/libsync/creds/abstractcredentials.cpp b/src/libsync/creds/abstractcredentials.cpp index 2dadc0793..f64b51fd2 100644 --- a/src/libsync/creds/abstractcredentials.cpp +++ b/src/libsync/creds/abstractcredentials.cpp @@ -15,7 +15,7 @@ #include #include -#include "asserts.h" +#include "common/asserts.h" #include "creds/abstractcredentials.h" namespace OCC { diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 9ebcd7fa8..a4345c58a 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -16,7 +16,7 @@ #include "account.h" #include "theme.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index c8bd46cd4..346d61e86 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -14,8 +14,8 @@ */ #include "owncloudpropagator.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "propagatedownload.h" #include "propagateupload.h" #include "propagateremotedelete.h" @@ -25,7 +25,7 @@ #include "configfile.h" #include "common/utility.h" #include "account.h" -#include "asserts.h" +#include "common/asserts.h" #ifdef Q_OS_WIN #include diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 093fd8aac..f62fd58a9 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -27,7 +27,7 @@ #include "csync_util.h" #include "syncfileitem.h" -#include "syncjournaldb.h" +#include "common/syncjournaldb.h" #include "bandwidthmanager.h" #include "accountfwd.h" #include "discoveryphase.h" diff --git a/src/libsync/ownsql.cpp b/src/libsync/ownsql.cpp deleted file mode 100644 index e66fdfb3e..000000000 --- a/src/libsync/ownsql.cpp +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - - -#include -#include -#include -#include -#include -#include - -#include "ownsql.h" -#include "common/utility.h" -#include "asserts.h" - -#define SQLITE_SLEEP_TIME_USEC 100000 -#define SQLITE_REPEAT_COUNT 20 - -#define SQLITE_DO(A) \ - if (1) { \ - _errId = (A); \ - if (_errId != SQLITE_OK) { \ - _error = QString::fromUtf8(sqlite3_errmsg(_db)); \ - } \ - } - -namespace OCC { - -Q_LOGGING_CATEGORY(lcSql, "sync.database.sql", QtInfoMsg) - -SqlDatabase::SqlDatabase() - : _db(0) - , _errId(0) -{ -} - -bool SqlDatabase::isOpen() -{ - return _db != 0; -} - -bool SqlDatabase::openHelper(const QString &filename, int sqliteFlags) -{ - if (isOpen()) { - return true; - } - - sqliteFlags |= SQLITE_OPEN_NOMUTEX; - - SQLITE_DO(sqlite3_open_v2(filename.toUtf8().constData(), &_db, sqliteFlags, 0)); - - if (_errId != SQLITE_OK) { - qCWarning(lcSql) << "Error:" << _error << "for" << filename; - if (_errId == SQLITE_CANTOPEN) { - qCWarning(lcSql) << "CANTOPEN extended errcode: " << sqlite3_extended_errcode(_db); -#if SQLITE_VERSION_NUMBER >= 3012000 - qCWarning(lcSql) << "CANTOPEN system errno: " << sqlite3_system_errno(_db); -#endif - } - close(); - return false; - } - - if (!_db) { - qCWarning(lcSql) << "Error: no database for" << filename; - return false; - } - - sqlite3_busy_timeout(_db, 5000); - - return true; -} - -bool SqlDatabase::checkDb() -{ - // quick_check can fail with a disk IO error when diskspace is low - SqlQuery quick_check(*this); - quick_check.prepare("PRAGMA quick_check;", /*allow_failure=*/true); - if (!quick_check.exec()) { - qCWarning(lcSql) << "Error running quick_check on database"; - return false; - } - - quick_check.next(); - QString result = quick_check.stringValue(0); - if (result != "ok") { - qCWarning(lcSql) << "quick_check returned failure:" << result; - return false; - } - - return true; -} - -bool SqlDatabase::openOrCreateReadWrite(const QString &filename) -{ - if (isOpen()) { - return true; - } - - if (!openHelper(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE)) { - return false; - } - - if (!checkDb()) { - // When disk space is low, checking the db may fail even though it's fine. - qint64 freeSpace = Utility::freeDiskSpace(QFileInfo(filename).dir().absolutePath()); - if (freeSpace != -1 && freeSpace < 1000000) { - qCWarning(lcSql) << "Consistency check failed, disk space is low, aborting" << freeSpace; - close(); - return false; - } - - qCCritical(lcSql) << "Consistency check failed, removing broken db" << filename; - close(); - QFile::remove(filename); - - return openHelper(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE); - } - - return true; -} - -bool SqlDatabase::openReadOnly(const QString &filename) -{ - if (isOpen()) { - return true; - } - - if (!openHelper(filename, SQLITE_OPEN_READONLY)) { - return false; - } - - if (!checkDb()) { - qCWarning(lcSql) << "Consistency check failed in readonly mode, giving up" << filename; - close(); - return false; - } - - return true; -} - -QString SqlDatabase::error() const -{ - const QString err(_error); - // _error.clear(); - return err; -} - -void SqlDatabase::close() -{ - if (_db) { - SQLITE_DO(sqlite3_close(_db)); - // Fatal because reopening an unclosed db might be problematic. - ENFORCE(_errId == SQLITE_OK, "Error when closing DB"); - _db = 0; - } -} - -bool SqlDatabase::transaction() -{ - if (!_db) { - return false; - } - SQLITE_DO(sqlite3_exec(_db, "BEGIN", 0, 0, 0)); - return _errId == SQLITE_OK; -} - -bool SqlDatabase::commit() -{ - if (!_db) { - return false; - } - SQLITE_DO(sqlite3_exec(_db, "COMMIT", 0, 0, 0)); - return _errId == SQLITE_OK; -} - -sqlite3 *SqlDatabase::sqliteDb() -{ - return _db; -} - -/* =========================================================================================== */ - -SqlQuery::SqlQuery(SqlDatabase &db) - : _db(db.sqliteDb()) - , _stmt(0) - , _errId(0) -{ -} - -SqlQuery::~SqlQuery() -{ - if (_stmt) { - finish(); - } -} - -SqlQuery::SqlQuery(const QString &sql, SqlDatabase &db) - : _db(db.sqliteDb()) - , _stmt(0) - , _errId(0) -{ - prepare(sql); -} - -int SqlQuery::prepare(const QString &sql, bool allow_failure) -{ - QString s(sql); - _sql = s.trimmed(); - if (_stmt) { - finish(); - } - if (!_sql.isEmpty()) { - int n = 0; - int rc; - do { - rc = sqlite3_prepare_v2(_db, _sql.toUtf8().constData(), -1, &_stmt, 0); - if ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED)) { - n++; - OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); - } - } while ((n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); - _errId = rc; - - if (_errId != SQLITE_OK) { - _error = QString::fromUtf8(sqlite3_errmsg(_db)); - qCWarning(lcSql) << "Sqlite prepare statement error:" << _error << "in" << _sql; - ENFORCE(allow_failure, "SQLITE Prepare error"); - } - } - return _errId; -} - -bool SqlQuery::isSelect() -{ - return (!_sql.isEmpty() && _sql.startsWith("SELECT", Qt::CaseInsensitive)); -} - -bool SqlQuery::isPragma() -{ - return (!_sql.isEmpty() && _sql.startsWith("PRAGMA", Qt::CaseInsensitive)); -} - -bool SqlQuery::exec() -{ - qCDebug(lcSql) << "SQL exec" << _sql; - - if (!_stmt) { - qCWarning(lcSql) << "Can't exec query, statement unprepared."; - return false; - } - - // Don't do anything for selects, that is how we use the lib :-| - if (!isSelect() && !isPragma()) { - int rc, n = 0; - do { - rc = sqlite3_step(_stmt); - if (rc == SQLITE_LOCKED) { - rc = sqlite3_reset(_stmt); /* This will also return SQLITE_LOCKED */ - n++; - OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); - } else if (rc == SQLITE_BUSY) { - OCC::Utility::usleep(SQLITE_SLEEP_TIME_USEC); - n++; - } - } while ((n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); - _errId = rc; - - if (_errId != SQLITE_DONE && _errId != SQLITE_ROW) { - _error = QString::fromUtf8(sqlite3_errmsg(_db)); - qCWarning(lcSql) << "Sqlite exec statement error:" << _errId << _error << "in" << _sql; - if (_errId == SQLITE_IOERR) { - qCWarning(lcSql) << "IOERR extended errcode: " << sqlite3_extended_errcode(_db); -#if SQLITE_VERSION_NUMBER >= 3012000 - qCWarning(lcSql) << "IOERR system errno: " << sqlite3_system_errno(_db); -#endif - } - } else { - qCDebug(lcSql) << "Last exec affected" << numRowsAffected() << "rows."; - } - return (_errId == SQLITE_DONE); // either SQLITE_ROW or SQLITE_DONE - } - - return true; -} - -bool SqlQuery::next() -{ - SQLITE_DO(sqlite3_step(_stmt)); - return _errId == SQLITE_ROW; -} - -void SqlQuery::bindValue(int pos, const QVariant &value) -{ - qCDebug(lcSql) << "SQL bind" << pos << value; - - int res = -1; - if (!_stmt) { - ASSERT(false); - return; - } - - switch (value.type()) { - case QVariant::Int: - case QVariant::Bool: - res = sqlite3_bind_int(_stmt, pos, value.toInt()); - break; - case QVariant::Double: - res = sqlite3_bind_double(_stmt, pos, value.toDouble()); - break; - case QVariant::UInt: - case QVariant::LongLong: - res = sqlite3_bind_int64(_stmt, pos, value.toLongLong()); - break; - case QVariant::DateTime: { - const QDateTime dateTime = value.toDateTime(); - const QString str = dateTime.toString(QLatin1String("yyyy-MM-ddThh:mm:ss.zzz")); - res = sqlite3_bind_text16(_stmt, pos, str.utf16(), - str.size() * sizeof(ushort), SQLITE_TRANSIENT); - break; - } - case QVariant::Time: { - const QTime time = value.toTime(); - const QString str = time.toString(QLatin1String("hh:mm:ss.zzz")); - res = sqlite3_bind_text16(_stmt, pos, str.utf16(), - str.size() * sizeof(ushort), SQLITE_TRANSIENT); - break; - } - case QVariant::String: { - if (!value.toString().isNull()) { - // lifetime of string == lifetime of its qvariant - const QString *str = static_cast(value.constData()); - res = sqlite3_bind_text16(_stmt, pos, str->utf16(), - (str->size()) * sizeof(QChar), SQLITE_TRANSIENT); - } else { - res = sqlite3_bind_null(_stmt, pos); - } - break; - } - case QVariant::ByteArray: { - auto ba = value.toByteArray(); - res = sqlite3_bind_text(_stmt, pos, ba.constData(), ba.size(), SQLITE_TRANSIENT); - break; - } - default: { - QString str = value.toString(); - // SQLITE_TRANSIENT makes sure that sqlite buffers the data - res = sqlite3_bind_text16(_stmt, pos, str.utf16(), - (str.size()) * sizeof(QChar), SQLITE_TRANSIENT); - break; - } - } - if (res != SQLITE_OK) { - qCWarning(lcSql) << "ERROR binding SQL value:" << value << "error:" << res; - } - ASSERT(res == SQLITE_OK); -} - -bool SqlQuery::nullValue(int index) -{ - return sqlite3_column_type(_stmt, index) == SQLITE_NULL; -} - -QString SqlQuery::stringValue(int index) -{ - return QString::fromUtf16(static_cast(sqlite3_column_text16(_stmt, index))); -} - -int SqlQuery::intValue(int index) -{ - return sqlite3_column_int(_stmt, index); -} - -quint64 SqlQuery::int64Value(int index) -{ - return sqlite3_column_int64(_stmt, index); -} - -QByteArray SqlQuery::baValue(int index) -{ - return QByteArray(static_cast(sqlite3_column_blob(_stmt, index)), - sqlite3_column_bytes(_stmt, index)); -} - -QString SqlQuery::error() const -{ - return _error; -} - -int SqlQuery::errorId() const -{ - return _errId; -} - -QString SqlQuery::lastQuery() const -{ - return _sql; -} - -int SqlQuery::numRowsAffected() -{ - return sqlite3_changes(_db); -} - -void SqlQuery::finish() -{ - SQLITE_DO(sqlite3_finalize(_stmt)); - _stmt = 0; -} - -void SqlQuery::reset_and_clear_bindings() -{ - if (_stmt) { - SQLITE_DO(sqlite3_reset(_stmt)); - SQLITE_DO(sqlite3_clear_bindings(_stmt)); - } -} - -} // namespace OCC diff --git a/src/libsync/ownsql.h b/src/libsync/ownsql.h deleted file mode 100644 index 72c749091..000000000 --- a/src/libsync/ownsql.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#ifndef OWNSQL_H -#define OWNSQL_H - -#include - -#include -#include - -#include "owncloudlib.h" - -namespace OCC { - -/** - * @brief The SqlDatabase class - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SqlDatabase -{ - Q_DISABLE_COPY(SqlDatabase) -public: - explicit SqlDatabase(); - - bool isOpen(); - bool openOrCreateReadWrite(const QString &filename); - bool openReadOnly(const QString &filename); - bool transaction(); - bool commit(); - void close(); - QString error() const; - sqlite3 *sqliteDb(); - -private: - bool openHelper(const QString &filename, int sqliteFlags); - bool checkDb(); - - sqlite3 *_db; - QString _error; // last error string - int _errId; -}; - -/** - * @brief The SqlQuery class - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SqlQuery -{ - Q_DISABLE_COPY(SqlQuery) -public: - explicit SqlQuery(SqlDatabase &db); - explicit SqlQuery(const QString &sql, SqlDatabase &db); - - ~SqlQuery(); - QString error() const; - int errorId() const; - - /// Checks whether the value at the given column index is NULL - bool nullValue(int index); - - - QString stringValue(int index); - int intValue(int index); - quint64 int64Value(int index); - QByteArray baValue(int index); - - bool isSelect(); - bool isPragma(); - bool exec(); - int prepare(const QString &sql, bool allow_failure = false); - bool next(); - void bindValue(int pos, const QVariant &value); - QString lastQuery() const; - int numRowsAffected(); - void reset_and_clear_bindings(); - void finish(); - -private: - sqlite3 *_db; - sqlite3_stmt *_stmt; - QString _error; - int _errId; - QString _sql; -}; - -} // namespace OCC - -#endif // OWNSQL_H diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 21af4d45c..3e6b8898a 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -17,13 +17,13 @@ #include "propagatedownload.h" #include "networkjobs.h" #include "account.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "common/utility.h" #include "filesystem.h" #include "propagatorjobs.h" -#include "checksums.h" -#include "asserts.h" +#include "common/checksums.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/propagateremotedelete.cpp b/src/libsync/propagateremotedelete.cpp index 73f5c8240..0ace61790 100644 --- a/src/libsync/propagateremotedelete.cpp +++ b/src/libsync/propagateremotedelete.cpp @@ -15,7 +15,7 @@ #include "propagateremotedelete.h" #include "owncloudpropagator_p.h" #include "account.h" -#include "asserts.h" +#include "common/asserts.h" #include diff --git a/src/libsync/propagateremotemkdir.cpp b/src/libsync/propagateremotemkdir.cpp index 92ba2b9a5..4772f42eb 100644 --- a/src/libsync/propagateremotemkdir.cpp +++ b/src/libsync/propagateremotemkdir.cpp @@ -15,9 +15,9 @@ #include "propagateremotemkdir.h" #include "owncloudpropagator_p.h" #include "account.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "propagateremotedelete.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp index 7f0d8785b..b347b0ab0 100644 --- a/src/libsync/propagateremotemove.cpp +++ b/src/libsync/propagateremotemove.cpp @@ -16,9 +16,9 @@ #include "propagatorjobs.h" #include "owncloudpropagator_p.h" #include "account.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "filesystem.h" -#include "asserts.h" +#include "common/asserts.h" #include #include #include diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index c49be0785..05ec97a00 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -17,15 +17,15 @@ #include "owncloudpropagator_p.h" #include "networkjobs.h" #include "account.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "common/utility.h" #include "filesystem.h" #include "propagatorjobs.h" -#include "checksums.h" +#include "common/checksums.h" #include "syncengine.h" #include "propagateremotedelete.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index edeb43516..4afdad38b 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -17,15 +17,15 @@ #include "owncloudpropagator_p.h" #include "networkjobs.h" #include "account.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "common/utility.h" #include "filesystem.h" #include "propagatorjobs.h" #include "syncengine.h" #include "propagateremotemove.h" #include "propagateremotedelete.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index a5396c700..790023a64 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -17,15 +17,15 @@ #include "owncloudpropagator_p.h" #include "networkjobs.h" #include "account.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "common/utility.h" #include "filesystem.h" #include "propagatorjobs.h" -#include "checksums.h" +#include "common/checksums.h" #include "syncengine.h" #include "propagateremotedelete.h" -#include "asserts.h" +#include "common/asserts.h" #include #include diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp index 1d841abb4..ecf46f285 100644 --- a/src/libsync/propagatorjobs.cpp +++ b/src/libsync/propagatorjobs.cpp @@ -17,8 +17,8 @@ #include "owncloudpropagator_p.h" #include "propagateremotemove.h" #include "common/utility.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "filesystem.h" #include #include diff --git a/src/libsync/propagatorjobs.h b/src/libsync/propagatorjobs.h index 04c09cd48..424685ab4 100644 --- a/src/libsync/propagatorjobs.h +++ b/src/libsync/propagatorjobs.h @@ -21,16 +21,10 @@ namespace OCC { /** - * Tags for checksum headers. - * They are here for being shared between Upload- and Download Job + * Tags for checksum header. + * It's here for being shared between Upload- and Download Job */ - -// the header itself static const char checkSumHeaderC[] = "OC-Checksum"; -// ...and it's values -static const char checkSumMD5C[] = "MD5"; -static const char checkSumSHA1C[] = "SHA1"; -static const char checkSumAdlerC[] = "Adler32"; /** * @brief Declaration of the other propagation jobs diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 5a0a56cf9..4b4836ad3 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -16,8 +16,8 @@ #include "syncengine.h" #include "account.h" #include "owncloudpropagator.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" #include "discoveryphase.h" #include "creds/abstractcredentials.h" #include "syncfilestatus.h" @@ -25,7 +25,7 @@ #include "filesystem.h" #include "propagateremotedelete.h" #include "propagatedownload.h" -#include "asserts.h" +#include "common/asserts.h" #ifdef Q_OS_WIN #include diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 3bc6da984..7f96afef9 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -38,7 +38,7 @@ #include "syncfilestatustracker.h" #include "accountfwd.h" #include "discoveryphase.h" -#include "checksums.h" +#include "common/checksums.h" class QProcess; diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 681d703c9..0e6d8cb2a 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -13,7 +13,7 @@ */ #include "syncfileitem.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournalfilerecord.h" #include "common/utility.h" #include diff --git a/src/libsync/syncfilestatustracker.cpp b/src/libsync/syncfilestatustracker.cpp index c0b315372..7b40b6071 100644 --- a/src/libsync/syncfilestatustracker.cpp +++ b/src/libsync/syncfilestatustracker.cpp @@ -15,9 +15,9 @@ #include "syncfilestatustracker.h" #include "syncengine.h" -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" -#include "asserts.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" +#include "common/asserts.h" #include diff --git a/src/libsync/syncfilestatustracker.h b/src/libsync/syncfilestatustracker.h index 48ecfd5c2..f4b31a130 100644 --- a/src/libsync/syncfilestatustracker.h +++ b/src/libsync/syncfilestatustracker.h @@ -16,7 +16,7 @@ #ifndef SYNCFILESTATUSTRACKER_H #define SYNCFILESTATUSTRACKER_H -#include "ownsql.h" +// #include "ownsql.h" #include "syncfileitem.h" #include "syncfilestatus.h" #include diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp deleted file mode 100644 index 5f344fa08..000000000 --- a/src/libsync/syncjournaldb.cpp +++ /dev/null @@ -1,1882 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "ownsql.h" - -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" -#include "common/utility.h" -#include "version.h" -#include "filesystem.h" -#include "asserts.h" -#include "checksums.h" - -#include "std/c_jhash.h" - -namespace OCC { - -Q_LOGGING_CATEGORY(lcDb, "sync.database", QtInfoMsg) - -static QString defaultJournalMode(const QString &dbPath) -{ -#ifdef Q_OS_WIN - // See #2693: Some exFAT file systems seem unable to cope with the - // WAL journaling mode. They work fine with DELETE. - QString fileSystem = FileSystem::fileSystemForPath(dbPath); - qCInfo(lcDb) << "Detected filesystem" << fileSystem << "for" << dbPath; - if (fileSystem.contains("FAT")) { - qCInfo(lcDb) << "Filesystem contains FAT - using DELETE journal mode"; - return "DELETE"; - } -#else - Q_UNUSED(dbPath) -#endif - return "WAL"; -} - -SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent) - : QObject(parent) - , _dbFile(dbFilePath) - , _transaction(0) -{ - // Allow forcing the journal mode for debugging - static QString envJournalMode = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE")); - _journalMode = envJournalMode; - if (_journalMode.isEmpty()) { - _journalMode = defaultJournalMode(_dbFile); - } -} - -QString SyncJournalDb::makeDbName(const QString &localPath, - const QUrl &remoteUrl, - const QString &remotePath, - const QString &user) -{ - QString journalPath = QLatin1String("._sync_"); - - QString key = QString::fromUtf8("%1@%2:%3").arg(user, remoteUrl.toString(), remotePath); - - QByteArray ba = QCryptographicHash::hash(key.toUtf8(), QCryptographicHash::Md5); - journalPath.append(ba.left(6).toHex()); - journalPath.append(".db"); - - // If the journal doesn't exist and we can't create a file - // at that location, try again with a journal name that doesn't - // have the ._ prefix. - // - // The disadvantage of that filename is that it will only be ignored - // by client versions >2.3.2. - // - // See #5633: "._*" is often forbidden on samba shared folders. - - // If it exists already, the path is clearly usable - QFile file(QDir(localPath).filePath(journalPath)); - if (file.exists()) { - return journalPath; - } - - // Try to create a file there - if (file.open(QIODevice::ReadWrite)) { - // Ok, all good. - file.close(); - file.remove(); - return journalPath; - } - - // Can we create it if we drop the underscore? - QString alternateJournalPath = journalPath.mid(2).prepend("."); - QFile file2(QDir(localPath).filePath(alternateJournalPath)); - if (file2.open(QIODevice::ReadWrite)) { - // The alternative worked, use it - qCInfo(lcDb) << "Using alternate database path" << alternateJournalPath; - file2.close(); - file2.remove(); - return alternateJournalPath; - } - - // Neither worked, just keep the original and throw errors later - qCWarning(lcDb) << "Could not find a writable database path" << file.fileName(); - return journalPath; -} - -bool SyncJournalDb::maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath) -{ - const QString oldDbName = localPath + QLatin1String(".csync_journal.db"); - if (!FileSystem::fileExists(oldDbName)) { - return true; - } - const QString oldDbNameShm = oldDbName + "-shm"; - const QString oldDbNameWal = oldDbName + "-wal"; - - const QString newDbName = absoluteJournalPath; - const QString newDbNameShm = newDbName + "-shm"; - const QString newDbNameWal = newDbName + "-wal"; - - // Whenever there is an old db file, migrate it to the new db path. - // This is done to make switching from older versions to newer versions - // work correctly even if the user had previously used a new version - // and therefore already has an (outdated) new-style db file. - QString error; - - if (FileSystem::fileExists(newDbName)) { - if (!FileSystem::remove(newDbName, &error)) { - qCWarning(lcDb) << "Database migration: Could not remove db file" << newDbName - << "due to" << error; - return false; - } - } - if (FileSystem::fileExists(newDbNameWal)) { - if (!FileSystem::remove(newDbNameWal, &error)) { - qCWarning(lcDb) << "Database migration: Could not remove db WAL file" << newDbNameWal - << "due to" << error; - return false; - } - } - if (FileSystem::fileExists(newDbNameShm)) { - if (!FileSystem::remove(newDbNameShm, &error)) { - qCWarning(lcDb) << "Database migration: Could not remove db SHM file" << newDbNameShm - << "due to" << error; - return false; - } - } - - if (!FileSystem::rename(oldDbName, newDbName, &error)) { - qCWarning(lcDb) << "Database migration: could not rename " << oldDbName - << "to" << newDbName << ":" << error; - return false; - } - if (!FileSystem::rename(oldDbNameWal, newDbNameWal, &error)) { - qCWarning(lcDb) << "Database migration: could not rename " << oldDbNameWal - << "to" << newDbNameWal << ":" << error; - return false; - } - if (!FileSystem::rename(oldDbNameShm, newDbNameShm, &error)) { - qCWarning(lcDb) << "Database migration: could not rename " << oldDbNameShm - << "to" << newDbNameShm << ":" << error; - return false; - } - - qCInfo(lcDb) << "Journal successfully migrated from" << oldDbName << "to" << newDbName; - return true; -} - -bool SyncJournalDb::exists() -{ - QMutexLocker locker(&_mutex); - return (!_dbFile.isEmpty() && QFile::exists(_dbFile)); -} - -QString SyncJournalDb::databaseFilePath() const -{ - return _dbFile; -} - -// Note that this does not change the size of the -wal file, but it is supposed to make -// the normal .db faster since the changes from the wal will be incorporated into it. -// Then the next sync (and the SocketAPI) will have a faster access. -void SyncJournalDb::walCheckpoint() -{ - QElapsedTimer t; - t.start(); - SqlQuery pragma1(_db); - pragma1.prepare("PRAGMA wal_checkpoint(FULL);"); - if (pragma1.exec()) { - qCDebug(lcDb) << "took" << t.elapsed() << "msec"; - } -} - -void SyncJournalDb::startTransaction() -{ - if (_transaction == 0) { - if (!_db.transaction()) { - qCWarning(lcDb) << "ERROR starting transaction: " << _db.error(); - return; - } - _transaction = 1; - } else { - qCDebug(lcDb) << "Database Transaction is running, not starting another one!"; - } -} - -void SyncJournalDb::commitTransaction() -{ - if (_transaction == 1) { - if (!_db.commit()) { - qCWarning(lcDb) << "ERROR committing to the database: " << _db.error(); - return; - } - _transaction = 0; - } else { - qCDebug(lcDb) << "No database Transaction to commit"; - } -} - -bool SyncJournalDb::sqlFail(const QString &log, const SqlQuery &query) -{ - commitTransaction(); - qCWarning(lcDb) << "SQL Error" << log << query.error(); - ASSERT(false); - _db.close(); - return false; -} - -bool SyncJournalDb::checkConnect() -{ - if (_db.isOpen()) { - return true; - } - - if (_dbFile.isEmpty()) { - qCWarning(lcDb) << "Database filename" + _dbFile + " is empty"; - return false; - } - - // The database file is created by this call (SQLITE_OPEN_CREATE) - if (!_db.openOrCreateReadWrite(_dbFile)) { - QString error = _db.error(); - qCWarning(lcDb) << "Error opening the db: " << error; - return false; - } - - if (!QFile::exists(_dbFile)) { - qCWarning(lcDb) << "Database file" + _dbFile + " does not exist"; - return false; - } - - SqlQuery pragma1(_db); - pragma1.prepare("SELECT sqlite_version();"); - if (!pragma1.exec()) { - return sqlFail("SELECT sqlite_version()", pragma1); - } else { - pragma1.next(); - qCInfo(lcDb) << "sqlite3 version" << pragma1.stringValue(0); - } - - pragma1.prepare(QString("PRAGMA journal_mode=%1;").arg(_journalMode)); - if (!pragma1.exec()) { - return sqlFail("Set PRAGMA journal_mode", pragma1); - } else { - pragma1.next(); - qCInfo(lcDb) << "sqlite3 journal_mode=" << pragma1.stringValue(0); - } - - // For debugging purposes, allow temp_store to be set - static QString env_temp_store = QString::fromLocal8Bit(qgetenv("OWNCLOUD_SQLITE_TEMP_STORE")); - if (!env_temp_store.isEmpty()) { - pragma1.prepare(QString("PRAGMA temp_store = %1;").arg(env_temp_store)); - if (!pragma1.exec()) { - return sqlFail("Set PRAGMA temp_store", pragma1); - } - qCInfo(lcDb) << "sqlite3 with temp_store =" << env_temp_store; - } - - pragma1.prepare("PRAGMA synchronous = 1;"); - if (!pragma1.exec()) { - return sqlFail("Set PRAGMA synchronous", pragma1); - } - pragma1.prepare("PRAGMA case_sensitive_like = ON;"); - if (!pragma1.exec()) { - return sqlFail("Set PRAGMA case_sensitivity", pragma1); - } - - /* Because insert is so slow, we do everything in a transaction, and only need one call to commit */ - startTransaction(); - - SqlQuery createQuery(_db); - createQuery.prepare("CREATE TABLE IF NOT EXISTS metadata(" - "phash INTEGER(8)," - "pathlen INTEGER," - "path VARCHAR(4096)," - "inode INTEGER," - "uid INTEGER," - "gid INTEGER," - "mode INTEGER," - "modtime INTEGER(8)," - "type INTEGER," - "md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */ - // updateDatabaseStructure() will add - // fileid - // remotePerm - // filesize - // ignoredChildrenRemote - // contentChecksum - // contentChecksumTypeId - "PRIMARY KEY(phash)" - ");"); - - if (!createQuery.exec()) { - // In certain situations the io error can be avoided by switching - // to the DELETE journal mode, see #5723 - if (_journalMode != "DELETE" - && createQuery.errorId() == SQLITE_IOERR - && sqlite3_extended_errcode(_db.sqliteDb()) == SQLITE_IOERR_SHMMAP) { - qCWarning(lcDb) << "IO error SHMMAP on table creation, attempting with DELETE journal mode"; - - _journalMode = "DELETE"; - createQuery.finish(); - pragma1.finish(); - commitTransaction(); - _db.close(); - return checkConnect(); - } - - return sqlFail("Create table metadata", createQuery); - } - - createQuery.prepare("CREATE TABLE IF NOT EXISTS downloadinfo(" - "path VARCHAR(4096)," - "tmpfile VARCHAR(4096)," - "etag VARCHAR(32)," - "errorcount INTEGER," - "PRIMARY KEY(path)" - ");"); - - if (!createQuery.exec()) { - return sqlFail("Create table downloadinfo", createQuery); - } - - createQuery.prepare("CREATE TABLE IF NOT EXISTS uploadinfo(" - "path VARCHAR(4096)," - "chunk INTEGER," - "transferid INTEGER," - "errorcount INTEGER," - "size INTEGER(8)," - "modtime INTEGER(8)," - "PRIMARY KEY(path)" - ");"); - - if (!createQuery.exec()) { - return sqlFail("Create table uploadinfo", createQuery); - } - - // create the blacklist table. - createQuery.prepare("CREATE TABLE IF NOT EXISTS blacklist (" - "path VARCHAR(4096)," - "lastTryEtag VARCHAR[32]," - "lastTryModtime INTEGER[8]," - "retrycount INTEGER," - "errorstring VARCHAR[4096]," - "PRIMARY KEY(path)" - ");"); - - if (!createQuery.exec()) { - return sqlFail("Create table blacklist", createQuery); - } - - createQuery.prepare("CREATE TABLE IF NOT EXISTS poll(" - "path VARCHAR(4096)," - "modtime INTEGER(8)," - "pollpath VARCHAR(4096));"); - if (!createQuery.exec()) { - return sqlFail("Create table poll", createQuery); - } - - // create the selectivesync table. - createQuery.prepare("CREATE TABLE IF NOT EXISTS selectivesync (" - "path VARCHAR(4096)," - "type INTEGER" - ");"); - - if (!createQuery.exec()) { - return sqlFail("Create table selectivesync", createQuery); - } - - // create the checksumtype table. - createQuery.prepare("CREATE TABLE IF NOT EXISTS checksumtype(" - "id INTEGER PRIMARY KEY," - "name TEXT UNIQUE" - ");"); - if (!createQuery.exec()) { - return sqlFail("Create table version", createQuery); - } - - // create the checksumtype table. - createQuery.prepare("CREATE TABLE IF NOT EXISTS datafingerprint(" - "fingerprint TEXT UNIQUE" - ");"); - if (!createQuery.exec()) { - return sqlFail("Create table datafingerprint", createQuery); - } - - createQuery.prepare("CREATE TABLE IF NOT EXISTS version(" - "major INTEGER(8)," - "minor INTEGER(8)," - "patch INTEGER(8)," - "custom VARCHAR(256)" - ");"); - if (!createQuery.exec()) { - return sqlFail("Create table version", createQuery); - } - - bool forceRemoteDiscovery = false; - - SqlQuery versionQuery("SELECT major, minor, patch FROM version;", _db); - if (!versionQuery.next()) { - // If there was no entry in the table, it means we are likely upgrading from 1.5 - qCInfo(lcDb) << "possibleUpgradeFromMirall_1_5 detected!"; - forceRemoteDiscovery = true; - - createQuery.prepare("INSERT INTO version VALUES (?1, ?2, ?3, ?4);"); - createQuery.bindValue(1, MIRALL_VERSION_MAJOR); - createQuery.bindValue(2, MIRALL_VERSION_MINOR); - createQuery.bindValue(3, MIRALL_VERSION_PATCH); - createQuery.bindValue(4, MIRALL_VERSION_BUILD); - if (!createQuery.exec()) { - return sqlFail("Update version", createQuery); - } - - } else { - int major = versionQuery.intValue(0); - int minor = versionQuery.intValue(1); - int patch = versionQuery.intValue(2); - - if (major == 1 && minor == 8 && (patch == 0 || patch == 1)) { - qCInfo(lcDb) << "possibleUpgradeFromMirall_1_8_0_or_1 detected!"; - forceRemoteDiscovery = true; - } - - // There was a bug in versions <2.3.0 that could lead to stale - // local files and a remote discovery will fix them. - // See #5190 #5242. - if (major == 2 && minor < 3) { - qCInfo(lcDb) << "upgrade form client < 2.3.0 detected! forcing remote discovery"; - forceRemoteDiscovery = true; - } - - // Not comparing the BUILD id here, correct? - if (!(major == MIRALL_VERSION_MAJOR && minor == MIRALL_VERSION_MINOR && patch == MIRALL_VERSION_PATCH)) { - createQuery.prepare("UPDATE version SET major=?1, minor=?2, patch =?3, custom=?4 " - "WHERE major=?5 AND minor=?6 AND patch=?7;"); - createQuery.bindValue(1, MIRALL_VERSION_MAJOR); - createQuery.bindValue(2, MIRALL_VERSION_MINOR); - createQuery.bindValue(3, MIRALL_VERSION_PATCH); - createQuery.bindValue(4, MIRALL_VERSION_BUILD); - createQuery.bindValue(5, major); - createQuery.bindValue(6, minor); - createQuery.bindValue(7, patch); - if (!createQuery.exec()) { - return sqlFail("Update version", createQuery); - } - } - } - - commitInternal("checkConnect"); - - bool rc = updateDatabaseStructure(); - if (!rc) { - qCWarning(lcDb) << "Failed to update the database structure!"; - } - - /* - * If we are upgrading from a client version older than 1.5, - * we cannot read from the database because we need to fetch the files id and etags. - * - * If 1.8.0 caused missing data in the local tree, so we also don't read from DB - * to get back the files that were gone. - * In 1.8.1 we had a fix to re-get the data, but this one here is better - */ - if (forceRemoteDiscovery) { - forceRemoteDiscoveryNextSyncLocked(); - } - - _getFileRecordQuery.reset(new SqlQuery(_db)); - if (_getFileRecordQuery->prepare( - "SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize," - " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" - " FROM metadata" - " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" - " WHERE phash=?1")) { - return sqlFail("prepare _getFileRecordQuery", *_getFileRecordQuery); - } - - _setFileRecordQuery.reset(new SqlQuery(_db)); - if (_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " - "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " - "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);")) { - return sqlFail("prepare _setFileRecordQuery", *_setFileRecordQuery); - } - - _setFileRecordChecksumQuery.reset(new SqlQuery(_db)); - if (_setFileRecordChecksumQuery->prepare( - "UPDATE metadata" - " SET contentChecksum = ?2, contentChecksumTypeId = ?3" - " WHERE phash == ?1;")) { - return sqlFail("prepare _setFileRecordChecksumQuery", *_setFileRecordChecksumQuery); - } - - _setFileRecordLocalMetadataQuery.reset(new SqlQuery(_db)); - if (_setFileRecordLocalMetadataQuery->prepare( - "UPDATE metadata" - " SET inode=?2, modtime=?3, filesize=?4" - " WHERE phash == ?1;")) { - return sqlFail("prepare _setFileRecordLocalMetadataQuery", *_setFileRecordLocalMetadataQuery); - } - - _getDownloadInfoQuery.reset(new SqlQuery(_db)); - if (_getDownloadInfoQuery->prepare("SELECT tmpfile, etag, errorcount FROM " - "downloadinfo WHERE path=?1")) { - return sqlFail("prepare _getDownloadInfoQuery", *_getDownloadInfoQuery); - } - - _setDownloadInfoQuery.reset(new SqlQuery(_db)); - if (_setDownloadInfoQuery->prepare("INSERT OR REPLACE INTO downloadinfo " - "(path, tmpfile, etag, errorcount) " - "VALUES ( ?1 , ?2, ?3, ?4 )")) { - return sqlFail("prepare _setDownloadInfoQuery", *_setDownloadInfoQuery); - } - - _deleteDownloadInfoQuery.reset(new SqlQuery(_db)); - if (_deleteDownloadInfoQuery->prepare("DELETE FROM downloadinfo WHERE path=?1")) { - return sqlFail("prepare _deleteDownloadInfoQuery", *_deleteDownloadInfoQuery); - } - - _getUploadInfoQuery.reset(new SqlQuery(_db)); - if (_getUploadInfoQuery->prepare("SELECT chunk, transferid, errorcount, size, modtime FROM " - "uploadinfo WHERE path=?1")) { - return sqlFail("prepare _getUploadInfoQuery", *_getUploadInfoQuery); - } - - _setUploadInfoQuery.reset(new SqlQuery(_db)); - if (_setUploadInfoQuery->prepare("INSERT OR REPLACE INTO uploadinfo " - "(path, chunk, transferid, errorcount, size, modtime) " - "VALUES ( ?1 , ?2, ?3 , ?4 , ?5, ?6 )")) { - return sqlFail("prepare _setUploadInfoQuery", *_setUploadInfoQuery); - } - - _deleteUploadInfoQuery.reset(new SqlQuery(_db)); - if (_deleteUploadInfoQuery->prepare("DELETE FROM uploadinfo WHERE path=?1")) { - return sqlFail("prepare _deleteUploadInfoQuery", *_deleteUploadInfoQuery); - } - - - _deleteFileRecordPhash.reset(new SqlQuery(_db)); - if (_deleteFileRecordPhash->prepare("DELETE FROM metadata WHERE phash=?1")) { - return sqlFail("prepare _deleteFileRecordPhash", *_deleteFileRecordPhash); - } - - _deleteFileRecordRecursively.reset(new SqlQuery(_db)); - if (_deleteFileRecordRecursively->prepare("DELETE FROM metadata WHERE path LIKE(?||'/%')")) { - return sqlFail("prepare _deleteFileRecordRecursively", *_deleteFileRecordRecursively); - } - - QString sql("SELECT lastTryEtag, lastTryModtime, retrycount, errorstring, lastTryTime, ignoreDuration, renameTarget, errorCategory " - "FROM blacklist WHERE path=?1"); - if (Utility::fsCasePreserving()) { - // if the file system is case preserving we have to check the blacklist - // case insensitively - sql += QLatin1String(" COLLATE NOCASE"); - } - _getErrorBlacklistQuery.reset(new SqlQuery(_db)); - if (_getErrorBlacklistQuery->prepare(sql)) { - return sqlFail("prepare _getErrorBlacklistQuery", *_getErrorBlacklistQuery); - } - - _setErrorBlacklistQuery.reset(new SqlQuery(_db)); - if (_setErrorBlacklistQuery->prepare("INSERT OR REPLACE INTO blacklist " - "(path, lastTryEtag, lastTryModtime, retrycount, errorstring, lastTryTime, ignoreDuration, renameTarget, errorCategory) " - "VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)")) { - return sqlFail("prepare _setErrorBlacklistQuery", *_setErrorBlacklistQuery); - } - - _getSelectiveSyncListQuery.reset(new SqlQuery(_db)); - if (_getSelectiveSyncListQuery->prepare("SELECT path FROM selectivesync WHERE type=?1")) { - return sqlFail("prepare _getSelectiveSyncListQuery", *_getSelectiveSyncListQuery); - } - - _getChecksumTypeIdQuery.reset(new SqlQuery(_db)); - if (_getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1")) { - return sqlFail("prepare _getChecksumTypeIdQuery", *_getChecksumTypeIdQuery); - } - - _getChecksumTypeQuery.reset(new SqlQuery(_db)); - if (_getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1")) { - return sqlFail("prepare _getChecksumTypeQuery", *_getChecksumTypeQuery); - } - - _insertChecksumTypeQuery.reset(new SqlQuery(_db)); - if (_insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)")) { - return sqlFail("prepare _insertChecksumTypeQuery", *_insertChecksumTypeQuery); - } - - _getDataFingerprintQuery.reset(new SqlQuery(_db)); - if (_getDataFingerprintQuery->prepare("SELECT fingerprint FROM datafingerprint")) { - return sqlFail("prepare _getDataFingerprintQuery", *_getDataFingerprintQuery); - } - - _setDataFingerprintQuery1.reset(new SqlQuery(_db)); - if (_setDataFingerprintQuery1->prepare("DELETE FROM datafingerprint;")) { - return sqlFail("prepare _setDataFingerprintQuery1", *_setDataFingerprintQuery1); - } - _setDataFingerprintQuery2.reset(new SqlQuery(_db)); - if (_setDataFingerprintQuery2->prepare("INSERT INTO datafingerprint (fingerprint) VALUES (?1);")) { - return sqlFail("prepare _setDataFingerprintQuery2", *_setDataFingerprintQuery2); - } - - // don't start a new transaction now - commitInternal(QString("checkConnect End"), false); - - // Hide 'em all! - FileSystem::setFileHidden(databaseFilePath(), true); - FileSystem::setFileHidden(databaseFilePath() + "-wal", true); - FileSystem::setFileHidden(databaseFilePath() + "-shm", true); - FileSystem::setFileHidden(databaseFilePath() + "-journal", true); - - return rc; -} - -void SyncJournalDb::close() -{ - QMutexLocker locker(&_mutex); - qCInfo(lcDb) << "Closing DB" << _dbFile; - - commitTransaction(); - - _getFileRecordQuery.reset(0); - _setFileRecordQuery.reset(0); - _setFileRecordChecksumQuery.reset(0); - _setFileRecordLocalMetadataQuery.reset(0); - _getDownloadInfoQuery.reset(0); - _setDownloadInfoQuery.reset(0); - _deleteDownloadInfoQuery.reset(0); - _getUploadInfoQuery.reset(0); - _setUploadInfoQuery.reset(0); - _deleteUploadInfoQuery.reset(0); - _deleteFileRecordPhash.reset(0); - _deleteFileRecordRecursively.reset(0); - _getErrorBlacklistQuery.reset(0); - _setErrorBlacklistQuery.reset(0); - _getSelectiveSyncListQuery.reset(0); - _getChecksumTypeIdQuery.reset(0); - _getChecksumTypeQuery.reset(0); - _insertChecksumTypeQuery.reset(0); - _getDataFingerprintQuery.reset(0); - _setDataFingerprintQuery1.reset(0); - _setDataFingerprintQuery2.reset(0); - - _db.close(); - _avoidReadFromDbOnNextSyncFilter.clear(); -} - - -bool SyncJournalDb::updateDatabaseStructure() -{ - if (!updateMetadataTableStructure()) - return false; - if (!updateErrorBlacklistTableStructure()) - return false; - return true; -} - -bool SyncJournalDb::updateMetadataTableStructure() -{ - QStringList columns = tableColumns("metadata"); - bool re = true; - - // check if the file_id column is there and create it if not - if (!checkConnect()) { - return false; - } - - if (columns.indexOf(QLatin1String("fileid")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN fileid VARCHAR(128);"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: Add column fileid", query); - re = false; - } - - query.prepare("CREATE INDEX metadata_file_id ON metadata(fileid);"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: create index fileid", query); - re = false; - } - commitInternal("update database structure: add fileid col"); - } - if (columns.indexOf(QLatin1String("remotePerm")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN remotePerm VARCHAR(128);"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: add column remotePerm", query); - re = false; - } - commitInternal("update database structure (remotePerm)"); - } - if (columns.indexOf(QLatin1String("filesize")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN filesize BIGINT;"); - if (!query.exec()) { - sqlFail("updateDatabaseStructure: add column filesize", query); - re = false; - } - commitInternal("update database structure: add filesize col"); - } - - if (1) { - SqlQuery query(_db); - query.prepare("CREATE INDEX IF NOT EXISTS metadata_inode ON metadata(inode);"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: create index inode", query); - re = false; - } - commitInternal("update database structure: add inode index"); - } - - if (1) { - SqlQuery query(_db); - query.prepare("CREATE INDEX IF NOT EXISTS metadata_path ON metadata(path);"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: create index path", query); - re = false; - } - commitInternal("update database structure: add path index"); - } - - if (columns.indexOf(QLatin1String("ignoredChildrenRemote")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN ignoredChildrenRemote INT;"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: add ignoredChildrenRemote column", query); - re = false; - } - commitInternal("update database structure: add ignoredChildrenRemote col"); - } - - if (columns.indexOf(QLatin1String("contentChecksum")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: add contentChecksum column", query); - re = false; - } - commitInternal("update database structure: add contentChecksum col"); - } - if (columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;"); - if (!query.exec()) { - sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query); - re = false; - } - commitInternal("update database structure: add contentChecksumTypeId col"); - } - - - return re; -} - -bool SyncJournalDb::updateErrorBlacklistTableStructure() -{ - QStringList columns = tableColumns("blacklist"); - bool re = true; - - // check if the file_id column is there and create it if not - if (!checkConnect()) { - return false; - } - - if (columns.indexOf(QLatin1String("lastTryTime")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE blacklist ADD COLUMN lastTryTime INTEGER(8);"); - if (!query.exec()) { - sqlFail("updateBlacklistTableStructure: Add lastTryTime fileid", query); - re = false; - } - query.prepare("ALTER TABLE blacklist ADD COLUMN ignoreDuration INTEGER(8);"); - if (!query.exec()) { - sqlFail("updateBlacklistTableStructure: Add ignoreDuration fileid", query); - re = false; - } - commitInternal("update database structure: add lastTryTime, ignoreDuration cols"); - } - if (columns.indexOf(QLatin1String("renameTarget")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE blacklist ADD COLUMN renameTarget VARCHAR(4096);"); - if (!query.exec()) { - sqlFail("updateBlacklistTableStructure: Add renameTarget", query); - re = false; - } - commitInternal("update database structure: add renameTarget col"); - } - - if (columns.indexOf(QLatin1String("errorCategory")) == -1) { - SqlQuery query(_db); - query.prepare("ALTER TABLE blacklist ADD COLUMN errorCategory INTEGER(8);"); - if (!query.exec()) { - sqlFail("updateBlacklistTableStructure: Add errorCategory", query); - re = false; - } - commitInternal("update database structure: add errorCategory col"); - } - - SqlQuery query(_db); - query.prepare("CREATE INDEX IF NOT EXISTS blacklist_index ON blacklist(path collate nocase);"); - if (!query.exec()) { - sqlFail("updateErrorBlacklistTableStructure: create index blacklit", query); - re = false; - } - - return re; -} - -QStringList SyncJournalDb::tableColumns(const QString &table) -{ - QStringList columns; - if (!table.isEmpty()) { - if (checkConnect()) { - QString q = QString("PRAGMA table_info('%1');").arg(table); - SqlQuery query(_db); - query.prepare(q); - - if (!query.exec()) { - return columns; - } - - while (query.next()) { - columns.append(query.stringValue(1)); - } - } - } - qCDebug(lcDb) << "Columns in the current journal: " << columns; - - return columns; -} - -qint64 SyncJournalDb::getPHash(const QString &file) -{ - QByteArray utf8File = file.toUtf8(); - int64_t h; - - if (file.isEmpty()) { - return -1; - } - - int len = utf8File.length(); - - h = c_jhash64((uint8_t *)utf8File.data(), len, 0); - return h; -} - -bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) -{ - SyncJournalFileRecord record = _record; - QMutexLocker locker(&_mutex); - - if (!_avoidReadFromDbOnNextSyncFilter.isEmpty()) { - // If we are a directory that should not be read from db next time, don't write the etag - QString prefix = record._path + "/"; - foreach (const QString &it, _avoidReadFromDbOnNextSyncFilter) { - if (it.startsWith(prefix)) { - qCInfo(lcDb) << "Filtered writing the etag of" << prefix << "because it is a prefix of" << it; - record._etag = "_invalid_"; - break; - } - } - } - - qCInfo(lcDb) << "Updating file record for path:" << record._path << "inode:" << record._inode - << "modtime:" << record._modtime << "type:" << record._type - << "etag:" << record._etag << "fileId:" << record._fileId << "remotePerm:" << record._remotePerm - << "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader; - - qlonglong phash = getPHash(record._path); - if (checkConnect()) { - QByteArray arr = record._path.toUtf8(); - int plen = arr.length(); - - QString etag(record._etag); - if (etag.isEmpty()) - etag = ""; - QString fileId(record._fileId); - if (fileId.isEmpty()) - fileId = ""; - QString remotePerm(record._remotePerm); - if (remotePerm.isEmpty()) - remotePerm = QString(); // have NULL in DB (vs empty) - QByteArray checksumType, checksum; - parseChecksumHeader(record._checksumHeader, &checksumType, &checksum); - int contentChecksumTypeId = mapChecksumType(checksumType); - _setFileRecordQuery->reset_and_clear_bindings(); - _setFileRecordQuery->bindValue(1, QString::number(phash)); - _setFileRecordQuery->bindValue(2, plen); - _setFileRecordQuery->bindValue(3, record._path); - _setFileRecordQuery->bindValue(4, record._inode); - _setFileRecordQuery->bindValue(5, 0); // uid Not used - _setFileRecordQuery->bindValue(6, 0); // gid Not used - _setFileRecordQuery->bindValue(7, 0); // mode Not used - _setFileRecordQuery->bindValue(8, QString::number(Utility::qDateTimeToTime_t(record._modtime))); - _setFileRecordQuery->bindValue(9, QString::number(record._type)); - _setFileRecordQuery->bindValue(10, etag); - _setFileRecordQuery->bindValue(11, fileId); - _setFileRecordQuery->bindValue(12, remotePerm); - _setFileRecordQuery->bindValue(13, record._fileSize); - _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1 : 0); - _setFileRecordQuery->bindValue(15, checksum); - _setFileRecordQuery->bindValue(16, contentChecksumTypeId); - - if (!_setFileRecordQuery->exec()) { - return false; - } - - _setFileRecordQuery->reset_and_clear_bindings(); - return true; - } else { - qCWarning(lcDb) << "Failed to connect database."; - return false; // checkConnect failed. - } -} - -bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively) -{ - QMutexLocker locker(&_mutex); - - if (checkConnect()) { - // if (!recursively) { - // always delete the actual file. - - qlonglong phash = getPHash(filename); - _deleteFileRecordPhash->reset_and_clear_bindings(); - _deleteFileRecordPhash->bindValue(1, QString::number(phash)); - - if (!_deleteFileRecordPhash->exec()) { - return false; - } - - _deleteFileRecordPhash->reset_and_clear_bindings(); - if (recursively) { - _deleteFileRecordRecursively->reset_and_clear_bindings(); - _deleteFileRecordRecursively->bindValue(1, filename); - if (!_deleteFileRecordRecursively->exec()) { - return false; - } - - _deleteFileRecordRecursively->reset_and_clear_bindings(); - } - return true; - } else { - qCWarning(lcDb) << "Failed to connect database."; - return false; // checkConnect failed. - } -} - - -SyncJournalFileRecord SyncJournalDb::getFileRecord(const QString &filename) -{ - QMutexLocker locker(&_mutex); - - qlonglong phash = getPHash(filename); - SyncJournalFileRecord rec; - - if (!filename.isEmpty() && checkConnect()) { - _getFileRecordQuery->reset_and_clear_bindings(); - _getFileRecordQuery->bindValue(1, QString::number(phash)); - - if (!_getFileRecordQuery->exec()) { - locker.unlock(); - close(); - return rec; - } - - if (_getFileRecordQuery->next()) { - rec._path = _getFileRecordQuery->stringValue(0); - rec._inode = _getFileRecordQuery->intValue(1); - //rec._uid = _getFileRecordQuery->value(2).toInt(&ok); Not Used - //rec._gid = _getFileRecordQuery->value(3).toInt(&ok); Not Used - //rec._mode = _getFileRecordQuery->intValue(4); - rec._modtime = Utility::qDateTimeFromTime_t(_getFileRecordQuery->int64Value(5)); - rec._type = _getFileRecordQuery->intValue(6); - rec._etag = _getFileRecordQuery->baValue(7); - rec._fileId = _getFileRecordQuery->baValue(8); - rec._remotePerm = _getFileRecordQuery->baValue(9); - rec._fileSize = _getFileRecordQuery->int64Value(10); - rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0); - rec._checksumHeader = _getFileRecordQuery->baValue(12); - _getFileRecordQuery->reset_and_clear_bindings(); - } else { - int errId = _getFileRecordQuery->errorId(); - if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE - QString err = _getFileRecordQuery->error(); - qCWarning(lcDb) << "No journal entry found for " << filename << "Error: " << err; - locker.unlock(); - close(); - locker.relock(); - } - } - if (_getFileRecordQuery) { - _getFileRecordQuery->reset_and_clear_bindings(); - } - } - return rec; -} - -bool SyncJournalDb::postSyncCleanup(const QSet &filepathsToKeep, - const QSet &prefixesToKeep) -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return false; - } - - SqlQuery query(_db); - query.prepare("SELECT phash, path FROM metadata order by path"); - - if (!query.exec()) { - return false; - } - - QStringList superfluousItems; - - while (query.next()) { - const QString file = query.stringValue(1); - bool keep = filepathsToKeep.contains(file); - if (!keep) { - foreach (const QString &prefix, prefixesToKeep) { - if (file.startsWith(prefix)) { - keep = true; - break; - } - } - } - if (!keep) { - superfluousItems.append(query.stringValue(0)); - } - } - - if (superfluousItems.count()) { - QString sql = "DELETE FROM metadata WHERE phash in (" + superfluousItems.join(",") + ")"; - qCInfo(lcDb) << "Sync Journal cleanup for" << superfluousItems; - SqlQuery delQuery(_db); - delQuery.prepare(sql); - if (!delQuery.exec()) { - return false; - } - } - - // Incorporate results back into main DB - walCheckpoint(); - - return true; -} - -int SyncJournalDb::getFileRecordCount() -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return -1; - } - - SqlQuery query(_db); - query.prepare("SELECT COUNT(*) FROM metadata"); - - if (!query.exec()) { - return 0; - } - - if (query.next()) { - int count = query.intValue(0); - return count; - } - - return 0; -} - -bool SyncJournalDb::updateFileRecordChecksum(const QString &filename, - const QByteArray &contentChecksum, - const QByteArray &contentChecksumType) -{ - QMutexLocker locker(&_mutex); - - qCInfo(lcDb) << "Updating file checksum" << filename << contentChecksum << contentChecksumType; - - qlonglong phash = getPHash(filename); - if (!checkConnect()) { - qCWarning(lcDb) << "Failed to connect database."; - return false; - } - - int checksumTypeId = mapChecksumType(contentChecksumType); - auto &query = _setFileRecordChecksumQuery; - - query->reset_and_clear_bindings(); - query->bindValue(1, QString::number(phash)); - query->bindValue(2, contentChecksum); - query->bindValue(3, checksumTypeId); - - if (!query->exec()) { - return false; - } - - query->reset_and_clear_bindings(); - return true; -} - -bool SyncJournalDb::updateLocalMetadata(const QString &filename, - qint64 modtime, quint64 size, quint64 inode) - -{ - QMutexLocker locker(&_mutex); - - qCInfo(lcDb) << "Updating local metadata for:" << filename << modtime << size << inode; - - qlonglong phash = getPHash(filename); - if (!checkConnect()) { - qCWarning(lcDb) << "Failed to connect database."; - return false; - } - - auto &query = _setFileRecordLocalMetadataQuery; - - query->reset_and_clear_bindings(); - query->bindValue(1, QString::number(phash)); - query->bindValue(2, inode); - query->bindValue(3, modtime); - query->bindValue(4, size); - - if (!query->exec()) { - return false; - } - - query->reset_and_clear_bindings(); - return true; -} - -bool SyncJournalDb::setFileRecordMetadata(const SyncJournalFileRecord &record) -{ - SyncJournalFileRecord existing = getFileRecord(record._path); - - // If there's no existing record, just insert the new one. - if (existing._path.isEmpty()) { - return setFileRecord(record); - } - - // Update the metadata on the existing record. - existing._inode = record._inode; - existing._modtime = record._modtime; - existing._type = record._type; - existing._etag = record._etag; - existing._fileId = record._fileId; - existing._remotePerm = record._remotePerm; - existing._fileSize = record._fileSize; - existing._serverHasIgnoredFiles = record._serverHasIgnoredFiles; - return setFileRecord(existing); -} - -static void toDownloadInfo(SqlQuery &query, SyncJournalDb::DownloadInfo *res) -{ - bool ok = true; - res->_tmpfile = query.stringValue(0); - res->_etag = query.baValue(1); - res->_errorCount = query.intValue(2); - res->_valid = ok; -} - -static bool deleteBatch(SqlQuery &query, const QStringList &entries, const QString &name) -{ - if (entries.isEmpty()) - return true; - - qCDebug(lcDb) << "Removing stale " << qPrintable(name) << " entries: " << entries.join(", "); - // FIXME: Was ported from execBatch, check if correct! - foreach (const QString &entry, entries) { - query.reset_and_clear_bindings(); - query.bindValue(1, entry); - if (!query.exec()) { - return false; - } - } - query.reset_and_clear_bindings(); // viel hilft viel ;-) - - return true; -} - -SyncJournalDb::DownloadInfo SyncJournalDb::getDownloadInfo(const QString &file) -{ - QMutexLocker locker(&_mutex); - - DownloadInfo res; - - if (checkConnect()) { - _getDownloadInfoQuery->reset_and_clear_bindings(); - _getDownloadInfoQuery->bindValue(1, file); - - if (!_getDownloadInfoQuery->exec()) { - return res; - } - - if (_getDownloadInfoQuery->next()) { - toDownloadInfo(*_getDownloadInfoQuery, &res); - } else { - res._valid = false; - } - _getDownloadInfoQuery->reset_and_clear_bindings(); - } - return res; -} - -void SyncJournalDb::setDownloadInfo(const QString &file, const SyncJournalDb::DownloadInfo &i) -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return; - } - - if (i._valid) { - _setDownloadInfoQuery->reset_and_clear_bindings(); - _setDownloadInfoQuery->bindValue(1, file); - _setDownloadInfoQuery->bindValue(2, i._tmpfile); - _setDownloadInfoQuery->bindValue(3, i._etag); - _setDownloadInfoQuery->bindValue(4, i._errorCount); - - if (!_setDownloadInfoQuery->exec()) { - return; - } - - _setDownloadInfoQuery->reset_and_clear_bindings(); - - } else { - _deleteDownloadInfoQuery->reset_and_clear_bindings(); - _deleteDownloadInfoQuery->bindValue(1, file); - - if (!_deleteDownloadInfoQuery->exec()) { - return; - } - - _deleteDownloadInfoQuery->reset_and_clear_bindings(); - } -} - -QVector SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet &keep) -{ - QVector empty_result; - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return empty_result; - } - - SqlQuery query(_db); - // The selected values *must* match the ones expected by toDownloadInfo(). - query.prepare("SELECT tmpfile, etag, errorcount, path FROM downloadinfo"); - - if (!query.exec()) { - return empty_result; - } - - QStringList superfluousPaths; - QVector deleted_entries; - - while (query.next()) { - const QString file = query.stringValue(3); // path - if (!keep.contains(file)) { - superfluousPaths.append(file); - DownloadInfo info; - toDownloadInfo(query, &info); - deleted_entries.append(info); - } - } - - if (!deleteBatch(*_deleteDownloadInfoQuery, superfluousPaths, "downloadinfo")) - return empty_result; - - return deleted_entries; -} - -int SyncJournalDb::downloadInfoCount() -{ - int re = 0; - - QMutexLocker locker(&_mutex); - if (checkConnect()) { - SqlQuery query("SELECT count(*) FROM downloadinfo", _db); - - if (!query.exec()) { - sqlFail("Count number of downloadinfo entries failed", query); - } - if (query.next()) { - re = query.intValue(0); - } - } - return re; -} - -SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString &file) -{ - QMutexLocker locker(&_mutex); - - UploadInfo res; - - if (checkConnect()) { - _getUploadInfoQuery->reset_and_clear_bindings(); - _getUploadInfoQuery->bindValue(1, file); - - if (!_getUploadInfoQuery->exec()) { - return res; - } - - if (_getUploadInfoQuery->next()) { - bool ok = true; - res._chunk = _getUploadInfoQuery->intValue(0); - res._transferid = _getUploadInfoQuery->intValue(1); - res._errorCount = _getUploadInfoQuery->intValue(2); - res._size = _getUploadInfoQuery->int64Value(3); - res._modtime = Utility::qDateTimeFromTime_t(_getUploadInfoQuery->int64Value(4)); - res._valid = ok; - } - _getUploadInfoQuery->reset_and_clear_bindings(); - } - return res; -} - -void SyncJournalDb::setUploadInfo(const QString &file, const SyncJournalDb::UploadInfo &i) -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return; - } - - if (i._valid) { - _setUploadInfoQuery->reset_and_clear_bindings(); - _setUploadInfoQuery->bindValue(1, file); - _setUploadInfoQuery->bindValue(2, i._chunk); - _setUploadInfoQuery->bindValue(3, i._transferid); - _setUploadInfoQuery->bindValue(4, i._errorCount); - _setUploadInfoQuery->bindValue(5, i._size); - _setUploadInfoQuery->bindValue(6, Utility::qDateTimeToTime_t(i._modtime)); - - if (!_setUploadInfoQuery->exec()) { - return; - } - - _setUploadInfoQuery->reset_and_clear_bindings(); - } else { - _deleteUploadInfoQuery->reset_and_clear_bindings(); - _deleteUploadInfoQuery->bindValue(1, file); - - if (!_deleteUploadInfoQuery->exec()) { - return; - } - - _deleteUploadInfoQuery->reset_and_clear_bindings(); - } -} - -QVector SyncJournalDb::deleteStaleUploadInfos(const QSet &keep) -{ - QMutexLocker locker(&_mutex); - QVector ids; - - if (!checkConnect()) { - return ids; - } - - SqlQuery query(_db); - query.prepare("SELECT path,transferid FROM uploadinfo"); - - if (!query.exec()) { - return ids; - } - - QStringList superfluousPaths; - - while (query.next()) { - const QString file = query.stringValue(0); - if (!keep.contains(file)) { - superfluousPaths.append(file); - ids.append(query.intValue(1)); - } - } - - deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo"); - return ids; -} - -SyncJournalErrorBlacklistRecord SyncJournalDb::errorBlacklistEntry(const QString &file) -{ - QMutexLocker locker(&_mutex); - SyncJournalErrorBlacklistRecord entry; - - if (file.isEmpty()) - return entry; - - // SELECT lastTryEtag, lastTryModtime, retrycount, errorstring - - if (checkConnect()) { - _getErrorBlacklistQuery->reset_and_clear_bindings(); - _getErrorBlacklistQuery->bindValue(1, file); - if (_getErrorBlacklistQuery->exec()) { - if (_getErrorBlacklistQuery->next()) { - entry._lastTryEtag = _getErrorBlacklistQuery->baValue(0); - entry._lastTryModtime = _getErrorBlacklistQuery->int64Value(1); - entry._retryCount = _getErrorBlacklistQuery->intValue(2); - entry._errorString = _getErrorBlacklistQuery->stringValue(3); - entry._lastTryTime = _getErrorBlacklistQuery->int64Value(4); - entry._ignoreDuration = _getErrorBlacklistQuery->int64Value(5); - entry._renameTarget = _getErrorBlacklistQuery->stringValue(6); - entry._errorCategory = static_cast( - _getErrorBlacklistQuery->intValue(7)); - entry._file = file; - } - _getErrorBlacklistQuery->reset_and_clear_bindings(); - } - } - - return entry; -} - -bool SyncJournalDb::deleteStaleErrorBlacklistEntries(const QSet &keep) -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return false; - } - - SqlQuery query(_db); - query.prepare("SELECT path FROM blacklist"); - - if (!query.exec()) { - return false; - } - - QStringList superfluousPaths; - - while (query.next()) { - const QString file = query.stringValue(0); - if (!keep.contains(file)) { - superfluousPaths.append(file); - } - } - - SqlQuery delQuery(_db); - delQuery.prepare("DELETE FROM blacklist WHERE path = ?"); - return deleteBatch(delQuery, superfluousPaths, "blacklist"); -} - -int SyncJournalDb::errorBlackListEntryCount() -{ - int re = 0; - - QMutexLocker locker(&_mutex); - if (checkConnect()) { - SqlQuery query("SELECT count(*) FROM blacklist", _db); - - if (!query.exec()) { - sqlFail("Count number of blacklist entries failed", query); - } - if (query.next()) { - re = query.intValue(0); - } - } - return re; -} - -int SyncJournalDb::wipeErrorBlacklist() -{ - QMutexLocker locker(&_mutex); - if (checkConnect()) { - SqlQuery query(_db); - - query.prepare("DELETE FROM blacklist"); - - if (!query.exec()) { - sqlFail("Deletion of whole blacklist failed", query); - return -1; - } - return query.numRowsAffected(); - } - return -1; -} - -void SyncJournalDb::wipeErrorBlacklistEntry(const QString &file) -{ - if (file.isEmpty()) { - return; - } - - QMutexLocker locker(&_mutex); - if (checkConnect()) { - SqlQuery query(_db); - - query.prepare("DELETE FROM blacklist WHERE path=?1"); - query.bindValue(1, file); - if (!query.exec()) { - sqlFail("Deletion of blacklist item failed.", query); - } - } -} - -void SyncJournalDb::wipeErrorBlacklistCategory(SyncJournalErrorBlacklistRecord::Category category) -{ - QMutexLocker locker(&_mutex); - if (checkConnect()) { - SqlQuery query(_db); - - query.prepare("DELETE FROM blacklist WHERE errorCategory=?1"); - query.bindValue(1, category); - if (!query.exec()) { - sqlFail("Deletion of blacklist category failed.", query); - } - } -} - -void SyncJournalDb::setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item) -{ - QMutexLocker locker(&_mutex); - - qCInfo(lcDb) << "Setting blacklist entry for " << item._file << item._retryCount - << item._errorString << item._lastTryTime << item._ignoreDuration - << item._lastTryModtime << item._lastTryEtag << item._renameTarget - << item._errorCategory; - - if (!checkConnect()) { - return; - } - - _setErrorBlacklistQuery->bindValue(1, item._file); - _setErrorBlacklistQuery->bindValue(2, item._lastTryEtag); - _setErrorBlacklistQuery->bindValue(3, QString::number(item._lastTryModtime)); - _setErrorBlacklistQuery->bindValue(4, item._retryCount); - _setErrorBlacklistQuery->bindValue(5, item._errorString); - _setErrorBlacklistQuery->bindValue(6, QString::number(item._lastTryTime)); - _setErrorBlacklistQuery->bindValue(7, QString::number(item._ignoreDuration)); - _setErrorBlacklistQuery->bindValue(8, item._renameTarget); - _setErrorBlacklistQuery->bindValue(9, item._errorCategory); - _setErrorBlacklistQuery->exec(); - _setErrorBlacklistQuery->reset_and_clear_bindings(); -} - -QVector SyncJournalDb::getPollInfos() -{ - QMutexLocker locker(&_mutex); - - QVector res; - - if (!checkConnect()) - return res; - - SqlQuery query("SELECT path, modtime, pollpath FROM poll", _db); - - if (!query.exec()) { - return res; - } - - while (query.next()) { - PollInfo info; - info._file = query.stringValue(0); - info._modtime = query.int64Value(1); - info._url = query.stringValue(2); - res.append(info); - } - - query.finish(); - return res; -} - -void SyncJournalDb::setPollInfo(const SyncJournalDb::PollInfo &info) -{ - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return; - } - - if (info._url.isEmpty()) { - qCDebug(lcDb) << "Deleting Poll job" << info._file; - SqlQuery query("DELETE FROM poll WHERE path=?", _db); - query.bindValue(1, info._file); - query.exec(); - } else { - SqlQuery query("INSERT OR REPLACE INTO poll (path, modtime, pollpath) VALUES( ? , ? , ? )", _db); - query.bindValue(1, info._file); - query.bindValue(2, QString::number(info._modtime)); - query.bindValue(3, info._url); - query.exec(); - } -} - -QStringList SyncJournalDb::getSelectiveSyncList(SyncJournalDb::SelectiveSyncListType type, bool *ok) -{ - QStringList result; - ASSERT(ok); - - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - *ok = false; - return result; - } - - _getSelectiveSyncListQuery->reset_and_clear_bindings(); - _getSelectiveSyncListQuery->bindValue(1, int(type)); - if (!_getSelectiveSyncListQuery->exec()) { - *ok = false; - return result; - } - while (_getSelectiveSyncListQuery->next()) { - auto entry = _getSelectiveSyncListQuery->stringValue(0); - if (!entry.endsWith(QLatin1Char('/'))) { - entry.append(QLatin1Char('/')); - } - result.append(entry); - } - *ok = true; - - return result; -} - -void SyncJournalDb::setSelectiveSyncList(SyncJournalDb::SelectiveSyncListType type, const QStringList &list) -{ - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return; - } - - //first, delete all entries of this type - SqlQuery delQuery("DELETE FROM selectivesync WHERE type == ?1", _db); - delQuery.bindValue(1, int(type)); - if (!delQuery.exec()) { - qCWarning(lcDb) << "SQL error when deleting selective sync list" << list << delQuery.error(); - } - - SqlQuery insQuery("INSERT INTO selectivesync VALUES (?1, ?2)", _db); - foreach (const auto &path, list) { - insQuery.reset_and_clear_bindings(); - insQuery.bindValue(1, path); - insQuery.bindValue(2, int(type)); - if (!insQuery.exec()) { - qCWarning(lcDb) << "SQL error when inserting into selective sync" << type << path << delQuery.error(); - } - } -} - -void SyncJournalDb::avoidRenamesOnNextSync(const QString &path) -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return; - } - - SqlQuery query(_db); - query.prepare("UPDATE metadata SET fileid = '', inode = '0' WHERE path == ?1 OR path LIKE(?2||'/%')"); - query.bindValue(1, path); - query.bindValue(2, path); - query.exec(); - - // We also need to remove the ETags so the update phase refreshes the directory paths - // on the next sync - locker.unlock(); - avoidReadFromDbOnNextSync(path); -} - -void SyncJournalDb::avoidReadFromDbOnNextSync(const QString &fileName) -{ - // Make sure that on the next sync, fileName is not read from the DB but uses the PROPFIND to - // get the info from the server - // We achieve that by clearing the etag of the parents directory recursively - - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return; - } - - SqlQuery query(_db); - // This query will match entries for which the path is a prefix of fileName - // Note: CSYNC_FTW_TYPE_DIR == 2 - query.prepare("UPDATE metadata SET md5='_invalid_' WHERE ?1 LIKE(path||'/%') AND type == 2;"); - query.bindValue(1, fileName); - query.exec(); - - // Prevent future overwrite of the etag for this sync - _avoidReadFromDbOnNextSyncFilter.append(fileName); -} - -void SyncJournalDb::forceRemoteDiscoveryNextSync() -{ - QMutexLocker locker(&_mutex); - - if (!checkConnect()) { - return; - } - - forceRemoteDiscoveryNextSyncLocked(); -} - -void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked() -{ - qCInfo(lcDb) << "Forcing remote re-discovery by deleting folder Etags"; - SqlQuery deleteRemoteFolderEtagsQuery(_db); - deleteRemoteFolderEtagsQuery.prepare("UPDATE metadata SET md5='_invalid_' WHERE type=2;"); - deleteRemoteFolderEtagsQuery.exec(); -} - - -QByteArray SyncJournalDb::getChecksumType(int checksumTypeId) -{ - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return QByteArray(); - } - - // Retrieve the id - auto &query = *_getChecksumTypeQuery; - query.reset_and_clear_bindings(); - query.bindValue(1, checksumTypeId); - if (!query.exec()) { - return 0; - } - - if (!query.next()) { - qCWarning(lcDb) << "No checksum type mapping found for" << checksumTypeId; - return 0; - } - return query.baValue(0); -} - -int SyncJournalDb::mapChecksumType(const QByteArray &checksumType) -{ - if (checksumType.isEmpty()) { - return 0; - } - - // Ensure the checksum type is in the db - _insertChecksumTypeQuery->reset_and_clear_bindings(); - _insertChecksumTypeQuery->bindValue(1, checksumType); - if (!_insertChecksumTypeQuery->exec()) { - return 0; - } - - // Retrieve the id - _getChecksumTypeIdQuery->reset_and_clear_bindings(); - _getChecksumTypeIdQuery->bindValue(1, checksumType); - if (!_getChecksumTypeIdQuery->exec()) { - return 0; - } - - if (!_getChecksumTypeIdQuery->next()) { - qCWarning(lcDb) << "No checksum type mapping found for" << checksumType; - return 0; - } - return _getChecksumTypeIdQuery->intValue(0); -} - -QByteArray SyncJournalDb::dataFingerprint() -{ - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return QByteArray(); - } - - _getDataFingerprintQuery->reset_and_clear_bindings(); - if (!_getDataFingerprintQuery->exec()) { - return QByteArray(); - } - - if (!_getDataFingerprintQuery->next()) { - return QByteArray(); - } - return _getDataFingerprintQuery->baValue(0); -} - -void SyncJournalDb::setDataFingerprint(const QByteArray &dataFingerprint) -{ - QMutexLocker locker(&_mutex); - if (!checkConnect()) { - return; - } - - _setDataFingerprintQuery1->reset_and_clear_bindings(); - _setDataFingerprintQuery1->exec(); - - _setDataFingerprintQuery2->reset_and_clear_bindings(); - _setDataFingerprintQuery2->bindValue(1, dataFingerprint); - _setDataFingerprintQuery2->exec(); -} - -void SyncJournalDb::clearFileTable() -{ - SqlQuery query(_db); - query.prepare("DELETE FROM metadata;"); - query.exec(); -} - -void SyncJournalDb::commit(const QString &context, bool startTrans) -{ - QMutexLocker lock(&_mutex); - commitInternal(context, startTrans); -} - -void SyncJournalDb::commitIfNeededAndStartNewTransaction(const QString &context) -{ - QMutexLocker lock(&_mutex); - if (_transaction == 1) { - commitInternal(context, true); - } else { - startTransaction(); - } -} - - -void SyncJournalDb::commitInternal(const QString &context, bool startTrans) -{ - qCDebug(lcDb) << "Transaction commit " << context << (startTrans ? "and starting new transaction" : ""); - commitTransaction(); - - if (startTrans) { - startTransaction(); - } -} - -SyncJournalDb::~SyncJournalDb() -{ - close(); -} - -bool SyncJournalDb::isConnected() -{ - QMutexLocker lock(&_mutex); - return checkConnect(); -} - -bool operator==(const SyncJournalDb::DownloadInfo &lhs, - const SyncJournalDb::DownloadInfo &rhs) -{ - return lhs._errorCount == rhs._errorCount - && lhs._etag == rhs._etag - && lhs._tmpfile == rhs._tmpfile - && lhs._valid == rhs._valid; -} - -bool operator==(const SyncJournalDb::UploadInfo &lhs, - const SyncJournalDb::UploadInfo &rhs) -{ - return lhs._errorCount == rhs._errorCount - && lhs._chunk == rhs._chunk - && lhs._modtime == rhs._modtime - && lhs._valid == rhs._valid - && lhs._size == rhs._size - && lhs._transferid == rhs._transferid; -} - -} // namespace OCC diff --git a/src/libsync/syncjournaldb.h b/src/libsync/syncjournaldb.h deleted file mode 100644 index da8054109..000000000 --- a/src/libsync/syncjournaldb.h +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#ifndef SYNCJOURNALDB_H -#define SYNCJOURNALDB_H - -#include -#include -#include -#include - -#include "common/utility.h" -#include "ownsql.h" -#include "syncjournalfilerecord.h" - -namespace OCC { -class SyncJournalFileRecord; - -/** - * @brief Class that handles the sync database - * - * This class is thread safe. All public functions lock the mutex. - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SyncJournalDb : public QObject -{ - Q_OBJECT -public: - explicit SyncJournalDb(const QString &dbFilePath, QObject *parent = 0); - virtual ~SyncJournalDb(); - - /// Create a journal path for a specific configuration - static QString makeDbName(const QString &localPath, - const QUrl &remoteUrl, - const QString &remotePath, - const QString &user); - - /// Migrate a csync_journal to the new path, if necessary. Returns false on error - static bool maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath); - - // to verify that the record could be queried successfully check - // with SyncJournalFileRecord::isValid() - SyncJournalFileRecord getFileRecord(const QString &filename); - bool setFileRecord(const SyncJournalFileRecord &record); - - /// Like setFileRecord, but preserves checksums - bool setFileRecordMetadata(const SyncJournalFileRecord &record); - - bool deleteFileRecord(const QString &filename, bool recursively = false); - int getFileRecordCount(); - bool updateFileRecordChecksum(const QString &filename, - const QByteArray &contentChecksum, - const QByteArray &contentChecksumType); - bool updateLocalMetadata(const QString &filename, - qint64 modtime, quint64 size, quint64 inode); - bool exists(); - void walCheckpoint(); - - QString databaseFilePath() const; - - static qint64 getPHash(const QString &); - - void setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item); - void wipeErrorBlacklistEntry(const QString &file); - void wipeErrorBlacklistCategory(SyncJournalErrorBlacklistRecord::Category category); - int wipeErrorBlacklist(); - int errorBlackListEntryCount(); - - struct DownloadInfo - { - DownloadInfo() - : _errorCount(0) - , _valid(false) - { - } - QString _tmpfile; - QByteArray _etag; - int _errorCount; - bool _valid; - }; - struct UploadInfo - { - UploadInfo() - : _chunk(0) - , _transferid(0) - , _size(0) - , _errorCount(0) - , _valid(false) - { - } - int _chunk; - int _transferid; - quint64 _size; //currently unused - QDateTime _modtime; - int _errorCount; - bool _valid; - }; - - struct PollInfo - { - QString _file; - QString _url; - time_t _modtime; - }; - - DownloadInfo getDownloadInfo(const QString &file); - void setDownloadInfo(const QString &file, const DownloadInfo &i); - QVector getAndDeleteStaleDownloadInfos(const QSet &keep); - int downloadInfoCount(); - - UploadInfo getUploadInfo(const QString &file); - void setUploadInfo(const QString &file, const UploadInfo &i); - // Return the list of transfer ids that were removed. - QVector deleteStaleUploadInfos(const QSet &keep); - - SyncJournalErrorBlacklistRecord errorBlacklistEntry(const QString &); - bool deleteStaleErrorBlacklistEntries(const QSet &keep); - - void avoidRenamesOnNextSync(const QString &path); - void setPollInfo(const PollInfo &); - QVector getPollInfos(); - - enum SelectiveSyncListType { - /** The black list is the list of folders that are unselected in the selective sync dialog. - * For the sync engine, those folders are considered as if they were not there, so the local - * folders will be deleted */ - SelectiveSyncBlackList = 1, - /** When a shared folder has a size bigger than a configured size, it is by default not sync'ed - * Unless it is in the white list, in which case the folder is sync'ed and all its children. - * If a folder is both on the black and the white list, the black list wins */ - SelectiveSyncWhiteList = 2, - /** List of big sync folders that have not been confirmed by the user yet and that the UI - * should notify about */ - SelectiveSyncUndecidedList = 3 - }; - /* return the specified list from the database */ - QStringList getSelectiveSyncList(SelectiveSyncListType type, bool *ok); - /* Write the selective sync list (remove all other entries of that list */ - void setSelectiveSyncList(SelectiveSyncListType type, const QStringList &list); - - /** - * Make sure that on the next sync, fileName is not read from the DB but uses the PROPFIND to - * get the info from the server - * - * Specifically, this sets the md5 field of fileName and all its parents to _invalid_. - * That causes a metadata difference and a resulting discovery from the remote for the - * affected folders. - * - * Since folders in the selective sync list will not be rediscovered (csync_ftw, - * _csync_detect_update skip them), the _invalid_ marker will stay and it. And any - * child items in the db will be ignored when reading a remote tree from the database. - */ - void avoidReadFromDbOnNextSync(const QString &fileName); - - /** - * Ensures full remote discovery happens on the next sync. - * - * Equivalent to calling avoidReadFromDbOnNextSync() for all files. - */ - void forceRemoteDiscoveryNextSync(); - - bool postSyncCleanup(const QSet &filepathsToKeep, - const QSet &prefixesToKeep); - - /* Because sqlite transactions are really slow, we encapsulate everything in big transactions - * Commit will actually commit the transaction and create a new one. - */ - void commit(const QString &context, bool startTrans = true); - void commitIfNeededAndStartNewTransaction(const QString &context); - - void close(); - - /** - * return true if everything is correct - */ - bool isConnected(); - - /** - * Returns the checksum type for an id. - */ - QByteArray getChecksumType(int checksumTypeId); - - /** - * The data-fingerprint used to detect backup - */ - void setDataFingerprint(const QByteArray &dataFingerprint); - QByteArray dataFingerprint(); - - /** - * Delete any file entry. This will force the next sync to re-sync everything as if it was new, - * restoring everyfile on every remote. If a file is there both on the client and server side, - * it will be a conflict that will be automatically resolved if the file is the same. - */ - void clearFileTable(); - -private: - bool updateDatabaseStructure(); - bool updateMetadataTableStructure(); - bool updateErrorBlacklistTableStructure(); - bool sqlFail(const QString &log, const SqlQuery &query); - void commitInternal(const QString &context, bool startTrans = true); - void startTransaction(); - void commitTransaction(); - QStringList tableColumns(const QString &table); - bool checkConnect(); - - // Same as forceRemoteDiscoveryNextSync but without acquiring the lock - void forceRemoteDiscoveryNextSyncLocked(); - - // Returns the integer id of the checksum type - // - // Returns 0 on failure and for empty checksum types. - int mapChecksumType(const QByteArray &checksumType); - - SqlDatabase _db; - QString _dbFile; - QMutex _mutex; // Public functions are protected with the mutex. - int _transaction; - - // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close - QScopedPointer _getFileRecordQuery; - QScopedPointer _setFileRecordQuery; - QScopedPointer _setFileRecordChecksumQuery; - QScopedPointer _setFileRecordLocalMetadataQuery; - QScopedPointer _getDownloadInfoQuery; - QScopedPointer _setDownloadInfoQuery; - QScopedPointer _deleteDownloadInfoQuery; - QScopedPointer _getUploadInfoQuery; - QScopedPointer _setUploadInfoQuery; - QScopedPointer _deleteUploadInfoQuery; - QScopedPointer _deleteFileRecordPhash; - QScopedPointer _deleteFileRecordRecursively; - QScopedPointer _getErrorBlacklistQuery; - QScopedPointer _setErrorBlacklistQuery; - QScopedPointer _getSelectiveSyncListQuery; - QScopedPointer _getChecksumTypeIdQuery; - QScopedPointer _getChecksumTypeQuery; - QScopedPointer _insertChecksumTypeQuery; - QScopedPointer _getDataFingerprintQuery; - QScopedPointer _setDataFingerprintQuery1; - QScopedPointer _setDataFingerprintQuery2; - - /* This is the list of paths we called avoidReadFromDbOnNextSync on. - * It means that they should not be written to the DB in any case since doing - * that would write the etag and would void the purpose of avoidReadFromDbOnNextSync - */ - QList _avoidReadFromDbOnNextSyncFilter; - - /** The journal mode to use for the db. - * - * Typically WAL initially, but may be set to other modes via environment - * variable, for specific filesystems, or when WAL fails in a particular way. - */ - QString _journalMode; -}; - -bool OWNCLOUDSYNC_EXPORT -operator==(const SyncJournalDb::DownloadInfo &lhs, - const SyncJournalDb::DownloadInfo &rhs); -bool OWNCLOUDSYNC_EXPORT -operator==(const SyncJournalDb::UploadInfo &lhs, - const SyncJournalDb::UploadInfo &rhs); - -} // namespace OCC -#endif // SYNCJOURNALDB_H diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp deleted file mode 100644 index 5936f0f50..000000000 --- a/src/libsync/syncjournalfilerecord.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#include "syncjournalfilerecord.h" -#include "common/utility.h" - -namespace OCC { - -SyncJournalFileRecord::SyncJournalFileRecord() - : _inode(0) - , _type(0) - , _fileSize(0) - , _serverHasIgnoredFiles(false) -{ -} - -QByteArray SyncJournalFileRecord::numericFileId() const -{ - // Use the id up until the first non-numeric character - for (int i = 0; i < _fileId.size(); ++i) { - if (_fileId[i] < '0' || _fileId[i] > '9') { - return _fileId.left(i); - } - } - return _fileId; -} - -bool SyncJournalErrorBlacklistRecord::isValid() const -{ - return !_file.isEmpty() - && (!_lastTryEtag.isEmpty() || _lastTryModtime != 0) - && _lastTryTime > 0; -} - -bool operator==(const SyncJournalFileRecord &lhs, - const SyncJournalFileRecord &rhs) -{ - return lhs._path == rhs._path - && lhs._inode == rhs._inode - && lhs._modtime.toTime_t() == rhs._modtime.toTime_t() - && lhs._type == rhs._type - && lhs._etag == rhs._etag - && lhs._fileId == rhs._fileId - && lhs._fileSize == rhs._fileSize - && lhs._remotePerm == rhs._remotePerm - && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles - && lhs._checksumHeader == rhs._checksumHeader; -} -} diff --git a/src/libsync/syncjournalfilerecord.h b/src/libsync/syncjournalfilerecord.h deleted file mode 100644 index 86d214908..000000000 --- a/src/libsync/syncjournalfilerecord.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) by Klaas Freitag - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - */ - -#ifndef SYNCJOURNALFILERECORD_H -#define SYNCJOURNALFILERECORD_H - -#include -#include - -#include "owncloudlib.h" - -namespace OCC { - -class SyncFileItem; - -/** - * @brief The SyncJournalFileRecord class - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SyncJournalFileRecord -{ -public: - SyncJournalFileRecord(); - - bool isValid() - { - return !_path.isEmpty(); - } - - /** Returns the numeric part of the full id in _fileId. - * - * On the server this is sometimes known as the internal file id. - * - * It is used in the construction of private links. - */ - QByteArray numericFileId() const; - - QString _path; - quint64 _inode; - QDateTime _modtime; - int _type; - QByteArray _etag; - QByteArray _fileId; - qint64 _fileSize; - QByteArray _remotePerm; - bool _serverHasIgnoredFiles; - QByteArray _checksumHeader; -}; - -bool OWNCLOUDSYNC_EXPORT -operator==(const SyncJournalFileRecord &lhs, - const SyncJournalFileRecord &rhs); - -class SyncJournalErrorBlacklistRecord -{ -public: - enum Category { - /// Normal errors have no special behavior - Normal = 0, - /// These get a special summary message - InsufficientRemoteStorage - }; - - SyncJournalErrorBlacklistRecord() - : _retryCount(0) - , _errorCategory(Category::Normal) - , _lastTryModtime(0) - , _lastTryTime(0) - , _ignoreDuration(0) - { - } - - /// The number of times the operation was unsuccessful so far. - int _retryCount; - - /// The last error string. - QString _errorString; - /// The error category. Sometimes used for special actions. - Category _errorCategory; - - time_t _lastTryModtime; - QByteArray _lastTryEtag; - - /// The last time the operation was attempted (in s since epoch). - time_t _lastTryTime; - - /// The number of seconds the file shall be ignored. - time_t _ignoreDuration; - - QString _file; - QString _renameTarget; - - bool isValid() const; -}; -} - -#endif // SYNCJOURNALFILERECORD_H diff --git a/test/csync/std_tests/check_std_c_jhash.c b/test/csync/std_tests/check_std_c_jhash.c index 29dc32207..de7d8e7d5 100644 --- a/test/csync/std_tests/check_std_c_jhash.c +++ b/test/csync/std_tests/check_std_c_jhash.c @@ -6,7 +6,7 @@ */ #include "torture.h" -#include "std/c_jhash.h" +#include "common/c_jhash.h" #define HASHSTATE 1 #define HASHLEN 1 diff --git a/test/syncenginetestutils.h b/test/syncenginetestutils.h index 3fb03324f..cabac73bf 100644 --- a/test/syncenginetestutils.h +++ b/test/syncenginetestutils.h @@ -11,7 +11,7 @@ #include "logger.h" #include "filesystem.h" #include "syncengine.h" -#include "syncjournaldb.h" +#include "common/syncjournaldb.h" #include #include diff --git a/test/testchecksumvalidator.cpp b/test/testchecksumvalidator.cpp index d2d10096d..041dc193c 100644 --- a/test/testchecksumvalidator.cpp +++ b/test/testchecksumvalidator.cpp @@ -9,7 +9,7 @@ #include #include -#include "checksums.h" +#include "common/checksums.h" #include "networkjobs.h" #include "common/utility.h" #include "filesystem.h" diff --git a/test/testownsql.cpp b/test/testownsql.cpp index 0172468e8..77541794a 100644 --- a/test/testownsql.cpp +++ b/test/testownsql.cpp @@ -8,7 +8,7 @@ #include -#include "ownsql.h" +#include "common/ownsql.h" using namespace OCC; diff --git a/test/testsyncjournaldb.cpp b/test/testsyncjournaldb.cpp index e01a32f13..e3ea07cd9 100644 --- a/test/testsyncjournaldb.cpp +++ b/test/testsyncjournaldb.cpp @@ -8,8 +8,8 @@ #include -#include "syncjournaldb.h" -#include "syncjournalfilerecord.h" +#include "common/syncjournaldb.h" +#include "common/syncjournalfilerecord.h" using namespace OCC; diff --git a/test/testuploadreset.cpp b/test/testuploadreset.cpp index bf077ddee..9c92d9a1d 100644 --- a/test/testuploadreset.cpp +++ b/test/testuploadreset.cpp @@ -8,7 +8,7 @@ #include #include "syncenginetestutils.h" #include -#include +#include using namespace OCC;