trav.error_status = cur->error_status;
trav.has_ignored_files = cur->has_ignored_files;
- trav.checksum = cur->checksum;
- trav.checksumTypeId = cur->checksumTypeId;
+ trav.checksumHeader = cur->checksumHeader;
if( other_node ) {
csync_file_stat_t *other_stat = (csync_file_stat_t*)other_node->data;
SAFE_FREE(st->directDownloadCookies);
SAFE_FREE(st->etag);
SAFE_FREE(st->destpath);
- SAFE_FREE(st->checksum);
+ SAFE_FREE(st->checksumHeader);
SAFE_FREE(st);
}
}
enum csync_vio_file_flags_e flags;
char *original_name; // only set if locale conversion fails
+
+ // For remote file stats: the highest quality checksum the server provided
+ // in the "SHA1:324315da2143" form.
+ char *checksumHeader;
};
csync_vio_file_stat_t OCSYNC_EXPORT *csync_vio_file_stat_new(void);
char *directDownloadUrl;
char *directDownloadCookies;
- const char *checksum;
- uint32_t checksumTypeId;
+ const char *checksumHeader;
struct {
int64_t size;
void *userdata);
/* Compute the checksum of the given \a checksumTypeId for \a path. */
-typedef const char* (*csync_checksum_hook) (
- const char *path, uint32_t checksumTypeId, void *userdata);
+typedef const char *(*csync_checksum_hook)(
+ const char *path, const char *otherChecksumHeader, void *userdata);
/**
* @brief Allocate a csync context.
char *directDownloadCookies;
char remotePerm[REMOTE_PERM_BUF_SIZE+1];
- const char *checksum;
- uint32_t checksumTypeId;
+ // In the local tree, this can hold a checksum and its type if it is
+ // computed during discovery for some reason.
+ // In the remote tree, this will have the server checksum, if available.
+ // In both cases, the format is "SHA1:baff".
+ const char *checksumHeader;
CSYNC_STATUS error_status;
}
}
+/* Returns true if we're reasonably certain that hash equality
+ * for the header means content equality.
+ *
+ * Cryptographic safety is not required - this is mainly
+ * intended to rule out checksums like Adler32 that are not intended for
+ * hashing and have high likelihood of collision with particular inputs.
+ */
+static bool _csync_is_collision_safe_hash(const char *checksum_header)
+{
+ return strncmp(checksum_header, "SHA1:", 5) == 0
+ || strncmp(checksum_header, "MD5:", 4) == 0;
+}
+
/**
* The main function in the reconcile pass.
*
// Folders of the same path are always considered equals
is_conflict = false;
} else {
+ // If the size or mtime is different, it's definitely a conflict.
is_conflict = ((other->size != cur->size) || (other->modtime != cur->modtime));
- // FIXME: do a binary comparision of the file here because of the following
- // edge case:
- // The files could still have different content, even though the mtime
- // and size are the same.
+
+ // It could be a conflict even if size and mtime match!
+ //
+ // In older client versions we always treated these cases as a
+ // non-conflict. This behavior is preserved in case the server
+ // doesn't provide a suitable content hash.
+ //
+ // When it does have one, however, we do create a job, but the job
+ // will compare hashes and avoid the download if they are equal.
+ const char *remoteChecksumHeader =
+ (ctx->current == REMOTE_REPLICA ? cur->checksumHeader : other->checksumHeader);
+ if (remoteChecksumHeader) {
+ is_conflict |= _csync_is_collision_safe_hash(remoteChecksumHeader);
+ }
+
+ // SO: If there is no checksum, we can have !is_conflict here
+ // even though the files have different content! This is an
+ // intentional tradeoff. Downloading and comparing files would
+ // be technically correct in this situation but leads to too
+ // much waste.
+ // In particular this kind of NEW/NEW situation with identical
+ // sizes and mtimes pops up when the local database is lost for
+ // whatever reason.
}
if (ctx->current == REMOTE_REPLICA) {
// If the files are considered equal, only update the DB with the etag from remote
return rc;
}
-#define METADATA_COLUMNS "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId"
+#define METADATA_QUERY \
+ "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, " \
+ "filesize, ignoredChildrenRemote, " \
+ "contentchecksumtype.name || ':' || contentChecksum " \
+ "FROM metadata " \
+ "LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
// This funciton parses a line from the metadata table into the given csync_file_stat
// structure which it is also allocating.
if(column_count > 13) {
(*st)->has_ignored_files = sqlite3_column_int(stmt, 13);
}
- if(column_count > 15 && sqlite3_column_int(stmt, 15)) {
- (*st)->checksum = c_strdup( (char*) sqlite3_column_text(stmt, 14));
- (*st)->checksumTypeId = sqlite3_column_int(stmt, 15);
+ if (column_count > 14 && sqlite3_column_text(stmt, 14)) {
+ (*st)->checksumHeader = c_strdup((char *)sqlite3_column_text(stmt, 14));
}
}
}
if( ctx->statedb.by_hash_stmt == NULL ) {
- const char *hash_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE phash=?1";
+ const char *hash_query = "SELECT " METADATA_QUERY " WHERE phash=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
}
if( ctx->statedb.by_fileid_stmt == NULL ) {
- const char *query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE fileid=?1";
+ const char *query = "SELECT " METADATA_QUERY " WHERE fileid=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
}
if( ctx->statedb.by_inode_stmt == NULL ) {
- const char *inode_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE inode=?1";
+ const char *inode_query = "SELECT " METADATA_QUERY " WHERE inode=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
* In other words, anything that is between path+'/' and path+'0',
* (because '0' follows '/' in ascii)
*/
- const char *below_path_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE path > (?||'/') AND path < (?||'0') ORDER BY path||'/' ASC";
+ const char *below_path_query = "SELECT " METADATA_QUERY " WHERE path > (?||'/') AND path < (?||'0') ORDER BY path||'/' ASC";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL));
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_OK ) {
// Checksum comparison at this stage is only enabled for .eml files,
// check #4754 #4755
bool isEmlFile = csync_fnmatch("*.eml", file, FNM_CASEFOLD) == 0;
- if (isEmlFile && fs->size == tmp->size && tmp->checksumTypeId) {
+ if (isEmlFile && fs->size == tmp->size && tmp->checksumHeader) {
if (ctx->callbacks.checksum_hook) {
- st->checksum = ctx->callbacks.checksum_hook(
- file, tmp->checksumTypeId,
- ctx->callbacks.checksum_userdata);
+ st->checksumHeader = ctx->callbacks.checksum_hook(
+ file, tmp->checksumHeader,
+ ctx->callbacks.checksum_userdata);
}
bool checksumIdentical = false;
- if (st->checksum) {
- st->checksumTypeId = tmp->checksumTypeId;
- checksumIdentical = strncmp(st->checksum, tmp->checksum, 1000) == 0;
+ if (st->checksumHeader) {
+ checksumIdentical = strncmp(st->checksumHeader, tmp->checksumHeader, 1000) == 0;
}
if (checksumIdentical) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "NOTE: Checksums are identical, file did not actually change: %s", path);
// Verify the checksum where possible
- if (isRename && tmp->checksumTypeId && ctx->callbacks.checksum_hook
- && fs->type == CSYNC_VIO_FILE_TYPE_REGULAR) {
- st->checksum = ctx->callbacks.checksum_hook(
- file, tmp->checksumTypeId,
- ctx->callbacks.checksum_userdata);
- if (st->checksum) {
- CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "checking checksum of potential rename %s %s <-> %s", path, st->checksum, tmp->checksum);
- st->checksumTypeId = tmp->checksumTypeId;
- isRename = strncmp(st->checksum, tmp->checksum, 1000) == 0;
+ if (isRename && tmp->checksumHeader && ctx->callbacks.checksum_hook
+ && fs->type == CSYNC_VIO_FILE_TYPE_REGULAR) {
+ st->checksumHeader = ctx->callbacks.checksum_hook(
+ file, tmp->checksumHeader,
+ ctx->callbacks.checksum_userdata);
+ if (st->checksumHeader) {
+ CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "checking checksum of potential rename %s %s <-> %s", path, st->checksumHeader, tmp->checksumHeader);
+ isRename = strncmp(st->checksumHeader, tmp->checksumHeader, 1000) == 0;
}
}
strncpy(st->remotePerm, fs->remotePerm, REMOTE_PERM_BUF_SIZE);
}
+ // For the remote: propagate the discovered checksum
+ if (fs->checksumHeader && ctx->current == REMOTE_REPLICA) {
+ st->checksumHeader = c_strdup(fs->checksumHeader);
+ }
+
st->phash = h;
st->pathlen = len;
memcpy(st->path, (len ? path : ""), len + 1);
if (file_stat_cpy->directDownloadUrl) {
file_stat_cpy->directDownloadUrl = c_strdup(file_stat_cpy->directDownloadUrl);
}
+ if (file_stat_cpy->checksumHeader) {
+ file_stat_cpy->checksumHeader = c_strdup(file_stat_cpy->checksumHeader);
+ }
file_stat_cpy->name = c_strdup(file_stat_cpy->name);
return file_stat_cpy;
}
SAFE_FREE(file_stat->directDownloadCookies);
SAFE_FREE(file_stat->name);
SAFE_FREE(file_stat->original_name);
+ SAFE_FREE(file_stat->checksumHeader);
SAFE_FREE(file_stat);
}
QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum)
{
+ if (checksumType.isEmpty() || checksum.isEmpty())
+ return QByteArray();
QByteArray header = checksumType;
header.append(':');
header.append(checksum);
return true;
}
+
+QByteArray parseChecksumHeaderType(const QByteArray &header)
+{
+ const auto idx = header.indexOf(':');
+ if (idx < 0) {
+ return QByteArray();
+ }
+ return header.left(idx);
+}
+
bool uploadChecksumEnabled()
{
static bool enabled = qgetenv("OWNCLOUD_DISABLE_CHECKSUM_UPLOAD").isEmpty();
emit validated(checksumType, checksum);
}
-CSyncChecksumHook::CSyncChecksumHook(SyncJournalDb *journal)
- : _journal(journal)
+CSyncChecksumHook::CSyncChecksumHook()
{
}
-const char *CSyncChecksumHook::hook(const char *path, uint32_t checksumTypeId, void *this_obj)
+const char *CSyncChecksumHook::hook(const char *path, const char *otherChecksumHeader, void * /*this_obj*/)
{
- CSyncChecksumHook *checksumHook = static_cast<CSyncChecksumHook *>(this_obj);
- QByteArray checksum = checksumHook->compute(QString::fromUtf8(path), checksumTypeId);
+ QByteArray type = parseChecksumHeaderType(QByteArray(otherChecksumHeader));
+ if (type.isEmpty())
+ return NULL;
+
+ QByteArray checksum = ComputeChecksum::computeNow(path, type);
if (checksum.isNull()) {
+ qCWarning(lcChecksums) << "Failed to compute checksum" << type << "for" << path;
return NULL;
}
- char *result = (char *)malloc(checksum.size() + 1);
- memcpy(result, checksum.constData(), checksum.size());
- result[checksum.size()] = 0;
+ QByteArray checksumHeader = makeChecksumHeader(type, checksum);
+ char *result = (char *)malloc(checksumHeader.size() + 1);
+ memcpy(result, checksumHeader.constData(), checksumHeader.size());
+ result[checksumHeader.size()] = 0;
return result;
}
-QByteArray CSyncChecksumHook::compute(const QString &path, int checksumTypeId)
-{
- QByteArray checksumType = _journal->getChecksumType(checksumTypeId);
- if (checksumType.isEmpty()) {
- qCWarning(lcChecksums) << "Checksum type" << checksumTypeId << "not found";
- return QByteArray();
- }
-
- QByteArray checksum = ComputeChecksum::computeNow(path, checksumType);
- if (checksum.isNull()) {
- qCWarning(lcChecksums) << "Failed to compute checksum" << checksumType << "for" << path;
- return QByteArray();
- }
-
- return checksum;
-}
}
/// Parses a checksum header
bool parseChecksumHeader(const QByteArray &header, QByteArray *type, QByteArray *checksum);
+/// Convenience for getting the type from a checksum header, null if none
+QByteArray parseChecksumHeaderType(const QByteArray &header);
+
/// Checks OWNCLOUD_DISABLE_CHECKSUM_UPLOAD
bool uploadChecksumEnabled();
{
Q_OBJECT
public:
- explicit CSyncChecksumHook(SyncJournalDb *journal);
+ explicit CSyncChecksumHook();
/**
- * Returns the checksum value for \a path for the given \a checksumTypeId.
+ * Returns the checksum value for \a path that is comparable to \a otherChecksum.
*
* Called from csync, where a instance of CSyncChecksumHook has
* to be set as userdata.
* The return value will be owned by csync.
*/
- static const char *hook(const char *path, uint32_t checksumTypeId, void *this_obj);
-
- QByteArray compute(const QString &path, int checksumTypeId);
-
-private:
- SyncJournalDb *_journal;
+ static const char *hook(const char *path, const char *otherChecksumHeader, void *this_obj);
};
}
<< "http://owncloud.org/ns:id"
<< "http://owncloud.org/ns:downloadURL"
<< "http://owncloud.org/ns:dDC"
- << "http://owncloud.org/ns:permissions";
+ << "http://owncloud.org/ns:permissions"
+ << "http://owncloud.org/ns:checksums";
if (_isRootPath)
props << "http://owncloud.org/ns:data-fingerprint";
}
}
+/**
+ * Returns the highest-quality checksum in a 'checksums'
+ * property retrieved from the server.
+ *
+ * Example: "ADLER32:1231 SHA1:ab124124 MD5:2131affa21"
+ * -> "SHA1:ab124124"
+ */
+static QByteArray findBestChecksum(const QByteArray &checksums)
+{
+ int i = 0;
+ // The order of the searches here defines the preference ordering.
+ if (-1 != (i = checksums.indexOf("SHA1:"))
+ || -1 != (i = checksums.indexOf("MD5:"))
+ || -1 != (i = checksums.indexOf("Adler32:"))) {
+ // Now i is the start of the best checksum
+ // Grab it until the next space or end of string.
+ auto checksum = checksums.mid(i);
+ return checksum.mid(0, checksum.indexOf(" "));
+ }
+ return QByteArray();
+}
+
static csync_vio_file_stat_t *propertyMapToFileStat(const QMap<QString, QString> &map)
{
csync_vio_file_stat_t *file_stat = csync_vio_file_stat_new();
} else {
qCWarning(lcDiscovery) << "permissions too large" << v;
}
+ } else if (property == "checksums") {
+ QByteArray checksum = findBestChecksum(value.toUtf8());
+ if (!checksum.isEmpty()) {
+ file_stat->checksumHeader = strdup(checksum.constData());
+ }
}
}
}
}
+ // If we have a conflict where size and mtime are identical,
+ // compare the remote checksum to the local one.
+ // Maybe it's not a real conflict and no download is necessary!
+ if (_item->_instruction == CSYNC_INSTRUCTION_CONFLICT
+ && _item->_size == _item->log._other_size
+ && _item->_modtime == _item->log._other_modtime
+ && !_item->_checksumHeader.isEmpty()) {
+ qCDebug(lcPropagateDownload) << _item->_file << "may not need download, computing checksum";
+ auto computeChecksum = new ComputeChecksum(this);
+ computeChecksum->setChecksumType(parseChecksumHeaderType(_item->_checksumHeader));
+ connect(computeChecksum, SIGNAL(done(QByteArray, QByteArray)),
+ SLOT(conflictChecksumComputed(QByteArray, QByteArray)));
+ computeChecksum->start(propagator()->getFilePath(_item->_file));
+ return;
+ }
+
+ startDownload();
+}
+
+void PropagateDownloadFile::conflictChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum)
+{
+ if (makeChecksumHeader(checksumType, checksum) == _item->_checksumHeader) {
+ qCDebug(lcPropagateDownload) << _item->_file << "remote and local checksum match";
+ // No download necessary, just update metadata
+ updateMetadata(/*isConflict=*/false);
+ return;
+ }
+ startDownload();
+}
+
+void PropagateDownloadFile::startDownload()
+{
+ if (propagator()->_abortRequested.fetchAndAddRelaxed(0))
+ return;
+
// do a klaas' case clash check.
if (propagator()->localFileNameClash(_item->_file)) {
done(SyncFileItem::NormalError, tr("File %1 can not be downloaded because of a local file name clash!").arg(QDir::toNativeSeparators(_item->_file)));
continue;
}
- qCInfo(lcPropagateDownload) << "Recalling" << localRecalledFile << "Checksum:" << record._contentChecksumType << record._contentChecksum;
+ qCInfo(lcPropagateDownload) << "Recalling" << localRecalledFile << "Checksum:" << record._checksumHeader;
QString targetPath = makeRecallFileName(recalledFile);
void PropagateDownloadFile::contentChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum)
{
- _item->_contentChecksum = checksum;
- _item->_contentChecksumType = checksumType;
+ _item->_checksumHeader = makeChecksumHeader(checksumType, checksum);
downloadFinished();
}
// Get up to date information for the journal.
_item->_size = FileSystem::getSize(fn);
+ updateMetadata(isConflict);
+}
+
+void PropagateDownloadFile::updateMetadata(bool isConflict)
+{
+ QString fn = propagator()->getFilePath(_item->_file);
+
if (!propagator()->_journal->setFileRecord(SyncJournalFileRecord(*_item, fn))) {
done(SyncFileItem::FatalError, tr("Error writing metadata to the database"));
return;
/**
* @brief The PropagateDownloadFile class
* @ingroup libsync
+ *
+ * This is the flow:
+
+\code{.unparsed}
+ start()
+ |
+ | deleteExistingFolder() if enabled
+ |
+ +--> mtime and size identical?
+ | then compute the local checksum
+ | done?-> conflictChecksumComputed()
+ | |
+ | checksum differs? |
+ +-> startDownload() <--------------------------+
+ | |
+ +-> run a GETFileJob | checksum identical?
+ |
+ done?-> slotGetFinished() |
+ | |
+ +-> validate checksum header |
+ |
+ done?-> transmissionChecksumValidated() |
+ | |
+ +-> compute the content checksum |
+ |
+ done?-> contentChecksumComputed() |
+ | |
+ +-> downloadFinished() |
+ | |
+ +------------------+ |
+ | |
+ +-> updateMetadata() <-------------------------+
+
+\endcode
*/
class PropagateDownloadFile : public PropagateItemJob
{
void setDeleteExistingFolder(bool enabled);
private slots:
+ /// Called when ComputeChecksum on the local file finishes,
+ /// maybe the local and remote checksums are identical?
+ void conflictChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum);
+ /// Called to start downloading the remote file
+ void startDownload();
+ /// Called when the GETFileJob finishes
void slotGetFinished();
- void abort() Q_DECL_OVERRIDE;
+ /// Called when the download's checksum header was validated
void transmissionChecksumValidated(const QByteArray &checksumType, const QByteArray &checksum);
+ /// Called when the download's checksum computation is done
void contentChecksumComputed(const QByteArray &checksumType, const QByteArray &checksum);
void downloadFinished();
+ /// Called when it's time to update the db metadata
+ void updateMetadata(bool isConflict);
+
+ void abort() Q_DECL_OVERRIDE;
void slotDownloadProgress(qint64, qint64);
void slotChecksumFail(const QString &errMsg);
SyncJournalFileRecord record(*_item, propagator()->getFilePath(_item->_renameTarget));
record._path = _item->_renameTarget;
if (oldRecord.isValid()) {
- record._contentChecksum = oldRecord._contentChecksum;
- record._contentChecksumType = oldRecord._contentChecksumType;
+ record._checksumHeader = oldRecord._checksumHeader;
if (record._fileSize != oldRecord._fileSize) {
qCWarning(lcPropagateRemoteMove) << "File sizes differ on server vs sync journal: " << record._fileSize << oldRecord._fileSize;
QByteArray checksumType = contentChecksumType();
// Maybe the discovery already computed the checksum?
- if (_item->_contentChecksumType == checksumType
- && !_item->_contentChecksum.isEmpty()) {
- slotComputeTransmissionChecksum(checksumType, _item->_contentChecksum);
+ QByteArray existingChecksumType, existingChecksum;
+ parseChecksumHeader(_item->_checksumHeader, &existingChecksumType, &existingChecksum);
+ if (existingChecksumType == checksumType) {
+ slotComputeTransmissionChecksum(checksumType, existingChecksum);
return;
}
void PropagateUploadFileCommon::slotComputeTransmissionChecksum(const QByteArray &contentChecksumType, const QByteArray &contentChecksum)
{
- _item->_contentChecksum = contentChecksum;
- _item->_contentChecksumType = contentChecksumType;
+ _item->_checksumHeader = makeChecksumHeader(contentChecksumType, contentChecksum);
#ifdef WITH_TESTING
_stopWatch.addLapTime(QLatin1String("ContentChecksum"));
// When we start chunks, we will add it again, once for every chunks.
propagator()->_activeJobList.removeOne(this);
- _transmissionChecksum = transmissionChecksum;
- _transmissionChecksumType = transmissionChecksumType;
+ _transmissionChecksumHeader = makeChecksumHeader(transmissionChecksumType, transmissionChecksum);
- if (_item->_contentChecksum.isEmpty() && _item->_contentChecksumType.isEmpty()) {
- // If the _contentChecksum was not set, reuse the transmission checksum as the content checksum.
- _item->_contentChecksum = transmissionChecksum;
- _item->_contentChecksumType = transmissionChecksumType;
+ // If no checksum header was not set, reuse the transmission checksum as the content checksum.
+ if (_item->_checksumHeader.isEmpty()) {
+ _item->_checksumHeader = _transmissionChecksumHeader;
}
const QString fullFilePath = propagator()->getFilePath(_item->_file);
Utility::StopWatch _stopWatch;
#endif
- QByteArray _transmissionChecksum;
- QByteArray _transmissionChecksumType;
+ QByteArray _transmissionChecksumHeader;
public:
PropagateUploadFileCommon(OwncloudPropagator *propagator, const SyncFileItemPtr &item)
if (!ifMatch.isEmpty()) {
headers["If"] = "<" + destination.toUtf8() + "> ([" + ifMatch + "])";
}
- if (!_transmissionChecksumType.isEmpty()) {
- headers[checkSumHeaderC] = makeChecksumHeader(
- _transmissionChecksumType, _transmissionChecksum);
+ if (!_transmissionChecksumHeader.isEmpty()) {
+ headers[checkSumHeaderC] = _transmissionChecksumHeader;
}
headers["OC-Total-Length"] = QByteArray::number(fileSize);
}
qCDebug(lcPropagateUpload) << _chunkCount << isFinalChunk << chunkStart << currentChunkSize;
- if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
- headers[checkSumHeaderC] = makeChecksumHeader(
- _transmissionChecksumType, _transmissionChecksum);
+ if (isFinalChunk && !_transmissionChecksumHeader.isEmpty()) {
+ headers[checkSumHeaderC] = _transmissionChecksumHeader;
}
const QString fileName = propagator()->getFilePath(_item->_file);
SyncJournalFileRecord record(*_item, targetFile);
record._path = _item->_renameTarget;
if (oldRecord.isValid()) {
- record._contentChecksum = oldRecord._contentChecksum;
- record._contentChecksumType = oldRecord._contentChecksumType;
+ record._checksumHeader = oldRecord._checksumHeader;
}
if (!_item->_isDirectory) { // Directories are saved at the end
, _backInTimeFiles(0)
, _uploadLimit(0)
, _downloadLimit(0)
- , _checksum_hook(journal)
, _anotherSyncNeeded(NoFollowUpSync)
{
qRegisterMetaType<SyncFileItem>("SyncFileItem");
}
// Sometimes the discovery computes checksums for local files
- if (!remote && file->checksum && file->checksumTypeId) {
- item->_contentChecksum = QByteArray(file->checksum);
- item->_contentChecksumType = _journal->getChecksumType(file->checksumTypeId);
+ if (!remote && file->checksumHeader) {
+ item->_checksumHeader = QByteArray(file->checksumHeader);
+ }
+ // For conflicts, store the remote checksum there
+ if (remote && item->_instruction == CSYNC_INSTRUCTION_CONFLICT && file->checksumHeader) {
+ item->_checksumHeader = QByteArray(file->checksumHeader);
}
// record the seen files to be able to clean the journal later
quint64 _inode;
QByteArray _fileId;
QByteArray _remotePerm;
- QByteArray _contentChecksum;
- QByteArray _contentChecksumType;
+
+ // When is this set, and is it the local or the remote checksum?
+ // - if mtime or size changed locally for *.eml files (local checksum)
+ // - for potential renames of local files (local checksum)
+ // - for conflicts (remote checksum) (what about eval_rename/new reconcile?)
+ QByteArray _checksumHeader;
+
QString _directDownloadUrl;
QString _directDownloadCookies;
#include "version.h"
#include "filesystem.h"
#include "asserts.h"
+#include "checksums.h"
#include "../../csync/src/std/c_jhash.h"
_getFileRecordQuery.reset(new SqlQuery(_db));
if (_getFileRecordQuery->prepare(
"SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize,"
- " ignoredChildrenRemote, contentChecksum, contentchecksumtype.name"
+ " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum"
" FROM metadata"
" LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
" WHERE phash=?1")) {
qCInfo(lcDb) << "Updating file record for path:" << record._path << "inode:" << record._inode
<< "modtime:" << record._modtime << "type:" << record._type
<< "etag:" << record._etag << "fileId:" << record._fileId << "remotePerm:" << record._remotePerm
- << "fileSize:" << record._fileSize << "checksum:" << record._contentChecksum << record._contentChecksumType;
+ << "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader;
qlonglong phash = getPHash(record._path);
if (checkConnect()) {
QString remotePerm(record._remotePerm);
if (remotePerm.isEmpty())
remotePerm = QString(); // have NULL in DB (vs empty)
- int contentChecksumTypeId = mapChecksumType(record._contentChecksumType);
+ QByteArray checksumType, checksum;
+ parseChecksumHeader(record._checksumHeader, &checksumType, &checksum);
+ int contentChecksumTypeId = mapChecksumType(checksumType);
_setFileRecordQuery->reset_and_clear_bindings();
_setFileRecordQuery->bindValue(1, QString::number(phash));
_setFileRecordQuery->bindValue(2, plen);
_setFileRecordQuery->bindValue(12, remotePerm);
_setFileRecordQuery->bindValue(13, record._fileSize);
_setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1 : 0);
- _setFileRecordQuery->bindValue(15, record._contentChecksum);
+ _setFileRecordQuery->bindValue(15, checksum);
_setFileRecordQuery->bindValue(16, contentChecksumTypeId);
if (!_setFileRecordQuery->exec()) {
rec._remotePerm = _getFileRecordQuery->baValue(9);
rec._fileSize = _getFileRecordQuery->int64Value(10);
rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0);
- rec._contentChecksum = _getFileRecordQuery->baValue(12);
- if (!_getFileRecordQuery->nullValue(13)) {
- rec._contentChecksumType = _getFileRecordQuery->baValue(13);
- }
+ rec._checksumHeader = _getFileRecordQuery->baValue(12);
_getFileRecordQuery->reset_and_clear_bindings();
} else {
int errId = _getFileRecordQuery->errorId();
, _fileSize(item._size)
, _remotePerm(item._remotePerm)
, _serverHasIgnoredFiles(item._serverHasIgnoredFiles)
- , _contentChecksum(item._contentChecksum)
- , _contentChecksumType(item._contentChecksumType)
+ , _checksumHeader(item._checksumHeader)
{
// use the "old" inode coming with the item for the case where the
// filesystem stat fails. That can happen if the the file was removed
item._size = _fileSize;
item._remotePerm = _remotePerm;
item._serverHasIgnoredFiles = _serverHasIgnoredFiles;
- item._contentChecksum = _contentChecksum;
- item._contentChecksumType = _contentChecksumType;
+ item._checksumHeader = _checksumHeader;
return item;
}
&& lhs._fileSize == rhs._fileSize
&& lhs._remotePerm == rhs._remotePerm
&& lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles
- && lhs._contentChecksum == rhs._contentChecksum
- && lhs._contentChecksumType == rhs._contentChecksumType;
+ && lhs._checksumHeader == rhs._checksumHeader;
}
}
qint64 _fileSize;
QByteArray _remotePerm;
bool _serverHasIgnoredFiles;
- QByteArray _contentChecksum;
- QByteArray _contentChecksumType;
+ QByteArray _checksumHeader;
};
bool OWNCLOUDSYNC_EXPORT
}
}
+ void testFakeConflict()
+ {
+ FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
+
+ int nGET = 0;
+ fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &) {
+ if (op == QNetworkAccessManager::GetOperation)
+ ++nGET;
+ return nullptr;
+ });
+
+ // For directly editing the remote checksum
+ FileInfo &remoteInfo = dynamic_cast<FileInfo &>(fakeFolder.remoteModifier());
+
+ // Base mtime with no ms content (filesystem is seconds only)
+ auto mtime = QDateTime::currentDateTime().addDays(-4);
+ mtime.setMSecsSinceEpoch(mtime.toMSecsSinceEpoch() / 1000 * 1000);
+
+ // Conflict: Same content, mtime, but no server checksum
+ // -> ignored in reconcile
+ fakeFolder.localModifier().setContents("A/a1", 'C');
+ fakeFolder.localModifier().setModTime("A/a1", mtime);
+ fakeFolder.remoteModifier().setContents("A/a1", 'C');
+ fakeFolder.remoteModifier().setModTime("A/a1", mtime);
+ QVERIFY(fakeFolder.syncOnce());
+ QCOMPARE(nGET, 0);
+
+ // Conflict: Same content, mtime, but weak server checksum
+ // -> ignored in reconcile
+ mtime = mtime.addDays(1);
+ fakeFolder.localModifier().setContents("A/a1", 'D');
+ fakeFolder.localModifier().setModTime("A/a1", mtime);
+ fakeFolder.remoteModifier().setContents("A/a1", 'D');
+ fakeFolder.remoteModifier().setModTime("A/a1", mtime);
+ remoteInfo.find("A/a1")->checksums = "Adler32:bad";
+ QVERIFY(fakeFolder.syncOnce());
+ QCOMPARE(nGET, 0);
+
+ // Conflict: Same content, mtime, but server checksum differs
+ // -> downloaded
+ mtime = mtime.addDays(1);
+ fakeFolder.localModifier().setContents("A/a1", 'W');
+ fakeFolder.localModifier().setModTime("A/a1", mtime);
+ fakeFolder.remoteModifier().setContents("A/a1", 'W');
+ fakeFolder.remoteModifier().setModTime("A/a1", mtime);
+ remoteInfo.find("A/a1")->checksums = "SHA1:bad";
+ QVERIFY(fakeFolder.syncOnce());
+ QCOMPARE(nGET, 1);
+
+ // Conflict: Same content, mtime, matching checksums
+ // -> PropagateDownload, but it skips the download
+ mtime = mtime.addDays(1);
+ fakeFolder.localModifier().setContents("A/a1", 'C');
+ fakeFolder.localModifier().setModTime("A/a1", mtime);
+ fakeFolder.remoteModifier().setContents("A/a1", 'C');
+ fakeFolder.remoteModifier().setModTime("A/a1", mtime);
+ remoteInfo.find("A/a1")->checksums = "SHA1:56900fb1d337cf7237ff766276b9c1e8ce507427";
+ QVERIFY(fakeFolder.syncOnce());
+ QCOMPARE(nGET, 1);
+
+ // Extra sync reads from db, no difference
+ QVERIFY(fakeFolder.syncOnce());
+ QCOMPARE(nGET, 1);
+ }
};
QTEST_GUILESS_MAIN(TestSyncEngine)
record._fileId = "abcd";
record._remotePerm = "744";
record._fileSize = 213089055;
- record._contentChecksum = "mychecksum";
- record._contentChecksumType = "MD5";
+ record._checksumHeader = "MD5:mychecksum";
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
// Update checksum
- record._contentChecksum = "newchecksum";
- record._contentChecksumType = "Adler32";
- _db.updateFileRecordChecksum("foo", record._contentChecksum, record._contentChecksumType);
+ record._checksumHeader = "Adler32:newchecksum";
+ _db.updateFileRecordChecksum("foo", "newchecksum", "Adler32");
storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
SyncJournalFileRecord record;
record._path = "foo-checksum";
record._remotePerm = "744";
- record._contentChecksum = "mychecksum";
- record._contentChecksumType = "MD5";
+ record._checksumHeader = "MD5:mychecksum";
record._modtime = QDateTime::currentDateTimeUtc();
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo-checksum");
QVERIFY(storedRecord._path == record._path);
QVERIFY(storedRecord._remotePerm == record._remotePerm);
- QVERIFY(storedRecord._contentChecksum == record._contentChecksum);
- QVERIFY(storedRecord._contentChecksumType == record._contentChecksumType);
+ QVERIFY(storedRecord._checksumHeader == record._checksumHeader);
// qDebug()<< "OOOOO " << storedRecord._modtime.toTime_t() << record._modtime.toTime_t();
projects / nextcloud-desktop.git / commitdiff