[Pkg-owncloud-commits] [owncloud-client] 389/484: Checksums: Don't reupload if size and checksum are unchanged #3235
Sandro Knauß
hefee-guest at moszumanska.debian.org
Wed Dec 16 00:38:10 UTC 2015
This is an automated email from the git hooks/post-receive script.
hefee-guest pushed a commit to branch master
in repository owncloud-client.
commit a25f094c4ccdf7a9b5f5291a6e13bdc232e30ab9
Author: Christian Kamm <mail at ckamm.de>
Date: Mon Nov 23 11:53:06 2015 +0100
Checksums: Don't reupload if size and checksum are unchanged #3235
* Compute the content checksum (in addition to the optional
transmission checksum) during upload (.eml files only)
* Add hook to compute and compare the checksum in csync_update
* Add content checksum to database, remove transmission checksum
---
csync/src/csync.c | 1 +
csync/src/csync.h | 6 ++
csync/src/csync_private.h | 8 +++
csync/src/csync_statedb.c | 15 +++--
csync/src/csync_update.c | 52 ++++++++++-----
csync/tests/csync_tests/check_csync_update.c | 6 ++
src/libsync/propagatedownload.cpp | 14 +---
src/libsync/propagatedownload.h | 2 +-
src/libsync/propagateremotemove.cpp | 4 +-
src/libsync/propagateupload.cpp | 70 +++++++++++---------
src/libsync/propagateupload.h | 6 +-
src/libsync/propagatorjobs.cpp | 4 +-
src/libsync/syncengine.cpp | 37 +++++++----
src/libsync/syncengine.h | 4 ++
src/libsync/syncfileitem.h | 4 +-
src/libsync/syncjournaldb.cpp | 89 ++++++++++++++++++--------
src/libsync/syncjournaldb.h | 10 ++-
src/libsync/syncjournalfilerecord.cpp | 12 ++--
src/libsync/syncjournalfilerecord.h | 4 +-
src/libsync/transmissionchecksumvalidator.cpp | 62 +++++++++++++-----
src/libsync/transmissionchecksumvalidator.h | 32 +++++++++
test/test_journal.db | Bin 57344 -> 57344 bytes
test/testsyncjournaldb.h | 14 ++--
23 files changed, 315 insertions(+), 141 deletions(-)
diff --git a/csync/src/csync.c b/csync/src/csync.c
index fe67048..7d04708 100644
--- a/csync/src/csync.c
+++ b/csync/src/csync.c
@@ -742,6 +742,7 @@ void csync_file_stat_free(csync_file_stat_t *st)
SAFE_FREE(st->directDownloadCookies);
SAFE_FREE(st->etag);
SAFE_FREE(st->destpath);
+ SAFE_FREE(st->checksum);
SAFE_FREE(st);
}
}
diff --git a/csync/src/csync.h b/csync/src/csync.h
index e2ab1ae..892d3e6 100644
--- a/csync/src/csync.h
+++ b/csync/src/csync.h
@@ -301,6 +301,12 @@ typedef void (*csync_vio_closedir_hook) (csync_vio_handle_t *dhhandle,
typedef int (*csync_vio_stat_hook) (csync_vio_handle_t *dhhandle,
void *userdata);
+/* compute the checksum of the given \a checksumTypeId for \a path
+ * and return true if it's the same as \a checksum */
+typedef bool (*csync_checksum_hook) (const char *path,
+ uint32_t checksumTypeId, const char *checksum,
+ void *userdata);
+
/**
* @brief Allocate a csync context.
*
diff --git a/csync/src/csync_private.h b/csync/src/csync_private.h
index e8f7c4b..291482d 100644
--- a/csync/src/csync_private.h
+++ b/csync/src/csync_private.h
@@ -96,6 +96,11 @@ struct csync_s {
csync_vio_readdir_hook remote_readdir_hook;
csync_vio_closedir_hook remote_closedir_hook;
void *vio_userdata;
+
+ /* hook for comparing checksums of files during discovery */
+ csync_checksum_hook checksum_hook;
+ void *checksum_userdata;
+
} callbacks;
c_strlist_t *excludes;
@@ -192,6 +197,9 @@ struct csync_file_stat_s {
char *directDownloadCookies;
char remotePerm[REMOTE_PERM_BUF_SIZE+1];
+ char *checksum;
+ uint32_t checksumTypeId;
+
CSYNC_STATUS error_status;
enum csync_instructions_e instruction; /* u32 */
diff --git a/csync/src/csync_statedb.c b/csync/src/csync_statedb.c
index 61fc897..38aec0a 100644
--- a/csync/src/csync_statedb.c
+++ b/csync/src/csync_statedb.c
@@ -226,6 +226,8 @@ int csync_statedb_close(CSYNC *ctx) {
return rc;
}
+#define METADATA_COLUMNS "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId"
+
// This funciton parses a line from the metadata table into the given csync_file_stat
// structure which it is also allocating.
// Note that this function calls laso sqlite3_step to actually get the info from db and
@@ -286,6 +288,11 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
if(column_count > 13) {
(*st)->has_ignored_files = sqlite3_column_int(stmt, 13);
}
+ if(column_count > 15 && sqlite3_column_int(stmt, 15)) {
+ (*st)->checksum = c_strdup( (char*) sqlite3_column_text(stmt, 14));
+ (*st)->checksumTypeId = sqlite3_column_int(stmt, 15);
+ }
+
}
} else {
if( rc != SQLITE_DONE ) {
@@ -307,7 +314,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
}
if( ctx->statedb.by_hash_stmt == NULL ) {
- const char *hash_query = "SELECT * FROM metadata WHERE phash=?1";
+ const char *hash_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE phash=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -350,7 +357,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
}
if( ctx->statedb.by_fileid_stmt == NULL ) {
- const char *query = "SELECT * FROM metadata WHERE fileid=?1";
+ const char *query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE fileid=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -390,7 +397,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
}
if( ctx->statedb.by_inode_stmt == NULL ) {
- const char *inode_query = "SELECT * FROM metadata WHERE inode=?1";
+ const char *inode_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE inode=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -433,7 +440,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
* In other words, anything that is between path+'/' and path+'0',
* (because '0' follows '/' in ascii)
*/
- const char *below_path_query = "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
+ const char *below_path_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL));
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_OK ) {
diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c
index e7b215b..28d7d0d 100644
--- a/csync/src/csync_update.c
+++ b/csync/src/csync_update.c
@@ -270,25 +270,43 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
((int64_t) fs->mtime), ((int64_t) tmp->modtime),
fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode,
(uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files );
- if((ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag ))
- || (ctx->current == LOCAL_REPLICA && (!_csync_mtime_equal(fs->mtime, tmp->modtime)
- // zero size in statedb can happen during migration
- || (tmp->size != 0 && fs->size != tmp->size)
+ if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) {
+ st->instruction = CSYNC_INSTRUCTION_EVAL;
+ goto out;
+ }
+ if (ctx->current == LOCAL_REPLICA &&
+ (!_csync_mtime_equal(fs->mtime, tmp->modtime)
+ // zero size in statedb can happen during migration
+ || (tmp->size != 0 && fs->size != tmp->size)
#if 0
- || fs->inode != tmp->inode
+ /* Comparison of the local inode is disabled because people reported problems
+ * on windows with flacky inode values, see github bug #779
+ *
+ * The inode needs to be observed because:
+ * $> echo a > a.txt ; echo b > b.txt
+ * both files have the same mtime
+ * sync them.
+ * $> rm a.txt && mv b.txt a.txt
+ * makes b.txt appearing as a.txt yet a sync is not performed because
+ * both have the same modtime as mv does not change that.
+ */
+ || fs->inode != tmp->inode
#endif
- ))) {
- /* Comparison of the local inode is disabled because people reported problems
- * on windows with flacky inode values, see github bug #779
- *
- * The inode needs to be observed because:
- * $> echo a > a.txt ; echo b > b.txt
- * both files have the same mtime
- * sync them.
- * $> rm a.txt && mv b.txt a.txt
- * makes b.txt appearing as a.txt yet a sync is not performed because
- * both have the same modtime as mv does not change that.
- */
+ )) {
+
+ if (fs->size == tmp->size && tmp->checksumTypeId) {
+ bool checksumIdentical = false;
+ if (ctx->callbacks.checksum_hook) {
+ checksumIdentical = ctx->callbacks.checksum_hook(
+ file, tmp->checksumTypeId, tmp->checksum,
+ ctx->callbacks.checksum_userdata);
+ }
+ if (checksumIdentical) {
+ st->instruction = CSYNC_INSTRUCTION_NONE;
+ st->should_update_metadata = true;
+ goto out;
+ }
+ }
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
diff --git a/csync/tests/csync_tests/check_csync_update.c b/csync/tests/csync_tests/check_csync_update.c
index ced053e..c3a7eaf 100644
--- a/csync/tests/csync_tests/check_csync_update.c
+++ b/csync/tests/csync_tests/check_csync_update.c
@@ -41,6 +41,12 @@ static void statedb_create_metadata_table(sqlite3 *db)
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32),"
+ "fileid VARCHAR(128),"
+ "remotePerm VARCHAR(128),"
+ "filesize BIGINT,"
+ "ignoredChildrenRemote INT,"
+ "contentChecksum TEXT,"
+ "contentChecksumTypeId INTEGER,"
"PRIMARY KEY(phash));";
rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp
index a0fde8a..ccf2595 100644
--- a/src/libsync/propagatedownload.cpp
+++ b/src/libsync/propagatedownload.cpp
@@ -351,10 +351,7 @@ void PropagateDownloadFileQNAM::start()
if (_resumeStart == _item->_size) {
qDebug() << "File is already complete, no need to download";
_tmpFile.close();
-
- // Unfortunately we lost the checksum header, if any...
- QByteArray noChecksumData;
- downloadFinished(noChecksumData, noChecksumData);
+ downloadFinished();
return;
}
}
@@ -537,7 +534,7 @@ void PropagateDownloadFileQNAM::slotGetFinished()
// as this is (still) also correct.
ValidateChecksumHeader *validator = new ValidateChecksumHeader(this);
connect(validator, SIGNAL(validated(QByteArray,QByteArray)),
- SLOT(downloadFinished(QByteArray,QByteArray)));
+ SLOT(downloadFinished()));
connect(validator, SIGNAL(validationFailed(QString)),
SLOT(slotChecksumFail(QString)));
auto checksumHeader = job->reply()->rawHeader(checkSumHeaderC);
@@ -621,13 +618,8 @@ static void handleRecallFile(const QString &fn)
}
} // end namespace
-void PropagateDownloadFileQNAM::downloadFinished(const QByteArray& checksumType, const QByteArray& checksum)
+void PropagateDownloadFileQNAM::downloadFinished()
{
- if (!checksumType.isEmpty()) {
- _item->_transmissionChecksum = checksum;
- _item->_transmissionChecksumType = checksumType;
- }
-
QString fn = _propagator->getFilePath(_item->_file);
// In case of file name clash, report an error
diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h
index 2a8877e..32e05db 100644
--- a/src/libsync/propagatedownload.h
+++ b/src/libsync/propagatedownload.h
@@ -117,7 +117,7 @@ public:
private slots:
void slotGetFinished();
void abort() Q_DECL_OVERRIDE;
- void downloadFinished(const QByteArray& checksumType, const QByteArray& checksum);
+ void downloadFinished();
void slotDownloadProgress(qint64,qint64);
void slotChecksumFail( const QString& errMsg );
diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp
index 33501aa..f41fd13 100644
--- a/src/libsync/propagateremotemove.cpp
+++ b/src/libsync/propagateremotemove.cpp
@@ -158,8 +158,8 @@ void PropagateRemoteMove::finalize()
SyncJournalFileRecord record(*_item, _propagator->getFilePath(_item->_renameTarget));
record._path = _item->_renameTarget;
- record._transmissionChecksum = oldRecord._transmissionChecksum;
- record._transmissionChecksumType = oldRecord._transmissionChecksumType;
+ record._contentChecksum = oldRecord._contentChecksum;
+ record._contentChecksumType = oldRecord._contentChecksumType;
_propagator->_journal->setFileRecord(record);
_propagator->_journal->commit("Remote Rename");
diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp
index 2ad23dd..859c499 100644
--- a/src/libsync/propagateupload.cpp
+++ b/src/libsync/propagateupload.cpp
@@ -216,24 +216,38 @@ void PropagateUploadFileQNAM::start()
_stopWatch.start();
- auto supportedChecksumTypes = _propagator->account()->capabilities().supportedChecksumTypes();
-
- // If we already have a checksum header and the checksum type is supported
- // by the server, we keep that - otherwise recompute.
- //
- // Note: Currently we *always* recompute because we usually only upload
- // files that have changed and thus have a new checksum. But if an earlier
- // phase computed a checksum, this is where we would make use of it.
- if (!_item->_transmissionChecksumType.isEmpty()) {
- if (supportedChecksumTypes.contains(_item->_transmissionChecksumType)) {
- // TODO: We could validate the old checksum and thereby determine whether
- // an upload is necessary or not.
- slotStartUpload(_item->_transmissionChecksumType, _item->_transmissionChecksum);
- return;
- }
+ // Compute the content checksum.
+ auto computeChecksum = new ComputeChecksum(this);
+ // We currently only do content checksums for the particular .eml case
+ // This should be done more generally in the future!
+ if (filePath.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive)) {
+ computeChecksum->setChecksumType("MD5");
+ } else {
+ computeChecksum->setChecksumType(QByteArray());
+ }
+
+ connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
+ SLOT(slotComputeTransmissionChecksum(QByteArray,QByteArray)));
+ computeChecksum->start(filePath);
+}
+
+void PropagateUploadFileQNAM::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum)
+{
+ _item->_contentChecksum = contentChecksum;
+ _item->_contentChecksumType = contentChecksumType;
+
+ _stopWatch.addLapTime(QLatin1String("ContentChecksum"));
+ _stopWatch.start();
+
+ // Reuse the content checksum as the transmission checksum if possible
+ const auto supportedTransmissionChecksums =
+ _propagator->account()->capabilities().supportedChecksumTypes();
+ if (supportedTransmissionChecksums.contains(contentChecksumType)) {
+ slotStartUpload(contentChecksumType, contentChecksum);
+ return;
}
- // Compute a new checksum.
+ // Compute the transmission checksum.
auto computeChecksum = new ComputeChecksum(this);
if (uploadChecksumEnabled()) {
computeChecksum->setChecksumType(_propagator->account()->capabilities().preferredChecksumType());
@@ -243,19 +257,14 @@ void PropagateUploadFileQNAM::start()
connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
SLOT(slotStartUpload(QByteArray,QByteArray)));
+ const QString filePath = _propagator->getFilePath(_item->_file);
computeChecksum->start(filePath);
}
-void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum)
+void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
{
- // Store the computed checksum in the database, if different
- if (checksumType != _item->_transmissionChecksumType
- || checksum != _item->_transmissionChecksum) {
- _item->_transmissionChecksum = checksum;
- _item->_transmissionChecksumType = checksumType;
- _propagator->_journal->updateFileRecordChecksum(
- _item->_file, checksum, checksumType);
- }
+ _transmissionChecksum = transmissionChecksum;
+ _transmissionChecksumType = transmissionChecksumType;
const QString fullFilePath = _propagator->getFilePath(_item->_file);
@@ -263,7 +272,7 @@ void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, co
done(SyncFileItem::SoftError, tr("File Removed"));
return;
}
- _stopWatch.addLapTime(QLatin1String("Checksum"));
+ _stopWatch.addLapTime(QLatin1String("TransmissionChecksum"));
time_t prevModtime = _item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start()
// but a potential checksum calculation could have taken some time during which the file could
@@ -511,9 +520,9 @@ void PropagateUploadFileQNAM::startNextChunk()
isFinalChunk = true;
}
- if (isFinalChunk && !_item->_transmissionChecksumType.isEmpty()) {
+ if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
headers[checkSumHeaderC] = makeChecksumHeader(
- _item->_transmissionChecksumType, _item->_transmissionChecksum);
+ _transmissionChecksumType, _transmissionChecksum);
}
if (! device->prepareAndOpen(_propagator->getFilePath(_item->_file), chunkStart, currentChunkSize)) {
@@ -724,7 +733,10 @@ void PropagateUploadFileQNAM::slotPutFinished()
// performance logging
_item->_requestDuration = _stopWatch.stop();
- qDebug() << "*==* duration UPLOAD" << _item->_size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item->_requestDuration;
+ qDebug() << "*==* duration UPLOAD" << _item->_size
+ << _stopWatch.durationOfLap(QLatin1String("ContentChecksum"))
+ << _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum"))
+ << _item->_requestDuration;
finalize(*_item);
}
diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h
index 9451e12..1b4bc74 100644
--- a/src/libsync/propagateupload.h
+++ b/src/libsync/propagateupload.h
@@ -183,6 +183,9 @@ private:
// measure the performance of checksum calc and upload
Utility::StopWatch _stopWatch;
+ QByteArray _transmissionChecksum;
+ QByteArray _transmissionChecksumType;
+
public:
PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItemPtr& item)
: PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {}
@@ -195,7 +198,8 @@ private slots:
void startNextChunk();
void finalize(const SyncFileItem&);
void slotJobDestroyed(QObject *job);
- void slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum);
+ void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
+ void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum);
private:
void startPollJob(const QString& path);
diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp
index ddb17ef..5b70b09 100644
--- a/src/libsync/propagatorjobs.cpp
+++ b/src/libsync/propagatorjobs.cpp
@@ -209,8 +209,8 @@ void PropagateLocalRename::start()
SyncJournalFileRecord record(*_item, targetFile);
record._path = _item->_renameTarget;
- record._transmissionChecksum = oldRecord._transmissionChecksum;
- record._transmissionChecksumType = oldRecord._transmissionChecksumType;
+ record._contentChecksum = oldRecord._contentChecksum;
+ record._contentChecksumType = oldRecord._contentChecksumType;
if (!_item->_isDirectory) { // Directories are saved at the end
_propagator->_journal->setFileRecord(record);
diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp
index 624c83b..d1e70cf 100644
--- a/src/libsync/syncengine.cpp
+++ b/src/libsync/syncengine.cpp
@@ -71,6 +71,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath,
, _uploadLimit(0)
, _downloadLimit(0)
, _newBigFolderSizeLimit(-1)
+ , _checksum_hook(journal)
, _anotherSyncNeeded(false)
{
qRegisterMetaType<SyncFileItem>("SyncFileItem");
@@ -466,10 +467,11 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
int re = 0;
switch(file->instruction) {
- case CSYNC_INSTRUCTION_NONE:
+ case CSYNC_INSTRUCTION_NONE: {
if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) {
- // Update the database now already: New fileid or Etag or RemotePerm
+ // Update the database now already: New remote fileid or Etag or RemotePerm
// Or for files that were detected as "resolved conflict".
+ // Or a local inode/mtime change (see localMetadataUpdate below)
// In case of "resolved conflict": there should have been a conflict because they
// both were new, or both had their local mtime or remote etag modified, but the
@@ -496,21 +498,28 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
_journal->setFileRecordMetadata(SyncJournalFileRecord(*item, filePath));
item->_should_update_metadata = false;
+
+ // Technically we're done with this item. See localMetadataUpdate hack below.
+ _syncItemMap.remove(key);
+ }
+ // Any files that are instruction NONE?
+ if (!item->_isDirectory && file->other.instruction == CSYNC_INSTRUCTION_NONE) {
+ _hasNoneFiles = true;
}
- if (item->_isDirectory && file->should_update_metadata) {
- // Because we want to still update etags of directories
- dir = SyncFileItem::None;
- } else {
- // No need to do anything.
- if (file->other.instruction == CSYNC_INSTRUCTION_NONE
- // Directories with ignored files does not count as 'None'
- && (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) {
- _hasNoneFiles = true;
+ // We want to still update etags of directories, other NONE
+ // items can be ignored.
+ bool directoryEtagUpdate = item->_isDirectory && file->should_update_metadata;
+ bool localMetadataUpdate = !remote && file->should_update_metadata;
+ if (!directoryEtagUpdate) {
+ if (localMetadataUpdate) {
+ // Hack, we want a local metadata update to happen, but only if the
+ // remote tree doesn't ask us to do some kind of propagation.
+ _syncItemMap.insert(key, item);
}
-
return re;
}
break;
+ }
case CSYNC_INSTRUCTION_RENAME:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
item->_renameTarget = renameTarget;
@@ -686,6 +695,10 @@ void SyncEngine::startSync()
csync_set_userdata(_csync_ctx, this);
+ // Set up checksumming hook
+ _csync_ctx->callbacks.checksum_hook = &CSyncChecksumHook::hook;
+ _csync_ctx->callbacks.checksum_userdata = &_checksum_hook;
+
_stopWatch.start();
qDebug() << "#### Discovery start #################################################### >>";
diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h
index fce1e1c..beedf84 100644
--- a/src/libsync/syncengine.h
+++ b/src/libsync/syncengine.h
@@ -37,6 +37,7 @@
#include "syncfilestatus.h"
#include "accountfwd.h"
#include "discoveryphase.h"
+#include "transmissionchecksumvalidator.h"
class QProcess;
@@ -216,6 +217,9 @@ private:
// hash containing the permissions on the remote directory
QHash<QString, QByteArray> _remotePerms;
+ /// Hook for computing checksums from csync_update
+ CSyncChecksumHook _checksum_hook;
+
bool _anotherSyncNeeded;
};
diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h
index b3901bd..1642ab2 100644
--- a/src/libsync/syncfileitem.h
+++ b/src/libsync/syncfileitem.h
@@ -165,8 +165,8 @@ public:
quint64 _inode;
QByteArray _fileId;
QByteArray _remotePerm;
- QByteArray _transmissionChecksum;
- QByteArray _transmissionChecksumType;
+ QByteArray _contentChecksum;
+ QByteArray _contentChecksumType;
QString _directDownloadUrl;
QString _directDownloadCookies;
diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp
index eb52d67..1fa8c54 100644
--- a/src/libsync/syncjournaldb.cpp
+++ b/src/libsync/syncjournaldb.cpp
@@ -204,10 +204,13 @@ bool SyncJournalDb::checkConnect()
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */
- // updateDatabaseStructure() will add a fileid column
- // updateDatabaseStructure() will add a remotePerm column
- // updateDatabaseStructure() will add a transmissionChecksum column
- // updateDatabaseStructure() will add a transmissionChecksumTypeId column
+ // updateDatabaseStructure() will add
+ // fileid
+ // remotePerm
+ // filesize
+ // ignoredChildrenRemote
+ // contentChecksum
+ // contentChecksumTypeId
"PRIMARY KEY(phash)"
");");
@@ -358,20 +361,20 @@ bool SyncJournalDb::checkConnect()
_getFileRecordQuery.reset(new SqlQuery(_db));
_getFileRecordQuery->prepare(
"SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize,"
- " ignoredChildrenRemote, transmissionChecksum, checksumtype.name"
+ " ignoredChildrenRemote, contentChecksum, contentchecksumtype.name"
" FROM metadata"
- " LEFT JOIN checksumtype ON metadata.transmissionChecksumTypeId == checksumtype.id"
+ " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
" WHERE phash=?1" );
_setFileRecordQuery.reset(new SqlQuery(_db) );
_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata "
- "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, transmissionChecksum, transmissionChecksumTypeId) "
+ "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) "
"VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);" );
_setFileRecordChecksumQuery.reset(new SqlQuery(_db) );
_setFileRecordChecksumQuery->prepare(
"UPDATE metadata"
- " SET transmissionChecksum = ?2, transmissionChecksumTypeId = ?3"
+ " SET contentChecksum = ?2, contentChecksumTypeId = ?3"
" WHERE phash == ?1;");
_getDownloadInfoQuery.reset(new SqlQuery(_db) );
@@ -426,6 +429,9 @@ bool SyncJournalDb::checkConnect()
_getChecksumTypeIdQuery.reset(new SqlQuery(_db));
_getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1");
+ _getChecksumTypeQuery.reset(new SqlQuery(_db));
+ _getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1");
+
_insertChecksumTypeQuery.reset(new SqlQuery(_db));
_insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)");
@@ -463,6 +469,7 @@ void SyncJournalDb::close()
_setErrorBlacklistQuery.reset(0);
_getSelectiveSyncListQuery.reset(0);
_getChecksumTypeIdQuery.reset(0);
+ _getChecksumTypeQuery.reset(0);
_insertChecksumTypeQuery.reset(0);
_db.close();
@@ -557,23 +564,23 @@ bool SyncJournalDb::updateMetadataTableStructure()
commitInternal("update database structure: add ignoredChildrenRemote col");
}
- if( columns.indexOf(QLatin1String("transmissionChecksum")) == -1 ) {
+ if( columns.indexOf(QLatin1String("contentChecksum")) == -1 ) {
SqlQuery query(_db);
- query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksum TEXT;");
+ query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;");
if( !query.exec()) {
- sqlFail("updateMetadataTableStructure: add transmissionChecksum column", query);
+ sqlFail("updateMetadataTableStructure: add contentChecksum column", query);
re = false;
}
- commitInternal("update database structure: add transmissionChecksum col");
+ commitInternal("update database structure: add contentChecksum col");
}
- if( columns.indexOf(QLatin1String("transmissionChecksumTypeId")) == -1 ) {
+ if( columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1 ) {
SqlQuery query(_db);
- query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksumTypeId INTEGER;");
+ query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;");
if( !query.exec()) {
- sqlFail("updateMetadataTableStructure: add transmissionChecksumTypeId column", query);
+ sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query);
re = false;
}
- commitInternal("update database structure: add transmissionChecksumTypeId col");
+ commitInternal("update database structure: add contentChecksumTypeId col");
}
@@ -677,7 +684,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
if( fileId.isEmpty() ) fileId = "";
QString remotePerm (record._remotePerm);
if (remotePerm.isEmpty()) remotePerm = QString(); // have NULL in DB (vs empty)
- int checksumTypeId = mapChecksumType(record._transmissionChecksumType);
+ int contentChecksumTypeId = mapChecksumType(record._contentChecksumType);
_setFileRecordQuery->reset();
_setFileRecordQuery->bindValue(1, QString::number(phash));
_setFileRecordQuery->bindValue(2, plen);
@@ -693,8 +700,8 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
_setFileRecordQuery->bindValue(12, remotePerm );
_setFileRecordQuery->bindValue(13, record._fileSize );
_setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1:0);
- _setFileRecordQuery->bindValue(15, record._transmissionChecksum );
- _setFileRecordQuery->bindValue(16, checksumTypeId );
+ _setFileRecordQuery->bindValue(15, record._contentChecksum );
+ _setFileRecordQuery->bindValue(16, contentChecksumTypeId );
if( !_setFileRecordQuery->exec() ) {
qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() << " :"
@@ -705,7 +712,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
qDebug() << _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode
<< QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type)
<< record._etag << record._fileId << record._remotePerm << record._fileSize << (record._serverHasIgnoredFiles ? 1:0)
- << record._transmissionChecksum << record._transmissionChecksumType << checksumTypeId;
+ << record._contentChecksum << record._contentChecksumType << contentChecksumTypeId;
_setFileRecordQuery->reset();
return true;
@@ -785,9 +792,9 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename )
rec._remotePerm = _getFileRecordQuery->baValue(9);
rec._fileSize = _getFileRecordQuery->int64Value(10);
rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0);
- rec._transmissionChecksum = _getFileRecordQuery->baValue(12);
+ rec._contentChecksum = _getFileRecordQuery->baValue(12);
if( !_getFileRecordQuery->nullValue(13) ) {
- rec._transmissionChecksumType = _getFileRecordQuery->baValue(13);
+ rec._contentChecksumType = _getFileRecordQuery->baValue(13);
}
} else {
QString err = _getFileRecordQuery->error();
@@ -878,8 +885,8 @@ int SyncJournalDb::getFileRecordCount()
}
bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
- const QByteArray& transmisisonChecksum,
- const QByteArray& transmissionChecksumType)
+ const QByteArray& contentChecksum,
+ const QByteArray& contentChecksumType)
{
QMutexLocker locker(&_mutex);
@@ -889,12 +896,12 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
return false;
}
- int checksumTypeId = mapChecksumType(transmissionChecksumType);
+ int checksumTypeId = mapChecksumType(contentChecksumType);
auto & query = _setFileRecordChecksumQuery;
query->reset();
query->bindValue(1, QString::number(phash));
- query->bindValue(2, transmisisonChecksum);
+ query->bindValue(2, contentChecksum);
query->bindValue(3, checksumTypeId);
if( !query->exec() ) {
@@ -904,8 +911,8 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
return false;
}
- qDebug() << query->lastQuery() << phash << transmisisonChecksum
- << transmissionChecksumType << checksumTypeId;
+ qDebug() << query->lastQuery() << phash << contentChecksum
+ << contentChecksumType << checksumTypeId;
query->reset();
return true;
@@ -1501,6 +1508,32 @@ void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked()
}
}
+
+QByteArray SyncJournalDb::getChecksumType(int checksumTypeId)
+{
+ QMutexLocker locker(&_mutex);
+ if( !checkConnect() ) {
+ return QByteArray();
+ }
+
+ // Retrieve the id
+ auto & query = *_getChecksumTypeQuery;
+ query.reset();
+ query.bindValue(1, checksumTypeId);
+ if( !query.exec() ) {
+ qWarning() << "Error SQL statement getChecksumType: "
+ << query.lastQuery() << " :"
+ << query.error();
+ return 0;
+ }
+
+ if( !query.next() ) {
+ qDebug() << "No checksum type mapping found for" << checksumTypeId;
+ return 0;
+ }
+ return query.baValue(0);
+}
+
int SyncJournalDb::mapChecksumType(const QByteArray& checksumType)
{
if (checksumType.isEmpty()) {
diff --git a/src/libsync/syncjournaldb.h b/src/libsync/syncjournaldb.h
index b5521ff..a56e41b 100644
--- a/src/libsync/syncjournaldb.h
+++ b/src/libsync/syncjournaldb.h
@@ -47,8 +47,8 @@ public:
bool deleteFileRecord( const QString& filename, bool recursively = false );
int getFileRecordCount();
bool updateFileRecordChecksum(const QString& filename,
- const QByteArray& transmisisonChecksum,
- const QByteArray& transmissionChecksumType);
+ const QByteArray& contentChecksum,
+ const QByteArray& contentChecksumType);
bool exists();
void walCheckpoint();
@@ -146,6 +146,11 @@ public:
*/
bool isConnected();
+ /**
+ * Returns the checksum type for an id.
+ */
+ QByteArray getChecksumType(int checksumTypeId);
+
private:
bool updateDatabaseStructure();
bool updateMetadataTableStructure();
@@ -186,6 +191,7 @@ private:
QScopedPointer<SqlQuery> _setErrorBlacklistQuery;
QScopedPointer<SqlQuery> _getSelectiveSyncListQuery;
QScopedPointer<SqlQuery> _getChecksumTypeIdQuery;
+ QScopedPointer<SqlQuery> _getChecksumTypeQuery;
QScopedPointer<SqlQuery> _insertChecksumTypeQuery;
/* This is the list of paths we called avoidReadFromDbOnNextSync on.
diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp
index 57081c1..c0bbd63 100644
--- a/src/libsync/syncjournalfilerecord.cpp
+++ b/src/libsync/syncjournalfilerecord.cpp
@@ -36,8 +36,8 @@ SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QSt
: _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)),
_type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size),
_remotePerm(item._remotePerm), _serverHasIgnoredFiles(item._serverHasIgnoredFiles),
- _transmissionChecksum(item._transmissionChecksum),
- _transmissionChecksumType(item._transmissionChecksumType)
+ _contentChecksum(item._contentChecksum),
+ _contentChecksumType(item._contentChecksumType)
{
// use the "old" inode coming with the item for the case where the
// filesystem stat fails. That can happen if the the file was removed
@@ -97,8 +97,8 @@ SyncFileItem SyncJournalFileRecord::toSyncFileItem()
item._size = _fileSize;
item._remotePerm = _remotePerm;
item._serverHasIgnoredFiles = _serverHasIgnoredFiles;
- item._transmissionChecksum = _transmissionChecksum;
- item._transmissionChecksumType = _transmissionChecksumType;
+ item._contentChecksum = _contentChecksum;
+ item._contentChecksumType = _contentChecksumType;
return item;
}
@@ -176,8 +176,8 @@ bool operator==(const SyncJournalFileRecord & lhs,
&& lhs._fileSize == rhs._fileSize
&& lhs._remotePerm == rhs._remotePerm
&& lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles
- && lhs._transmissionChecksum == rhs._transmissionChecksum
- && lhs._transmissionChecksumType == rhs._transmissionChecksumType;
+ && lhs._contentChecksum == rhs._contentChecksum
+ && lhs._contentChecksumType == rhs._contentChecksumType;
}
}
diff --git a/src/libsync/syncjournalfilerecord.h b/src/libsync/syncjournalfilerecord.h
index 251e96b..ec2ec9c 100644
--- a/src/libsync/syncjournalfilerecord.h
+++ b/src/libsync/syncjournalfilerecord.h
@@ -55,8 +55,8 @@ public:
qint64 _fileSize;
QByteArray _remotePerm;
bool _serverHasIgnoredFiles;
- QByteArray _transmissionChecksum;
- QByteArray _transmissionChecksumType;
+ QByteArray _contentChecksum;
+ QByteArray _contentChecksumType;
};
bool OWNCLOUDSYNC_EXPORT
diff --git a/src/libsync/transmissionchecksumvalidator.cpp b/src/libsync/transmissionchecksumvalidator.cpp
index 15923bd..9edd70d 100644
--- a/src/libsync/transmissionchecksumvalidator.cpp
+++ b/src/libsync/transmissionchecksumvalidator.cpp
@@ -77,36 +77,40 @@ QByteArray ComputeChecksum::checksumType() const
void ComputeChecksum::start(const QString& filePath)
{
- const QString csType = checksumType();
-
// Calculate the checksum in a different thread first.
connect( &_watcher, SIGNAL(finished()),
this, SLOT(slotCalculationDone()),
Qt::UniqueConnection );
- if( csType == checkSumMD5C ) {
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, filePath));
+ _watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType()));
+}
- } else if( csType == checkSumSHA1C ) {
- _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, filePath));
+QByteArray ComputeChecksum::computeNow(const QString& filePath, const QByteArray& checksumType)
+{
+ if( checksumType == checkSumMD5C ) {
+ return FileSystem::calcMd5(filePath);
+ } else if( checksumType == checkSumSHA1C ) {
+ return FileSystem::calcSha1(filePath);
}
#ifdef ZLIB_FOUND
- else if( csType == checkSumAdlerC) {
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, filePath));
+ else if( checksumType == checkSumAdlerC) {
+ return FileSystem::calcAdler32(filePath);
}
#endif
- else {
- // for an unknown checksum or no checksum, we're done right now
- if( !csType.isEmpty() ) {
- qDebug() << "Unknown checksum type:" << csType;
- }
- emit done(QByteArray(), QByteArray());
+ // for an unknown checksum or no checksum, we're done right now
+ if( !checksumType.isEmpty() ) {
+ qDebug() << "Unknown checksum type:" << checksumType;
}
+ return QByteArray();
}
void ComputeChecksum::slotCalculationDone()
{
QByteArray checksum = _watcher.future().result();
- emit done(_checksumType, checksum);
+ if (!checksum.isNull()) {
+ emit done(_checksumType, checksum);
+ } else {
+ emit done(QByteArray(), QByteArray());
+ }
}
@@ -151,4 +155,32 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray& checksumTy
emit validated(checksumType, checksum);
}
+CSyncChecksumHook::CSyncChecksumHook(SyncJournalDb *journal)
+ : _journal(journal)
+{
+}
+
+bool CSyncChecksumHook::hook(const char* path, uint32_t checksumTypeId, const char* checksum, void *this_obj)
+{
+ CSyncChecksumHook* checksumHook = static_cast<CSyncChecksumHook*>(this_obj);
+ return checksumHook->check(QString::fromUtf8(path), checksumTypeId, QByteArray(checksum));
+}
+
+bool CSyncChecksumHook::check(const QString& path, int checksumTypeId, const QByteArray& checksum)
+{
+ QByteArray checksumType = _journal->getChecksumType(checksumTypeId);
+ if (checksumType.isEmpty()) {
+ qDebug() << "Checksum type" << checksumTypeId << "not found";
+ return false;
+ }
+
+ QByteArray newChecksum = ComputeChecksum::computeNow(path, checksumType);
+ if (newChecksum.isNull()) {
+ qDebug() << "Failed to compute checksum" << checksumType << "for" << path;
+ return false;
+ }
+ return newChecksum == checksum;
+}
+
+
}
diff --git a/src/libsync/transmissionchecksumvalidator.h b/src/libsync/transmissionchecksumvalidator.h
index 3a37c69..bee78cd 100644
--- a/src/libsync/transmissionchecksumvalidator.h
+++ b/src/libsync/transmissionchecksumvalidator.h
@@ -23,6 +23,8 @@
namespace OCC {
+class SyncJournalDb;
+
/// Creates a checksum header from type and value.
QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum);
@@ -59,6 +61,11 @@ public:
*/
void start(const QString& filePath);
+ /**
+ * Computes the checksum synchronously.
+ */
+ static QByteArray computeNow(const QString& filePath, const QByteArray& checksumType);
+
signals:
void done(const QByteArray& checksumType, const QByteArray& checksum);
@@ -103,4 +110,29 @@ private:
QByteArray _expectedChecksum;
};
+/**
+ * Hooks checksum computations into csync.
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT CSyncChecksumHook : public QObject
+{
+ Q_OBJECT
+public:
+ explicit CSyncChecksumHook(SyncJournalDb* journal);
+
+ /**
+ * Returns true if the checksum for \a path is the same as the one provided.
+ *
+ * Called from csync, where a instance of CSyncChecksumHook
+ * has to be set as userdata.
+ */
+ static bool hook(const char* path, uint32_t checksumTypeId, const char* checksum,
+ void* this_obj);
+
+ bool check(const QString& path, int checksumTypeId, const QByteArray& checksum);
+
+private:
+ SyncJournalDb* _journal;
+};
+
}
diff --git a/test/test_journal.db b/test/test_journal.db
index 30905b6..2b58c07 100644
Binary files a/test/test_journal.db and b/test/test_journal.db differ
diff --git a/test/testsyncjournaldb.h b/test/testsyncjournaldb.h
index 758180e..1f9ee6d 100644
--- a/test/testsyncjournaldb.h
+++ b/test/testsyncjournaldb.h
@@ -59,17 +59,17 @@ private slots:
record._fileId = "abcd";
record._remotePerm = "744";
record._fileSize = 213089055;
- record._transmissionChecksum = "mychecksum";
- record._transmissionChecksumType = "MD5";
+ record._contentChecksum = "mychecksum";
+ record._contentChecksumType = "MD5";
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
// Update checksum
- record._transmissionChecksum = "newchecksum";
- record._transmissionChecksumType = "Adler32";
- _db.updateFileRecordChecksum("foo", record._transmissionChecksum, record._transmissionChecksumType);
+ record._contentChecksum = "newchecksum";
+ record._contentChecksumType = "Adler32";
+ _db.updateFileRecordChecksum("foo", record._contentChecksum, record._contentChecksumType);
storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
@@ -97,8 +97,8 @@ private slots:
SyncJournalFileRecord record;
record._path = "foo-checksum";
record._remotePerm = "744";
- record._transmissionChecksum = "mychecksum";
- record._transmissionChecksumType = "MD5";
+ record._contentChecksum = "mychecksum";
+ record._contentChecksumType = "MD5";
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo-checksum");
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud-client.git
More information about the Pkg-owncloud-commits
mailing list