[Pkg-owncloud-commits] [owncloud-client] 60/175: PropagateUpload: Add checksum calculation if required by config.
Sandro Knauß
hefee-guest at moszumanska.debian.org
Sat Aug 8 10:36:27 UTC 2015
This is an automated email from the git hooks/post-receive script.
hefee-guest pushed a commit to branch master
in repository owncloud-client.
commit 3701fbcbfeab4bc4d4a725948c66d6eb49947fc2
Author: Klaas Freitag <freitag at owncloud.com>
Date: Tue May 12 16:36:40 2015 +0200
PropagateUpload: Add checksum calculation if required by config.
If the config file has an transmissionChecksum entry, a checksum
is added to the PUT requests in a header.
---
src/libsync/networkjobs.cpp | 1 -
src/libsync/propagateupload.cpp | 90 +++++++++++++++++++++++++++++++++++++++++
src/libsync/propagateupload.h | 13 ++++++
3 files changed, 103 insertions(+), 1 deletion(-)
diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp
index 8a43e5c..4b42378 100644
--- a/src/libsync/networkjobs.cpp
+++ b/src/libsync/networkjobs.cpp
@@ -43,7 +43,6 @@ namespace OCC {
AbstractNetworkJob::AbstractNetworkJob(AccountPtr account, const QString &path, QObject *parent)
: QObject(parent)
- , _duration(0)
, _timedout(false)
, _followRedirects(false)
, _ignoreCredentialFailure(false)
diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp
index 887731e..eb5346c 100644
--- a/src/libsync/propagateupload.cpp
+++ b/src/libsync/propagateupload.cpp
@@ -21,12 +21,15 @@
#include "utility.h"
#include "filesystem.h"
#include "propagatorjobs.h"
+#include "configfile.h"
+
#include <json.h>
#include <QNetworkAccessManager>
#include <QFileInfo>
#include <QDir>
#include <cmath>
#include <cstring>
+#include <QtConcurrent>
#ifdef USE_NEON
#include "propagator_legacy.h"
@@ -41,6 +44,14 @@ const char owncloudShouldSoftCancelPropertyName[] = "owncloud-should-soft-cancel
namespace OCC {
/**
+ * Tags for checksum headers.
+ */
+static const char checkSumMD5C[] = "MD5";
+static const char checkSumSHA1C[] = "SHA1";
+static const char checkSumAdlerC[] = "Adler32";
+static const char checkSumAdlerUpperC[] = "ADLER32";
+
+/**
* We do not want to upload files that are currently being modified.
* To avoid that, we don't upload files that have a modification time
* that is too close to the current time.
@@ -196,16 +207,87 @@ void PropagateUploadFileQNAM::start()
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
}
+ ConfigFile cfg; // FIXME: Do not open it for each and every propagation.
+
+ const QString filePath = _propagator->getFilePath(_item._file);
+ _item._modtime = FileSystem::getModTime(filePath);
+ // calculate the files checksum
+ const QString transChecksum = cfg.transmissionChecksum();
+ _stopWatch.start();
+
+ if( transChecksum.isEmpty() ) {
+ // no checksumming required, jump to really start the uplaod
+ startUpload();
+ } else {
+ // Calculate the checksum in a different thread first.
+ connect( &_watcher, SIGNAL(finished()),
+ this, SLOT(slotChecksumCalculated()));
+ bool haveFuture = true;
+ if( transChecksum == checkSumMD5C ) {
+ _item._checksum = checkSumMD5C;
+ _item._checksum += ":";
+ _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5Worker,filePath));
+
+ } else if( transChecksum == checkSumSHA1C ) {
+ _item._checksum = checkSumSHA1C;
+ _item._checksum += ":";
+ _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1Worker, filePath));
+ }
+#ifdef ZLIB_FOUND
+ else if( transChecksum == checkSumAdlerC) {
+ _item._checksum = checkSumAdlerC;
+ _item._checksum += ":";
+ _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32Worker, filePath));
+ }
+#endif
+ else {
+ haveFuture = false;
+ }
+ // in case there is a wrong checksum header, let continue without
+ if( !haveFuture ) {
+ startUpload();
+ }
+ }
+
+}
+
+void PropagateUploadFileQNAM::slotChecksumCalculated( )
+{
+ QByteArray checksum = _watcher.future().result();
+
+ if( !checksum.isEmpty() ) {
+ _item._checksum.append(checksum);
+ } else {
+ _item._checksum.clear();
+ }
+
+ startUpload();
+}
+
+void PropagateUploadFileQNAM::startUpload()
+{
const QString fullFilePath(_propagator->getFilePath(_item._file));
if (!FileSystem::fileExists(fullFilePath)) {
done(SyncFileItem::SoftError, tr("File Removed"));
return;
}
+ _stopWatch.addLapTime(QLatin1String("Checksum"));
+
+ // Update the mtime and size, it might have changed since discovery.
+ time_t prevModtime = _item._modtime; // the value was set in PropagateUploadFileQNAM::start()
+ // but a potential checksum calculation could have taken some time during which the file could
+ // have been changed again, so better check again here.
// Update the mtime and size, it might have changed since discovery.
_item._modtime = FileSystem::getModTime(fullFilePath);
+ if( prevModtime != _item._modtime ) {
+ _propagator->_anotherSyncNeeded = true;
+ done(SyncFileItem::SoftError, tr("Local filei changed while calculating the checksum."));
+ return;
+ }
+
quint64 fileSize = FileSystem::getSize(fullFilePath);
_item._size = fileSize;
@@ -420,6 +502,9 @@ void PropagateUploadFileQNAM::startNextChunk()
if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
+ if( !_item._checksum.isEmpty() ) {
+ headers["OC-Checksum"] = _item._checksum;
+ }
}
}
@@ -640,6 +725,11 @@ void PropagateUploadFileQNAM::slotPutFinished()
// Well, the mtime was not set
#endif
}
+
+ // performance logging
+ _item._requestDuration = _stopWatch.stop();
+ qDebug() << "*==* duration UPLOAD" << _item._size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item._requestDuration;
+
finalize(_item);
}
diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h
index 4bfbe30..1e3b6f2 100644
--- a/src/libsync/propagateupload.h
+++ b/src/libsync/propagateupload.h
@@ -19,6 +19,8 @@
#include <QBuffer>
#include <QFile>
#include <QDebug>
+#include <QFutureWatcher>
+
namespace OCC {
class BandwidthManager;
@@ -75,6 +77,8 @@ protected slots:
class PUTFileJob : public AbstractNetworkJob {
Q_OBJECT
+
+private:
QScopedPointer<QIODevice> _device;
QMap<QByteArray, QByteArray> _headers;
QString _errorString;
@@ -146,6 +150,7 @@ signals:
class PropagateUploadFileQNAM : public PropagateItemJob {
Q_OBJECT
+private:
/**
* That's the start chunk that was stored in the database for resuming.
* In the non-resuming case it is 0.
@@ -163,6 +168,12 @@ class PropagateUploadFileQNAM : public PropagateItemJob {
QElapsedTimer _duration;
QVector<PUTFileJob*> _jobs; /// network jobs that are currently in transit
bool _finished; // Tells that all the jobs have been finished
+
+ // watcher for the checksum calculation thread
+ QFutureWatcher<QByteArray> _watcher;
+
+ // measure the performance of checksum calc and upload
+ Utility::StopWatch _stopWatch;
public:
PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItem& item)
: PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {}
@@ -175,6 +186,8 @@ private slots:
void startNextChunk();
void finalize(const SyncFileItem&);
void slotJobDestroyed(QObject *job);
+ void startUpload();
+ void slotChecksumCalculated();
private:
void startPollJob(const QString& path);
void abortWithError(SyncFileItem::Status status, const QString &error);
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud-client.git
More information about the Pkg-owncloud-commits
mailing list