[Pkg-owncloud-commits] [owncloud-client] 168/484: Checksums: Improvement in async computation and validation #3735
Sandro Knauß
hefee-guest at moszumanska.debian.org
Wed Dec 16 00:37:39 UTC 2015
This is an automated email from the git hooks/post-receive script.
hefee-guest pushed a commit to branch master
in repository owncloud-client.
commit 03542897952608667569dd32793804fc1c6c4b67
Author: Christian Kamm <mail at ckamm.de>
Date: Wed Oct 14 15:03:40 2015 +0200
Checksums: Improvement in async computation and validation #3735
---
src/libsync/capabilities.cpp | 4 +-
src/libsync/capabilities.h | 2 +-
src/libsync/propagatedownload.cpp | 5 +-
src/libsync/propagateupload.cpp | 25 ++---
src/libsync/propagateupload.h | 2 +-
src/libsync/syncfileitem.h | 2 +-
src/libsync/transmissionchecksumvalidator.cpp | 149 +++++++++++++-------------
src/libsync/transmissionchecksumvalidator.h | 83 ++++++++------
test/testtranschecksumvalidator.h | 58 +++++-----
9 files changed, 174 insertions(+), 156 deletions(-)
diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp
index 2e4ef80..c6887c8 100644
--- a/src/libsync/capabilities.cpp
+++ b/src/libsync/capabilities.cpp
@@ -63,9 +63,9 @@ bool Capabilities::shareResharing() const
return _capabilities["files_sharing"].toMap()["resharing"].toBool();
}
-QStringList Capabilities::supportedChecksumTypes() const
+QList<QByteArray> Capabilities::supportedChecksumTypes() const
{
- return QStringList();
+ return QList<QByteArray>();
}
}
diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h
index 621ab48..28bcca6 100644
--- a/src/libsync/capabilities.h
+++ b/src/libsync/capabilities.h
@@ -39,7 +39,7 @@ public:
bool sharePublicLinkEnforceExpireDate() const;
int sharePublicLinkExpireDateDays() const;
bool shareResharing() const;
- QStringList supportedChecksumTypes() const;
+ QList<QByteArray> supportedChecksumTypes() const;
private:
QVariantMap _capabilities;
diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp
index efb5f66..b37e6bd 100644
--- a/src/libsync/propagatedownload.cpp
+++ b/src/libsync/propagatedownload.cpp
@@ -531,11 +531,10 @@ void PropagateDownloadFileQNAM::slotGetFinished()
// Do checksum validation for the download. If there is no checksum header, the validator
// will also emit the validated() signal to continue the flow in slot downloadFinished()
// as this is (still) also correct.
- TransmissionChecksumValidator *validator = new TransmissionChecksumValidator(_tmpFile.fileName(), this);
+ ValidateChecksumHeader *validator = new ValidateChecksumHeader(this);
connect(validator, SIGNAL(validated(QByteArray)), this, SLOT(downloadFinished()));
connect(validator, SIGNAL(validationFailed(QString)), this, SLOT(slotChecksumFail(QString)));
- validator->downloadValidation(job->reply()->rawHeader(checkSumHeaderC));
-
+ validator->start(_tmpFile.fileName(), job->reply()->rawHeader(checkSumHeaderC));
}
void PropagateDownloadFileQNAM::slotChecksumFail( const QString& errMsg )
diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp
index 88cf153..edcfc88 100644
--- a/src/libsync/propagateupload.cpp
+++ b/src/libsync/propagateupload.cpp
@@ -212,27 +212,28 @@ void PropagateUploadFileQNAM::start()
// do whatever is needed to add a checksum to the http upload request.
// in any case, the validator will emit signal startUpload to let the flow
// continue in slotStartUpload here.
- TransmissionChecksumValidator *validator = new TransmissionChecksumValidator(filePath, this);
+ auto computeChecksum = new ComputeChecksum(this);
// If the config file does not specify a checksum type but the
// server supports it choose a type based on that.
- if (validator->checksumType().isEmpty()) {
- QStringList checksumTypes = _propagator->account()->capabilities().supportedChecksumTypes();
+ if (computeChecksum->checksumType().isEmpty()) {
+ auto checksumTypes = _propagator->account()->capabilities().supportedChecksumTypes();
if (!checksumTypes.isEmpty()) {
// TODO: We might want to prefer some types over others instead
// of choosing the first.
- validator->setChecksumType(checksumTypes.first());
+ computeChecksum->setChecksumType(checksumTypes.first());
}
}
- connect(validator, SIGNAL(validated(QByteArray)), this, SLOT(slotStartUpload(QByteArray)));
- validator->uploadValidation();
+ connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
+ SLOT(slotStartUpload(QByteArray,QByteArray)));
+ computeChecksum->start(filePath);
}
-void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksum)
+void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum)
{
const QString fullFilePath = _propagator->getFilePath(_item->_file);
- _item->_checksum = checksum;
+ _item->_checksumHeader = makeChecksumHeader(checksumType, checksum);
if (!FileSystem::fileExists(fullFilePath)) {
done(SyncFileItem::SoftError, tr("File Removed"));
@@ -478,14 +479,14 @@ void PropagateUploadFileQNAM::startNextChunk()
if( currentChunkSize == 0 ) { // if the last chunk pretends to be 0, its actually the full chunk size.
currentChunkSize = chunkSize();
}
- if( !_item->_checksum.isEmpty() ) {
- headers[checkSumHeaderC] = _item->_checksum;
+ if( !_item->_checksumHeader.isEmpty() ) {
+ headers[checkSumHeaderC] = _item->_checksumHeader;
}
}
} else {
// checksum if its only one chunk
- if( !_item->_checksum.isEmpty() ) {
- headers[checkSumHeaderC] = _item->_checksum;
+ if( !_item->_checksumHeader.isEmpty() ) {
+ headers[checkSumHeaderC] = _item->_checksumHeader;
}
}
diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h
index c7a9740..9451e12 100644
--- a/src/libsync/propagateupload.h
+++ b/src/libsync/propagateupload.h
@@ -195,7 +195,7 @@ private slots:
void startNextChunk();
void finalize(const SyncFileItem&);
void slotJobDestroyed(QObject *job);
- void slotStartUpload(const QByteArray &checksum);
+ void slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum);
private:
void startPollJob(const QString& path);
diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h
index e6b1835..b6a6558 100644
--- a/src/libsync/syncfileitem.h
+++ b/src/libsync/syncfileitem.h
@@ -157,7 +157,7 @@ public:
quint64 _inode;
QByteArray _fileId;
QByteArray _remotePerm;
- QByteArray _checksum;
+ QByteArray _checksumHeader;
QString _directDownloadUrl;
QString _directDownloadCookies;
diff --git a/src/libsync/transmissionchecksumvalidator.cpp b/src/libsync/transmissionchecksumvalidator.cpp
index 43b0518..b598149 100644
--- a/src/libsync/transmissionchecksumvalidator.cpp
+++ b/src/libsync/transmissionchecksumvalidator.cpp
@@ -23,123 +23,124 @@
namespace OCC {
-TransmissionChecksumValidator::TransmissionChecksumValidator(const QString& filePath, QObject *parent)
- : QObject(parent),
- _filePath(filePath)
+QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum)
+{
+ QByteArray header = checksumType;
+ header.append(':');
+ header.append(checksum);
+ return header;
+}
+
+bool parseChecksumHeader(const QByteArray& header, QByteArray* type, QByteArray* checksum)
+{
+ if (header.isEmpty()) {
+ type->clear();
+ checksum->clear();
+ return true;
+ }
+
+ const auto idx = header.indexOf(':');
+ if (idx < 0) {
+ return false;
+ }
+
+ *type = header.left(idx);
+ *checksum = header.mid(idx + 1);
+ return true;
+}
+
+ComputeChecksum::ComputeChecksum(QObject* parent)
+ : QObject(parent)
{
// If the config file specifies a checksum type, use that.
ConfigFile cfg;
- _checksumType = cfg.transmissionChecksum();
+ _checksumType = cfg.transmissionChecksum().toLatin1();
}
-void TransmissionChecksumValidator::setChecksumType(const QString& type)
+void ComputeChecksum::setChecksumType(const QByteArray& type)
{
_checksumType = type;
}
-QString TransmissionChecksumValidator::checksumType() const
+QByteArray ComputeChecksum::checksumType() const
{
return _checksumType;
}
-void TransmissionChecksumValidator::uploadValidation()
+void ComputeChecksum::start(const QString& filePath)
{
const QString csType = checksumType();
- if( csType.isEmpty() ) {
- // if there is no checksum defined, continue to upload
- emit validated(QByteArray());
- } else {
- // Calculate the checksum in a different thread first.
-
- connect( &_watcher, SIGNAL(finished()),
- this, SLOT(slotUploadChecksumCalculated()));
- if( csType == checkSumMD5C ) {
- _checksumHeader = checkSumMD5C;
- _checksumHeader += ":";
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, _filePath));
-
- } else if( csType == checkSumSHA1C ) {
- _checksumHeader = checkSumSHA1C;
- _checksumHeader += ":";
- _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, _filePath));
- }
+ // Calculate the checksum in a different thread first.
+ connect( &_watcher, SIGNAL(finished()),
+ this, SLOT(slotCalculationDone()),
+ Qt::UniqueConnection );
+ if( csType == checkSumMD5C ) {
+ _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, filePath));
+
+ } else if( csType == checkSumSHA1C ) {
+ _watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, filePath));
+ }
#ifdef ZLIB_FOUND
- else if( csType == checkSumAdlerC) {
- _checksumHeader = checkSumAdlerC;
- _checksumHeader += ":";
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, _filePath));
- }
+ else if( csType == checkSumAdlerC) {
+ _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, filePath));
+ }
#endif
- else {
- // for an unknown checksum, continue to upload
- emit validated(QByteArray());
+ else {
+ // for an unknown checksum or no checksum, we're done right now
+ if( !csType.isEmpty() ) {
+ qDebug() << "Unknown checksum type:" << csType;
}
+ emit done(QByteArray(), QByteArray());
}
}
-void TransmissionChecksumValidator::slotUploadChecksumCalculated( )
+void ComputeChecksum::slotCalculationDone()
{
QByteArray checksum = _watcher.future().result();
+ emit done(_checksumType, checksum);
+}
- if( !checksum.isEmpty() ) {
- checksum.prepend( _checksumHeader );
- }
- emit validated(checksum);
+ValidateChecksumHeader::ValidateChecksumHeader(QObject *parent)
+ : QObject(parent)
+{
}
-
-void TransmissionChecksumValidator::downloadValidation( const QByteArray& checksumHeader )
+void ValidateChecksumHeader::start(const QString& filePath, const QByteArray& checksumHeader)
{
- // if the incoming header is empty, there was no checksum header, and
- // no validation can happen. Just continue.
+ // If the incoming header is empty no validation can happen. Just continue.
if( checksumHeader.isEmpty() ) {
emit validated(QByteArray());
return;
}
- int indx = checksumHeader.indexOf(':');
- if( indx < 0 ) {
+ if( !parseChecksumHeader(checksumHeader, &_expectedChecksumType, &_expectedChecksum) ) {
qDebug() << "Checksum header malformed:" << checksumHeader;
- emit validationFailed(tr("The checksum header is malformed.")); // show must go on - even not validated.
- return;
- }
-
- const QByteArray type = checksumHeader.left(indx).toUpper();
- _expectedHash = checksumHeader.mid(indx+1);
-
- connect( &_watcher, SIGNAL(finished()), this, SLOT(slotDownloadChecksumCalculated()) );
-
- // start the calculation in different thread
- if( type == checkSumMD5C ) {
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, _filePath));
- } else if( type == checkSumSHA1C ) {
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcSha1, _filePath));
- }
-#ifdef ZLIB_FOUND
- else if( type == checkSumAdlerUpperC ) {
- _watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, _filePath));
- }
-#endif
- else {
- qDebug() << "Unknown checksum type" << type;
emit validationFailed(tr("The checksum header is malformed."));
return;
}
+
+ auto calculator = new ComputeChecksum(this);
+ calculator->setChecksumType(_expectedChecksumType);
+ connect(calculator, SIGNAL(done(QByteArray,QByteArray)),
+ SLOT(slotChecksumCalculated(QByteArray,QByteArray)));
+ calculator->start(filePath);
}
-void TransmissionChecksumValidator::slotDownloadChecksumCalculated()
+void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray& checksumType,
+ const QByteArray& checksum)
{
- const QByteArray hash = _watcher.future().result();
-
- if( hash != _expectedHash ) {
+ if( checksumType != _expectedChecksumType ) {
+ emit validationFailed(tr("The checksum header contained an unknown checksum type '%1'").arg(
+ QString::fromLatin1(_expectedChecksumType)));
+ return;
+ }
+ if( checksum != _expectedChecksum ) {
emit validationFailed(tr("The downloaded file does not match the checksum, it will be resumed."));
- } else {
- // qDebug() << "Checksum checked and matching: " << _expectedHash;
- emit validated(hash);
+ return;
}
+ emit validated(makeChecksumHeader(checksumType, checksum));
}
-
}
diff --git a/src/libsync/transmissionchecksumvalidator.h b/src/libsync/transmissionchecksumvalidator.h
index 73f0fbd..ef1792d 100644
--- a/src/libsync/transmissionchecksumvalidator.h
+++ b/src/libsync/transmissionchecksumvalidator.h
@@ -23,62 +23,79 @@
namespace OCC {
+/// Creates a checksum header from type and value.
+QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum);
+
+/// Parses a checksum header
+bool parseChecksumHeader(const QByteArray& header, QByteArray* type, QByteArray* checksum);
+
/**
- * @brief The TransmissionChecksumValidator class
- * @ingroup libsync
+ * Computes the checksum of a file.
+ * \ingroup libsync
*/
-class OWNCLOUDSYNC_EXPORT TransmissionChecksumValidator : public QObject
+class OWNCLOUDSYNC_EXPORT ComputeChecksum : public QObject
{
Q_OBJECT
public:
- explicit TransmissionChecksumValidator(const QString& filePath, QObject *parent = 0);
+ explicit ComputeChecksum(QObject* parent = 0);
+
+ /**
+ * By default the checksum type is read from the config file, but can be overridden
+ * with this method.
+ */
+ void setChecksumType(const QByteArray& type);
+
+ QByteArray checksumType() const;
/**
- * method to prepare a checksum for transmission and save it to the _checksum
- * member of the SyncFileItem *item.
- * The kind of requested checksum is taken from config. No need to set from outside.
+ * Computes the checksum for the given file path.
*
- * In any case of processing (checksum set, no checksum required and also unusual error)
- * the object will emit the signal validated(). The item->_checksum is than either
- * set to a proper value or empty.
+ * done() is emitted when the calculation finishes.
*/
- void uploadValidation();
+ void start(const QString& filePath);
+
+signals:
+ void done(const QByteArray& checksumType, const QByteArray& checksum);
+
+private slots:
+ void slotCalculationDone();
+
+private:
+ QByteArray _checksumType;
+
+ // watcher for the checksum calculation thread
+ QFutureWatcher<QByteArray> _watcher;
+};
+
+/**
+ * Checks whether a file's checksum matches the expected value.
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT ValidateChecksumHeader : public QObject
+{
+ Q_OBJECT
+public:
+ explicit ValidateChecksumHeader(QObject *parent = 0);
/**
- * method to verify the checksum coming with requests in a checksum header. The required
- * checksum method is read from config.
+ * Check a file's actual checksum against the provided checksumHeader
*
* If no checksum is there, or if a correct checksum is there, the signal validated()
* will be emitted. In case of any kind of error, the signal validationFailed() will
* be emitted.
*/
- void downloadValidation( const QByteArray& checksumHeader );
-
- /**
- * By default the checksum type is read from the config file, but can be overridden
- * with this method.
- */
- void setChecksumType(const QString& type);
-
- QString checksumType() const;
+ void start(const QString& filePath, const QByteArray& checksumHeader);
signals:
- void validated(const QByteArray& checksum);
+ void validated(const QByteArray& checksumHeader);
void validationFailed( const QString& errMsg );
private slots:
- void slotUploadChecksumCalculated();
- void slotDownloadChecksumCalculated();
+ void slotChecksumCalculated(const QByteArray& checksumType, const QByteArray& checksum);
private:
- QString _checksumType;
- QByteArray _expectedHash;
- QByteArray _checksumHeader;
-
- QString _filePath;
-
- // watcher for the checksum calculation thread
- QFutureWatcher<QByteArray> _watcher;
+ QByteArray _expectedChecksumType;
+ QByteArray _expectedChecksum;
};
}
diff --git a/test/testtranschecksumvalidator.h b/test/testtranschecksumvalidator.h
index e815c90..2e08333 100644
--- a/test/testtranschecksumvalidator.h
+++ b/test/testtranschecksumvalidator.h
@@ -33,14 +33,16 @@ using namespace OCC;
QString _testfile;
QString _expectedError;
QByteArray _expected;
+ QByteArray _expectedType;
bool _successDown;
bool _errorSeen;
public slots:
- void slotUpValidated(const QByteArray& checksum) {
+ void slotUpValidated(const QByteArray& type, const QByteArray& checksum) {
qDebug() << "Checksum: " << checksum;
QVERIFY(_expected == checksum );
+ QVERIFY(_expectedType == type );
}
void slotDownValidated() {
@@ -62,23 +64,22 @@ using namespace OCC;
rootDir.mkpath(_root );
_testfile = _root+"/csFile";
Utility::writeRandomFile( _testfile);
-
}
void testUploadChecksummingAdler() {
- TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this);
- vali->setChecksumType("Adler32");
+ ComputeChecksum *vali = new ComputeChecksum(this);
+ _expectedType = "Adler32";
+ vali->setChecksumType(_expectedType);
- connect(vali, SIGNAL(validated(QByteArray)), this, SLOT(slotUpValidated(QByteArray)));
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), SLOT(slotUpValidated(QByteArray,QByteArray)));
- QString testfile = _testfile;
- _expected = "Adler32:"+FileSystem::calcAdler32( testfile );
+ _expected = FileSystem::calcAdler32( _testfile );
qDebug() << "XX Expected Checksum: " << _expected;
- vali->uploadValidation();
+ vali->start(_testfile);
QEventLoop loop;
- connect(vali, SIGNAL(validated(QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
loop.exec();
delete vali;
@@ -86,16 +87,16 @@ using namespace OCC;
void testUploadChecksummingMd5() {
- TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this);
- vali->setChecksumType( OCC::checkSumMD5C );
- connect(vali, SIGNAL(validated(QByteArray)), this, SLOT(slotUpValidated(QByteArray)));
+ ComputeChecksum *vali = new ComputeChecksum(this);
+ _expectedType = OCC::checkSumMD5C;
+ vali->setChecksumType(_expectedType);
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
- _expected = checkSumMD5C;
- _expected.append(":"+FileSystem::calcMd5( _testfile ));
- vali->uploadValidation();
+ _expected = FileSystem::calcMd5( _testfile );
+ vali->start(_testfile);
QEventLoop loop;
- connect(vali, SIGNAL(validated(QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
loop.exec();
delete vali;
@@ -103,17 +104,17 @@ using namespace OCC;
void testUploadChecksummingSha1() {
- TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this);
- vali->setChecksumType( OCC::checkSumSHA1C );
- connect(vali, SIGNAL(validated(QByteArray)), this, SLOT(slotUpValidated(QByteArray)));
+ ComputeChecksum *vali = new ComputeChecksum(this);
+ _expectedType = OCC::checkSumSHA1C;
+ vali->setChecksumType(_expectedType);
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), this, SLOT(slotUpValidated(QByteArray,QByteArray)));
- _expected = checkSumSHA1C;
- _expected.append(":"+FileSystem::calcSha1( _testfile ));
+ _expected = FileSystem::calcSha1( _testfile );
- vali->uploadValidation();
+ vali->start(_testfile);
QEventLoop loop;
- connect(vali, SIGNAL(validated(QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
+ connect(vali, SIGNAL(done(QByteArray,QByteArray)), &loop, SLOT(quit()), Qt::QueuedConnection);
loop.exec();
delete vali;
@@ -126,22 +127,21 @@ using namespace OCC;
adler.append(FileSystem::calcAdler32( _testfile ));
_successDown = false;
- TransmissionChecksumValidator *vali = new TransmissionChecksumValidator(_testfile, this);
- vali->setChecksumType("Adler32");
+ ValidateChecksumHeader *vali = new ValidateChecksumHeader(this);
connect(vali, SIGNAL(validated(QByteArray)), this, SLOT(slotDownValidated()));
connect(vali, SIGNAL(validationFailed(QString)), this, SLOT(slotDownError(QString)));
- vali->downloadValidation(adler);
+ vali->start(_testfile, adler);
QTRY_VERIFY(_successDown);
_expectedError = QLatin1String("The downloaded file does not match the checksum, it will be resumed.");
_errorSeen = false;
- vali->downloadValidation("Adler32:543345");
+ vali->start(_testfile, "Adler32:543345");
QTRY_VERIFY(_errorSeen);
- _expectedError = QLatin1String("The checksum header is malformed.");
+ _expectedError = QLatin1String("The checksum header contained an unknown checksum type 'Klaas32'");
_errorSeen = false;
- vali->downloadValidation("Klaas32:543345");
+ vali->start(_testfile, "Klaas32:543345");
QTRY_VERIFY(_errorSeen);
delete vali;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud-client.git
More information about the Pkg-owncloud-commits
mailing list