[Pkg-owncloud-commits] [owncloud-client] 67/219: Clean up stale journal entries and temporaries. #2057
Sandro Knauß
hefee-guest at moszumanska.debian.org
Sat Oct 11 14:43:11 UTC 2014
This is an automated email from the git hooks/post-receive script.
hefee-guest pushed a commit to branch master
in repository owncloud-client.
commit 5d36a278932c869d69302bb279a2c7c3adb28695
Author: Christian Kamm <kamm at incasoftware.de>
Date: Wed Sep 3 12:11:03 2014 +0200
Clean up stale journal entries and temporaries. #2057
* Downloadinfo entries for files that no longer need to be downloaded
are useless and can be removed. In particular, the temporary files
holding partially retrieved files are now deleted when no longer
necessary.
* The same is true for blacklist entries for paths that are no longer
being discovered.
* Same for uploadinfos for files that no longer need to be uploaded.
---
src/mirall/owncloudpropagator.cpp | 5 ++
src/mirall/owncloudpropagator.h | 1 +
src/mirall/propagator_legacy.cpp | 10 +--
src/mirall/propagator_qnam.cpp | 12 ++--
src/mirall/syncengine.cpp | 56 +++++++++++++++++
src/mirall/syncengine.h | 10 +++
src/mirall/syncjournaldb.cpp | 129 ++++++++++++++++++++++++++++++++++++--
src/mirall/syncjournaldb.h | 6 ++
8 files changed, 213 insertions(+), 16 deletions(-)
diff --git a/src/mirall/owncloudpropagator.cpp b/src/mirall/owncloudpropagator.cpp
index 46985c4..dd0a859 100644
--- a/src/mirall/owncloudpropagator.cpp
+++ b/src/mirall/owncloudpropagator.cpp
@@ -399,6 +399,11 @@ bool OwncloudPropagator::localFileNameClash( const QString& relFile )
return re;
}
+QString OwncloudPropagator::getFilePath(const QString& tmp_file_name) const
+{
+ return _localDir + tmp_file_name;
+}
+
// ================================================================================
void PropagateDirectory::start()
diff --git a/src/mirall/owncloudpropagator.h b/src/mirall/owncloudpropagator.h
index cfec665..185ee60 100644
--- a/src/mirall/owncloudpropagator.h
+++ b/src/mirall/owncloudpropagator.h
@@ -220,6 +220,7 @@ public:
bool isInSharedDirectory(const QString& file);
bool localFileNameClash(const QString& relfile);
+ QString getFilePath(const QString& tmp_file_name) const;
void abort() {
_abortRequested.fetchAndStoreOrdered(true);
diff --git a/src/mirall/propagator_legacy.cpp b/src/mirall/propagator_legacy.cpp
index e1750f9..d2a243c 100644
--- a/src/mirall/propagator_legacy.cpp
+++ b/src/mirall/propagator_legacy.cpp
@@ -62,7 +62,7 @@ void PropagateUploadFileLegacy::start()
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
- QFile file(_propagator->_localDir + _item._file);
+ QFile file(_propagator->getFilePath(_item._file));
if (!file.open(QIODevice::ReadOnly)) {
done(SyncFileItem::NormalError, file.errorString());
return;
@@ -189,7 +189,7 @@ void PropagateUploadFileLegacy::start()
return;
}
- _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->_localDir + _item._file));
+ _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->getFilePath(_item._file)));
// Remove from the progress database:
_propagator->_journal->setUploadInfo(_item._file, SyncJournalDb::UploadInfo());
_propagator->_journal->commit("upload file start");
@@ -498,7 +498,7 @@ void PropagateDownloadFileLegacy::start()
if (progressInfo._valid) {
// if the etag has changed meanwhile, remove the already downloaded part.
if (progressInfo._etag != _item._etag) {
- QFile::remove(_propagator->_localDir + progressInfo._tmpfile);
+ QFile::remove(_propagator->getFilePath(progressInfo._tmpfile));
_propagator->_journal->setDownloadInfo(_item._file, SyncJournalDb::DownloadInfo());
} else {
tmpFileName = progressInfo._tmpfile;
@@ -516,7 +516,7 @@ void PropagateDownloadFileLegacy::start()
tmpFileName += ".~" + QString::number(uint(qrand()), 16);
}
- QFile tmpFile(_propagator->_localDir + tmpFileName);
+ QFile tmpFile(_propagator->getFilePath(tmpFileName));
_file = &tmpFile;
if (!tmpFile.open(QIODevice::Append | QIODevice::Unbuffered)) {
done(SyncFileItem::NormalError, tmpFile.errorString());
@@ -610,7 +610,7 @@ void PropagateDownloadFileLegacy::start()
tmpFile.close();
tmpFile.flush();
- QString fn = _propagator->_localDir + _item._file;
+ QString fn = _propagator->getFilePath(_item._file);
bool isConflict = _item._instruction == CSYNC_INSTRUCTION_CONFLICT
diff --git a/src/mirall/propagator_qnam.cpp b/src/mirall/propagator_qnam.cpp
index 68016fa..022d585 100644
--- a/src/mirall/propagator_qnam.cpp
+++ b/src/mirall/propagator_qnam.cpp
@@ -97,7 +97,7 @@ void PropagateUploadFileQNAM::start()
if (_propagator->_abortRequested.fetchAndAddRelaxed(0))
return;
- _file = new QFile(_propagator->_localDir + _item._file, this);
+ _file = new QFile(_propagator->getFilePath(_item._file), this);
if (!_file->open(QIODevice::ReadOnly)) {
done(SyncFileItem::NormalError, _file->errorString());
delete _file;
@@ -307,7 +307,7 @@ void PropagateUploadFileQNAM::slotPutFinished()
|| job->reply()->hasRawHeader("OC-ETag");
if (!finished) {
- QFileInfo fi(_propagator->_localDir + _item._file);
+ QFileInfo fi(_propagator->getFilePath(_item._file));
if( !fi.exists() ) {
_propagator->_activeJobs--;
done(SyncFileItem::SoftError, tr("The local file was removed during sync."));
@@ -381,7 +381,7 @@ void PropagateUploadFileQNAM::finalize(const SyncFileItem ©)
_item._requestDuration = _duration.elapsed();
- _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->_localDir + _item._file));
+ _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->getFilePath(_item._file)));
// Remove from the progress database:
_propagator->_journal->setUploadInfo(_item._file, SyncJournalDb::UploadInfo());
_propagator->_journal->commit("upload file start");
@@ -569,7 +569,7 @@ void PropagateDownloadFileQNAM::start()
if (progressInfo._valid) {
// if the etag has changed meanwhile, remove the already downloaded part.
if (progressInfo._etag != _item._etag) {
- QFile::remove(_propagator->_localDir + progressInfo._tmpfile);
+ QFile::remove(_propagator->getFilePath(progressInfo._tmpfile));
_propagator->_journal->setDownloadInfo(_item._file, SyncJournalDb::DownloadInfo());
} else {
tmpFileName = progressInfo._tmpfile;
@@ -587,7 +587,7 @@ void PropagateDownloadFileQNAM::start()
tmpFileName += ".~" + QString::number(uint(qrand()), 16);
}
- _tmpFile.setFileName(_propagator->_localDir + tmpFileName);
+ _tmpFile.setFileName(_propagator->getFilePath(tmpFileName));
if (!_tmpFile.open(QIODevice::Append | QIODevice::Unbuffered)) {
done(SyncFileItem::NormalError, _tmpFile.errorString());
return;
@@ -711,7 +711,7 @@ QString makeConflictFileName(const QString &fn, const QDateTime &dt)
void PropagateDownloadFileQNAM::downloadFinished()
{
- QString fn = _propagator->_localDir + _item._file;
+ QString fn = _propagator->getFilePath(_item._file);
bool isConflict = _item._instruction == CSYNC_INSTRUCTION_CONFLICT
diff --git a/src/mirall/syncengine.cpp b/src/mirall/syncengine.cpp
index d282df4..05d2f6e 100644
--- a/src/mirall/syncengine.cpp
+++ b/src/mirall/syncengine.cpp
@@ -244,6 +244,57 @@ bool SyncEngine::checkBlacklisting( SyncFileItem *item )
return re;
}
+void SyncEngine::deleteStaleDownloadInfos()
+{
+ // Find all downloadinfo paths that we want to preserve.
+ QSet<QString> download_file_paths;
+ foreach(const SyncFileItem& it, _syncedItems) {
+ if (it._direction == SyncFileItem::Down
+ && it._type == SyncFileItem::File)
+ {
+ download_file_paths.insert(it._file);
+ }
+ }
+
+ // Delete from journal and from filesystem.
+ const QVector<SyncJournalDb::DownloadInfo> deleted_infos =
+ _journal->getAndDeleteStaleDownloadInfos(download_file_paths);
+ foreach (const SyncJournalDb::DownloadInfo & deleted_info, deleted_infos) {
+ const QString tmppath = _propagator->getFilePath(deleted_info._tmpfile);
+ qDebug() << "Deleting stale temporary file: " << tmppath;
+ QFile::remove(tmppath);
+ }
+}
+
+void SyncEngine::deleteStaleUploadInfos()
+{
+ // Find all blacklisted paths that we want to preserve.
+ QSet<QString> upload_file_paths;
+ foreach(const SyncFileItem& it, _syncedItems) {
+ if (it._direction == SyncFileItem::Up
+ && it._type == SyncFileItem::File)
+ {
+ upload_file_paths.insert(it._file);
+ }
+ }
+
+ // Delete from journal.
+ _journal->deleteStaleUploadInfos(upload_file_paths);
+}
+
+void SyncEngine::deleteStaleBlacklistEntries()
+{
+ // Find all blacklisted paths that we want to preserve.
+ QSet<QString> blacklist_file_paths;
+ foreach(const SyncFileItem& it, _syncedItems) {
+ if (it._status == SyncFileItem::FileIgnored)
+ blacklist_file_paths.insert(it._file);
+ }
+
+ // Delete from journal.
+ _journal->deleteStaleBlacklistEntries(blacklist_file_paths);
+}
+
int SyncEngine::treewalkLocal( TREE_WALK_FILE* file, void *data )
{
return static_cast<SyncEngine*>(data)->treewalkFile( file, false );
@@ -647,6 +698,11 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
// apply the network limits to the propagator
setNetworkLimits(_uploadLimit, _downloadLimit);
+ deleteStaleDownloadInfos();
+ deleteStaleUploadInfos();
+ deleteStaleBlacklistEntries();
+ _journal->commit("post stale entry removal");
+
_propagator->start(_syncedItems);
}
diff --git a/src/mirall/syncengine.h b/src/mirall/syncengine.h
index a6e9c63..d1720c2 100644
--- a/src/mirall/syncengine.h
+++ b/src/mirall/syncengine.h
@@ -107,6 +107,16 @@ private:
int treewalkFile( TREE_WALK_FILE*, bool );
bool checkBlacklisting( SyncFileItem *item );
+ // Cleans up unnecessary downloadinfo entries in the journal as well
+ // as their temporary files.
+ void deleteStaleDownloadInfos();
+
+ // Removes stale uploadinfos from the journal.
+ void deleteStaleUploadInfos();
+
+ // Removes stale blacklist entries from the journal.
+ void deleteStaleBlacklistEntries();
+
// cleanup and emit the finished signal
void finalize();
diff --git a/src/mirall/syncjournaldb.cpp b/src/mirall/syncjournaldb.cpp
index 211792f..b41cf80 100644
--- a/src/mirall/syncjournaldb.cpp
+++ b/src/mirall/syncjournaldb.cpp
@@ -608,6 +608,32 @@ int SyncJournalDb::getFileRecordCount()
return 0;
}
+static void toDownloadInfo(const QSqlQuery & query, SyncJournalDb::DownloadInfo * res)
+{
+ bool ok = true;
+ res->_tmpfile = query.value(0).toString();
+ res->_etag = query.value(1).toByteArray();
+ res->_errorCount = query.value(2).toInt(&ok);
+ res->_valid = ok;
+}
+
+static bool deleteBatch(QSqlQuery & query, const QStringList & entries, const QString & name)
+{
+ if (entries.isEmpty())
+ return true;
+
+ qDebug() << "Removing stale " << qPrintable(name) << " entries: " << entries.join(", ");
+ query.bindValue(0, entries);
+ if (!query.execBatch()) {
+ QString err = query.lastError().text();
+ qDebug() << "Error removing stale " << qPrintable(name) << " entries: "
+ << query.lastQuery() << ", Error:" << err;
+ return false;
+ }
+ query.finish();
+ return true;
+}
+
SyncJournalDb::DownloadInfo SyncJournalDb::getDownloadInfo(const QString& file)
{
QMutexLocker locker(&_mutex);
@@ -624,11 +650,7 @@ SyncJournalDb::DownloadInfo SyncJournalDb::getDownloadInfo(const QString& file)
}
if( _getDownloadInfoQuery->next() ) {
- bool ok = true;
- res._tmpfile = _getDownloadInfoQuery->value(0).toString();
- res._etag = _getDownloadInfoQuery->value(1).toByteArray();
- res._errorCount = _getDownloadInfoQuery->value(2).toInt(&ok);
- res._valid = ok;
+ toDownloadInfo(*_getDownloadInfoQuery, &res);
}
_getDownloadInfoQuery->finish();
}
@@ -669,6 +691,43 @@ void SyncJournalDb::setDownloadInfo(const QString& file, const SyncJournalDb::Do
}
}
+QVector<SyncJournalDb::DownloadInfo> SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet<QString>& keep)
+{
+ QMutexLocker locker(&_mutex);
+
+ if (!checkConnect()) {
+ return {};
+ }
+
+ QSqlQuery query(_db);
+ // The selected values *must* match the ones expected by toDownloadInfo().
+ query.prepare("SELECT tmpfile, etag, errorcount, path FROM downloadinfo");
+
+ if (!query.exec()) {
+ QString err = query.lastError().text();
+ qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
+ return {};
+ }
+
+ QStringList superfluousPaths;
+ QVector<SyncJournalDb::DownloadInfo> deleted_entries;
+
+ while (query.next()) {
+ const QString file = query.value(3).toString(); // path
+ if (!keep.contains(file)) {
+ superfluousPaths.append(file);
+ DownloadInfo info;
+ toDownloadInfo(query, &info);
+ deleted_entries.append(info);
+ }
+ }
+
+ if (!deleteBatch(*_deleteDownloadInfoQuery, superfluousPaths, "downloadinfo"))
+ return {};
+
+ return deleted_entries;
+}
+
SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString& file)
{
QMutexLocker locker(&_mutex);
@@ -734,6 +793,35 @@ void SyncJournalDb::setUploadInfo(const QString& file, const SyncJournalDb::Uplo
}
}
+bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
+{
+ QMutexLocker locker(&_mutex);
+
+ if (!checkConnect()) {
+ return false;
+ }
+
+ QSqlQuery query(_db);
+ query.prepare("SELECT path FROM uploadinfo");
+
+ if (!query.exec()) {
+ QString err = query.lastError().text();
+ qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
+ return false;
+ }
+
+ QStringList superfluousPaths;
+
+ while (query.next()) {
+ const QString file = query.value(0).toString();
+ if (!keep.contains(file)) {
+ superfluousPaths.append(file);
+ }
+ }
+
+ return deleteBatch(*_deleteUploadInfoQuery, superfluousPaths, "uploadinfo");
+}
+
SyncJournalBlacklistRecord SyncJournalDb::blacklistEntry( const QString& file )
{
QMutexLocker locker(&_mutex);
@@ -764,6 +852,37 @@ SyncJournalBlacklistRecord SyncJournalDb::blacklistEntry( const QString& file )
return entry;
}
+bool SyncJournalDb::deleteStaleBlacklistEntries(const QSet<QString> &keep)
+{
+ QMutexLocker locker(&_mutex);
+
+ if (!checkConnect()) {
+ return false;
+ }
+
+ QSqlQuery query(_db);
+ query.prepare("SELECT path FROM blacklist");
+
+ if (!query.exec()) {
+ QString err = query.lastError().text();
+ qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
+ return false;
+ }
+
+ QStringList superfluousPaths;
+
+ while (query.next()) {
+ const QString file = query.value(0).toString();
+ if (!keep.contains(file)) {
+ superfluousPaths.append(file);
+ }
+ }
+
+ QSqlQuery delQuery(_db);
+ delQuery.prepare("DELETE FROM blacklist WHERE path = ?");
+ return deleteBatch(delQuery, superfluousPaths, "blacklist");
+}
+
int SyncJournalDb::blackListEntryCount()
{
int re = 0;
diff --git a/src/mirall/syncjournaldb.h b/src/mirall/syncjournaldb.h
index 1f12a1c..2660ec7 100644
--- a/src/mirall/syncjournaldb.h
+++ b/src/mirall/syncjournaldb.h
@@ -68,9 +68,15 @@ public:
DownloadInfo getDownloadInfo(const QString &file);
void setDownloadInfo(const QString &file, const DownloadInfo &i);
+ QVector<DownloadInfo> getAndDeleteStaleDownloadInfos(const QSet<QString>& keep);
+
UploadInfo getUploadInfo(const QString &file);
void setUploadInfo(const QString &file, const UploadInfo &i);
+ bool deleteStaleUploadInfos(const QSet<QString>& keep);
+
SyncJournalBlacklistRecord blacklistEntry( const QString& );
+ bool deleteStaleBlacklistEntries(const QSet<QString>& keep);
+
void avoidRenamesOnNextSync(const QString &path);
/**
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud-client.git
More information about the Pkg-owncloud-commits
mailing list