[Pkg-owncloud-commits] [owncloud-client] 37/498: Time estimate: Refactor remaining time guess. #2328
Sandro Knauß
hefee-guest at moszumanska.debian.org
Tue Aug 11 14:48:31 UTC 2015
This is an automated email from the git hooks/post-receive script.
hefee-guest pushed a commit to branch master
in repository owncloud-client.
commit 96ecdb866d0142619fbf17274d6b71cc2e91076e
Author: Christian Kamm <kamm at incasoftware.de>
Date: Fri Jan 30 13:36:20 2015 +0100
Time estimate: Refactor remaining time guess. #2328
---
src/cmd/cmd.cpp | 2 +-
src/gui/accountsettings.cpp | 25 ++--
src/gui/accountsettings.h | 2 +-
src/gui/application.cpp | 2 -
src/gui/folder.cpp | 10 +-
src/gui/folder.h | 2 +-
src/gui/owncloudgui.cpp | 22 +--
src/gui/owncloudgui.h | 2 +-
src/gui/protocolwidget.cpp | 12 +-
src/gui/protocolwidget.h | 2 +-
src/gui/settingsdialog.cpp | 4 +-
src/libsync/progressdispatcher.cpp | 191 +++++++++++++++++++++++++-
src/libsync/progressdispatcher.h | 275 ++++++++++++++++++++-----------------
src/libsync/syncengine.cpp | 31 ++---
src/libsync/syncengine.h | 4 +-
15 files changed, 391 insertions(+), 195 deletions(-)
diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp
index 55d924b..486cef3 100644
--- a/src/cmd/cmd.cpp
+++ b/src/cmd/cmd.cpp
@@ -471,7 +471,7 @@ restart_sync:
SyncEngine engine(account, _csync_ctx, options.source_dir, QUrl(options.target_url).path(), folder, &db);
QObject::connect(&engine, SIGNAL(finished()), &app, SLOT(quit()));
- QObject::connect(&engine, SIGNAL(transmissionProgress(Progress::Info)), &cmd, SLOT(transmissionProgressSlot()));
+ QObject::connect(&engine, SIGNAL(transmissionProgress(ProgressInfo)), &cmd, SLOT(transmissionProgressSlot()));
engine.setSelectiveSyncBlackList(selectiveSyncList);
diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp
index ac195cc..4a7baf9 100644
--- a/src/gui/accountsettings.cpp
+++ b/src/gui/accountsettings.cpp
@@ -593,7 +593,7 @@ QString AccountSettings::shortenFilename( const QString& folder, const QString&
return shortFile;
}
-void AccountSettings::slotSetProgress(const QString& folder, const Progress::Info &progress )
+void AccountSettings::slotSetProgress(const QString& folder, const ProgressInfo &progress )
{
if (!isVisible()) {
return; // for https://github.com/owncloud/client/issues/2648#issuecomment-71377909
@@ -630,10 +630,10 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
SyncFileItem curItem = progress._lastCompletedItem;
qint64 curItemProgress = -1; // -1 means finished
quint64 biggerItemSize = -1;
- foreach(const Progress::Info::ProgressItem &citm, progress._currentItems) {
- if (curItemProgress == -1 || (Progress::isSizeDependent(citm._item)
+ foreach(const ProgressInfo::ProgressItem &citm, progress._currentItems) {
+ if (curItemProgress == -1 || (ProgressInfo::isSizeDependent(citm._item)
&& biggerItemSize < citm._item._size)) {
- curItemProgress = citm._completedSize;
+ curItemProgress = citm._progress.completed();
curItem = citm._item;
biggerItemSize = citm._item._size;
}
@@ -648,15 +648,15 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
QString fileProgressString;
- if (Progress::isSizeDependent(curItem)) {
+ if (ProgressInfo::isSizeDependent(curItem)) {
QString s1 = Utility::octetsToString( curItemProgress );
QString s2 = Utility::octetsToString( curItem._size );
- quint64 estimatedBw = progress.getFileEstimate(curItem).getEstimatedBandwidth();
+ quint64 estimatedBw = progress.fileProgress(curItem).estimatedBandwidth;
if (estimatedBw) {
//: Example text: "uploading foobar.png (1MB of 2MB) time left 2 minutes at a rate of 24Kb/s"
fileProgressString = tr("%1 %2 (%3 of %4) %5 left at a rate of %6/s")
.arg(kindString, itemFileName, s1, s2,
- Utility::timeToDescriptiveString(progress.getFileEstimate(curItem).getEtaEstimate(), 3, " ", true),
+ Utility::timeToDescriptiveString(progress.fileProgress(curItem).estimatedEta, 3, " ", true),
Utility::octetsToString(estimatedBw) );
} else {
//: Example text: "uploading foobar.png (2MB of 2MB)"
@@ -670,11 +670,12 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
// overall progress
quint64 completedSize = progress.completedSize();
- quint64 currentFile = progress._completedFileCount + progress._currentItems.count();
+ quint64 completedFile = progress.completedFiles();
+ quint64 currentFile = progress.currentFile();
if (currentFile == ULLONG_MAX)
currentFile = 0;
- quint64 totalSize = qMax(completedSize, progress._totalSize);
- quint64 totalFileCount = qMax(currentFile, progress._totalFileCount);
+ quint64 totalSize = qMax(completedSize, progress.totalSize());
+ quint64 totalFileCount = qMax(currentFile, progress.totalFiles());
QString overallSyncString;
if (totalSize > 0) {
QString s1 = Utility::octetsToString( completedSize );
@@ -682,7 +683,7 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
overallSyncString = tr("%1 of %2, file %3 of %4\nTotal time left %5")
.arg(s1, s2)
.arg(currentFile).arg(totalFileCount)
- .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 3, " ", true) );
+ .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 3, " ", true) );
} else if (totalFileCount > 0) {
// Don't attemt to estimate the time left if there is no kb to transfer.
overallSyncString = tr("file %1 of %2") .arg(currentFile).arg(totalFileCount);
@@ -693,7 +694,7 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
int overallPercent = 0;
if( totalFileCount > 0 ) {
// Add one 'byte' for each files so the percentage is moving when deleting or renaming files
- overallPercent = qRound(double(completedSize + progress._completedFileCount)/double(totalSize + totalFileCount) * 100.0);
+ overallPercent = qRound(double(completedSize + completedFile)/double(totalSize + totalFileCount) * 100.0);
}
overallPercent = qBound(0, overallPercent, 100);
item->setData( overallPercent, FolderStatusDelegate::SyncProgressOverallPercent);
diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h
index 46b6fd9..a6811eb 100644
--- a/src/gui/accountsettings.h
+++ b/src/gui/accountsettings.h
@@ -61,7 +61,7 @@ public slots:
void slotOpenOC();
void slotUpdateFolderState( Folder* );
void slotDoubleClicked( const QModelIndex& );
- void slotSetProgress(const QString& folder, const Progress::Info& progress);
+ void slotSetProgress(const QString& folder, const ProgressInfo& progress);
void slotButtonsSetEnabled();
void slotUpdateQuota( qint64,qint64 );
diff --git a/src/gui/application.cpp b/src/gui/application.cpp
index 946384e..5898d76 100644
--- a/src/gui/application.cpp
+++ b/src/gui/application.cpp
@@ -136,8 +136,6 @@ Application::Application(int &argc, char **argv) :
setQuitOnLastWindowClosed(false);
- qRegisterMetaType<Progress::Info>("Progress::Info");
-
ConfigFile cfg;
_theme->setSystrayUseMonoIcons(cfg.monoIcons());
connect (_theme, SIGNAL(systrayUseMonoIconsChanged(bool)), SLOT(slotUseMonoIconsChanged(bool)));
diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp
index 96713d0..0e55b8c 100644
--- a/src/gui/folder.cpp
+++ b/src/gui/folder.cpp
@@ -804,7 +804,7 @@ void Folder::startSync(const QStringList &pathList)
connect(_engine.data(), SIGNAL(aboutToRemoveAllFiles(SyncFileItem::Direction,bool*)),
SLOT(slotAboutToRemoveAllFiles(SyncFileItem::Direction,bool*)));
connect(_engine.data(), SIGNAL(folderDiscovered(bool,QString)), this, SLOT(slotFolderDiscovered(bool,QString)));
- connect(_engine.data(), SIGNAL(transmissionProgress(Progress::Info)), this, SLOT(slotTransmissionProgress(Progress::Info)));
+ connect(_engine.data(), SIGNAL(transmissionProgress(ProgressInfo)), this, SLOT(slotTransmissionProgress(ProgressInfo)));
connect(_engine.data(), SIGNAL(jobCompleted(const SyncFileItem &)), this, SLOT(slotJobCompleted(const SyncFileItem &)));
connect(_engine.data(), SIGNAL(syncItemDiscovered(const SyncFileItem &)), this, SLOT(slotSyncItemDiscovered(const SyncFileItem &)));
@@ -959,7 +959,7 @@ void Folder::slotEmitFinishedDelayed()
void Folder::slotFolderDiscovered(bool, QString folderName)
{
- Progress::Info pi;
+ ProgressInfo pi;
pi._currentDiscoveredFolder = folderName;
ProgressDispatcher::instance()->setProgressInfo(alias(), pi);
}
@@ -967,10 +967,10 @@ void Folder::slotFolderDiscovered(bool, QString folderName)
// the progress comes without a folder and the valid path set. Add that here
// and hand the result over to the progress dispatcher.
-void Folder::slotTransmissionProgress(const Progress::Info &pi)
+void Folder::slotTransmissionProgress(const ProgressInfo &pi)
{
- if( pi._completedFileCount ) {
- // No job completed yet, this is the beginning of a sync, set the warning level to 0
+ if( !pi.hasStarted() ) {
+ // this is the beginning of a sync, set the warning level to 0
_syncResult.setWarnCount(0);
}
ProgressDispatcher::instance()->setProgressInfo(alias(), pi);
diff --git a/src/gui/folder.h b/src/gui/folder.h
index f07be95..ae29973 100644
--- a/src/gui/folder.h
+++ b/src/gui/folder.h
@@ -181,7 +181,7 @@ private slots:
void slotSyncFinished();
void slotFolderDiscovered(bool local, QString folderName);
- void slotTransmissionProgress(const Progress::Info& pi);
+ void slotTransmissionProgress(const ProgressInfo& pi);
void slotJobCompleted(const SyncFileItem&);
void slotSyncItemDiscovered(const SyncFileItem & item);
diff --git a/src/gui/owncloudgui.cpp b/src/gui/owncloudgui.cpp
index 48ec2b5..63bccad 100644
--- a/src/gui/owncloudgui.cpp
+++ b/src/gui/owncloudgui.cpp
@@ -79,8 +79,8 @@ ownCloudGui::ownCloudGui(Application *parent) :
this, SLOT(slotOpenPath(QString)));
ProgressDispatcher *pd = ProgressDispatcher::instance();
- connect( pd, SIGNAL(progressInfo(QString,Progress::Info)), this,
- SLOT(slotUpdateProgress(QString,Progress::Info)) );
+ connect( pd, SIGNAL(progressInfo(QString,ProgressInfo)), this,
+ SLOT(slotUpdateProgress(QString,ProgressInfo)) );
FolderMan *folderMan = FolderMan::instance();
connect( folderMan, SIGNAL(folderSyncStateChange(QString)),
@@ -469,24 +469,24 @@ void ownCloudGui::slotRebuildRecentMenus()
}
-void ownCloudGui::slotUpdateProgress(const QString &folder, const Progress::Info& progress)
+void ownCloudGui::slotUpdateProgress(const QString &folder, const ProgressInfo& progress)
{
Q_UNUSED(folder);
if (!progress._currentDiscoveredFolder.isEmpty()) {
_actionStatus->setText( tr("Discovering '%1'")
.arg( progress._currentDiscoveredFolder ));
- } else if (progress._totalSize == 0 ) {
- quint64 currentFile = progress._completedFileCount + progress._currentItems.count();
- quint64 totalFileCount = qMax(progress._totalFileCount, currentFile);
+ } else if (progress.totalSize() == 0 ) {
+ quint64 currentFile = progress.currentFile();
+ quint64 totalFileCount = qMax(progress.totalFiles(), currentFile);
_actionStatus->setText( tr("Syncing %1 of %2 (%3 left)")
.arg( currentFile ).arg( totalFileCount )
- .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 2, " ",true) ) );
+ .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 2, " ",true) ) );
} else {
- QString totalSizeStr = Utility::octetsToString( progress._totalSize );
+ QString totalSizeStr = Utility::octetsToString( progress.totalSize() );
_actionStatus->setText( tr("Syncing %1 (%2 left)")
.arg( totalSizeStr )
- .arg( Utility::timeToDescriptiveString(progress.totalEstimate().getEtaEstimate(), 2, " ",true) ) );
+ .arg( Utility::timeToDescriptiveString(progress.totalProgress().estimatedEta, 2, " ",true) ) );
}
@@ -524,8 +524,8 @@ void ownCloudGui::slotUpdateProgress(const QString &folder, const Progress::Info
slotRebuildRecentMenus();
}
- if (progress._completedFileCount != ULLONG_MAX
- && progress._completedFileCount >= progress._totalFileCount
+ if (progress.hasStarted()
+ && progress.completedFiles() >= progress.totalFiles()
&& progress._currentDiscoveredFolder.isEmpty()) {
QTimer::singleShot(2000, this, SLOT(slotDisplayIdle()));
}
diff --git a/src/gui/owncloudgui.h b/src/gui/owncloudgui.h
index 9dedcc6..4511116 100644
--- a/src/gui/owncloudgui.h
+++ b/src/gui/owncloudgui.h
@@ -56,7 +56,7 @@ public slots:
void slotFolderOpenAction( const QString& alias );
void slotRefreshQuotaDisplay( qint64 total, qint64 used );
void slotRebuildRecentMenus();
- void slotUpdateProgress(const QString &folder, const Progress::Info& progress);
+ void slotUpdateProgress(const QString &folder, const ProgressInfo& progress);
void slotShowGuiMessage(const QString &title, const QString &message);
void slotFoldersChanged();
void slotShowSettings();
diff --git a/src/gui/protocolwidget.cpp b/src/gui/protocolwidget.cpp
index 0bccd81..dc1ab29 100644
--- a/src/gui/protocolwidget.cpp
+++ b/src/gui/protocolwidget.cpp
@@ -40,8 +40,8 @@ ProtocolWidget::ProtocolWidget(QWidget *parent) :
{
_ui->setupUi(this);
- connect(ProgressDispatcher::instance(), SIGNAL(progressInfo(QString,Progress::Info)),
- this, SLOT(slotProgressInfo(QString,Progress::Info)));
+ connect(ProgressDispatcher::instance(), SIGNAL(progressInfo(QString,ProgressInfo)),
+ this, SLOT(slotProgressInfo(QString,ProgressInfo)));
connect(_ui->_treeWidget, SIGNAL(itemActivated(QTreeWidgetItem*,int)), SLOT(slotOpenFile(QTreeWidgetItem*,int)));
@@ -224,7 +224,7 @@ QTreeWidgetItem* ProtocolWidget::createCompletedTreewidgetItem(const QString& fo
icon = Theme::instance()->syncStateIcon(SyncResult::Problem);
}
- if (Progress::isSizeDependent(item)) {
+ if (ProgressInfo::isSizeDependent(item)) {
columns << Utility::octetsToString( item._size );
}
@@ -266,13 +266,13 @@ void ProtocolWidget::computeResyncButtonEnabled()
}
-void ProtocolWidget::slotProgressInfo( const QString& folder, const Progress::Info& progress )
+void ProtocolWidget::slotProgressInfo( const QString& folder, const ProgressInfo& progress )
{
- if( progress._completedFileCount == ULLONG_MAX ) {
+ if( !progress.hasStarted() ) {
// The sync is restarting, clean the old items
cleanIgnoreItems(folder);
computeResyncButtonEnabled();
- } else if (progress._completedFileCount >= progress._totalFileCount) {
+ } else if (progress.completedFiles() >= progress.totalFiles()) {
//Sync completed
computeResyncButtonEnabled();
}
diff --git a/src/gui/protocolwidget.h b/src/gui/protocolwidget.h
index 0227db3..91ae119 100644
--- a/src/gui/protocolwidget.h
+++ b/src/gui/protocolwidget.h
@@ -40,7 +40,7 @@ public:
~ProtocolWidget();
public slots:
- void slotProgressInfo( const QString& folder, const Progress::Info& progress );
+ void slotProgressInfo( const QString& folder, const ProgressInfo& progress );
void slotOpenFile( QTreeWidgetItem* item, int );
protected slots:
diff --git a/src/gui/settingsdialog.cpp b/src/gui/settingsdialog.cpp
index acd80d9..2aea2b8 100644
--- a/src/gui/settingsdialog.cpp
+++ b/src/gui/settingsdialog.cpp
@@ -112,8 +112,8 @@ SettingsDialog::SettingsDialog(ownCloudGui *gui, QWidget *parent) :
connect( _accountSettings, SIGNAL(openFolderAlias(const QString&)),
gui, SLOT(slotFolderOpenAction(QString)));
- connect( ProgressDispatcher::instance(), SIGNAL(progressInfo(QString, Progress::Info)),
- _accountSettings, SLOT(slotSetProgress(QString, Progress::Info)) );
+ connect( ProgressDispatcher::instance(), SIGNAL(progressInfo(QString, ProgressInfo)),
+ _accountSettings, SLOT(slotSetProgress(QString, ProgressInfo)) );
// default to Account
diff --git a/src/libsync/progressdispatcher.cpp b/src/libsync/progressdispatcher.cpp
index dc6b9d9..caaf8e0 100644
--- a/src/libsync/progressdispatcher.cpp
+++ b/src/libsync/progressdispatcher.cpp
@@ -113,7 +113,7 @@ ProgressDispatcher::~ProgressDispatcher()
}
-void ProgressDispatcher::setProgressInfo(const QString& folder, const Progress::Info& progress)
+void ProgressDispatcher::setProgressInfo(const QString& folder, const ProgressInfo& progress)
{
if( folder.isEmpty())
// The update phase now also has progress
@@ -125,5 +125,194 @@ void ProgressDispatcher::setProgressInfo(const QString& folder, const Progress::
emit progressInfo( folder, progress );
}
+void ProgressInfo::start()
+{
+ connect(&_updateEstimatesTimer, SIGNAL(timeout()), SLOT(updateEstimates()));
+ _updateEstimatesTimer.start(1000);
+}
+
+bool ProgressInfo::hasStarted() const
+{
+ return _updateEstimatesTimer.isActive();
+}
+
+void ProgressInfo::adjustTotalsForFile(const SyncFileItem &item)
+{
+ if (!item._isDirectory) {
+ _fileProgress._total++;
+ if (isSizeDependent(item)) {
+ _sizeProgress._total += item._size;
+ }
+ } else if (item._instruction != CSYNC_INSTRUCTION_NONE) {
+ // Added or removed directories certainly count.
+ _fileProgress._total++;
+ }
+}
+
+void ProgressInfo::adjustTotalSize(qint64 change)
+{
+ _sizeProgress._total += change;
+}
+
+quint64 ProgressInfo::totalFiles() const
+{
+ return _fileProgress._total;
+}
+
+quint64 ProgressInfo::completedFiles() const
+{
+ return _fileProgress._completed;
+}
+
+quint64 ProgressInfo::currentFile() const
+{
+ return completedFiles() + _currentItems.size();
+}
+
+quint64 ProgressInfo::totalSize() const
+{
+ return _sizeProgress._total;
+}
+
+quint64 ProgressInfo::completedSize() const
+{
+ return _sizeProgress._completed;
+}
+
+void ProgressInfo::setProgressComplete(const SyncFileItem &item)
+{
+ _currentItems.remove(item._file);
+ _fileProgress._completed += item._affectedItems;
+ if (ProgressInfo::isSizeDependent(item)) {
+ _totalSizeOfCompletedJobs += item._size;
+ }
+ recomputeCompletedSize();
+ _lastCompletedItem = item;
+}
+
+void ProgressInfo::setProgressItem(const SyncFileItem &item, quint64 size)
+{
+ _currentItems[item._file]._item = item;
+ _currentItems[item._file]._progress._completed = size;
+ _currentItems[item._file]._progress._total = item._size;
+ recomputeCompletedSize();
+
+ // This seems dubious!
+ _lastCompletedItem = SyncFileItem();
+}
+
+ProgressInfo::Estimates ProgressInfo::totalProgress() const
+{
+ Estimates file = _fileProgress.estimates();
+ if (_sizeProgress._total == 0) {
+ return file;
+ }
+
+ Estimates size = _sizeProgress.estimates();
+
+ // Ideally the remaining time would be modeled as:
+ // remaning_file_sizes / transfer_speed
+ // + remaining_file_count * per_file_overhead
+ // + remaining_chunked_file_sizes / chunked_reassembly_speed
+ // with us estimating the three parameters in conjunction.
+ //
+ // But we currently only model the bandwidth and the files per
+ // second independently, which leads to incorrect values. To slightly
+ // mitigate this problem, we combine the two models depending on
+ // which factor dominates (essentially big-file-upload vs.
+ // many-small-files)
+ //
+ // If we have size information, we prefer an estimate based
+ // on the upload speed. That's particularly relevant for large file
+ // up/downloads, where files per second will be close to 0.
+ //
+ // However, when many *small* files are transfered, the estimate
+ // can become very pessimistic as the transfered amount per second
+ // drops significantly.
+ //
+ // So, if we detect a high rate of files per second, we gradually prefer
+ // a file-per-second estimate and assume the remaining transfer will
+ // be done with the highest speed we've seen.
+ quint64 combinedEta = file.estimatedEta + _sizeProgress.remaining() / _maxBytesPerSecond * 1000;
+ if (combinedEta < size.estimatedEta) {
+ double filesPerSec = _fileProgress._progressPerSec;
+ // value between 0 (fps==5) and 1 (fps==20)
+ double scale = qBound(0.0, (filesPerSec - 5.0) / 15.0, 1.0);
+ size.estimatedEta = (1.0 - scale) * size.estimatedEta + scale * combinedEta;
+ }
+ return size;
+}
+
+ProgressInfo::Estimates ProgressInfo::fileProgress(const SyncFileItem &item) const
+{
+ return _currentItems[item._file]._progress.estimates();
+}
+
+void ProgressInfo::updateEstimates()
+{
+ _sizeProgress.update();
+ _fileProgress.update();
+
+ // Update progress of all running items.
+ QMutableHashIterator<QString, ProgressItem> it(_currentItems);
+ while (it.hasNext()) {
+ it.next();
+ it.value()._progress.update();
+ }
+
+ _maxFilesPerSecond = qMax(_fileProgress._progressPerSec,
+ _maxFilesPerSecond);
+ _maxBytesPerSecond = qMax(_sizeProgress._progressPerSec,
+ _maxBytesPerSecond);
+}
+
+void ProgressInfo::recomputeCompletedSize()
+{
+ quint64 r = _totalSizeOfCompletedJobs;
+ foreach(const ProgressItem &i, _currentItems) {
+ if (isSizeDependent(i._item))
+ r += i._progress._completed;
+ }
+ _sizeProgress._completed = r;
+}
+
+ProgressInfo::Estimates ProgressInfo::Progress::estimates() const
+{
+ Estimates est;
+ est.estimatedBandwidth = _progressPerSec;
+ if (_progressPerSec != 0) {
+ est.estimatedEta = (_total - _completed) / _progressPerSec * 1000.0;
+ } else {
+ est.estimatedEta = 0; // looks better than quint64 max
+ }
+ return est;
+}
+
+quint64 ProgressInfo::Progress::completed() const
+{
+ return _completed;
+}
+
+quint64 ProgressInfo::Progress::remaining() const
+{
+ return _total - _completed;
+}
+
+void ProgressInfo::Progress::update()
+{
+ // A good way to think about the smoothing factor:
+ // If we make progress P per sec and then stop making progress at all,
+ // after N calls to this function (and thus seconds) the _progressPerSec
+ // will have reduced to P*smoothing^N.
+ // With a value of 0.9, only 4% of the original value is left after 30s
+ //
+ // In the first few updates we want to go to the correct value quickly.
+ // Therefore, smoothing starts at 0 and ramps up to its final value over time.
+ const double smoothing = 0.9 * (1.0 - _initialSmoothing);
+ _initialSmoothing *= 0.7; // goes from 1 to 0.03 in 10s
+ _progressPerSec = smoothing * _progressPerSec + (1.0 - smoothing) * (_completed - _prevCompleted);
+ _prevCompleted = _completed;
+}
+
}
diff --git a/src/libsync/progressdispatcher.h b/src/libsync/progressdispatcher.h
index f4e657b..42ae64e 100644
--- a/src/libsync/progressdispatcher.h
+++ b/src/libsync/progressdispatcher.h
@@ -20,17 +20,60 @@
#include <QTime>
#include <QQueue>
#include <QElapsedTimer>
+#include <QTimer>
#include <QDebug>
#include "syncfileitem.h"
namespace OCC {
-/**
- * @brief The FolderScheduler class schedules folders for sync
- */
-namespace Progress
+class ProgressInfo : public QObject
{
+ Q_OBJECT
+public:
+ ProgressInfo()
+ : _totalSizeOfCompletedJobs(0)
+ , _maxFilesPerSecond(2.0)
+ , _maxBytesPerSecond(100000.0)
+ {}
+
+ /**
+ * Called when propagation starts.
+ *
+ * hasStarted() will return true afterwards.
+ */
+ void start();
+
+ /**
+ * Returns true when propagation has started (start() was called).
+ *
+ * This is used when the SyncEngine wants to indicate a new sync
+ * is about to start via the transmissionProgress() signal. The
+ * first ProgressInfo will have hasStarted() == false.
+ */
+ bool hasStarted() const;
+
+ /**
+ * Increase the file and size totals by the amount indicated in item.
+ */
+ void adjustTotalsForFile(const SyncFileItem & item);
+
+ /**
+ * Adjust the total size by some value.
+ *
+ * Deprecated. Used only in the legacy propagator.
+ */
+ void adjustTotalSize(qint64 change);
+
+ quint64 totalFiles() const;
+ quint64 completedFiles() const;
+
+ quint64 totalSize() const;
+ quint64 completedSize() const;
+
+ /** Number of a file that is currently in progress. */
+ quint64 currentFile() const;
+
/** Return true is the size need to be taken in account in the total amount of time */
static inline bool isSizeDependent(const SyncFileItem & item)
{
@@ -40,136 +83,110 @@ namespace Progress
|| item._instruction == CSYNC_INSTRUCTION_NEW);
}
+ /**
+ * Holds estimates about progress, returned to the user.
+ */
+ struct Estimates
+ {
+ /// Estimated completion amount per second. (of bytes or files)
+ quint64 estimatedBandwidth;
- struct Info {
- Info() : _totalFileCount(0), _totalSize(0), _completedFileCount(0), _completedSize(0) {}
-
- // Used during local and remote update phase
- QString _currentDiscoveredFolder;
-
- quint64 _totalFileCount;
- quint64 _totalSize;
- quint64 _completedFileCount;
- quint64 _completedSize;
- // Should this be in a separate file?
- struct EtaEstimate {
- EtaEstimate()
- : _startedTime(QDateTime::currentMSecsSinceEpoch())
- , _agvEtaMSecs(0)
- , _effectivProgressPerSec(0)
- , _sampleCount(1)
- {
- }
-
- static const int MAX_AVG_DIVIDER=60;
- static const int INITAL_WAIT_TIME=5;
-
- quint64 _startedTime ;
- quint64 _agvEtaMSecs;
- quint64 _effectivProgressPerSec;
- float _sampleCount;
-
- /**
- * reset the estiamte.
- */
- void reset() {
- _startedTime = QDateTime::currentMSecsSinceEpoch();
- _sampleCount =1;
- _effectivProgressPerSec = _agvEtaMSecs = 0;
- }
-
- /**
- * update the estimated eta time with more current data.
- * @param quint64 completed the amount the was completed.
- * @param quint64 total the total amout that should be completed.
- */
- void updateTime(quint64 completed, quint64 total) {
- quint64 elapsedTime = QDateTime::currentMSecsSinceEpoch() - this->_startedTime ;
- //don't start until you have some good data to process, prevents jittring estiamtes at the start of the syncing process
- if(total != 0 && completed != 0 && elapsedTime > INITAL_WAIT_TIME ) {
- if(_sampleCount < MAX_AVG_DIVIDER) { _sampleCount+=0.01f; }
- // (elapsedTime-1) is an hack to avoid float "rounding" issue (ie. 0.99999999999999999999....)
- _agvEtaMSecs = _agvEtaMSecs + (((static_cast<float>(total) / completed) * elapsedTime) - (elapsedTime-1)) - this->getEtaEstimate();
- _effectivProgressPerSec = ( total - completed ) / (1+this->getEtaEstimate()/1000);
- }
- }
-
- /**
- * Get the eta estimate in milliseconds
- * @return quint64 the estimate amount of milliseconds to end the process.
- */
- quint64 getEtaEstimate() const {
- return _agvEtaMSecs / _sampleCount;
- }
-
- /**
- * Get the estimated average bandwidth usage.
- * @return quint64 the estimated bandwidth usage in bytes.
- */
- quint64 getEstimatedBandwidth() const {
- return _effectivProgressPerSec;
- }
- };
- EtaEstimate _totalEtaEstimate;
-
- struct ProgressItem {
- ProgressItem() : _completedSize(0) {}
- SyncFileItem _item;
- quint64 _completedSize;
- EtaEstimate _etaEstimate;
- };
- QHash<QString, ProgressItem> _currentItems;
- SyncFileItem _lastCompletedItem;
-
- void setProgressComplete(const SyncFileItem &item) {
- _currentItems.remove(item._file);
- _completedFileCount += item._affectedItems;
- if (Progress::isSizeDependent(item)) {
- _completedSize += item._size;
- }
- _lastCompletedItem = item;
- this->updateEstimation();
- }
- void setProgressItem(const SyncFileItem &item, quint64 size) {
- _currentItems[item._file]._item = item;
- _currentItems[item._file]._completedSize = size;
- _lastCompletedItem = SyncFileItem();
- this->updateEstimation();
- _currentItems[item._file]._etaEstimate.updateTime(size,item._size);
- }
-
- void updateEstimation() {
- if(this->_totalSize > 0) {
- _totalEtaEstimate.updateTime(this->completedSize(),this->_totalSize);
- } else {
- _totalEtaEstimate.updateTime(this->_completedFileCount,this->_totalFileCount);
- }
- }
+ /// Estimated eta in milliseconds.
+ quint64 estimatedEta;
+ };
- quint64 completedSize() const {
- quint64 r = _completedSize;
- foreach(const ProgressItem &i, _currentItems) {
- if (Progress::isSizeDependent(i._item))
- r += i._completedSize;
- }
- return r;
- }
- /**
- * Get the total completion estimate structure
- * @return EtaEstimate a structure containing the total completion information.
- */
- EtaEstimate totalEstimate() const {
- return _totalEtaEstimate;
+ /**
+ * Holds the current state of something making progress and maintains an
+ * estimate of the current progress per second.
+ */
+ struct Progress
+ {
+ Progress()
+ : _progressPerSec(0)
+ , _completed(0)
+ , _prevCompleted(0)
+ , _total(0)
+ , _initialSmoothing(1.0)
+ {
}
+ /** Returns the estimates about progress per second and eta. */
+ Estimates estimates() const;
+
+ quint64 completed() const;
+ quint64 remaining() const;
+
+ private:
/**
- * Get the current file completion estimate structure
- * @return EtaEstimate a structure containing the current file completion information.
+ * Update the exponential moving average estimate of _progressPerSec.
*/
- EtaEstimate getFileEstimate(const SyncFileItem &item) const {
- return _currentItems[item._file]._etaEstimate;
- }
+ void update();
+
+ double _progressPerSec;
+ quint64 _completed;
+ quint64 _prevCompleted;
+ quint64 _total;
+
+ // Used to get to a good value faster when
+ // progress measurement stats. See update().
+ double _initialSmoothing;
+
+ friend class ProgressInfo;
+ };
+
+ struct ProgressItem
+ {
+ SyncFileItem _item;
+ Progress _progress;
};
+ QHash<QString, ProgressItem> _currentItems;
+
+ SyncFileItem _lastCompletedItem;
+
+ // Used during local and remote update phase
+ QString _currentDiscoveredFolder;
+
+ void setProgressComplete(const SyncFileItem &item);
+
+ void setProgressItem(const SyncFileItem &item, quint64 size);
+
+ /**
+ * Get the total completion estimate
+ */
+ Estimates totalProgress() const;
+
+ /**
+ * Get the current file completion estimate structure
+ */
+ Estimates fileProgress(const SyncFileItem &item) const;
+
+private slots:
+ /**
+ * Called every second once started, this function updates the
+ * estimates.
+ */
+ void updateEstimates();
+
+private:
+ // Sets the completed size by summing finished jobs with the progress
+ // of active ones.
+ void recomputeCompletedSize();
+
+ // Triggers the update() slot every second once propagation started.
+ QTimer _updateEstimatesTimer;
+
+ Progress _sizeProgress;
+ Progress _fileProgress;
+
+ // All size from completed jobs only.
+ quint64 _totalSizeOfCompletedJobs;
+
+ // The fastest observed rate of files per second in this sync.
+ double _maxFilesPerSecond;
+ double _maxBytesPerSecond;
+};
+
+namespace Progress {
OWNCLOUDSYNC_EXPORT QString asActionString( const SyncFileItem& item );
OWNCLOUDSYNC_EXPORT QString asResultString( const SyncFileItem& item );
@@ -205,7 +222,7 @@ signals:
@param[out] progress A struct with all progress info.
*/
- void progressInfo( const QString& folder, const Progress::Info& progress );
+ void progressInfo( const QString& folder, const ProgressInfo& progress );
/**
* @brief: the item's job is completed
*/
@@ -214,7 +231,7 @@ signals:
void syncItemDiscovered(const QString &folder, const SyncFileItem & item);
protected:
- void setProgressInfo(const QString& folder, const Progress::Info& progress);
+ void setProgressInfo(const QString& folder, const ProgressInfo& progress);
private:
ProgressDispatcher(QObject* parent = 0);
diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp
index ee15576..633319f 100644
--- a/src/libsync/syncengine.cpp
+++ b/src/libsync/syncengine.cpp
@@ -65,6 +65,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath,
, _remoteUrl(remoteURL)
, _remotePath(remotePath)
, _journal(journal)
+ , _progressInfo(new ProgressInfo)
, _hasNoneFiles(false)
, _hasRemoveFile(false)
, _uploadLimit(0)
@@ -73,7 +74,6 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath,
{
qRegisterMetaType<SyncFileItem>("SyncFileItem");
qRegisterMetaType<SyncFileItem::Status>("SyncFileItem::Status");
- qRegisterMetaType<Progress::Info>("Progress::Info");
_thread.setObjectName("CSync_Neon_Thread");
_thread.start();
@@ -494,15 +494,8 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
checkErrorBlacklisting( *item );
}
- if (!item->_isDirectory) {
- _progressInfo._totalFileCount++;
- if (Progress::isSizeDependent(item)) {
- _progressInfo._totalSize += file->size;
- }
- } else if (file->instruction != CSYNC_INSTRUCTION_NONE) {
- // Added or removed directories certainly count.
- _progressInfo._totalFileCount++;
- }
+ _progressInfo->adjustTotalsForFile(*item);
+
_needsUpdate = true;
item->log._etag = file->etag;
@@ -702,8 +695,6 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
qDebug() << "<<#### Reconcile end #################################################### " << _stopWatch.addLapTime(QLatin1String("Reconcile Finished"));
- _progressInfo = Progress::Info();
-
_hasNoneFiles = false;
_hasRemoveFile = false;
bool walkOk = true;
@@ -744,9 +735,9 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
// To announce the beginning of the sync
emit aboutToPropagate(_syncedItems);
- _progressInfo._completedFileCount = ULLONG_MAX; // indicate the start with max
- emit transmissionProgress(_progressInfo);
- _progressInfo._completedFileCount = 0;
+ // it's important to do this before ProgressInfo::start(), to announce start of new sync
+ emit transmissionProgress(*_progressInfo);
+ _progressInfo->start();
if (!_hasNoneFiles && _hasRemoveFile) {
qDebug() << Q_FUNC_INFO << "All the files are going to be changed, asking the user";
@@ -843,13 +834,13 @@ void SyncEngine::slotJobCompleted(const SyncFileItem &item)
const char * instruction_str = csync_instruction_str(item._instruction);
qDebug() << Q_FUNC_INFO << item._file << instruction_str << item._status << item._errorString;
- _progressInfo.setProgressComplete(item);
+ _progressInfo->setProgressComplete(item);
if (item._status == SyncFileItem::FatalError) {
emit csyncError(item._errorString);
}
- emit transmissionProgress(_progressInfo);
+ emit transmissionProgress(*_progressInfo);
emit jobCompleted(item);
}
@@ -891,14 +882,14 @@ void SyncEngine::finalize()
void SyncEngine::slotProgress(const SyncFileItem& item, quint64 current)
{
- _progressInfo.setProgressItem(item, current);
- emit transmissionProgress(_progressInfo);
+ _progressInfo->setProgressItem(item, current);
+ emit transmissionProgress(*_progressInfo);
}
void SyncEngine::slotAdjustTotalTransmissionSize(qint64 change)
{
- _progressInfo._totalSize += change;
+ _progressInfo->adjustTotalSize(change);
}
/* Given a path on the remote, give the path as it is when the rename is done */
diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h
index bd74066..baa3ffc 100644
--- a/src/libsync/syncengine.h
+++ b/src/libsync/syncengine.h
@@ -99,7 +99,7 @@ signals:
// after sync is done
void treeWalkResult(const SyncFileItemVector&);
- void transmissionProgress( const Progress::Info& progress );
+ void transmissionProgress( const ProgressInfo& progress );
void csyncStateDbFile( const QString& );
void wipeDb();
@@ -176,7 +176,7 @@ private:
QThread _thread;
- Progress::Info _progressInfo;
+ QScopedPointer<ProgressInfo> _progressInfo;
Utility::StopWatch _stopWatch;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-owncloud/owncloud-client.git
More information about the Pkg-owncloud-commits
mailing list