Merge pull request #4403 from glassez/fastresume

Improve .fastresume saving and torrents starting up. Closes #4315.

src/app/upgrade.h

@@ -36,6 +36,7 @@
 #include <QString>
 #include <QDir>
 #include <QFile>
+#include <QRegExp>
 #ifndef DISABLE_GUI
 #include <QMessageBox>
 #endif
@@ -76,7 +77,7 @@ bool userAcceptsUpgrade()
     return false;
 }
 
-bool upgradeResumeFile(const QString &filepath, const QVariantHash &oldTorrent, int &maxPrio)
+bool upgradeResumeFile(const QString &filepath, const QVariantHash &oldTorrent = QVariantHash())
 {
     QFile file1(filepath);
     if (!file1.open(QIODevice::ReadOnly))
@@ -93,18 +94,31 @@ bool upgradeResumeFile(const QString &filepath, const QVariantHash &oldTorrent,
     libtorrent::entry fastNew;
     fastNew = fastOld;
 
-    int priority = fastOld.dict_find_int_value("qBt-queuePosition");
+    bool v3_3 = false;
-    if (priority > maxPrio)
+    int queuePosition = 0;
-        maxPrio = priority;
+    QString outFilePath = filepath;
-
+    QRegExp rx(QLatin1String("([A-Fa-f0-9]{40})\\.fastresume\\.(\\d+)$"));
+    if (rx.indexIn(filepath) != -1) {
+        // old v3.3.x format
+        queuePosition = rx.cap(2).toInt();
+        v3_3 = true;
+        outFilePath.replace(QRegExp("\\.\\d+$"), "");
+    }
+    else {
+        queuePosition = fastOld.dict_find_int_value("qBt-queuePosition", 0);
         fastNew["qBt-name"] = Utils::String::toStdString(oldTorrent.value("name").toString());
         fastNew["qBt-tempPathDisabled"] = false;
+    }
 
-    QFile file2(QString("%1.%2").arg(filepath).arg(priority > 0 ? priority : 0));
+    // in versions < 3.3 we have -1 for seeding torrents, so we convert it to 0
+    fastNew["qBt-queuePosition"] = (queuePosition >= 0 ? queuePosition : 0);
+
+    QFile file2(outFilePath);
     QVector<char> out;
     libtorrent::bencode(std::back_inserter(out), fastNew);
     if (file2.open(QIODevice::WriteOnly)) {
         if (file2.write(&out[0], out.size()) == out.size()) {
+            if (v3_3)
                 Utils::Fs::forceRemove(filepath);
             return true;
         }
@@ -118,26 +132,35 @@ bool upgrade(bool ask = true)
     // Move RSS cookies to common storage
     Preferences::instance()->moveRSSCookies();
 
+    QString backupFolderPath = Utils::Fs::expandPathAbs(Utils::Fs::QDesktopServicesDataLocation() + "BT_backup");
+    QDir backupFolderDir(backupFolderPath);
+
+    // ****************************************************************************************
+    // Silently converts old v3.3.x .fastresume files
+    QStringList backupFiles_3_3 = backupFolderDir.entryList(
+                QStringList(QLatin1String("*.fastresume.*")), QDir::Files, QDir::Unsorted);
+    foreach (const QString &backupFile, backupFiles_3_3)
+        upgradeResumeFile(backupFolderDir.absoluteFilePath(backupFile));
+    // ****************************************************************************************
+
     QIniSettings *oldResumeSettings = new QIniSettings("qBittorrent", "qBittorrent-resume");
     QString oldResumeFilename = oldResumeSettings->fileName();
     QVariantHash oldResumeData = oldResumeSettings->value("torrents").toHash();
     delete oldResumeSettings;
-    bool oldResumeWasEmpty = oldResumeData.isEmpty();
+
-    if (oldResumeWasEmpty)
+    if (oldResumeData.isEmpty()) {
         Utils::Fs::forceRemove(oldResumeFilename);
+        return true;
+    }
 
-
-    QString backupFolderPath = Utils::Fs::expandPathAbs(Utils::Fs::QDesktopServicesDataLocation() + "BT_backup");
-    QDir backupFolderDir(backupFolderPath);
-    QStringList backupFiles = backupFolderDir.entryList(QStringList() << QLatin1String("*.fastresume"), QDir::Files, QDir::Unsorted);
-    if (backupFiles.isEmpty() && oldResumeWasEmpty) return true;
     if (ask && !userAcceptsUpgrade()) return false;
 
-    int maxPrio = 0;
+    QStringList backupFiles = backupFolderDir.entryList(
+                QStringList(QLatin1String("*.fastresume")), QDir::Files, QDir::Unsorted);
     QRegExp rx(QLatin1String("^([A-Fa-f0-9]{40})\\.fastresume$"));
     foreach (QString backupFile, backupFiles) {
         if (rx.indexIn(backupFile) != -1) {
-            if (upgradeResumeFile(backupFolderDir.absoluteFilePath(backupFile), oldResumeData[rx.cap(1)].toHash(), maxPrio))
+            if (upgradeResumeFile(backupFolderDir.absoluteFilePath(backupFile), oldResumeData[rx.cap(1)].toHash()))
                 oldResumeData.remove(rx.cap(1));
             else
                 Logger::instance()->addMessage(QObject::tr("Couldn't migrate torrent with hash: %1").arg(rx.cap(1)), Log::WARNING);
@@ -162,7 +185,10 @@ bool upgrade(bool ask = true)
             resumeData["qBt-seedStatus"] = oldTorrent.value("seed").toBool();
             resumeData["qBt-tempPathDisabled"] = false;
 
-            QString filename = QString("%1.fastresume.%2").arg(hash).arg(++maxPrio);
+            int queuePosition = oldTorrent.value("priority", 0).toInt();
+            resumeData["qBt-queuePosition"] = (queuePosition >= 0 ? queuePosition : 0);
+
+            QString filename = QString("%1.fastresume").arg(hash);
             QString filepath = backupFolderDir.absoluteFilePath(filename);
 
             QFile resumeFile(filepath);
@@ -173,17 +199,13 @@ bool upgrade(bool ask = true)
         }
     }
 
-    if (!oldResumeWasEmpty) {
     int counter = 0;
     QString backupResumeFilename = oldResumeFilename + ".bak";
-
     while (QFile::exists(backupResumeFilename)) {
         ++counter;
         backupResumeFilename = oldResumeFilename + ".bak" + QString::number(counter);
     }
-
     QFile::rename(oldResumeFilename, backupResumeFilename);
-    }
 
     return true;
 }

src/base/base.pri

@@ -36,6 +36,7 @@ HEADERS += \
     $$PWD/bittorrent/private/bandwidthscheduler.h \
     $$PWD/bittorrent/private/filterparserthread.h \
     $$PWD/bittorrent/private/statistics.h \
+    $$PWD/bittorrent/private/resumedatasavingmanager.h \
     $$PWD/rss/rssmanager.h \
     $$PWD/rss/rssfeed.h \
     $$PWD/rss/rssfolder.h \
@@ -87,6 +88,7 @@ SOURCES += \
     $$PWD/bittorrent/private/bandwidthscheduler.cpp \
     $$PWD/bittorrent/private/filterparserthread.cpp \
     $$PWD/bittorrent/private/statistics.cpp \
+    $$PWD/bittorrent/private/resumedatasavingmanager.cpp \
     $$PWD/rss/rssmanager.cpp \
     $$PWD/rss/rssfeed.cpp \
     $$PWD/rss/rssfolder.cpp \

src/base/bittorrent/private/resumedatasavingmanager.cpp

@@ -0,0 +1,65 @@
+/*
+ * Bittorrent Client using Qt and libtorrent.
+ * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * In addition, as a special exception, the copyright holders give permission to
+ * link this program with the OpenSSL project's "OpenSSL" library (or with
+ * modified versions of it that use the same license as the "OpenSSL" library),
+ * and distribute the linked executables. You must obey the GNU General Public
+ * License in all respects for all of the code used other than "OpenSSL".  If you
+ * modify file(s), you may extend this exception to your version of the file(s),
+ * but you are not obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ */
+
+#include <QDebug>
+#ifdef QBT_USES_QT5
+#include <QSaveFile>
+#else
+#include <QFile>
+#endif
+
+#include "base/logger.h"
+#include "base/utils/fs.h"
+#include "resumedatasavingmanager.h"
+
+ResumeDataSavingManager::ResumeDataSavingManager(const QString &resumeFolderPath)
+    : m_resumeDataDir(resumeFolderPath)
+{
+}
+
+void ResumeDataSavingManager::saveResumeData(QString infoHash, QByteArray data) const
+{
+    QString filename = QString("%1.fastresume").arg(infoHash);
+    QString filepath = m_resumeDataDir.absoluteFilePath(filename);
+
+    qDebug() << "Saving resume data in" << filepath;
+#ifdef QBT_USES_QT5
+    QSaveFile resumeFile(filepath);
+#else
+    QFile resumeFile(filepath);
+#endif
+    if (resumeFile.open(QIODevice::WriteOnly)) {
+        resumeFile.write(data);
+#ifdef QBT_USES_QT5
+        if (!resumeFile.commit()) {
+            Logger::instance()->addMessage(QString("Couldn't save resume data in %1. Error: %2")
+                                           .arg(filepath).arg(resumeFile.errorString()), Log::WARNING);
+        }
+#endif
+    }
+}

src/base/bittorrent/private/resumedatasavingmanager.h

@@ -0,0 +1,50 @@
+/*
+ * Bittorrent Client using Qt and libtorrent.
+ * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * In addition, as a special exception, the copyright holders give permission to
+ * link this program with the OpenSSL project's "OpenSSL" library (or with
+ * modified versions of it that use the same license as the "OpenSSL" library),
+ * and distribute the linked executables. You must obey the GNU General Public
+ * License in all respects for all of the code used other than "OpenSSL".  If you
+ * modify file(s), you may extend this exception to your version of the file(s),
+ * but you are not obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ */
+
+#ifndef RESUMEDATASAVINGMANAGER_H
+#define RESUMEDATASAVINGMANAGER_H
+
+#include <QObject>
+#include <QByteArray>
+#include <QDir>
+
+class ResumeDataSavingManager: public QObject
+{
+    Q_OBJECT
+
+public:
+    explicit ResumeDataSavingManager(const QString &resumeFolderPath);
+
+public slots:
+    void saveResumeData(QString infoHash, QByteArray data) const;
+
+private:
+    QDir m_resumeDataDir;
+};
+
+#endif // RESUMEDATASAVINGMANAGER_H

src/base/bittorrent/session.cpp

@@ -43,6 +43,7 @@ using namespace BitTorrent;
 #include <QTimer>
 #include <QProcess>
 #include <QCoreApplication>
+#include <QThread>
 
 #include <queue>
 #include <vector>
@@ -78,6 +79,7 @@ using namespace BitTorrent;
 #include "private/filterparserthread.h"
 #include "private/statistics.h"
 #include "private/bandwidthscheduler.h"
+#include "private/resumedatasavingmanager.h"
 #include "trackerentry.h"
 #include "tracker.h"
 #include "magneturi.h"
@@ -93,7 +95,7 @@ namespace libt = libtorrent;
 using namespace BitTorrent;
 
 static bool readFile(const QString &path, QByteArray &buf);
-static bool loadTorrentResumeData(const QByteArray &data, AddTorrentData &out, MagnetUri &magnetUri);
+static bool loadTorrentResumeData(const QByteArray &data, AddTorrentData &out, int &prio, MagnetUri &magnetUri);
 
 static void torrentQueuePositionUp(const libt::torrent_handle &handle);
 static void torrentQueuePositionDown(const libt::torrent_handle &handle);
@@ -193,6 +195,11 @@ Session::Session(QObject *parent)
     connect(&m_networkManager, SIGNAL(configurationRemoved(const QNetworkConfiguration&)), SLOT(networkConfigurationChange(const QNetworkConfiguration&)));
     connect(&m_networkManager, SIGNAL(configurationChanged(const QNetworkConfiguration&)), SLOT(networkConfigurationChange(const QNetworkConfiguration&)));
 
+    m_ioThread = new QThread(this);
+    m_resumeDataSavingManager = new ResumeDataSavingManager(m_resumeFolderPath);
+    m_resumeDataSavingManager->moveToThread(m_ioThread);
+    connect(m_ioThread, SIGNAL(finished()), m_resumeDataSavingManager, SLOT(deleteLater()));
+    m_ioThread->start();
     m_resumeDataTimer->start();
 
     // initialize PortForwarder instance
@@ -270,6 +277,9 @@ Session::~Session()
     qDebug("Deleting the session");
     delete m_nativeSession;
 
+    m_ioThread->quit();
+    m_ioThread->wait();
+
     m_resumeFolderLock.close();
     m_resumeFolderLock.remove();
 }
@@ -1701,7 +1711,16 @@ void Session::handleTorrentFinished(TorrentHandle *const torrent)
 void Session::handleTorrentResumeDataReady(TorrentHandle *const torrent, const libtorrent::entry &data)
 {
     --m_numResumeData;
-    writeResumeDataFile(torrent, data);
+
+    // Separated thread is used for the blocking IO which results in slow processing of many torrents.
+    // Encoding data in parallel while doing IO saves time. Copying libtorrent::entry objects around
+    // isn't cheap too.
+
+    QByteArray out;
+    libt::bencode(std::back_inserter(out), data);
+
+    QMetaObject::invokeMethod(m_resumeDataSavingManager, "saveResumeData",
+                              Q_ARG(QString, torrent->hash()), Q_ARG(QByteArray, out));
 }
 
 void Session::handleTorrentResumeDataFailed(TorrentHandle *const torrent)
@@ -1868,48 +1887,63 @@ void Session::startUpTorrents()
 
     const QDir resumeDataDir(m_resumeFolderPath);
     QStringList fastresumes = resumeDataDir.entryList(
-                QStringList(QLatin1String("*.fastresume.*")), QDir::Files, QDir::Unsorted);
+                QStringList(QLatin1String("*.fastresume")), QDir::Files, QDir::Unsorted);
 
-    typedef QPair<int, QString> PrioHashPair;
-    typedef std::vector<PrioHashPair> PrioHashVector;
-    typedef std::greater<PrioHashPair> PrioHashGreater;
-    std::priority_queue<PrioHashPair, PrioHashVector, PrioHashGreater> torrentQueue;
-    // Fastresume file name format:
-    //     <torrent_info_hash>.fastresume.<torrent_queue_position>
-    // E.g.:
-    //     fc8a15a2faf2734dbb1dc5f7afdc5c9beaeb1f59.fastresume.2
-    QRegExp rx(QLatin1String("^([A-Fa-f0-9]{40})\\.fastresume\\.(\\d+)$"));
-    foreach (const QString &fastresume, fastresumes) {
-        if (rx.indexIn(fastresume) != -1) {
-            PrioHashPair p = qMakePair(rx.cap(2).toInt(), rx.cap(1));
-            torrentQueue.push(p);
-        }
-    }
-
-    QString filePath;
     Logger *const logger = Logger::instance();
 
-    qDebug("Starting up torrents");
+    typedef struct
-    qDebug("Priority queue size: %ld", (long)torrentQueue.size());
+    {
-    // Resume downloads
+        QString hash;
-    while (!torrentQueue.empty()) {
+        MagnetUri magnetUri;
-        const int prio = torrentQueue.top().first;
+        AddTorrentData addTorrentData;
-        const QString hash = torrentQueue.top().second;
+        QByteArray data;
-        torrentQueue.pop();
+    } TorrentResumeData;
 
-        QString fastresumePath =
+    auto startupTorrent = [this, logger, resumeDataDir](const TorrentResumeData &params)
-                resumeDataDir.absoluteFilePath(QString("%1.fastresume.%2").arg(hash).arg(prio));
+    {
+        QString filePath = resumeDataDir.filePath(QString("%1.torrent").arg(params.hash));
+        qDebug() << "Starting up torrent" << params.hash << "...";
+        if (!addTorrent_impl(params.addTorrentData, params.magnetUri, TorrentInfo::loadFromFile(filePath), params.data))
+            logger->addMessage(tr("Unable to resume torrent '%1'.", "e.g: Unable to resume torrent 'hash'.")
+                               .arg(params.hash), Log::CRITICAL);
+    };
+
+    qDebug("Starting up torrents");
+    qDebug("Queue size: %d", fastresumes.size());
+    // Resume downloads
+    QMap<int, TorrentResumeData> queuedResumeData;
+    int nextQueuePosition = 1;
+    QRegExp rx(QLatin1String("^([A-Fa-f0-9]{40})\\.fastresume$"));
+    foreach (const QString &fastresumeName, fastresumes) {
+        if (rx.indexIn(fastresumeName) == -1) continue;
+
+        QString hash = rx.cap(1);
+        QString fastresumePath = resumeDataDir.absoluteFilePath(fastresumeName);
         QByteArray data;
         AddTorrentData resumeData;
         MagnetUri magnetUri;
-        if (readFile(fastresumePath, data) && loadTorrentResumeData(data, resumeData, magnetUri)) {
+        int queuePosition;
-            filePath = resumeDataDir.filePath(QString("%1.torrent").arg(hash));
+        if (readFile(fastresumePath, data) && loadTorrentResumeData(data, resumeData, queuePosition, magnetUri)) {
-            qDebug("Starting up torrent %s ...", qPrintable(hash));
+            if (queuePosition <= nextQueuePosition) {
-            if (!addTorrent_impl(resumeData, magnetUri, TorrentInfo::loadFromFile(filePath), data))
+                startupTorrent({ hash, magnetUri, resumeData, data });
-                logger->addMessage(tr("Unable to resume torrent '%1'.", "e.g: Unable to resume torrent 'hash'.")
+
-                                   .arg(Utils::Fs::toNativePath(hash)), Log::CRITICAL);
+                if (queuePosition == nextQueuePosition) {
+                    ++nextQueuePosition;
+                    while (queuedResumeData.contains(nextQueuePosition)) {
+                        startupTorrent(queuedResumeData.take(nextQueuePosition));
+                        ++nextQueuePosition;
                     }
                 }
+            }
+            else {
+                queuedResumeData[queuePosition] = { hash, magnetUri, resumeData, data };
+            }
+        }
+    }
+
+    // starting up downloading torrents (queue position > 0)
+    foreach (const TorrentResumeData &torrentResumeData, queuedResumeData)
+        startupTorrent(torrentResumeData);
 }
 
 quint64 Session::getAlltimeDL() const
@@ -2338,7 +2372,7 @@ bool readFile(const QString &path, QByteArray &buf)
     return true;
 }
 
-bool loadTorrentResumeData(const QByteArray &data, AddTorrentData &out, MagnetUri &magnetUri)
+bool loadTorrentResumeData(const QByteArray &data, AddTorrentData &out, int &prio,  MagnetUri &magnetUri)
 {
     out = AddTorrentData();
     out.resumed = true;
@@ -2365,31 +2399,11 @@ bool loadTorrentResumeData(const QByteArray &data, AddTorrentData &out, MagnetUr
     out.addPaused = fast.dict_find_int_value("qBt-paused");
     out.addForced = fast.dict_find_int_value("qBt-forced");
 
+    prio = fast.dict_find_int_value("qBt-queuePosition");
+
     return true;
 }
 
-bool Session::writeResumeDataFile(TorrentHandle *const torrent, const libt::entry &data)
-{
-    const QDir resumeDataDir(m_resumeFolderPath);
-
-    QStringList filters(QString("%1.fastresume.*").arg(torrent->hash()));
-    const QStringList files = resumeDataDir.entryList(filters, QDir::Files, QDir::Unsorted);
-    foreach (const QString &file, files)
-        Utils::Fs::forceRemove(resumeDataDir.absoluteFilePath(file));
-
-    QString filename = QString("%1.fastresume.%2").arg(torrent->hash()).arg(torrent->queuePosition());
-    QString filepath = resumeDataDir.absoluteFilePath(filename);
-
-    qDebug("Saving resume data in %s", qPrintable(filepath));
-    QFile resumeFile(filepath);
-    QVector<char> out;
-    libt::bencode(std::back_inserter(out), data);
-    if (resumeFile.open(QIODevice::WriteOnly))
-        return (resumeFile.write(&out[0], out.size()) == out.size());
-
-    return false;
-}
-
 void torrentQueuePositionUp(const libt::torrent_handle &handle)
 {
     try {

src/base/bittorrent/session.h

@@ -99,6 +99,7 @@ namespace libtorrent
     struct external_ip_alert;
 }
 
+class QThread;
 class QTimer;
 class QStringList;
 class QString;
@@ -108,6 +109,7 @@ template<typename T> class QList;
 class FilterParserThread;
 class BandwidthScheduler;
 class Statistics;
+class ResumeDataSavingManager;
 
 typedef QPair<QString, QString> QStringPair;
 
@@ -327,7 +329,6 @@ namespace BitTorrent
         void createTorrentHandle(const libtorrent::torrent_handle &nativeHandle);
 
         void saveResumeData();
-        bool writeResumeDataFile(TorrentHandle *const torrent, const libtorrent::entry &data);
 
         void dispatchAlerts(std::auto_ptr<libtorrent::alert> alertPtr);
         void getPendingAlerts(QVector<libtorrent::alert *> &out, ulong time = 0);
@@ -366,6 +367,9 @@ namespace BitTorrent
         QPointer<BandwidthScheduler> m_bwScheduler;
         // Tracker
         QPointer<Tracker> m_tracker;
+        // fastresume data writing thread
+        QThread *m_ioThread;
+        ResumeDataSavingManager *m_resumeDataSavingManager;
 
         QHash<InfoHash, TorrentInfo> m_loadedMetadata;
         QHash<InfoHash, TorrentHandle *> m_torrents;

src/base/bittorrent/torrenthandle.cpp

@@ -1475,6 +1475,7 @@ void TorrentHandle::handleSaveResumeDataAlert(libtorrent::save_resume_data_alert
     resumeData["qBt-name"] = Utils::String::toStdString(m_name);
     resumeData["qBt-seedStatus"] = m_hasSeedStatus;
     resumeData["qBt-tempPathDisabled"] = m_tempPathDisabled;
+    resumeData["qBt-queuePosition"] = queuePosition();
 
     m_session->handleTorrentResumeDataReady(this, resumeData);
 }