From ed4570cb4d157bb9dee53337841e2bb20ce73ed1 Mon Sep 17 00:00:00 2001 From: Vladimir Golovnev Date: Mon, 19 Jul 2021 07:59:06 +0300 Subject: [PATCH] Store minimal metadata for "restore torrent" purposes (#15191) We can no longer save valid torrent files in the general case, because for torrents of version 2, we need a full merkle tree to do it, but if a torrent is added from magnet link, full merkle tree may not be available even before the end of downloading all the data. Actually, we don't need the full torrent file for the purposes of resuming the torrent, so we can allow libtorrent to produce only a minimal part of the metadata as part complete resume data, but we still want to store it in a separate file, so we extract the resulting metadata from the complete resume data before saving and merge it together before loading. --- .../bittorrent/bencoderesumedatastorage.cpp | 55 +++++++++++-------- .../bittorrent/bencoderesumedatastorage.h | 4 +- src/base/bittorrent/dbresumedatastorage.cpp | 35 +++++++----- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/base/bittorrent/bencoderesumedatastorage.cpp b/src/base/bittorrent/bencoderesumedatastorage.cpp index 9c35c85bb..961d9c61d 100644 --- a/src/base/bittorrent/bencoderesumedatastorage.cpp +++ b/src/base/bittorrent/bencoderesumedatastorage.cpp @@ -30,9 +30,9 @@ #include #include -#include #include #include +#include #include #include @@ -51,7 +51,6 @@ #include "base/utils/string.h" #include "infohash.h" #include "loadtorrentparams.h" -#include "torrentinfo.h" namespace BitTorrent { @@ -151,25 +150,36 @@ std::optional BitTorrent::BencodeResumeDataStorag const QString fastresumePath = m_resumeDataDir.absoluteFilePath(QString::fromLatin1("%1.fastresume").arg(idString)); const QString torrentFilePath = m_resumeDataDir.absoluteFilePath(QString::fromLatin1("%1.torrent").arg(idString)); - QFile file {fastresumePath}; - if (!file.open(QIODevice::ReadOnly)) + QFile resumeDataFile {fastresumePath}; + if (!resumeDataFile.open(QIODevice::ReadOnly)) { - LogMsg(tr("Cannot read file %1: %2").arg(fastresumePath, file.errorString()), Log::WARNING); + LogMsg(tr("Cannot read file %1: %2").arg(fastresumePath, resumeDataFile.errorString()), Log::WARNING); return std::nullopt; } - const QByteArray data = file.readAll(); - const TorrentInfo metadata = TorrentInfo::loadFromFile(torrentFilePath); + QFile metadataFile {torrentFilePath}; + if (metadataFile.exists() && !metadataFile.open(QIODevice::ReadOnly)) + { + LogMsg(tr("Cannot read file %1: %2").arg(torrentFilePath, metadataFile.errorString()), Log::WARNING); + return std::nullopt; + } + + const QByteArray data = resumeDataFile.readAll(); + const QByteArray metadata = (metadataFile.isOpen() ? metadataFile.readAll() : ""); return loadTorrentResumeData(data, metadata); } std::optional BitTorrent::BencodeResumeDataStorage::loadTorrentResumeData( - const QByteArray &data, const TorrentInfo &metadata) const + const QByteArray &data, const QByteArray &metadata) const { + const QByteArray allData = ((metadata.isEmpty() || data.isEmpty()) + ? data : (data.chopped(1) + metadata.mid(1))); + lt::error_code ec; - const lt::bdecode_node root = lt::bdecode(data, ec); - if (ec || (root.type() != lt::bdecode_node::dict_t)) return std::nullopt; + const lt::bdecode_node root = lt::bdecode(allData, ec); + if (ec || (root.type() != lt::bdecode_node::dict_t)) + return std::nullopt; LoadTorrentParams torrentParams; torrentParams.restored = true; @@ -220,8 +230,6 @@ std::optional BitTorrent::BencodeResumeDataStorag p = lt::read_resume_data(root, ec); p.save_path = Profile::instance()->fromPortablePath(fromLTString(p.save_path)).toStdString(); - if (metadata.isValid()) - p.ti = metadata.nativeInfo(); if (p.flags & lt::torrent_flags::stop_when_ready) { @@ -333,15 +341,22 @@ void BitTorrent::BencodeResumeDataStorage::Worker::store(const TorrentID &id, co } } + lt::entry data = lt::write_resume_data(p); + // metadata is stored in separate .torrent file - const std::shared_ptr torrentInfo = std::move(p.ti); - if (torrentInfo) + if (p.ti) { + lt::entry::dictionary_type &dataDict = data.dict(); + lt::entry metadata {lt::entry::dictionary_t}; + lt::entry::dictionary_type &metadataDict = metadata.dict(); + metadataDict.insert(dataDict.extract("info")); + metadataDict.insert(dataDict.extract("creation date")); + metadataDict.insert(dataDict.extract("created by")); + metadataDict.insert(dataDict.extract("comment")); + const QString torrentFilepath = m_resumeDataDir.absoluteFilePath(QString::fromLatin1("%1.torrent").arg(id.toString())); try { - const auto torrentCreator = lt::create_torrent(*torrentInfo); - const lt::entry metadata = torrentCreator.generate(); writeEntryToFile(torrentFilepath, metadata); } catch (const RuntimeError &err) @@ -350,16 +365,8 @@ void BitTorrent::BencodeResumeDataStorage::Worker::store(const TorrentID &id, co .arg(torrentFilepath, err.message()), Log::CRITICAL); return; } - catch (const std::exception &err) - { - LogMsg(tr("Couldn't save torrent metadata to '%1'. Error: %2.") - .arg(torrentFilepath, QString::fromLocal8Bit(err.what())), Log::CRITICAL); - return; - } } - lt::entry data = lt::write_resume_data(p); - data["qBt-savePath"] = Profile::instance()->toPortablePath(resumeData.savePath).toStdString(); data["qBt-ratioLimit"] = static_cast(resumeData.ratioLimit * 1000); data["qBt-seedingTimeLimit"] = resumeData.seedingTimeLimit; diff --git a/src/base/bittorrent/bencoderesumedatastorage.h b/src/base/bittorrent/bencoderesumedatastorage.h index b8a81d1d0..41f3325d6 100644 --- a/src/base/bittorrent/bencoderesumedatastorage.h +++ b/src/base/bittorrent/bencoderesumedatastorage.h @@ -38,8 +38,6 @@ class QThread; namespace BitTorrent { - class TorrentInfo; - class BencodeResumeDataStorage final : public ResumeDataStorage { Q_OBJECT @@ -57,7 +55,7 @@ namespace BitTorrent private: void loadQueue(const QString &queueFilename); - std::optional loadTorrentResumeData(const QByteArray &data, const TorrentInfo &metadata) const; + std::optional loadTorrentResumeData(const QByteArray &data, const QByteArray &metadata) const; const QDir m_resumeDataDir; QVector m_registeredTorrents; diff --git a/src/base/bittorrent/dbresumedatastorage.cpp b/src/base/bittorrent/dbresumedatastorage.cpp index 397f5161f..78016dfb9 100644 --- a/src/base/bittorrent/dbresumedatastorage.cpp +++ b/src/base/bittorrent/dbresumedatastorage.cpp @@ -30,9 +30,9 @@ #include #include -#include #include #include +#include #include #include @@ -52,7 +52,6 @@ #include "base/utils/string.h" #include "infohash.h" #include "loadtorrentparams.h" -#include "torrentinfo.h" namespace { @@ -290,20 +289,19 @@ std::optional BitTorrent::DBResumeDataStorage::lo resumeData.stopped = query.value(DB_COLUMN_STOPPED.name).toBool(); const QByteArray bencodedResumeData = query.value(DB_COLUMN_RESUMEDATA.name).toByteArray(); + const QByteArray bencodedMetadata = query.value(DB_COLUMN_METADATA.name).toByteArray(); + const QByteArray allData = ((bencodedMetadata.isEmpty() || bencodedResumeData.isEmpty()) + ? bencodedResumeData + : (bencodedResumeData.chopped(1) + bencodedMetadata.mid(1))); lt::error_code ec; - const lt::bdecode_node root = lt::bdecode(bencodedResumeData, ec); + const lt::bdecode_node root = lt::bdecode(allData, ec); lt::add_torrent_params &p = resumeData.ltAddTorrentParams; p = lt::read_resume_data(root, ec); p.save_path = Profile::instance()->fromPortablePath(fromLTString(p.save_path)).toStdString(); - const QByteArray bencodedMetadata = query.value(DB_COLUMN_METADATA.name).toByteArray(); - auto metadata = TorrentInfo::load(bencodedMetadata); - if (metadata.isValid()) - p.ti = metadata.nativeInfo(); - return resumeData; } @@ -453,16 +451,23 @@ void BitTorrent::DBResumeDataStorage::Worker::store(const TorrentID &id, const L DB_COLUMN_RESUMEDATA }; + lt::entry data = lt::write_resume_data(p); + // metadata is stored in separate column QByteArray bencodedMetadata; - bencodedMetadata.reserve(512 * 1024); - const std::shared_ptr torrentInfo = std::move(p.ti); - if (torrentInfo) + if (p.ti) { + lt::entry::dictionary_type &dataDict = data.dict(); + lt::entry metadata {lt::entry::dictionary_t}; + lt::entry::dictionary_type &metadataDict = metadata.dict(); + metadataDict.insert(dataDict.extract("info")); + metadataDict.insert(dataDict.extract("creation date")); + metadataDict.insert(dataDict.extract("created by")); + metadataDict.insert(dataDict.extract("comment")); + try { - const auto torrentCreator = lt::create_torrent(*torrentInfo); - const lt::entry metadata = torrentCreator.generate(); + bencodedMetadata.reserve(512 * 1024); lt::bencode(std::back_inserter(bencodedMetadata), metadata); } catch (const std::exception &err) @@ -477,7 +482,7 @@ void BitTorrent::DBResumeDataStorage::Worker::store(const TorrentID &id, const L QByteArray bencodedResumeData; bencodedResumeData.reserve(256 * 1024); - lt::bencode(std::back_inserter(bencodedResumeData), lt::write_resume_data(p)); + lt::bencode(std::back_inserter(bencodedResumeData), data); const QString insertTorrentStatement = makeInsertStatement(DB_TABLE_TORRENTS, columns) + makeOnConflictUpdateStatement(DB_COLUMN_TORRENT_ID, columns); @@ -503,7 +508,7 @@ void BitTorrent::DBResumeDataStorage::Worker::store(const TorrentID &id, const L query.bindValue(DB_COLUMN_OPERATING_MODE.placeholder, Utils::String::fromEnum(resumeData.operatingMode)); query.bindValue(DB_COLUMN_STOPPED.placeholder, resumeData.stopped); query.bindValue(DB_COLUMN_RESUMEDATA.placeholder, bencodedResumeData); - if (torrentInfo) + if (!bencodedMetadata.isEmpty()) query.bindValue(DB_COLUMN_METADATA.placeholder, bencodedMetadata); if (!query.exec())