/* * Bittorrent Client using Qt and libtorrent. * Copyright (C) 2017-2023 Vladimir Golovnev * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * In addition, as a special exception, the copyright holders give permission to * link this program with the OpenSSL project's "OpenSSL" library (or with * modified versions of it that use the same license as the "OpenSSL" library), * and distribute the linked executables. You must obey the GNU General Public * License in all respects for all of the code used other than "OpenSSL". If you * modify file(s), you may extend this exception to your version of the file(s), * but you are not obligated to do so. If you do not wish to do so, delete this * exception statement from your version. */ #include "rss_autodownloader.h" #include #include #include #include #include #include #include #include #include #include #include #include "base/addtorrentmanager.h" #include "base/asyncfilestorage.h" #include "base/bittorrent/addtorrenterror.h" #include "base/bittorrent/session.h" #include "base/bittorrent/torrentdescriptor.h" #include "base/global.h" #include "base/interfaces/iapplication.h" #include "base/logger.h" #include "base/profile.h" #include "base/utils/fs.h" #include "base/utils/io.h" #include "rss_article.h" #include "rss_autodownloadrule.h" #include "rss_feed.h" #include "rss_folder.h" #include "rss_session.h" struct ProcessingJob { QString feedURL; QVariantHash articleData; }; const QString CONF_FOLDER_NAME = u"rss"_s; const QString RULES_FILE_NAME = u"download_rules.json"_s; namespace { QList rulesFromJSON(const QByteArray &jsonData) { QJsonParseError jsonError; const QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonData, &jsonError); if (jsonError.error != QJsonParseError::NoError) throw RSS::ParsingError(jsonError.errorString()); if (!jsonDoc.isObject()) throw RSS::ParsingError(RSS::AutoDownloader::tr("Invalid data format.")); const QJsonObject jsonObj {jsonDoc.object()}; QList rules; for (auto it = jsonObj.begin(); it != jsonObj.end(); ++it) { const QJsonValue jsonVal {it.value()}; if (!jsonVal.isObject()) throw RSS::ParsingError(RSS::AutoDownloader::tr("Invalid data format.")); rules.append(RSS::AutoDownloadRule::fromJsonObject(jsonVal.toObject(), it.key())); } return rules; } } using namespace RSS; QPointer AutoDownloader::m_instance = nullptr; QString computeSmartFilterRegex(const QStringList &filters) { return u"(?:_|\\b)(?:%1)(?:_|\\b)"_s.arg(filters.join(u")|(?:")); } AutoDownloader::AutoDownloader(IApplication *app) : ApplicationComponent(app) , m_storeProcessingEnabled {u"RSS/AutoDownloader/EnableProcessing"_s, false} , m_storeSmartEpisodeFilter {u"RSS/AutoDownloader/SmartEpisodeFilter"_s} , m_storeDownloadRepacks {u"RSS/AutoDownloader/DownloadRepacks"_s} , m_processingTimer {new QTimer(this)} , m_ioThread {new QThread} { Q_ASSERT(!m_instance); // only one instance is allowed m_instance = this; m_fileStorage = new AsyncFileStorage(specialFolderLocation(SpecialFolder::Config) / Path(CONF_FOLDER_NAME)); m_fileStorage->moveToThread(m_ioThread.get()); connect(m_ioThread.get(), &QThread::finished, m_fileStorage, &AsyncFileStorage::deleteLater); connect(m_fileStorage, &AsyncFileStorage::failed, [](const Path &fileName, const QString &errorString) { LogMsg(tr("Couldn't save RSS AutoDownloader data in %1. Error: %2") .arg(fileName.toString(), errorString), Log::CRITICAL); }); m_ioThread->setObjectName("RSS::AutoDownloader m_ioThread"); m_ioThread->start(); connect(app->addTorrentManager(), &AddTorrentManager::torrentAdded , this, &AutoDownloader::handleTorrentAdded); connect(app->addTorrentManager(), &AddTorrentManager::addTorrentFailed , this, &AutoDownloader::handleAddTorrentFailed); // initialise the smart episode regex const QString regex = computeSmartFilterRegex(smartEpisodeFilters()); m_smartEpisodeRegex = QRegularExpression(regex , QRegularExpression::CaseInsensitiveOption | QRegularExpression::ExtendedPatternSyntaxOption | QRegularExpression::UseUnicodePropertiesOption); load(); connect(Session::instance(), &Session::feedURLChanged, this, &AutoDownloader::handleFeedURLChanged); m_processingTimer->setSingleShot(true); connect(m_processingTimer, &QTimer::timeout, this, &AutoDownloader::process); const auto *btSession = BitTorrent::Session::instance(); if (btSession->isRestored()) { if (isProcessingEnabled()) startProcessing(); } else { connect(btSession, &BitTorrent::Session::restored, this, [this]() { if (isProcessingEnabled()) startProcessing(); }); } } AutoDownloader::~AutoDownloader() { store(); } AutoDownloader *AutoDownloader::instance() { return m_instance; } bool AutoDownloader::hasRule(const QString &ruleName) const { return m_rulesByName.contains(ruleName); } AutoDownloadRule AutoDownloader::ruleByName(const QString &ruleName) const { const auto index = m_rulesByName.value(ruleName, -1); return m_rules.value(index, AutoDownloadRule(u"Unknown Rule"_s)); } QList AutoDownloader::rules() const { return m_rules; } void AutoDownloader::setRule(const AutoDownloadRule &rule) { if (!hasRule(rule.name())) { // Insert new rule setRule_impl(rule); sortRules(); m_dirty = true; store(); emit ruleAdded(rule.name()); resetProcessingQueue(); } else if (ruleByName(rule.name()) != rule) { // Update existing rule setRule_impl(rule); sortRules(); m_dirty = true; storeDeferred(); emit ruleChanged(rule.name()); resetProcessingQueue(); } } bool AutoDownloader::renameRule(const QString &ruleName, const QString &newRuleName) { if (!hasRule(ruleName) || hasRule(newRuleName)) return false; const auto index = m_rulesByName.take(ruleName); m_rules[index].setName(newRuleName); m_rulesByName.insert(newRuleName, index); m_dirty = true; store(); emit ruleRenamed(newRuleName, ruleName); return true; } void AutoDownloader::removeRule(const QString &ruleName) { if (!hasRule(ruleName)) return; emit ruleAboutToBeRemoved(ruleName); const auto index = m_rulesByName.take(ruleName); m_rules.removeAt(index); for (qsizetype i = index; i < m_rules.size(); ++i) { const AutoDownloadRule &rule = m_rules[i]; m_rulesByName[rule.name()] = i; } m_dirty = true; store(); } QByteArray AutoDownloader::exportRules(AutoDownloader::RulesFileFormat format) const { switch (format) { case RulesFileFormat::Legacy: return exportRulesToLegacyFormat(); default: return exportRulesToJSONFormat(); } } void AutoDownloader::importRules(const QByteArray &data, const AutoDownloader::RulesFileFormat format) { switch (format) { case RulesFileFormat::Legacy: importRulesFromLegacyFormat(data); break; default: importRulesFromJSONFormat(data); } } QByteArray AutoDownloader::exportRulesToJSONFormat() const { QJsonObject jsonObj; for (const auto &rule : asConst(rules())) jsonObj.insert(rule.name(), rule.toJsonObject()); return QJsonDocument(jsonObj).toJson(); } void AutoDownloader::importRulesFromJSONFormat(const QByteArray &data) { for (const auto &rule : asConst(rulesFromJSON(data))) setRule(rule); } QByteArray AutoDownloader::exportRulesToLegacyFormat() const { QVariantHash dict; for (const auto &rule : asConst(rules())) dict[rule.name()] = rule.toLegacyDict(); QByteArray data; QDataStream out(&data, QIODevice::WriteOnly); out.setVersion(QDataStream::Qt_4_5); out << dict; return data; } void AutoDownloader::importRulesFromLegacyFormat(const QByteArray &data) { QDataStream in(data); in.setVersion(QDataStream::Qt_4_5); QVariantHash dict; in >> dict; if (in.status() != QDataStream::Ok) throw ParsingError(tr("Invalid data format")); for (const QVariant &val : asConst(dict)) setRule(AutoDownloadRule::fromLegacyDict(val.toHash())); } QStringList AutoDownloader::smartEpisodeFilters() const { const QVariant filter = m_storeSmartEpisodeFilter.get(); if (filter.isNull()) { const QStringList defaultFilters = { u"s(\\d+)e(\\d+)"_s, // Format 1: s01e01 u"(\\d+)x(\\d+)"_s, // Format 2: 01x01 u"(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})"_s, // Format 3: 2017.01.01 u"(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})"_s // Format 4: 01.01.2017 }; return defaultFilters; } return filter.toStringList(); } QRegularExpression AutoDownloader::smartEpisodeRegex() const { return m_smartEpisodeRegex; } void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters) { m_storeSmartEpisodeFilter = filters; const QString regex = computeSmartFilterRegex(filters); m_smartEpisodeRegex.setPattern(regex); } bool AutoDownloader::downloadRepacks() const { return m_storeDownloadRepacks.get(true); } void AutoDownloader::setDownloadRepacks(const bool enabled) { m_storeDownloadRepacks = enabled; } void AutoDownloader::process() { if (m_processingQueue.isEmpty()) // processing was disabled return; processJob(m_processingQueue.takeFirst()); if (!m_processingQueue.isEmpty()) { // Schedule to process the next torrent (if any) m_processingTimer->start(); } } void AutoDownloader::handleTorrentAdded(const QString &source) { const auto job = m_waitingJobs.take(source); if (!job) return; if (Feed *feed = Session::instance()->feedByURL(job->feedURL)) { if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString())) article->markAsRead(); } } void AutoDownloader::handleAddTorrentFailed(const QString &source, const BitTorrent::AddTorrentError &error) { const auto job = m_waitingJobs.take(source); if (!job) return; if (error.kind == BitTorrent::AddTorrentError::DuplicateTorrent) { if (Feed *feed = Session::instance()->feedByURL(job->feedURL)) { if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString())) article->markAsRead(); } } else { // TODO: Re-schedule job here. } } void AutoDownloader::handleNewArticle(const Article *article) { if (!article->isRead() && !article->torrentUrl().isEmpty()) addJobForArticle(article); } void AutoDownloader::handleFeedURLChanged(Feed *feed, const QString &oldURL) { for (AutoDownloadRule &rule : m_rules) { if (const auto i = rule.feedURLs().indexOf(oldURL); i >= 0) { auto feedURLs = rule.feedURLs(); feedURLs.replace(i, feed->url()); rule.setFeedURLs(feedURLs); m_dirty = true; } } for (const QSharedPointer &job : asConst(m_processingQueue)) { if (job->feedURL == oldURL) job->feedURL = feed->url(); } for (const QSharedPointer &job : asConst(m_waitingJobs)) { if (job->feedURL == oldURL) job->feedURL = feed->url(); } store(); } void AutoDownloader::setRule_impl(const AutoDownloadRule &rule) { const QString ruleName = rule.name(); const auto index = m_rulesByName.value(ruleName, -1); if (index < 0) { m_rules.append(rule); m_rulesByName[ruleName] = m_rules.size() - 1; } else { m_rules[index] = rule; } } void AutoDownloader::sortRules() { std::sort(m_rules.begin(), m_rules.end(), [](const AutoDownloadRule &lhs, const AutoDownloadRule &rhs) { return (lhs.priority() < rhs.priority()); }); for (qsizetype i = 0; i < m_rules.size(); ++i) { const AutoDownloadRule &rule = m_rules[i]; m_rulesByName[rule.name()] = i; } } void AutoDownloader::addJobForArticle(const Article *article) { const QString torrentURL = article->torrentUrl(); if (m_waitingJobs.contains(torrentURL)) return; auto job = QSharedPointer::create(); job->feedURL = article->feed()->url(); job->articleData = article->data(); m_processingQueue.append(job); if (!m_processingTimer->isActive()) m_processingTimer->start(); } void AutoDownloader::processJob(const QSharedPointer &job) { for (AutoDownloadRule &rule : m_rules) { if (!rule.isEnabled()) continue; if (!rule.feedURLs().contains(job->feedURL)) continue; if (!rule.accepts(job->articleData)) continue; m_dirty = true; storeDeferred(); LogMsg(tr("RSS article '%1' is accepted by rule '%2'. Trying to add torrent...") .arg(job->articleData.value(Article::KeyTitle).toString(), rule.name())); const auto torrentURL = job->articleData.value(Article::KeyTorrentURL).toString(); app()->addTorrentManager()->addTorrent(torrentURL, rule.addTorrentParams()); if (BitTorrent::TorrentDescriptor::parse(torrentURL)) { if (Feed *feed = Session::instance()->feedByURL(job->feedURL)) { if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString())) article->markAsRead(); } } else { // waiting for torrent file downloading m_waitingJobs.insert(torrentURL, job); } return; } } void AutoDownloader::load() { const qint64 maxFileSize = 10 * 1024 * 1024; const auto readResult = Utils::IO::readFile((m_fileStorage->storageDir() / Path(RULES_FILE_NAME)), maxFileSize); if (!readResult) { if (readResult.error().status == Utils::IO::ReadError::NotExist) { loadRulesLegacy(); return; } LogMsg((tr("Failed to read RSS AutoDownloader rules. %1").arg(readResult.error().message)), Log::WARNING); return; } loadRules(readResult.value()); } void AutoDownloader::loadRules(const QByteArray &data) { try { const auto rules = rulesFromJSON(data); for (const auto &rule : rules) setRule_impl(rule); sortRules(); } catch (const ParsingError &error) { LogMsg(tr("Couldn't load RSS AutoDownloader rules. Reason: %1") .arg(error.message()), Log::CRITICAL); } } void AutoDownloader::loadRulesLegacy() { const std::unique_ptr settings = Profile::instance()->applicationSettings(u"qBittorrent-rss"_s); const QVariantHash rules = settings->value(u"download_rules"_s).toHash(); for (const QVariant &ruleVar : rules) { const auto rule = AutoDownloadRule::fromLegacyDict(ruleVar.toHash()); if (!rule.name().isEmpty()) setRule(rule); } } void AutoDownloader::store() { if (!m_dirty) return; m_dirty = false; m_savingTimer.stop(); QJsonObject jsonObj; for (const auto &rule : asConst(m_rules)) jsonObj.insert(rule.name(), rule.toJsonObject()); m_fileStorage->store(Path(RULES_FILE_NAME), QJsonDocument(jsonObj).toJson()); } void AutoDownloader::storeDeferred() { if (!m_savingTimer.isActive()) m_savingTimer.start(5 * 1000, this); } bool AutoDownloader::isProcessingEnabled() const { return m_storeProcessingEnabled; } void AutoDownloader::resetProcessingQueue() { m_processingQueue.clear(); if (!isProcessingEnabled()) return; for (Article *article : asConst(Session::instance()->rootFolder()->articles())) { if (!article->isRead() && !article->torrentUrl().isEmpty()) addJobForArticle(article); } } void AutoDownloader::startProcessing() { resetProcessingQueue(); const RSS::Folder *rootFolder = Session::instance()->rootFolder(); for (const Article *article : asConst(rootFolder->articles())) handleNewArticle(article); connect(rootFolder, &Folder::newArticle, this, &AutoDownloader::handleNewArticle); } void AutoDownloader::setProcessingEnabled(const bool enabled) { if (m_storeProcessingEnabled != enabled) { m_storeProcessingEnabled = enabled; if (enabled) { if (BitTorrent::Session::instance()->isRestored()) startProcessing(); } else { m_processingQueue.clear(); disconnect(Session::instance()->rootFolder(), &Folder::newArticle, this, &AutoDownloader::handleNewArticle); } emit processingStateChanged(enabled); } } void AutoDownloader::timerEvent([[maybe_unused]] QTimerEvent *event) { store(); }