Initial implementation of Smart Filter feature

src/base/rss/rss_autodownloader.cpp

@@ -333,6 +333,8 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
         }
 
         rule.setLastMatch(articleDate);
+        rule.appendLastComputedEpisode();
+
         m_dirty = true;
         storeDeferred();
 

src/base/rss/rss_autodownloadrule.cpp

@@ -34,7 +34,6 @@
 #include <QHash>
 #include <QJsonArray>
 #include <QJsonObject>
-#include <QRegExp>
 #include <QRegularExpression>
 #include <QSharedData>
 #include <QString>
@@ -100,6 +99,8 @@ const QString Str_AssignedCategory(QStringLiteral("assignedCategory"));
 const QString Str_LastMatch(QStringLiteral("lastMatch"));
 const QString Str_IgnoreDays(QStringLiteral("ignoreDays"));
 const QString Str_AddPaused(QStringLiteral("addPaused"));
+const QString Str_SmartFilter(QStringLiteral("smartFilter"));
+const QString Str_PreviouslyMatched(QStringLiteral("previouslyMatchedEpisodes"));
 
 namespace RSS
 {
@@ -120,6 +121,10 @@ namespace RSS
         QString category;
         TriStateBool addPaused = TriStateBool::Undefined;
 
+        bool smartFilter = false;
+        QStringList previouslyMatchedEpisodes;
+
+        mutable QString lastComputedEpisode;
         mutable QHash<QString, QRegularExpression> cachedRegexes;
 
         bool operator==(const AutoDownloadRuleData &other) const
@@ -135,13 +140,52 @@ namespace RSS
                     && (lastMatch == other.lastMatch)
                     && (savePath == other.savePath)
                     && (category == other.category)
-                    && (addPaused == other.addPaused);
+                    && (addPaused == other.addPaused)
+                    && (smartFilter == other.smartFilter);
         }
     };
 }
 
 using namespace RSS;
 
+QString computeEpisodeName(const QString &article)
+{
+    const QRegularExpression episodeRegex(
+                    "(?:^|[^a-z0-9])(?:"
+
+                    //Format 1: s01e01
+                    "(?:s(\\d+)e(\\d+))|"
+
+                    //Format 2: 01x01
+                    "(?:(\\d+)x(\\d+))|"
+
+                    //Format 3: 2017.01.01
+                    "((?:\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})|"
+
+                    //Format 4: 01.01.2017
+                    "(?:\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4}))"
+
+                    ")(?:[^a-z0-9]|$)",
+                    QRegularExpression::CaseInsensitiveOption
+                    | QRegularExpression::ExtendedPatternSyntaxOption);
+
+    QRegularExpressionMatch match = episodeRegex.match(article);
+
+    // See if we can extract an season/episode number or date from the title
+    if (!match.hasMatch()) {
+        return QString();
+    }
+
+    int lastCapturedIndex = match.lastCapturedIndex();
+    if (lastCapturedIndex == 5) {
+        return match.captured(5);
+    }
+    else {
+        return QString("%1x%2").arg(match.captured(lastCapturedIndex - 1).toInt())
+                               .arg(match.captured(lastCapturedIndex).toInt());
+    }
+}
+
 AutoDownloadRule::AutoDownloadRule(const QString &name)
     : m_dataPtr(new AutoDownloadRuleData)
 {
@@ -197,6 +241,9 @@ bool AutoDownloadRule::matches(const QString &articleTitle, const QString &expre
 
 bool AutoDownloadRule::matches(const QString &articleTitle) const
 {
+    // Reset the lastComputedEpisode, we don't want to leak it between matches
+    m_dataPtr->lastComputedEpisode.clear();
+
     if (!m_dataPtr->mustContain.empty()) {
         bool logged = false;
         bool foundMustContain = false;
@@ -334,6 +381,20 @@ bool AutoDownloadRule::matches(const QString &articleTitle) const
         return false;
     }
 
+    if (useSmartFilter()) {
+        // now see if this episode has been downloaded before
+        QString episodeStr = computeEpisodeName(articleTitle);
+
+        if (!episodeStr.isEmpty()) {
+            bool previouslyMatched = m_dataPtr->previouslyMatchedEpisodes.contains(episodeStr);
+            bool isRepack = articleTitle.contains("REPACK", Qt::CaseInsensitive) || articleTitle.contains("PROPER", Qt::CaseInsensitive);
+            if (previouslyMatched && !isRepack) {
+                return false;
+            }
+            m_dataPtr->lastComputedEpisode = episodeStr;
+        }
+    }
+
 //    qDebug() << "Matched article:" << articleTitle;
     return true;
 }
@@ -367,7 +428,9 @@ QJsonObject AutoDownloadRule::toJsonObject() const
         , {Str_AssignedCategory, assignedCategory()}
         , {Str_LastMatch, lastMatch().toString(Qt::RFC2822Date)}
         , {Str_IgnoreDays, ignoreDays()}
-        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}};
+        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}
+        , {Str_SmartFilter, useSmartFilter()}
+        , {Str_PreviouslyMatched, QJsonArray::fromStringList(previouslyMatchedEpisodes())}};
 }
 
 AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, const QString &name)
@@ -384,6 +447,7 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
     rule.setAddPaused(jsonValueToTriStateBool(jsonObj.value(Str_AddPaused)));
     rule.setLastMatch(QDateTime::fromString(jsonObj.value(Str_LastMatch).toString(), Qt::RFC2822Date));
     rule.setIgnoreDays(jsonObj.value(Str_IgnoreDays).toInt());
+    rule.setUseSmartFilter(jsonObj.value(Str_SmartFilter).toBool(false));
 
     const QJsonValue feedsVal = jsonObj.value(Str_AffectedFeeds);
     QStringList feedURLs;
@@ -393,6 +457,17 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
         feedURLs << urlVal.toString();
     rule.setFeedURLs(feedURLs);
 
+    const QJsonValue previouslyMatchedVal = jsonObj.value(Str_PreviouslyMatched);
+    QStringList previouslyMatched;
+    if (previouslyMatchedVal.isString()) {
+        previouslyMatched << previouslyMatchedVal.toString();
+    }
+    else {
+        foreach (const QJsonValue &val, previouslyMatchedVal.toArray())
+            previouslyMatched << val.toString();
+    }
+    rule.setPreviouslyMatchedEpisodes(previouslyMatched);
+
     return rule;
 }
 
@@ -549,6 +624,16 @@ QString AutoDownloadRule::mustNotContain() const
     return m_dataPtr->mustNotContain.join("|");
 }
 
+bool AutoDownloadRule::useSmartFilter() const
+{
+    return m_dataPtr->smartFilter;
+}
+
+void AutoDownloadRule::setUseSmartFilter(bool enabled)
+{
+    m_dataPtr->smartFilter = enabled;
+}
+
 bool AutoDownloadRule::useRegex() const
 {
     return m_dataPtr->useRegex;
@@ -560,6 +645,25 @@ void AutoDownloadRule::setUseRegex(bool enabled)
     m_dataPtr->cachedRegexes.clear();
 }
 
+QStringList AutoDownloadRule::previouslyMatchedEpisodes() const
+{
+    return m_dataPtr->previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes)
+{
+    m_dataPtr->previouslyMatchedEpisodes = previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::appendLastComputedEpisode()
+{
+    if (!m_dataPtr->lastComputedEpisode.isEmpty()) {
+        // TODO: probably need to add a marker for PROPER/REPACK to avoid duplicate downloads
+        m_dataPtr->previouslyMatchedEpisodes.append(m_dataPtr->lastComputedEpisode);
+        m_dataPtr->lastComputedEpisode.clear();
+    }
+}
+
 QString AutoDownloadRule::episodeFilter() const
 {
     return m_dataPtr->episodeFilter;

src/base/rss/rss_autodownloadrule.h

@@ -66,9 +66,15 @@ namespace RSS
         void setLastMatch(const QDateTime &lastMatch);
         bool useRegex() const;
         void setUseRegex(bool enabled);
+        bool useSmartFilter() const;
+        void setUseSmartFilter(bool enabled);
         QString episodeFilter() const;
         void setEpisodeFilter(const QString &e);
 
+        void appendLastComputedEpisode();
+        QStringList previouslyMatchedEpisodes() const;
+        void setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes);
+
         QString savePath() const;
         void setSavePath(const QString &savePath);
         TriStateBool addPaused() const;

src/gui/rss/automatedrssdownloader.cpp

@@ -113,6 +113,7 @@ AutomatedRssDownloader::AutomatedRssDownloader(QWidget *parent)
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustLineValidity);
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustNotLineValidity);
+    connect(m_ui->checkSmart, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
 
     connect(m_ui->listFeeds, &QListWidget::itemChanged, this, &AutomatedRssDownloader::handleFeedCheckStateChange);
 
@@ -255,6 +256,9 @@ void AutomatedRssDownloader::updateRuleDefinitionBox()
         m_ui->checkRegex->blockSignals(true);
         m_ui->checkRegex->setChecked(m_currentRule.useRegex());
         m_ui->checkRegex->blockSignals(false);
+        m_ui->checkSmart->blockSignals(true);
+        m_ui->checkSmart->setChecked(m_currentRule.useSmartFilter());
+        m_ui->checkSmart->blockSignals(false);
         m_ui->comboCategory->setCurrentIndex(m_ui->comboCategory->findText(m_currentRule.assignedCategory()));
         if (m_currentRule.assignedCategory().isEmpty())
             m_ui->comboCategory->clearEditText();
@@ -299,6 +303,7 @@ void AutomatedRssDownloader::clearRuleDefinitionBox()
     m_ui->comboCategory->clearEditText();
     m_ui->comboCategory->setCurrentIndex(-1);
     m_ui->checkRegex->setChecked(false);
+    m_ui->checkSmart->setChecked(false);
     m_ui->spinIgnorePeriod->setValue(0);
     m_ui->comboAddPaused->clearEditText();
     m_ui->comboAddPaused->setCurrentIndex(-1);
@@ -323,6 +328,7 @@ void AutomatedRssDownloader::updateEditedRule()
 
     m_currentRule.setEnabled(m_currentRuleItem->checkState() != Qt::Unchecked);
     m_currentRule.setUseRegex(m_ui->checkRegex->isChecked());
+    m_currentRule.setUseSmartFilter(m_ui->checkSmart->isChecked());
     m_currentRule.setMustContain(m_ui->lineContains->text());
     m_currentRule.setMustNotContain(m_ui->lineNotContains->text());
     m_currentRule.setEpisodeFilter(m_ui->lineEFilter->text());
@@ -465,7 +471,9 @@ void AutomatedRssDownloader::displayRulesListMenu()
     QAction *addAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-add"), tr("Add new rule..."));
     QAction *delAct = nullptr;
     QAction *renameAct = nullptr;
+    QAction *clearAct = nullptr;
     const QList<QListWidgetItem *> selection = m_ui->listRules->selectedItems();
+
     if (!selection.isEmpty()) {
         if (selection.count() == 1) {
             delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete rule"));
@@ -475,6 +483,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
         else {
             delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete selected rules"));
         }
+        menu.addSeparator();
+        clearAct = menu.addAction(GuiIconProvider::instance()->getIcon("edit-clear"), tr("Clear downloaded episodes..."));
     }
 
     QAction *act = menu.exec(QCursor::pos());
@@ -486,6 +496,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
         on_removeRuleBtn_clicked();
     else if (act == renameAct)
         renameSelectedRule();
+    else if (act == clearAct)
+        clearSelectedRuleDownloadedEpisodeList();
 }
 
 void AutomatedRssDownloader::renameSelectedRule()
@@ -518,6 +530,20 @@ void AutomatedRssDownloader::handleRuleCheckStateChange(QListWidgetItem *ruleIte
     m_ui->listRules->setCurrentItem(ruleItem);
 }
 
+void AutomatedRssDownloader::clearSelectedRuleDownloadedEpisodeList()
+{
+    QMessageBox::StandardButton reply = QMessageBox::question(
+                this,
+                tr("Clear downloaded episodes"),
+                tr("Are you sure you want to clear the list of downloaded episodes for the selected rule?"),
+                QMessageBox::Yes | QMessageBox::No);
+
+    if (reply == QMessageBox::Yes) {
+        m_currentRule.setPreviouslyMatchedEpisodes(QStringList());
+        handleRuleDefinitionChanged();
+    }
+}
+
 void AutomatedRssDownloader::handleFeedCheckStateChange(QListWidgetItem *feedItem)
 {
     const QString feedURL = feedItem->data(Qt::UserRole).toString();
@@ -750,7 +776,7 @@ void AutomatedRssDownloader::handleRuleRenamed(const QString &ruleName, const QS
 void AutomatedRssDownloader::handleRuleChanged(const QString &ruleName)
 {
     auto item = m_itemsByRuleName.value(ruleName);
-    if (item != m_currentRuleItem)
+    if (item && (item != m_currentRuleItem))
         item->setCheckState(RSS::AutoDownloader::instance()->ruleByName(ruleName).isEnabled() ? Qt::Checked : Qt::Unchecked);
 }
 

src/gui/rss/automatedrssdownloader.h

@@ -71,6 +71,7 @@ private slots:
     void displayRulesListMenu();
     void renameSelectedRule();
     void updateRuleDefinitionBox();
+    void clearSelectedRuleDownloadedEpisodeList();
     void updateFieldsToolTips(bool regex);
     void updateMustLineValidity();
     void updateMustNotLineValidity();

src/gui/rss/automatedrssdownloader.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>816</width>
+    <width>818</width>
-    <height>523</height>
+    <height>571</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -106,6 +106,17 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QCheckBox" name="checkSmart">
+            <property name="toolTip">
+		    <string>Smart Episode Filter will check the episode number to prevent downloading of duplicates.
+Supports the formats: S01E01, 1x1, 2017.01.01 and 01.01.2017 (Date formats also support - as a separator)</string>
+            </property>
+            <property name="text">
+             <string>Use Smart Episode Filter</string>
+            </property>
+           </widget>
+          </item>
           <item>
            <layout class="QGridLayout" name="gridLayout">
             <item row="0" column="0">
@@ -405,6 +416,9 @@
      </item>
      <item>
       <widget class="QDialogButtonBox" name="buttonBox">
+       <property name="focusPolicy">
+        <enum>Qt::StrongFocus</enum>
+       </property>
        <property name="standardButtons">
         <set>QDialogButtonBox::Close</set>
        </property>
@@ -414,6 +428,26 @@
    </item>
   </layout>
  </widget>
+ <tabstops>
+  <tabstop>removeRuleBtn</tabstop>
+  <tabstop>addRuleBtn</tabstop>
+  <tabstop>listRules</tabstop>
+  <tabstop>checkRegex</tabstop>
+  <tabstop>checkSmart</tabstop>
+  <tabstop>lineContains</tabstop>
+  <tabstop>lineNotContains</tabstop>
+  <tabstop>lineEFilter</tabstop>
+  <tabstop>comboCategory</tabstop>
+  <tabstop>saveDiffDir_check</tabstop>
+  <tabstop>lineSavePath</tabstop>
+  <tabstop>browseSP</tabstop>
+  <tabstop>spinIgnorePeriod</tabstop>
+  <tabstop>comboAddPaused</tabstop>
+  <tabstop>listFeeds</tabstop>
+  <tabstop>treeMatchingArticles</tabstop>
+  <tabstop>importBtn</tabstop>
+  <tabstop>exportBtn</tabstop>
+ </tabstops>
  <resources/>
  <connections>
   <connection>