Improve merge database import method and remove append method

API.md

@@ -2561,7 +2561,7 @@ Import a Tautulli, PlexWatch, or Plexivity database into Tautulli.
 Required parameters:
     app (str):                      "tautulli" or "plexwatch" or "plexivity"
     database_path (str):            The full path to the plexwatch database file
-    method (str):                   For Tautulli only, "merge" or "append" or "overwrite"
+    method (str):                   For Tautulli only, "merge" or "overwrite"
     table_name (str):               For PlexWatch or Plexivity only, "processed" or "grouped"
 
 

data/interfaces/default/app_import.html

@@ -5,6 +5,7 @@
             <h4 class="modal-title">Import ${app} Database</h4>
         </div>
         <div class="modal-body" id="modal-text">
+            <input type="hidden" id="import_app" name="import_app" value="${app.lower()}" />
             % if app in ('PlexWatch', 'Plexivity'):
             <p class="help-block">
                 <%
@@ -33,23 +34,20 @@
                     <div class="col-xs-4">
                         <select class="form-control" id="import_method" name="import_method">
                             <option value="merge">Merge</option>
-                            <option value="append">Append</option>
                             <option value="overwrite">Oerwrite</option>
                         </select>
                     </div>
                 </div>
                 <p class="help-block">Select how you would like to import the Tautulli history.</p>
                 <ul class="help-block" style="padding-inline-start: 15px;">
-                    <li><strong>Merge</strong> will only add missing history from the imported database into the current database.</li>
+                    <li><strong>Merge</strong> will add all history and remove any duplicates from the imported database into the current database.</li>
-                    <li><strong>Append</strong> will add all history from the imported database into the current database.
-                        <br>Note: History will be duplicated if it is present in both databases.</li>
                     <li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
                 </ul>
-                <p class="help-block">Note: Other settings such as notification and newsletter agents will also be imported.</p>
+                <p class="help-block">Note: Libraries, users, notification agents, newsletter agents, and registered mobile devices will also be imported</p>
             </div>
             <div class="checkbox">
                 <label>
-                    <input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Database
+                    <input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Current Database
                 </label>
                 <p class="help-block">Automatically create a backup of the current database before importing.</p>
             </div>
@@ -88,6 +86,7 @@
 <script>
     // Send database path to import script
     $("#import_db").click(function() {
+        var import_app = $("#import_app").val();
         var database_path = $("#import_database_path").val();
         var import_method = $("#import_method").val();
         var import_backup_db = $("#import_backup_db").is(':checked');
@@ -96,7 +95,7 @@
         $.ajax({
             url: 'import_database',
             data: {
-                app: "${app}",
+                app: import_app,
                 database_path: database_path,
                 method: import_method,
                 backup: import_backup_db,

plexpy/database.py

@@ -58,31 +58,38 @@ def import_tautulli_db(database=None, method=None, backup=True):
         logger.error("Tautulli Database :: Failed to import Tautulli database: %s", db_validate)
         return False
 
-    if method not in ('merge', 'append', 'overwrite'):
+    if method not in ('merge', 'overwrite'):
         logger.error("Tautulli Database :: Failed to import Tautulli database: invalid import method '%s'", method)
         return False
 
-    # Make a backup of the current database first
     if backup:
+        # Make a backup of the current database first
         logger.info("Tautulli Database :: Creating a database backup before importing.")
         if not make_backup():
             logger.error("Tautulli Database :: Failed to import Tautulli database: failed to create database backup")
             return False
 
-    logger.info("Tautulli Database :: Tautulli database import started with method '%s'...", method)
+    logger.info("Tautulli Database :: Importing Tautulli database '%s' with import method '%s'...", database, method)
 
     db = MonitorDatabase()
     db.connection.execute('BEGIN IMMEDIATE')
     db.connection.execute('ATTACH ? AS import_db', [database])
 
-    # Create a temporary table so we can reindex session_history.reference_id after merging
+    # Get the current number of used ids in the session_history table
-    if method == 'append':
+    session_history_seq = db.select_single('SELECT seq FROM sqlite_sequence WHERE name = "session_history"')
+
+    if method == 'merge':
+        # Create a temporary table so we can reindex session_history.reference_id after merging
         logger.info("Tautulli Database :: Creating temporary database table to re-index grouped session history.")
-        session_history_seq = db.select_single('SELECT seq FROM sqlite_sequence WHERE name = "session_history" ')
         db.action('CREATE TABLE IF NOT EXISTS temp (id INTEGER PRIMARY KEY, old_id, new_id)')
         db.action('INSERT INTO temp (old_id) SELECT id FROM import_db.session_history')
         db.action('UPDATE temp SET new_id = id + ?', [session_history_seq['seq']])
 
+    # Keep track of all table columns so that duplicates can be removed after importing
+    session_history_tables = ('session_history', 'session_history_metadata', 'session_history_media_info')
+    session_history_columns = []
+    table_columns = {}
+
     tables = db.select('SELECT name FROM import_db.sqlite_master '
                        'WHERE type = "table" AND name NOT LIKE "sqlite_%"')
     for table in tables:
@@ -91,28 +98,50 @@ def import_tautulli_db(database=None, method=None, backup=True):
             # Skip temporary sessions table
             continue
 
-        logger.info("Tautulli Database :: Importing database table '%s'", table_name)
+        logger.info("Tautulli Database :: Importing database table '%s'.", table_name)
 
         if method == 'overwrite':
+            # Clear the table and reset the autoincrement ids
             db.action('DELETE FROM {table}'.format(table=table_name))
             db.action('DELETE FROM sqlite_sequence WHERE name = ?', [table_name])
 
+        # Get the list of columns to import
         columns = db.select('PRAGMA import_db.table_info({table})'.format(table=table_name))
-
+        import_columns = [c['name'] for c in columns if not c['pk']]
-        if method == 'merge':
-            import_columns = [c['name'] for c in columns]
-        else:
-            import_columns = [c['name'] for c in columns if not c['pk']]
-
         insert_columns = ', '.join(import_columns)
+
+        table_columns[table_name] = insert_columns
+        if table == 'session_history':
+            session_history_columns = import_columns
+
+        # Insert the data with ignore instead of replace to be safe
         db.action('INSERT OR IGNORE INTO {table} ({columns}) '
                   'SELECT {columns} FROM import_db.{table}'.format(table=table_name, columns=insert_columns))
 
-    # Reindex session_history.reference_id and delete the temp table
+    db.connection.execute('DETACH import_db')
-    if method == 'append':
+
+    if method == 'merge':
+        # Reindex session_history.reference_id
         logger.info("Tautulli Database :: Re-indexing grouped session history.")
         db.action('UPDATE session_history SET reference_id = (SELECT new_id FROM temp WHERE old_id = reference_id)'
                   'WHERE id > ?', [session_history_seq['seq']])
+
+        if session_history_columns:
+            # Remove reference_id column from the list of session_history table columns to get unique rows
+            columns = ', '.join([c for c in session_history_columns if c != 'reference_id'])
+            # Remove session_history ids to KEEP from the temp table
+            db.action('DELETE FROM temp WHERE new_id IN ('
+                      'SELECT MIN(id) FROM session_history '
+                      'GROUP BY {columns})'.format(columns=columns))
+
+        for table, columns in table_columns.items():
+            logger.info("Tautulli Database :: Removing duplicate rows from database table '%s'.", table)
+            if table in session_history_tables:
+                db.action('DELETE FROM {table} WHERE id IN (SELECT new_id FROM temp)'.format(table=table))
+            else:
+                db.action('DELETE FROM {table} WHERE id NOT IN '
+                          '(SELECT MIN(id) FROM {table} GROUP BY {columns})'.format(table=table, columns=columns))
+
         logger.info("Tautulli Database :: Deleting temporary database table.")
         db.action('DROP TABLE temp')
 

plexpy/webserve.py

@@ -3748,7 +3748,7 @@ class WebInterface(object):
             Required parameters:
                 app (str):                      "tautulli" or "plexwatch" or "plexivity"
                 database_path (str):            The full path to the plexwatch database file
-                method (str):                   For Tautulli only, "merge" or "append" or "overwrite"
+                method (str):                   For Tautulli only, "merge" or "overwrite"
                 table_name (str):               For PlexWatch or Plexivity only, "processed" or "grouped"