mirror of
https://github.com/Tautulli/Tautulli.git
synced 2025-07-11 07:46:07 -07:00
Improve merge database import method and remove append method
This commit is contained in:
parent
e698bcb375
commit
a5653e365e
4 changed files with 52 additions and 24 deletions
2
API.md
2
API.md
|
@ -2561,7 +2561,7 @@ Import a Tautulli, PlexWatch, or Plexivity database into Tautulli.
|
||||||
Required parameters:
|
Required parameters:
|
||||||
app (str): "tautulli" or "plexwatch" or "plexivity"
|
app (str): "tautulli" or "plexwatch" or "plexivity"
|
||||||
database_path (str): The full path to the plexwatch database file
|
database_path (str): The full path to the plexwatch database file
|
||||||
method (str): For Tautulli only, "merge" or "append" or "overwrite"
|
method (str): For Tautulli only, "merge" or "overwrite"
|
||||||
table_name (str): For PlexWatch or Plexivity only, "processed" or "grouped"
|
table_name (str): For PlexWatch or Plexivity only, "processed" or "grouped"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
<h4 class="modal-title">Import ${app} Database</h4>
|
<h4 class="modal-title">Import ${app} Database</h4>
|
||||||
</div>
|
</div>
|
||||||
<div class="modal-body" id="modal-text">
|
<div class="modal-body" id="modal-text">
|
||||||
|
<input type="hidden" id="import_app" name="import_app" value="${app.lower()}" />
|
||||||
% if app in ('PlexWatch', 'Plexivity'):
|
% if app in ('PlexWatch', 'Plexivity'):
|
||||||
<p class="help-block">
|
<p class="help-block">
|
||||||
<%
|
<%
|
||||||
|
@ -33,23 +34,20 @@
|
||||||
<div class="col-xs-4">
|
<div class="col-xs-4">
|
||||||
<select class="form-control" id="import_method" name="import_method">
|
<select class="form-control" id="import_method" name="import_method">
|
||||||
<option value="merge">Merge</option>
|
<option value="merge">Merge</option>
|
||||||
<option value="append">Append</option>
|
|
||||||
<option value="overwrite">Oerwrite</option>
|
<option value="overwrite">Oerwrite</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<p class="help-block">Select how you would like to import the Tautulli history.</p>
|
<p class="help-block">Select how you would like to import the Tautulli history.</p>
|
||||||
<ul class="help-block" style="padding-inline-start: 15px;">
|
<ul class="help-block" style="padding-inline-start: 15px;">
|
||||||
<li><strong>Merge</strong> will only add missing history from the imported database into the current database.</li>
|
<li><strong>Merge</strong> will add all history and remove any duplicates from the imported database into the current database.</li>
|
||||||
<li><strong>Append</strong> will add all history from the imported database into the current database.
|
|
||||||
<br>Note: History will be duplicated if it is present in both databases.</li>
|
|
||||||
<li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
|
<li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
|
||||||
</ul>
|
</ul>
|
||||||
<p class="help-block">Note: Other settings such as notification and newsletter agents will also be imported.</p>
|
<p class="help-block">Note: Libraries, users, notification agents, newsletter agents, and registered mobile devices will also be imported</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="checkbox">
|
<div class="checkbox">
|
||||||
<label>
|
<label>
|
||||||
<input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Database
|
<input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Current Database
|
||||||
</label>
|
</label>
|
||||||
<p class="help-block">Automatically create a backup of the current database before importing.</p>
|
<p class="help-block">Automatically create a backup of the current database before importing.</p>
|
||||||
</div>
|
</div>
|
||||||
|
@ -88,6 +86,7 @@
|
||||||
<script>
|
<script>
|
||||||
// Send database path to import script
|
// Send database path to import script
|
||||||
$("#import_db").click(function() {
|
$("#import_db").click(function() {
|
||||||
|
var import_app = $("#import_app").val();
|
||||||
var database_path = $("#import_database_path").val();
|
var database_path = $("#import_database_path").val();
|
||||||
var import_method = $("#import_method").val();
|
var import_method = $("#import_method").val();
|
||||||
var import_backup_db = $("#import_backup_db").is(':checked');
|
var import_backup_db = $("#import_backup_db").is(':checked');
|
||||||
|
@ -96,7 +95,7 @@
|
||||||
$.ajax({
|
$.ajax({
|
||||||
url: 'import_database',
|
url: 'import_database',
|
||||||
data: {
|
data: {
|
||||||
app: "${app}",
|
app: import_app,
|
||||||
database_path: database_path,
|
database_path: database_path,
|
||||||
method: import_method,
|
method: import_method,
|
||||||
backup: import_backup_db,
|
backup: import_backup_db,
|
||||||
|
|
|
@ -58,31 +58,38 @@ def import_tautulli_db(database=None, method=None, backup=True):
|
||||||
logger.error("Tautulli Database :: Failed to import Tautulli database: %s", db_validate)
|
logger.error("Tautulli Database :: Failed to import Tautulli database: %s", db_validate)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if method not in ('merge', 'append', 'overwrite'):
|
if method not in ('merge', 'overwrite'):
|
||||||
logger.error("Tautulli Database :: Failed to import Tautulli database: invalid import method '%s'", method)
|
logger.error("Tautulli Database :: Failed to import Tautulli database: invalid import method '%s'", method)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Make a backup of the current database first
|
|
||||||
if backup:
|
if backup:
|
||||||
|
# Make a backup of the current database first
|
||||||
logger.info("Tautulli Database :: Creating a database backup before importing.")
|
logger.info("Tautulli Database :: Creating a database backup before importing.")
|
||||||
if not make_backup():
|
if not make_backup():
|
||||||
logger.error("Tautulli Database :: Failed to import Tautulli database: failed to create database backup")
|
logger.error("Tautulli Database :: Failed to import Tautulli database: failed to create database backup")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info("Tautulli Database :: Tautulli database import started with method '%s'...", method)
|
logger.info("Tautulli Database :: Importing Tautulli database '%s' with import method '%s'...", database, method)
|
||||||
|
|
||||||
db = MonitorDatabase()
|
db = MonitorDatabase()
|
||||||
db.connection.execute('BEGIN IMMEDIATE')
|
db.connection.execute('BEGIN IMMEDIATE')
|
||||||
db.connection.execute('ATTACH ? AS import_db', [database])
|
db.connection.execute('ATTACH ? AS import_db', [database])
|
||||||
|
|
||||||
# Create a temporary table so we can reindex session_history.reference_id after merging
|
# Get the current number of used ids in the session_history table
|
||||||
if method == 'append':
|
session_history_seq = db.select_single('SELECT seq FROM sqlite_sequence WHERE name = "session_history"')
|
||||||
|
|
||||||
|
if method == 'merge':
|
||||||
|
# Create a temporary table so we can reindex session_history.reference_id after merging
|
||||||
logger.info("Tautulli Database :: Creating temporary database table to re-index grouped session history.")
|
logger.info("Tautulli Database :: Creating temporary database table to re-index grouped session history.")
|
||||||
session_history_seq = db.select_single('SELECT seq FROM sqlite_sequence WHERE name = "session_history" ')
|
|
||||||
db.action('CREATE TABLE IF NOT EXISTS temp (id INTEGER PRIMARY KEY, old_id, new_id)')
|
db.action('CREATE TABLE IF NOT EXISTS temp (id INTEGER PRIMARY KEY, old_id, new_id)')
|
||||||
db.action('INSERT INTO temp (old_id) SELECT id FROM import_db.session_history')
|
db.action('INSERT INTO temp (old_id) SELECT id FROM import_db.session_history')
|
||||||
db.action('UPDATE temp SET new_id = id + ?', [session_history_seq['seq']])
|
db.action('UPDATE temp SET new_id = id + ?', [session_history_seq['seq']])
|
||||||
|
|
||||||
|
# Keep track of all table columns so that duplicates can be removed after importing
|
||||||
|
session_history_tables = ('session_history', 'session_history_metadata', 'session_history_media_info')
|
||||||
|
session_history_columns = []
|
||||||
|
table_columns = {}
|
||||||
|
|
||||||
tables = db.select('SELECT name FROM import_db.sqlite_master '
|
tables = db.select('SELECT name FROM import_db.sqlite_master '
|
||||||
'WHERE type = "table" AND name NOT LIKE "sqlite_%"')
|
'WHERE type = "table" AND name NOT LIKE "sqlite_%"')
|
||||||
for table in tables:
|
for table in tables:
|
||||||
|
@ -91,28 +98,50 @@ def import_tautulli_db(database=None, method=None, backup=True):
|
||||||
# Skip temporary sessions table
|
# Skip temporary sessions table
|
||||||
continue
|
continue
|
||||||
|
|
||||||
logger.info("Tautulli Database :: Importing database table '%s'", table_name)
|
logger.info("Tautulli Database :: Importing database table '%s'.", table_name)
|
||||||
|
|
||||||
if method == 'overwrite':
|
if method == 'overwrite':
|
||||||
|
# Clear the table and reset the autoincrement ids
|
||||||
db.action('DELETE FROM {table}'.format(table=table_name))
|
db.action('DELETE FROM {table}'.format(table=table_name))
|
||||||
db.action('DELETE FROM sqlite_sequence WHERE name = ?', [table_name])
|
db.action('DELETE FROM sqlite_sequence WHERE name = ?', [table_name])
|
||||||
|
|
||||||
|
# Get the list of columns to import
|
||||||
columns = db.select('PRAGMA import_db.table_info({table})'.format(table=table_name))
|
columns = db.select('PRAGMA import_db.table_info({table})'.format(table=table_name))
|
||||||
|
import_columns = [c['name'] for c in columns if not c['pk']]
|
||||||
if method == 'merge':
|
|
||||||
import_columns = [c['name'] for c in columns]
|
|
||||||
else:
|
|
||||||
import_columns = [c['name'] for c in columns if not c['pk']]
|
|
||||||
|
|
||||||
insert_columns = ', '.join(import_columns)
|
insert_columns = ', '.join(import_columns)
|
||||||
|
|
||||||
|
table_columns[table_name] = insert_columns
|
||||||
|
if table == 'session_history':
|
||||||
|
session_history_columns = import_columns
|
||||||
|
|
||||||
|
# Insert the data with ignore instead of replace to be safe
|
||||||
db.action('INSERT OR IGNORE INTO {table} ({columns}) '
|
db.action('INSERT OR IGNORE INTO {table} ({columns}) '
|
||||||
'SELECT {columns} FROM import_db.{table}'.format(table=table_name, columns=insert_columns))
|
'SELECT {columns} FROM import_db.{table}'.format(table=table_name, columns=insert_columns))
|
||||||
|
|
||||||
# Reindex session_history.reference_id and delete the temp table
|
db.connection.execute('DETACH import_db')
|
||||||
if method == 'append':
|
|
||||||
|
if method == 'merge':
|
||||||
|
# Reindex session_history.reference_id
|
||||||
logger.info("Tautulli Database :: Re-indexing grouped session history.")
|
logger.info("Tautulli Database :: Re-indexing grouped session history.")
|
||||||
db.action('UPDATE session_history SET reference_id = (SELECT new_id FROM temp WHERE old_id = reference_id)'
|
db.action('UPDATE session_history SET reference_id = (SELECT new_id FROM temp WHERE old_id = reference_id)'
|
||||||
'WHERE id > ?', [session_history_seq['seq']])
|
'WHERE id > ?', [session_history_seq['seq']])
|
||||||
|
|
||||||
|
if session_history_columns:
|
||||||
|
# Remove reference_id column from the list of session_history table columns to get unique rows
|
||||||
|
columns = ', '.join([c for c in session_history_columns if c != 'reference_id'])
|
||||||
|
# Remove session_history ids to KEEP from the temp table
|
||||||
|
db.action('DELETE FROM temp WHERE new_id IN ('
|
||||||
|
'SELECT MIN(id) FROM session_history '
|
||||||
|
'GROUP BY {columns})'.format(columns=columns))
|
||||||
|
|
||||||
|
for table, columns in table_columns.items():
|
||||||
|
logger.info("Tautulli Database :: Removing duplicate rows from database table '%s'.", table)
|
||||||
|
if table in session_history_tables:
|
||||||
|
db.action('DELETE FROM {table} WHERE id IN (SELECT new_id FROM temp)'.format(table=table))
|
||||||
|
else:
|
||||||
|
db.action('DELETE FROM {table} WHERE id NOT IN '
|
||||||
|
'(SELECT MIN(id) FROM {table} GROUP BY {columns})'.format(table=table, columns=columns))
|
||||||
|
|
||||||
logger.info("Tautulli Database :: Deleting temporary database table.")
|
logger.info("Tautulli Database :: Deleting temporary database table.")
|
||||||
db.action('DROP TABLE temp')
|
db.action('DROP TABLE temp')
|
||||||
|
|
||||||
|
|
|
@ -3748,7 +3748,7 @@ class WebInterface(object):
|
||||||
Required parameters:
|
Required parameters:
|
||||||
app (str): "tautulli" or "plexwatch" or "plexivity"
|
app (str): "tautulli" or "plexwatch" or "plexivity"
|
||||||
database_path (str): The full path to the plexwatch database file
|
database_path (str): The full path to the plexwatch database file
|
||||||
method (str): For Tautulli only, "merge" or "append" or "overwrite"
|
method (str): For Tautulli only, "merge" or "overwrite"
|
||||||
table_name (str): For PlexWatch or Plexivity only, "processed" or "grouped"
|
table_name (str): For PlexWatch or Plexivity only, "processed" or "grouped"
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue