From 214ad21ea1206bc613cec684cc091550f6c857a5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:25:18 -0500 Subject: [PATCH 01/22] Refactor autoProcess -> auto_process --- core/__init__.py | 10 +++++----- core/{autoProcess => auto_process}/__init__.py | 0 .../{autoProcess => auto_process}/autoProcessComics.py | 0 core/{autoProcess => auto_process}/autoProcessGames.py | 0 core/{autoProcess => auto_process}/autoProcessMovie.py | 0 core/{autoProcess => auto_process}/autoProcessMusic.py | 0 core/{autoProcess => auto_process}/autoProcessTV.py | 0 nzbToMedia.py | 10 +++++----- 8 files changed, 10 insertions(+), 10 deletions(-) rename core/{autoProcess => auto_process}/__init__.py (100%) rename core/{autoProcess => auto_process}/autoProcessComics.py (100%) rename core/{autoProcess => auto_process}/autoProcessGames.py (100%) rename core/{autoProcess => auto_process}/autoProcessMovie.py (100%) rename core/{autoProcess => auto_process}/autoProcessMusic.py (100%) rename core/{autoProcess => auto_process}/autoProcessTV.py (100%) diff --git a/core/__init__.py b/core/__init__.py index 605f4dd7..b78b157b 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -43,11 +43,11 @@ import six from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck -from core.autoProcess.autoProcessComics import Comic -from core.autoProcess.autoProcessGames import Game -from core.autoProcess.autoProcessMovie import Movie -from core.autoProcess.autoProcessMusic import Music -from core.autoProcess.autoProcessTV import TV +from core.auto_process.autoProcessComics import Comic +from core.auto_process.autoProcessGames import Game +from core.auto_process.autoProcessMovie import Movie +from core.auto_process.autoProcessMusic import Music +from core.auto_process.autoProcessTV import TV from core.databases import mainDB from core.nzbToMediaConfig import config from core.nzbToMediaUtil import ( diff --git a/core/autoProcess/__init__.py b/core/auto_process/__init__.py similarity index 100% rename from core/autoProcess/__init__.py rename to core/auto_process/__init__.py diff --git a/core/autoProcess/autoProcessComics.py b/core/auto_process/autoProcessComics.py similarity index 100% rename from core/autoProcess/autoProcessComics.py rename to core/auto_process/autoProcessComics.py diff --git a/core/autoProcess/autoProcessGames.py b/core/auto_process/autoProcessGames.py similarity index 100% rename from core/autoProcess/autoProcessGames.py rename to core/auto_process/autoProcessGames.py diff --git a/core/autoProcess/autoProcessMovie.py b/core/auto_process/autoProcessMovie.py similarity index 100% rename from core/autoProcess/autoProcessMovie.py rename to core/auto_process/autoProcessMovie.py diff --git a/core/autoProcess/autoProcessMusic.py b/core/auto_process/autoProcessMusic.py similarity index 100% rename from core/autoProcess/autoProcessMusic.py rename to core/auto_process/autoProcessMusic.py diff --git a/core/autoProcess/autoProcessTV.py b/core/auto_process/autoProcessTV.py similarity index 100% rename from core/autoProcess/autoProcessTV.py rename to core/auto_process/autoProcessTV.py diff --git a/nzbToMedia.py b/nzbToMedia.py index 0a855443..94feda31 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -629,11 +629,11 @@ import sys import core from core import logger, nzbToMediaDB -from core.autoProcess.autoProcessComics import Comic -from core.autoProcess.autoProcessGames import Game -from core.autoProcess.autoProcessMovie import Movie -from core.autoProcess.autoProcessMusic import Music -from core.autoProcess.autoProcessTV import TV +from core.auto_process.autoProcessComics import Comic +from core.auto_process.autoProcessGames import Game +from core.auto_process.autoProcessMovie import Movie +from core.auto_process.autoProcessMusic import Music +from core.auto_process.autoProcessTV import TV from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status From 088f23ad3a5279b4e6c74be466336c9ceac5f483 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:26:02 -0500 Subject: [PATCH 02/22] Refactor autoProcessComics -> comics --- core/__init__.py | 2 +- core/auto_process/{autoProcessComics.py => comics.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/auto_process/{autoProcessComics.py => comics.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index b78b157b..6f673b0c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -43,7 +43,7 @@ import six from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck -from core.auto_process.autoProcessComics import Comic +from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie from core.auto_process.autoProcessMusic import Music diff --git a/core/auto_process/autoProcessComics.py b/core/auto_process/comics.py similarity index 100% rename from core/auto_process/autoProcessComics.py rename to core/auto_process/comics.py diff --git a/nzbToMedia.py b/nzbToMedia.py index 94feda31..2f488cb1 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -629,7 +629,7 @@ import sys import core from core import logger, nzbToMediaDB -from core.auto_process.autoProcessComics import Comic +from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie from core.auto_process.autoProcessMusic import Music From 95c67978caf62e9915f11f449732cfdbca9d79d3 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:26:32 -0500 Subject: [PATCH 03/22] Refactor autoProcessTV -> tv --- core/__init__.py | 2 +- core/auto_process/{autoProcessTV.py => tv.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/auto_process/{autoProcessTV.py => tv.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index 6f673b0c..37ea06d7 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -47,7 +47,7 @@ from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie from core.auto_process.autoProcessMusic import Music -from core.auto_process.autoProcessTV import TV +from core.auto_process.tv import TV from core.databases import mainDB from core.nzbToMediaConfig import config from core.nzbToMediaUtil import ( diff --git a/core/auto_process/autoProcessTV.py b/core/auto_process/tv.py similarity index 100% rename from core/auto_process/autoProcessTV.py rename to core/auto_process/tv.py diff --git a/nzbToMedia.py b/nzbToMedia.py index 2f488cb1..75f19156 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -633,7 +633,7 @@ from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie from core.auto_process.autoProcessMusic import Music -from core.auto_process.autoProcessTV import TV +from core.auto_process.tv import TV from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status From cbff62a08cf382150604c9522206a9d696e0be80 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:26:56 -0500 Subject: [PATCH 04/22] Refactor autoProcessMusic -> music --- core/__init__.py | 2 +- core/auto_process/{autoProcessMusic.py => music.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/auto_process/{autoProcessMusic.py => music.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index 37ea06d7..63ed5da7 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -46,7 +46,7 @@ from core import logger, nzbToMediaDB, versionCheck from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie -from core.auto_process.autoProcessMusic import Music +from core.auto_process.music import Music from core.auto_process.tv import TV from core.databases import mainDB from core.nzbToMediaConfig import config diff --git a/core/auto_process/autoProcessMusic.py b/core/auto_process/music.py similarity index 100% rename from core/auto_process/autoProcessMusic.py rename to core/auto_process/music.py diff --git a/nzbToMedia.py b/nzbToMedia.py index 75f19156..f3054c9b 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -632,7 +632,7 @@ from core import logger, nzbToMediaDB from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game from core.auto_process.autoProcessMovie import Movie -from core.auto_process.autoProcessMusic import Music +from core.auto_process.music import Music from core.auto_process.tv import TV from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status From 4bd6e6251a313d4393f6a4487eba41815fbf6cd1 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:27:30 -0500 Subject: [PATCH 05/22] Refactor autoProcessMovie -> movies --- core/__init__.py | 2 +- core/auto_process/{autoProcessMovie.py => movies.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/auto_process/{autoProcessMovie.py => movies.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index 63ed5da7..66d6c5a1 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -45,7 +45,7 @@ from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game -from core.auto_process.autoProcessMovie import Movie +from core.auto_process.movies import Movie from core.auto_process.music import Music from core.auto_process.tv import TV from core.databases import mainDB diff --git a/core/auto_process/autoProcessMovie.py b/core/auto_process/movies.py similarity index 100% rename from core/auto_process/autoProcessMovie.py rename to core/auto_process/movies.py diff --git a/nzbToMedia.py b/nzbToMedia.py index f3054c9b..1de15362 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -631,7 +631,7 @@ import core from core import logger, nzbToMediaDB from core.auto_process.comics import Comic from core.auto_process.autoProcessGames import Game -from core.auto_process.autoProcessMovie import Movie +from core.auto_process.movies import Movie from core.auto_process.music import Music from core.auto_process.tv import TV from core.nzbToMediaUserScript import external_script From 5839257f9ba72876171e14cb9132019047bd5f95 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:27:57 -0500 Subject: [PATCH 06/22] Refactor autoProcessGames -> games --- core/__init__.py | 2 +- core/auto_process/{autoProcessGames.py => games.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/auto_process/{autoProcessGames.py => games.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index 66d6c5a1..f4826687 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -44,7 +44,7 @@ from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck from core.auto_process.comics import Comic -from core.auto_process.autoProcessGames import Game +from core.auto_process.games import Game from core.auto_process.movies import Movie from core.auto_process.music import Music from core.auto_process.tv import TV diff --git a/core/auto_process/autoProcessGames.py b/core/auto_process/games.py similarity index 100% rename from core/auto_process/autoProcessGames.py rename to core/auto_process/games.py diff --git a/nzbToMedia.py b/nzbToMedia.py index 1de15362..20360404 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -630,7 +630,7 @@ import sys import core from core import logger, nzbToMediaDB from core.auto_process.comics import Comic -from core.auto_process.autoProcessGames import Game +from core.auto_process.games import Game from core.auto_process.movies import Movie from core.auto_process.music import Music from core.auto_process.tv import TV From 3b429dcc7e59ad929af5be91ccb3f253eaa3864d Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 17:32:03 -0500 Subject: [PATCH 07/22] Add Comic, Game, Movie, Music, and TV to core.auto_process --- core/__init__.py | 6 +----- core/auto_process/__init__.py | 6 ++++++ nzbToMedia.py | 6 +----- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/core/__init__.py b/core/__init__.py index f4826687..008871f9 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -43,11 +43,7 @@ import six from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck -from core.auto_process.comics import Comic -from core.auto_process.games import Game -from core.auto_process.movies import Movie -from core.auto_process.music import Music -from core.auto_process.tv import TV +from core.auto_process import Comic, Game, Movie, Music, TV from core.databases import mainDB from core.nzbToMediaConfig import config from core.nzbToMediaUtil import ( diff --git a/core/auto_process/__init__.py b/core/auto_process/__init__.py index 9bad5790..8ce70527 100644 --- a/core/auto_process/__init__.py +++ b/core/auto_process/__init__.py @@ -1 +1,7 @@ # coding=utf-8 + +from core.auto_process.comics import Comic +from core.auto_process.games import Game +from core.auto_process.movies import Movie +from core.auto_process.music import Music +from core.auto_process.tv import TV diff --git a/nzbToMedia.py b/nzbToMedia.py index 20360404..9ad63722 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -629,11 +629,7 @@ import sys import core from core import logger, nzbToMediaDB -from core.auto_process.comics import Comic -from core.auto_process.games import Game -from core.auto_process.movies import Movie -from core.auto_process.music import Music -from core.auto_process.tv import TV +from core.auto_process import Comic, Game, Movie, Music, TV from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status From 7d1f2fb2d56b0db32610e504e9858088be74418b Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 18:49:59 -0500 Subject: [PATCH 08/22] Refactor databases.mainDB -> databases --- core/__init__.py | 5 ++--- core/{databases/mainDB.py => databases.py} | 0 core/databases/__init__.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) rename core/{databases/mainDB.py => databases.py} (100%) delete mode 100644 core/databases/__init__.py diff --git a/core/__init__.py b/core/__init__.py index 008871f9..d0362fd5 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -42,9 +42,8 @@ MYAPP = None import six from six.moves import reload_module -from core import logger, nzbToMediaDB, versionCheck +from core import logger, nzbToMediaDB, versionCheck, databases from core.auto_process import Comic, Game, Movie, Music, TV -from core.databases import mainDB from core.nzbToMediaConfig import config from core.nzbToMediaUtil import ( RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, @@ -322,7 +321,7 @@ def initialize(section=None): logger.info("{0}: {1}".format(item, os.environ[item]), "ENVIRONMENT") # initialize the main SB database - nzbToMediaDB.upgrade_database(nzbToMediaDB.DBConnection(), mainDB.InitialSchema) + nzbToMediaDB.upgrade_database(nzbToMediaDB.DBConnection(), databases.InitialSchema) # Set Version and GIT variables NZBTOMEDIA_VERSION = '11.06' diff --git a/core/databases/mainDB.py b/core/databases.py similarity index 100% rename from core/databases/mainDB.py rename to core/databases.py diff --git a/core/databases/__init__.py b/core/databases/__init__.py deleted file mode 100644 index 9bad5790..00000000 --- a/core/databases/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 From d3ce2a10f5bafe99eeac6a0a97361c8e802de519 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 18:52:19 -0500 Subject: [PATCH 09/22] Refactor transcoder.transcoder -> transcoder --- core/__init__.py | 4 +--- core/auto_process/movies.py | 3 +-- core/auto_process/tv.py | 3 +-- core/nzbToMediaUserScript.py | 3 +-- core/{transcoder => }/transcoder.py | 2 ++ core/transcoder/__init__.py | 2 -- tests/general.py | 2 +- 7 files changed, 7 insertions(+), 12 deletions(-) rename core/{transcoder => }/transcoder.py (99%) delete mode 100644 core/transcoder/__init__.py diff --git a/core/__init__.py b/core/__init__.py index d0362fd5..b665d255 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -42,7 +42,7 @@ MYAPP = None import six from six.moves import reload_module -from core import logger, nzbToMediaDB, versionCheck, databases +from core import logger, nzbToMediaDB, versionCheck, databases, transcoder from core.auto_process import Comic, Game, Movie, Music, TV from core.nzbToMediaConfig import config from core.nzbToMediaUtil import ( @@ -51,8 +51,6 @@ from core.nzbToMediaUtil import ( list_media_files, make_dir, parse_args, pause_torrent, remove_torrent, resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, ) -from core.transcoder import transcoder - # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 9a3ed3db..6f86a706 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -7,10 +7,9 @@ import time import requests import core -from core import logger +from core import logger, transcoder from core.nzbToMediaSceneExceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding -from core.transcoder import transcoder requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 1636e55b..d93f1d53 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -9,11 +9,10 @@ import time import requests import core -from core import logger +from core import logger, transcoder from core.nzbToMediaAutoFork import auto_fork from core.nzbToMediaSceneExceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding -from core.transcoder import transcoder requests.packages.urllib3.disable_warnings() diff --git a/core/nzbToMediaUserScript.py b/core/nzbToMediaUserScript.py index c4885444..4e0e966a 100644 --- a/core/nzbToMediaUserScript.py +++ b/core/nzbToMediaUserScript.py @@ -4,9 +4,8 @@ import os from subprocess import Popen import core -from core import logger +from core import logger, transcoder from core.nzbToMediaUtil import import_subs, list_media_files, remove_dir -from core.transcoder import transcoder def external_script(output_destination, torrent_name, torrent_label, settings): diff --git a/core/transcoder/transcoder.py b/core/transcoder.py similarity index 99% rename from core/transcoder/transcoder.py rename to core/transcoder.py index 49ca1ace..bc0e2d2a 100644 --- a/core/transcoder/transcoder.py +++ b/core/transcoder.py @@ -15,6 +15,8 @@ import core from core import logger from core.nzbToMediaUtil import make_dir +__author__ = 'Justin' + def is_video_good(videofile, status): file_name_ext = os.path.basename(videofile) diff --git a/core/transcoder/__init__.py b/core/transcoder/__init__.py deleted file mode 100644 index b1629751..00000000 --- a/core/transcoder/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -__author__ = 'Justin' diff --git a/tests/general.py b/tests/general.py index e0410c14..d67c9b81 100755 --- a/tests/general.py +++ b/tests/general.py @@ -5,9 +5,9 @@ import guessit import requests import core +from core import transcoder from core.nzbToMediaAutoFork import auto_fork from core.nzbToMediaUtil import server_responding -from core.transcoder import transcoder # Initialize the config core.initialize() From 23cd600a6cfd8c2385c0bf793d1e7af793420964 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:00:14 -0500 Subject: [PATCH 10/22] Refactor nzbToMediaAutoFork -> forks --- core/auto_process/tv.py | 2 +- core/{nzbToMediaAutoFork.py => forks.py} | 0 tests/general.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/{nzbToMediaAutoFork.py => forks.py} (100%) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index d93f1d53..bdb9c981 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -10,7 +10,7 @@ import requests import core from core import logger, transcoder -from core.nzbToMediaAutoFork import auto_fork +from core.forks import auto_fork from core.nzbToMediaSceneExceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding diff --git a/core/nzbToMediaAutoFork.py b/core/forks.py similarity index 100% rename from core/nzbToMediaAutoFork.py rename to core/forks.py diff --git a/tests/general.py b/tests/general.py index d67c9b81..42a8034c 100755 --- a/tests/general.py +++ b/tests/general.py @@ -6,7 +6,7 @@ import requests import core from core import transcoder -from core.nzbToMediaAutoFork import auto_fork +from core.forks import auto_fork from core.nzbToMediaUtil import server_responding # Initialize the config From 5d423b0f3801dcc8caf970ce76307e387f4a7544 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:04:44 -0500 Subject: [PATCH 11/22] Refactor nzbToMediaConfig -> configuration --- core/__init__.py | 2 +- core/{nzbToMediaConfig.py => configuration.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename core/{nzbToMediaConfig.py => configuration.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index b665d255..dd4268c2 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -44,7 +44,7 @@ from six.moves import reload_module from core import logger, nzbToMediaDB, versionCheck, databases, transcoder from core.auto_process import Comic, Game, Movie, Music, TV -from core.nzbToMediaConfig import config +from core.configuration import config from core.nzbToMediaUtil import ( RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, create_torrent_class, extract_files, flatten, get_dirs, get_download_info, diff --git a/core/nzbToMediaConfig.py b/core/configuration.py similarity index 100% rename from core/nzbToMediaConfig.py rename to core/configuration.py From 9d9abffdb688d3635812c6e51ff94a07c13a392f Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:08:09 -0500 Subject: [PATCH 12/22] Refactor nzbToMediaDB -> main_db --- TorrentToMedia.py | 4 ++-- core/__init__.py | 4 ++-- core/databases.py | 6 +++--- core/{nzbToMediaDB.py => main_db.py} | 0 core/nzbToMediaUtil.py | 6 +++--- nzbToMedia.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) rename core/{nzbToMediaDB.py => main_db.py} (100%) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index e040a61e..1fee99d9 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -5,7 +5,7 @@ import os import sys import core -from core import logger, nzbToMediaDB +from core import logger, main_db from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, convert_to_ascii, plex_update, replace_links from six import text_type @@ -19,7 +19,7 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp if client_agent != 'manual' and not core.DOWNLOADINFO: logger.debug('Adding TORRENT download info for directory {0} to database'.format(input_directory)) - my_db = nzbToMediaDB.DBConnection() + my_db = main_db.DBConnection() input_directory1 = input_directory input_name1 = input_name diff --git a/core/__init__.py b/core/__init__.py index dd4268c2..0fe35dc6 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -42,7 +42,7 @@ MYAPP = None import six from six.moves import reload_module -from core import logger, nzbToMediaDB, versionCheck, databases, transcoder +from core import logger, main_db, versionCheck, databases, transcoder from core.auto_process import Comic, Game, Movie, Music, TV from core.configuration import config from core.nzbToMediaUtil import ( @@ -319,7 +319,7 @@ def initialize(section=None): logger.info("{0}: {1}".format(item, os.environ[item]), "ENVIRONMENT") # initialize the main SB database - nzbToMediaDB.upgrade_database(nzbToMediaDB.DBConnection(), databases.InitialSchema) + main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) # Set Version and GIT variables NZBTOMEDIA_VERSION = '11.06' diff --git a/core/databases.py b/core/databases.py index d6c1e1c3..2cec59d8 100644 --- a/core/databases.py +++ b/core/databases.py @@ -1,6 +1,6 @@ # coding=utf-8 -from core import logger, nzbToMediaDB +from core import logger, main_db from core.nzbToMediaUtil import backup_versioned_file MIN_DB_VERSION = 1 # oldest db version we support migrating from @@ -9,7 +9,7 @@ MAX_DB_VERSION = 2 def backup_database(version): logger.info("Backing up database before upgrade") - if not backup_versioned_file(nzbToMediaDB.db_filename(), version): + if not backup_versioned_file(main_db.db_filename(), version): logger.log_error_and_exit("Database backup failed, abort upgrading database") else: logger.info("Proceeding with upgrade") @@ -20,7 +20,7 @@ def backup_database(version): # ====================== # Add new migrations at the bottom of the list; subclass the previous migration. -class InitialSchema(nzbToMediaDB.SchemaUpgrade): +class InitialSchema(main_db.SchemaUpgrade): def test(self): no_update = False if self.has_table("db_version"): diff --git a/core/nzbToMediaDB.py b/core/main_db.py similarity index 100% rename from core/nzbToMediaDB.py rename to core/main_db.py diff --git a/core/nzbToMediaUtil.py b/core/nzbToMediaUtil.py index 21985623..97e23e40 100644 --- a/core/nzbToMediaUtil.py +++ b/core/nzbToMediaUtil.py @@ -26,7 +26,7 @@ from transmissionrpc.client import Client as TransmissionClient from utorrent.client import UTorrentClient import core -from core import logger, nzbToMediaDB +from core import logger, main_db from core.extractor import extractor requests.packages.urllib3.disable_warnings() @@ -1284,7 +1284,7 @@ def backup_versioned_file(old_file, version): def update_download_info_status(input_name, status): logger.db("Updating status of our download {0} in the DB to {1}".format(input_name, status)) - my_db = nzbToMediaDB.DBConnection() + my_db = main_db.DBConnection() my_db.action("UPDATE downloads SET status=?, last_update=? WHERE input_name=?", [status, datetime.date.today().toordinal(), text_type(input_name)]) @@ -1292,7 +1292,7 @@ def update_download_info_status(input_name, status): def get_download_info(input_name, status): logger.db("Getting download info for {0} from the DB".format(input_name)) - my_db = nzbToMediaDB.DBConnection() + my_db = main_db.DBConnection() sql_results = my_db.select("SELECT * FROM downloads WHERE input_name=? AND status=?", [text_type(input_name), status]) diff --git a/nzbToMedia.py b/nzbToMedia.py index 9ad63722..e3fb2910 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -628,7 +628,7 @@ import os import sys import core -from core import logger, nzbToMediaDB +from core import logger, main_db from core.auto_process import Comic, Game, Movie, Music, TV from core.nzbToMediaUserScript import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status @@ -653,7 +653,7 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d if client_agent != 'manual' and not core.DOWNLOADINFO: logger.debug('Adding NZB download info for directory {0} to database'.format(input_directory)) - my_db = nzbToMediaDB.DBConnection() + my_db = main_db.DBConnection() input_directory1 = input_directory input_name1 = input_name From c7398b95508ea889be13a1e5449c08553562f7e2 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:10:57 -0500 Subject: [PATCH 13/22] Refactor nzbToMediaSceneExceptions -> scene_exceptions --- core/auto_process/movies.py | 2 +- core/auto_process/music.py | 2 +- core/auto_process/tv.py | 2 +- core/{nzbToMediaSceneExceptions.py => scene_exceptions.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename core/{nzbToMediaSceneExceptions.py => scene_exceptions.py} (100%) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index 6f86a706..f3bde909 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -8,7 +8,7 @@ import requests import core from core import logger, transcoder -from core.nzbToMediaSceneExceptions import process_all_exceptions +from core.scene_exceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/music.py b/core/auto_process/music.py index f32ab0ae..f2aa7de9 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -8,7 +8,7 @@ import requests import core from core import logger -from core.nzbToMediaSceneExceptions import process_all_exceptions +from core.scene_exceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, list_media_files, remote_dir, remove_dir, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index bdb9c981..e1183ea8 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -11,7 +11,7 @@ import requests import core from core import logger, transcoder from core.forks import auto_fork -from core.nzbToMediaSceneExceptions import process_all_exceptions +from core.scene_exceptions import process_all_exceptions from core.nzbToMediaUtil import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/nzbToMediaSceneExceptions.py b/core/scene_exceptions.py similarity index 100% rename from core/nzbToMediaSceneExceptions.py rename to core/scene_exceptions.py From 8176c2f007c0085977c5478b428681b3332a8bcd Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:12:15 -0500 Subject: [PATCH 14/22] Refactor nzbToMediaUserScripts -> user_scripts --- TorrentToMedia.py | 2 +- core/{nzbToMediaUserScript.py => user_scripts.py} | 0 nzbToMedia.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename core/{nzbToMediaUserScript.py => user_scripts.py} (100%) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 1fee99d9..728afe61 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -6,7 +6,7 @@ import sys import core from core import logger, main_db -from core.nzbToMediaUserScript import external_script +from core.user_scripts import external_script from core.nzbToMediaUtil import char_replace, convert_to_ascii, plex_update, replace_links from six import text_type diff --git a/core/nzbToMediaUserScript.py b/core/user_scripts.py similarity index 100% rename from core/nzbToMediaUserScript.py rename to core/user_scripts.py diff --git a/nzbToMedia.py b/nzbToMedia.py index e3fb2910..edeef057 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -630,7 +630,7 @@ import sys import core from core import logger, main_db from core.auto_process import Comic, Game, Movie, Music, TV -from core.nzbToMediaUserScript import external_script +from core.user_scripts import external_script from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status try: From c6240298464015eb54a06d0b5e15d0c1384ec648 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:13:38 -0500 Subject: [PATCH 15/22] Refactor versionCheck -> version_check --- core/__init__.py | 8 ++++---- core/{versionCheck.py => version_check.py} | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename core/{versionCheck.py => version_check.py} (100%) diff --git a/core/__init__.py b/core/__init__.py index 0fe35dc6..58b474c5 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -42,7 +42,7 @@ MYAPP = None import six from six.moves import reload_module -from core import logger, main_db, versionCheck, databases, transcoder +from core import logger, main_db, version_check, databases, transcoder from core.auto_process import Comic, Game, Movie, Music, TV from core.configuration import config from core.nzbToMediaUtil import ( @@ -336,10 +336,10 @@ def initialize(section=None): NOEXTRACTFAILED = int(CFG["General"]["no_extract_failed"]) # Check for updates via GitHUB - if versionCheck.CheckVersion().check_for_new_version(): + if version_check.CheckVersion().check_for_new_version(): if AUTO_UPDATE == 1: logger.info("Auto-Updating nzbToMedia, Please wait ...") - updated = versionCheck.CheckVersion().update() + updated = version_check.CheckVersion().update() if updated: # restart nzbToMedia try: @@ -845,7 +845,7 @@ def initialize(section=None): def restart(): - install_type = versionCheck.CheckVersion().install_type + install_type = version_check.CheckVersion().install_type status = 0 popen_list = [] diff --git a/core/versionCheck.py b/core/version_check.py similarity index 100% rename from core/versionCheck.py rename to core/version_check.py From 094ae574a2c9107320b6509066e19d75e137a860 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:18:09 -0500 Subject: [PATCH 16/22] Refactor gh_api -> github_api --- core/{gh_api.py => github_api.py} | 0 core/version_check.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename core/{gh_api.py => github_api.py} (100%) diff --git a/core/gh_api.py b/core/github_api.py similarity index 100% rename from core/gh_api.py rename to core/github_api.py diff --git a/core/version_check.py b/core/version_check.py index 7876c208..41c3ccb9 100644 --- a/core/version_check.py +++ b/core/version_check.py @@ -14,7 +14,7 @@ import traceback from six.moves.urllib.request import urlretrieve import core -from core import gh_api as github, logger +from core import github_api as github, logger class CheckVersion(object): From 34904ee892653510b7aaebc294c69d5f7c6e32e9 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:21:57 -0500 Subject: [PATCH 17/22] Refactor extractor.extractor -> extractor --- core/extractor/__init__.py | 180 +++++++++++++++++++++++++++++++++++ core/extractor/extractor.py | 181 ------------------------------------ core/nzbToMediaUtil.py | 3 +- 3 files changed, 181 insertions(+), 183 deletions(-) delete mode 100644 core/extractor/extractor.py diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index 9bad5790..4796bb3e 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -1 +1,181 @@ # coding=utf-8 + +import os +import platform +import shutil +import stat +import subprocess +from subprocess import Popen, call +from time import sleep + +import core + + +def extract(file_path, output_destination): + success = 0 + # Using Windows + if platform.system() == 'Windows': + if not os.path.exists(core.SEVENZIP): + core.logger.error("EXTRACTOR: Could not find 7-zip, Exiting") + return False + wscriptlocation = os.path.join(os.environ['WINDIR'], 'system32', 'wscript.exe') + invislocation = os.path.join(core.PROGRAM_DIR, 'core', 'extractor', 'bin', 'invisible.vbs') + cmd_7zip = [wscriptlocation, invislocation, str(core.SHOWEXTRACT), core.SEVENZIP, "x", "-y"] + ext_7zip = [".rar", ".zip", ".tar.gz", "tgz", ".tar.bz2", ".tbz", ".tar.lzma", ".tlz", ".7z", ".xz"] + extract_commands = dict.fromkeys(ext_7zip, cmd_7zip) + # Using unix + else: + required_cmds = ["unrar", "unzip", "tar", "unxz", "unlzma", "7zr", "bunzip2"] + # ## Possible future suport: + # gunzip: gz (cmd will delete original archive) + # ## the following do not extract to dest dir + # ".xz": ["xz", "-d --keep"], + # ".lzma": ["xz", "-d --format=lzma --keep"], + # ".bz2": ["bzip2", "-d --keep"], + + extract_commands = { + ".rar": ["unrar", "x", "-o+", "-y"], + ".tar": ["tar", "-xf"], + ".zip": ["unzip"], + ".tar.gz": ["tar", "-xzf"], ".tgz": ["tar", "-xzf"], + ".tar.bz2": ["tar", "-xjf"], ".tbz": ["tar", "-xjf"], + ".tar.lzma": ["tar", "--lzma", "-xf"], ".tlz": ["tar", "--lzma", "-xf"], + ".tar.xz": ["tar", "--xz", "-xf"], ".txz": ["tar", "--xz", "-xf"], + ".7z": ["7zr", "x"], + } + # Test command exists and if not, remove + if not os.getenv('TR_TORRENT_DIR'): + devnull = open(os.devnull, 'w') + for cmd in required_cmds: + if call(['which', cmd], stdout=devnull, + stderr=devnull): # note, returns 0 if exists, or 1 if doesn't exist. + for k, v in extract_commands.items(): + if cmd in v[0]: + if not call(["which", "7zr"], stdout=devnull, stderr=devnull): # we do have "7zr" + extract_commands[k] = ["7zr", "x", "-y"] + elif not call(["which", "7z"], stdout=devnull, stderr=devnull): # we do have "7z" + extract_commands[k] = ["7z", "x", "-y"] + elif not call(["which", "7za"], stdout=devnull, stderr=devnull): # we do have "7za" + extract_commands[k] = ["7za", "x", "-y"] + else: + core.logger.error("EXTRACTOR: {cmd} not found, " + "disabling support for {feature}".format + (cmd=cmd, feature=k)) + del extract_commands[k] + devnull.close() + else: + core.logger.warning("EXTRACTOR: Cannot determine which tool to use when called from Transmission") + + if not extract_commands: + core.logger.warning("EXTRACTOR: No archive extracting programs found, plugin will be disabled") + + ext = os.path.splitext(file_path) + cmd = [] + if ext[1] in (".gz", ".bz2", ".lzma"): + # Check if this is a tar + if os.path.splitext(ext[0])[1] == ".tar": + cmd = extract_commands[".tar{ext}".format(ext=ext[1])] + elif ext[1] in (".1", ".01", ".001") and os.path.splitext(ext[0])[1] in (".rar", ".zip", ".7z"): + cmd = extract_commands[os.path.splitext(ext[0])[1]] + elif ext[1] in (".cb7", ".cba", ".cbr", ".cbt", ".cbz"): # don't extract these comic book archives. + return False + else: + if ext[1] in extract_commands: + cmd = extract_commands[ext[1]] + else: + core.logger.debug("EXTRACTOR: Unknown file type: {ext}".format + (ext=ext[1])) + return False + + # Create outputDestination folder + core.make_dir(output_destination) + + if core.PASSWORDSFILE and os.path.isfile(os.path.normpath(core.PASSWORDSFILE)): + passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDSFILE))] + else: + passwords = [] + + core.logger.info("Extracting {file} to {destination}".format + (file=file_path, destination=output_destination)) + core.logger.debug("Extracting {cmd} {file} {destination}".format + (cmd=cmd, file=file_path, destination=output_destination)) + + orig_files = [] + orig_dirs = [] + for dir, subdirs, files in os.walk(output_destination): + for subdir in subdirs: + orig_dirs.append(os.path.join(dir, subdir)) + for file in files: + orig_files.append(os.path.join(dir, file)) + + pwd = os.getcwd() # Get our Present Working Directory + os.chdir(output_destination) # Not all unpack commands accept full paths, so just extract into this directory + devnull = open(os.devnull, 'w') + + try: # now works same for nt and *nix + info = None + cmd.append(file_path) # add filePath to final cmd arg. + if platform.system() == 'Windows': + info = subprocess.STARTUPINFO() + info.dwFlags |= subprocess.STARTF_USESHOWWINDOW + else: + cmd = core.NICENESS + cmd + cmd2 = cmd + cmd2.append("-p-") # don't prompt for password. + p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + res = p.wait() + if res == 0: # Both Linux and Windows return 0 for successful. + core.logger.info("EXTRACTOR: Extraction was successful for {file} to {destination}".format + (file=file_path, destination=output_destination)) + success = 1 + elif len(passwords) > 0: + core.logger.info("EXTRACTOR: Attempting to extract with passwords") + for password in passwords: + if password == "": # if edited in windows or otherwise if blank lines. + continue + cmd2 = cmd + # append password here. + passcmd = "-p{pwd}".format(pwd=password) + cmd2.append(passcmd) + p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + res = p.wait() + if (res >= 0 and platform == 'Windows') or res == 0: + core.logger.info("EXTRACTOR: Extraction was successful " + "for {file} to {destination} using password: {pwd}".format + (file=file_path, destination=output_destination, pwd=password)) + success = 1 + break + else: + continue + except: + core.logger.error("EXTRACTOR: Extraction failed for {file}. " + "Could not call command {cmd}".format + (file=file_path, cmd=cmd)) + os.chdir(pwd) + return False + + devnull.close() + os.chdir(pwd) # Go back to our Original Working Directory + if success: + # sleep to let files finish writing to disk + sleep(3) + perms = stat.S_IMODE(os.lstat(os.path.split(file_path)[0]).st_mode) + for dir, subdirs, files in os.walk(output_destination): + for subdir in subdirs: + if not os.path.join(dir, subdir) in orig_files: + try: + os.chmod(os.path.join(dir, subdir), perms) + except: + pass + for file in files: + if not os.path.join(dir, file) in orig_files: + try: + shutil.copymode(file_path, os.path.join(dir, file)) + except: + pass + return True + else: + core.logger.error("EXTRACTOR: Extraction failed for {file}. " + "Result was {result}".format + (file=file_path, result=res)) + return False diff --git a/core/extractor/extractor.py b/core/extractor/extractor.py deleted file mode 100644 index 4796bb3e..00000000 --- a/core/extractor/extractor.py +++ /dev/null @@ -1,181 +0,0 @@ -# coding=utf-8 - -import os -import platform -import shutil -import stat -import subprocess -from subprocess import Popen, call -from time import sleep - -import core - - -def extract(file_path, output_destination): - success = 0 - # Using Windows - if platform.system() == 'Windows': - if not os.path.exists(core.SEVENZIP): - core.logger.error("EXTRACTOR: Could not find 7-zip, Exiting") - return False - wscriptlocation = os.path.join(os.environ['WINDIR'], 'system32', 'wscript.exe') - invislocation = os.path.join(core.PROGRAM_DIR, 'core', 'extractor', 'bin', 'invisible.vbs') - cmd_7zip = [wscriptlocation, invislocation, str(core.SHOWEXTRACT), core.SEVENZIP, "x", "-y"] - ext_7zip = [".rar", ".zip", ".tar.gz", "tgz", ".tar.bz2", ".tbz", ".tar.lzma", ".tlz", ".7z", ".xz"] - extract_commands = dict.fromkeys(ext_7zip, cmd_7zip) - # Using unix - else: - required_cmds = ["unrar", "unzip", "tar", "unxz", "unlzma", "7zr", "bunzip2"] - # ## Possible future suport: - # gunzip: gz (cmd will delete original archive) - # ## the following do not extract to dest dir - # ".xz": ["xz", "-d --keep"], - # ".lzma": ["xz", "-d --format=lzma --keep"], - # ".bz2": ["bzip2", "-d --keep"], - - extract_commands = { - ".rar": ["unrar", "x", "-o+", "-y"], - ".tar": ["tar", "-xf"], - ".zip": ["unzip"], - ".tar.gz": ["tar", "-xzf"], ".tgz": ["tar", "-xzf"], - ".tar.bz2": ["tar", "-xjf"], ".tbz": ["tar", "-xjf"], - ".tar.lzma": ["tar", "--lzma", "-xf"], ".tlz": ["tar", "--lzma", "-xf"], - ".tar.xz": ["tar", "--xz", "-xf"], ".txz": ["tar", "--xz", "-xf"], - ".7z": ["7zr", "x"], - } - # Test command exists and if not, remove - if not os.getenv('TR_TORRENT_DIR'): - devnull = open(os.devnull, 'w') - for cmd in required_cmds: - if call(['which', cmd], stdout=devnull, - stderr=devnull): # note, returns 0 if exists, or 1 if doesn't exist. - for k, v in extract_commands.items(): - if cmd in v[0]: - if not call(["which", "7zr"], stdout=devnull, stderr=devnull): # we do have "7zr" - extract_commands[k] = ["7zr", "x", "-y"] - elif not call(["which", "7z"], stdout=devnull, stderr=devnull): # we do have "7z" - extract_commands[k] = ["7z", "x", "-y"] - elif not call(["which", "7za"], stdout=devnull, stderr=devnull): # we do have "7za" - extract_commands[k] = ["7za", "x", "-y"] - else: - core.logger.error("EXTRACTOR: {cmd} not found, " - "disabling support for {feature}".format - (cmd=cmd, feature=k)) - del extract_commands[k] - devnull.close() - else: - core.logger.warning("EXTRACTOR: Cannot determine which tool to use when called from Transmission") - - if not extract_commands: - core.logger.warning("EXTRACTOR: No archive extracting programs found, plugin will be disabled") - - ext = os.path.splitext(file_path) - cmd = [] - if ext[1] in (".gz", ".bz2", ".lzma"): - # Check if this is a tar - if os.path.splitext(ext[0])[1] == ".tar": - cmd = extract_commands[".tar{ext}".format(ext=ext[1])] - elif ext[1] in (".1", ".01", ".001") and os.path.splitext(ext[0])[1] in (".rar", ".zip", ".7z"): - cmd = extract_commands[os.path.splitext(ext[0])[1]] - elif ext[1] in (".cb7", ".cba", ".cbr", ".cbt", ".cbz"): # don't extract these comic book archives. - return False - else: - if ext[1] in extract_commands: - cmd = extract_commands[ext[1]] - else: - core.logger.debug("EXTRACTOR: Unknown file type: {ext}".format - (ext=ext[1])) - return False - - # Create outputDestination folder - core.make_dir(output_destination) - - if core.PASSWORDSFILE and os.path.isfile(os.path.normpath(core.PASSWORDSFILE)): - passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDSFILE))] - else: - passwords = [] - - core.logger.info("Extracting {file} to {destination}".format - (file=file_path, destination=output_destination)) - core.logger.debug("Extracting {cmd} {file} {destination}".format - (cmd=cmd, file=file_path, destination=output_destination)) - - orig_files = [] - orig_dirs = [] - for dir, subdirs, files in os.walk(output_destination): - for subdir in subdirs: - orig_dirs.append(os.path.join(dir, subdir)) - for file in files: - orig_files.append(os.path.join(dir, file)) - - pwd = os.getcwd() # Get our Present Working Directory - os.chdir(output_destination) # Not all unpack commands accept full paths, so just extract into this directory - devnull = open(os.devnull, 'w') - - try: # now works same for nt and *nix - info = None - cmd.append(file_path) # add filePath to final cmd arg. - if platform.system() == 'Windows': - info = subprocess.STARTUPINFO() - info.dwFlags |= subprocess.STARTF_USESHOWWINDOW - else: - cmd = core.NICENESS + cmd - cmd2 = cmd - cmd2.append("-p-") # don't prompt for password. - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. - res = p.wait() - if res == 0: # Both Linux and Windows return 0 for successful. - core.logger.info("EXTRACTOR: Extraction was successful for {file} to {destination}".format - (file=file_path, destination=output_destination)) - success = 1 - elif len(passwords) > 0: - core.logger.info("EXTRACTOR: Attempting to extract with passwords") - for password in passwords: - if password == "": # if edited in windows or otherwise if blank lines. - continue - cmd2 = cmd - # append password here. - passcmd = "-p{pwd}".format(pwd=password) - cmd2.append(passcmd) - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. - res = p.wait() - if (res >= 0 and platform == 'Windows') or res == 0: - core.logger.info("EXTRACTOR: Extraction was successful " - "for {file} to {destination} using password: {pwd}".format - (file=file_path, destination=output_destination, pwd=password)) - success = 1 - break - else: - continue - except: - core.logger.error("EXTRACTOR: Extraction failed for {file}. " - "Could not call command {cmd}".format - (file=file_path, cmd=cmd)) - os.chdir(pwd) - return False - - devnull.close() - os.chdir(pwd) # Go back to our Original Working Directory - if success: - # sleep to let files finish writing to disk - sleep(3) - perms = stat.S_IMODE(os.lstat(os.path.split(file_path)[0]).st_mode) - for dir, subdirs, files in os.walk(output_destination): - for subdir in subdirs: - if not os.path.join(dir, subdir) in orig_files: - try: - os.chmod(os.path.join(dir, subdir), perms) - except: - pass - for file in files: - if not os.path.join(dir, file) in orig_files: - try: - shutil.copymode(file_path, os.path.join(dir, file)) - except: - pass - return True - else: - core.logger.error("EXTRACTOR: Extraction failed for {file}. " - "Result was {result}".format - (file=file_path, result=res)) - return False diff --git a/core/nzbToMediaUtil.py b/core/nzbToMediaUtil.py index 97e23e40..aa454348 100644 --- a/core/nzbToMediaUtil.py +++ b/core/nzbToMediaUtil.py @@ -26,8 +26,7 @@ from transmissionrpc.client import Client as TransmissionClient from utorrent.client import UTorrentClient import core -from core import logger, main_db -from core.extractor import extractor +from core import extractor, logger, main_db requests.packages.urllib3.disable_warnings() From f61a17b8a0e520f8d652ec42119808b978a645ae Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:27:47 -0500 Subject: [PATCH 18/22] Refactor nzbToMediaUtil -> utils --- TorrentToMedia.py | 2 +- core/__init__.py | 2 +- core/auto_process/comics.py | 2 +- core/auto_process/games.py | 2 +- core/auto_process/movies.py | 2 +- core/auto_process/music.py | 2 +- core/auto_process/tv.py | 2 +- core/databases.py | 2 +- core/scene_exceptions.py | 2 +- core/transcoder.py | 4 ++-- core/user_scripts.py | 2 +- core/{nzbToMediaUtil.py => utils.py} | 0 nzbToMedia.py | 2 +- tests/general.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) rename core/{nzbToMediaUtil.py => utils.py} (100%) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index 728afe61..a1208b93 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -7,7 +7,7 @@ import sys import core from core import logger, main_db from core.user_scripts import external_script -from core.nzbToMediaUtil import char_replace, convert_to_ascii, plex_update, replace_links +from core.utils import char_replace, convert_to_ascii, plex_update, replace_links from six import text_type diff --git a/core/__init__.py b/core/__init__.py index 58b474c5..8e726e48 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -45,7 +45,7 @@ from six.moves import reload_module from core import logger, main_db, version_check, databases, transcoder from core.auto_process import Comic, Game, Movie, Music, TV from core.configuration import config -from core.nzbToMediaUtil import ( +from core.utils import ( RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, create_torrent_class, extract_files, flatten, get_dirs, get_download_info, list_media_files, make_dir, parse_args, pause_torrent, remove_torrent, diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 47a2cb56..383105e2 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -6,7 +6,7 @@ import requests import core from core import logger -from core.nzbToMediaUtil import convert_to_ascii, remote_dir, server_responding +from core.utils import convert_to_ascii, remote_dir, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/games.py b/core/auto_process/games.py index 258c0137..19ecc63a 100644 --- a/core/auto_process/games.py +++ b/core/auto_process/games.py @@ -7,7 +7,7 @@ import requests import core from core import logger -from core.nzbToMediaUtil import convert_to_ascii, server_responding +from core.utils import convert_to_ascii, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index f3bde909..ff6470b0 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -9,7 +9,7 @@ import requests import core from core import logger, transcoder from core.scene_exceptions import process_all_exceptions -from core.nzbToMediaUtil import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding +from core.utils import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/music.py b/core/auto_process/music.py index f2aa7de9..36383208 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -9,7 +9,7 @@ import requests import core from core import logger from core.scene_exceptions import process_all_exceptions -from core.nzbToMediaUtil import convert_to_ascii, list_media_files, remote_dir, remove_dir, server_responding +from core.utils import convert_to_ascii, list_media_files, remote_dir, remove_dir, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index e1183ea8..800776d6 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -12,7 +12,7 @@ import core from core import logger, transcoder from core.forks import auto_fork from core.scene_exceptions import process_all_exceptions -from core.nzbToMediaUtil import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, report_nzb, remove_dir, server_responding +from core.utils import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding requests.packages.urllib3.disable_warnings() diff --git a/core/databases.py b/core/databases.py index 2cec59d8..fce380a8 100644 --- a/core/databases.py +++ b/core/databases.py @@ -1,7 +1,7 @@ # coding=utf-8 from core import logger, main_db -from core.nzbToMediaUtil import backup_versioned_file +from core.utils import backup_versioned_file MIN_DB_VERSION = 1 # oldest db version we support migrating from MAX_DB_VERSION = 2 diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py index 6a93d514..90e745de 100644 --- a/core/scene_exceptions.py +++ b/core/scene_exceptions.py @@ -8,7 +8,7 @@ import subprocess import core from core import logger -from core.nzbToMediaUtil import list_media_files +from core.utils import list_media_files reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", diff --git a/core/transcoder.py b/core/transcoder.py index bc0e2d2a..223b1c20 100644 --- a/core/transcoder.py +++ b/core/transcoder.py @@ -9,11 +9,11 @@ import shutil import subprocess from babelfish import Language -from six import iteritems, text_type, string_types +from six import iteritems, string_types, text_type import core from core import logger -from core.nzbToMediaUtil import make_dir +from core.utils import make_dir __author__ = 'Justin' diff --git a/core/user_scripts.py b/core/user_scripts.py index 4e0e966a..6eae29f5 100644 --- a/core/user_scripts.py +++ b/core/user_scripts.py @@ -5,7 +5,7 @@ from subprocess import Popen import core from core import logger, transcoder -from core.nzbToMediaUtil import import_subs, list_media_files, remove_dir +from core.utils import import_subs, list_media_files, remove_dir def external_script(output_destination, torrent_name, torrent_label, settings): diff --git a/core/nzbToMediaUtil.py b/core/utils.py similarity index 100% rename from core/nzbToMediaUtil.py rename to core/utils.py diff --git a/nzbToMedia.py b/nzbToMedia.py index edeef057..bea35444 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -631,7 +631,7 @@ import core from core import logger, main_db from core.auto_process import Comic, Game, Movie, Music, TV from core.user_scripts import external_script -from core.nzbToMediaUtil import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status +from core.utils import char_replace, clean_dir, convert_to_ascii, extract_files, get_dirs, get_download_info, get_nzoid, plex_update, update_download_info_status try: text_type = unicode diff --git a/tests/general.py b/tests/general.py index 42a8034c..3736e772 100755 --- a/tests/general.py +++ b/tests/general.py @@ -7,7 +7,7 @@ import requests import core from core import transcoder from core.forks import auto_fork -from core.nzbToMediaUtil import server_responding +from core.utils import server_responding # Initialize the config core.initialize() From ad6f0b7bb6756a62f02f337786f7fc73fc9779c5 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 19:32:50 -0500 Subject: [PATCH 19/22] Optimize imports --- core/utils.py | 44 +++++++++++++++++++++++--------------------- tests/general.py | 4 ++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/core/utils.py b/core/utils.py index aa454348..08e11d34 100644 --- a/core/utils.py +++ b/core/utils.py @@ -3,24 +3,23 @@ from __future__ import print_function, unicode_literals import datetime +from functools import partial import os -import platform import re import shutil import socket import stat import struct import time -from functools import partial +from babelfish import Language import beets import guessit import linktastic -import requests -import subliminal -from babelfish import Language from qbittorrent import Client as qBittorrentClient +import requests from six import text_type +import subliminal from synchronousdeluge.client import DelugeClient from transmissionrpc.client import Client as TransmissionClient from utorrent.client import UTorrentClient @@ -28,6 +27,20 @@ from utorrent.client import UTorrentClient import core from core import extractor, logger, main_db +try: + from win32event import CreateMutex + from win32api import CloseHandle, GetLastError + from winerror import ERROR_ALREADY_EXISTS +except ImportError: + if os.name == 'nt': + raise + +try: + import jaraco +except ImportError: + if os.name == 'nt': + raise + requests.packages.urllib3.disable_warnings() # Monkey Patch shutil.copyfileobj() to adjust the buffer length to 512KB rather than 4KB @@ -285,7 +298,6 @@ def replace_links(link): n = 0 target = link if os.name == 'nt': - import jaraco if not jaraco.windows.filesystem.islink(link): logger.debug('{0} is not a link'.format(link)) return @@ -1302,7 +1314,7 @@ class RunningProcess(object): """ Limits application to single instance """ def __init__(self): - if platform.system() == 'Windows': + if os.name == 'nt': self.process = WindowsProcess() else: self.process = PosixProcess() @@ -1318,20 +1330,10 @@ class WindowsProcess(object): def __init__(self): self.mutex = None self.mutexname = "nzbtomedia_{pid}".format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}" - if platform.system() == 'Windows': - try: - from win32.win32event import CreateMutex - from win32.win32api import CloseHandle, GetLastError - from win32.lib.winerror import ERROR_ALREADY_EXISTS - except ImportError: - from win32event import CreateMutex - from win32api import CloseHandle, GetLastError - from winerror import ERROR_ALREADY_EXISTS - - self.CreateMutex = CreateMutex - self.CloseHandle = CloseHandle - self.GetLastError = GetLastError - self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS + self.CreateMutex = CreateMutex + self.CloseHandle = CloseHandle + self.GetLastError = GetLastError + self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS def alreadyrunning(self): self.mutex = self.CreateMutex(None, 0, self.mutexname) diff --git a/tests/general.py b/tests/general.py index 3736e772..8692d430 100755 --- a/tests/general.py +++ b/tests/general.py @@ -1,8 +1,10 @@ #! /usr/bin/env python2 from __future__ import print_function +from babelfish import Language import guessit import requests +import subliminal import core from core import transcoder @@ -38,7 +40,6 @@ if server_responding("http://127.0.0.1:8085"): if server_responding("http://127.0.0.1:8090"): print("Mylar Running") -from babelfish import Language lan = 'pt' lan = Language.fromalpha2(lan) print(lan.alpha3) @@ -59,7 +60,6 @@ if guess: results = r.json() print(results) -import subliminal subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) languages = set() languages.add(lan) From 7a46bfa55a667f0cce13c2aa37d8780cfa0e2085 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Tue, 18 Dec 2018 21:41:20 -0500 Subject: [PATCH 20/22] Remove RunningProcess class and replace with platform alias --- core/utils.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/core/utils.py b/core/utils.py index 08e11d34..ce8944ea 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1309,23 +1309,6 @@ def get_download_info(input_name, status): return sql_results - -class RunningProcess(object): - """ Limits application to single instance """ - - def __init__(self): - if os.name == 'nt': - self.process = WindowsProcess() - else: - self.process = PosixProcess() - - def alreadyrunning(self): - return self.process.alreadyrunning() - - # def __del__(self): - # self.process.__del__() - - class WindowsProcess(object): def __init__(self): self.mutex = None @@ -1401,3 +1384,8 @@ class PosixProcess(object): self.lock_socket.close() if os.path.isfile(self.pidpath): os.unlink(self.pidpath) + +if os.name == 'nt': + RunningProcess = WindowsProcess +else: + RunningProcess = PosixProcess From 6b52bb68d00f9d46e3e41e57d4a4213d2b4cfe7c Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Wed, 19 Dec 2018 17:24:04 -0500 Subject: [PATCH 21/22] Refactor process_episode -> process --- TorrentToMedia.py | 8 ++++---- core/auto_process/comics.py | 2 +- core/auto_process/tv.py | 2 +- nzbToMedia.py | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/TorrentToMedia.py b/TorrentToMedia.py index a1208b93..55840dd5 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -237,14 +237,14 @@ def process_torrent(input_directory, input_name, input_category, input_hash, inp elif section_name in ['SickBeard', 'NzbDrone', 'Sonarr']: if input_hash: input_hash = input_hash.upper() - result = core.TV().process_episode(section_name, output_destination, input_name, - status, client_agent, input_hash, input_category) + result = core.TV().process(section_name, output_destination, input_name, + status, client_agent, input_hash, input_category) elif section_name in ['HeadPhones', 'Lidarr']: result = core.Music().process(section_name, output_destination, input_name, status, client_agent, input_category) elif section_name == 'Mylar': - result = core.Comic().process_episode(section_name, output_destination, input_name, - status, client_agent, input_category) + result = core.Comic().process(section_name, output_destination, input_name, + status, client_agent, input_category) elif section_name == 'Gamez': result = core.Game().process(section_name, output_destination, input_name, status, client_agent, input_category) diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py index 383105e2..40accce1 100644 --- a/core/auto_process/comics.py +++ b/core/auto_process/comics.py @@ -12,7 +12,7 @@ requests.packages.urllib3.disable_warnings() class Comic(object): - def process_episode(self, section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + def process(self, section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): apc_version = "2.04" comicrn_version = "1.01" diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 800776d6..988ebade 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -51,7 +51,7 @@ class TV(object): # ValueError catches simplejson's JSONDecodeError and json's ValueError return False - def process_episode(self, section, dir_name, input_name=None, failed=False, client_agent="manual", download_id=None, input_category=None, failure_link=None): + def process(self, section, dir_name, input_name=None, failed=False, client_agent="manual", download_id=None, input_category=None, failure_link=None): cfg = dict(core.CFG[section][input_category]) diff --git a/nzbToMedia.py b/nzbToMedia.py index bea35444..c29a3de4 100755 --- a/nzbToMedia.py +++ b/nzbToMedia.py @@ -728,13 +728,13 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d result = Movie().process(section_name, input_directory, input_name, status, client_agent, download_id, input_category, failure_link) elif section_name in ["SickBeard", "NzbDrone", "Sonarr"]: - result = TV().process_episode(section_name, input_directory, input_name, status, client_agent, - download_id, input_category, failure_link) + result = TV().process(section_name, input_directory, input_name, status, client_agent, + download_id, input_category, failure_link) elif section_name in ["HeadPhones", "Lidarr"]: result = Music().process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == "Mylar": - result = Comic().process_episode(section_name, input_directory, input_name, status, client_agent, - input_category) + result = Comic().process(section_name, input_directory, input_name, status, client_agent, + input_category) elif section_name == "Gamez": result = Game().process(section_name, input_directory, input_name, status, client_agent, input_category) elif section_name == 'UserScript': From d546a7dcee221df298e110752cd99e2511358271 Mon Sep 17 00:00:00 2001 From: Labrys of Knossos Date: Wed, 19 Dec 2018 17:28:20 -0500 Subject: [PATCH 22/22] Refactor method order --- core/auto_process/movies.py | 258 ++++++++++++++++++------------------ core/auto_process/music.py | 164 +++++++++++------------ core/auto_process/tv.py | 66 ++++----- 3 files changed, 244 insertions(+), 244 deletions(-) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py index ff6470b0..dde29946 100644 --- a/core/auto_process/movies.py +++ b/core/auto_process/movies.py @@ -15,135 +15,6 @@ requests.packages.urllib3.disable_warnings() class Movie(object): - def get_release(self, base_url, imdb_id=None, download_id=None, release_id=None): - results = {} - params = {} - - # determine cmd and params to send to CouchPotato to get our results - section = 'movies' - cmd = "media.list" - if release_id or imdb_id: - section = 'media' - cmd = "media.get" - params['id'] = release_id or imdb_id - - if not (release_id or imdb_id or download_id): - logger.debug("No information available to filter CP results") - return results - - url = "{0}{1}".format(base_url, cmd) - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url)) - return results - - try: - result = r.json() - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("CouchPotato returned the following non-json data") - for line in r.iter_lines(): - logger.error("{0}".format(line)) - return results - - if not result['success']: - if 'error' in result: - logger.error('{0}'.format(result['error'])) - else: - logger.error("no media found for id {0}".format(params['id'])) - return results - - # Gather release info and return it back, no need to narrow results - if release_id: - try: - id = result[section]['_id'] - results[id] = result[section] - return results - except: - pass - - # Gather release info and proceed with trying to narrow results to one release choice - - movies = result[section] - if not isinstance(movies, list): - movies = [movies] - for movie in movies: - if movie['status'] not in ['active', 'done']: - continue - releases = movie['releases'] - if not releases: - continue - for release in releases: - try: - if release['status'] not in ['snatched', 'downloaded', 'done']: - continue - if download_id: - if download_id.lower() != release['download_info']['id'].lower(): - continue - - id = release['_id'] - results[id] = release - results[id]['title'] = movie['title'] - except: - continue - - # Narrow results by removing old releases by comparing their last_edit field - if len(results) > 1: - for id1, x1 in results.items(): - for id2, x2 in results.items(): - try: - if x2["last_edit"] > x1["last_edit"]: - results.pop(id1) - except: - continue - - # Search downloads on clients for a match to try and narrow our results down to 1 - if len(results) > 1: - for id, x in results.items(): - try: - if not find_download(str(x['download_info']['downloader']).lower(), x['download_info']['id']): - results.pop(id) - except: - continue - - return results - - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def completed_download_handling(self, url2, headers, section="MAIN"): - try: - r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url2), section) - return False - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return False - else: - try: - return r.json().get("enableCompletedDownloadHandling", False) - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return False - def process(self, section, dir_name, input_name=None, status=0, client_agent="manual", download_id="", input_category=None, failure_link=None): cfg = dict(core.CFG[section][input_category]) @@ -464,3 +335,132 @@ class Movie(object): "{0} does not appear to have changed status after {1} minutes, Please check your logs.".format(input_name, wait_for), section) return [1, "{0}: Failed to post-process - No change in status".format(section)] + + def command_complete(self, url, params, headers, section): + try: + r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL: {0}".format(url), section) + return None + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return None + else: + try: + return r.json()['state'] + except (ValueError, KeyError): + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error("{0} did not return expected json data.".format(section), section) + return None + + def completed_download_handling(self, url2, headers, section="MAIN"): + try: + r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL: {0}".format(url2), section) + return False + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return False + else: + try: + return r.json().get("enableCompletedDownloadHandling", False) + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + return False + + def get_release(self, base_url, imdb_id=None, download_id=None, release_id=None): + results = {} + params = {} + + # determine cmd and params to send to CouchPotato to get our results + section = 'movies' + cmd = "media.list" + if release_id or imdb_id: + section = 'media' + cmd = "media.get" + params['id'] = release_id or imdb_id + + if not (release_id or imdb_id or download_id): + logger.debug("No information available to filter CP results") + return results + + url = "{0}{1}".format(base_url, cmd) + logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL {0}".format(url)) + return results + + try: + result = r.json() + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error("CouchPotato returned the following non-json data") + for line in r.iter_lines(): + logger.error("{0}".format(line)) + return results + + if not result['success']: + if 'error' in result: + logger.error('{0}'.format(result['error'])) + else: + logger.error("no media found for id {0}".format(params['id'])) + return results + + # Gather release info and return it back, no need to narrow results + if release_id: + try: + id = result[section]['_id'] + results[id] = result[section] + return results + except: + pass + + # Gather release info and proceed with trying to narrow results to one release choice + + movies = result[section] + if not isinstance(movies, list): + movies = [movies] + for movie in movies: + if movie['status'] not in ['active', 'done']: + continue + releases = movie['releases'] + if not releases: + continue + for release in releases: + try: + if release['status'] not in ['snatched', 'downloaded', 'done']: + continue + if download_id: + if download_id.lower() != release['download_info']['id'].lower(): + continue + + id = release['_id'] + results[id] = release + results[id]['title'] = movie['title'] + except: + continue + + # Narrow results by removing old releases by comparing their last_edit field + if len(results) > 1: + for id1, x1 in results.items(): + for id2, x2 in results.items(): + try: + if x2["last_edit"] > x1["last_edit"]: + results.pop(id1) + except: + continue + + # Search downloads on clients for a match to try and narrow our results down to 1 + if len(results) > 1: + for id, x in results.items(): + try: + if not find_download(str(x['download_info']['downloader']).lower(), x['download_info']['id']): + results.pop(id) + except: + continue + + return results diff --git a/core/auto_process/music.py b/core/auto_process/music.py index 36383208..0e5daeca 100644 --- a/core/auto_process/music.py +++ b/core/auto_process/music.py @@ -15,87 +15,6 @@ requests.packages.urllib3.disable_warnings() class Music(object): - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def get_status(self, url, apikey, dir_name): - logger.debug("Attempting to get current status for release:{0}".format(os.path.basename(dir_name))) - - params = { - 'apikey': apikey, - 'cmd': "getHistory" - } - - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.RequestException: - logger.error("Unable to open URL") - return None - - try: - result = r.json() - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return None - - for album in result: - if os.path.basename(dir_name) == album['FolderName']: - return album["Status"].lower() - - def force_process(self, params, url, apikey, input_name, dir_name, section, wait_for): - release_status = self.get_status(url, apikey, dir_name) - if not release_status: - logger.error("Could not find a status for {0}, is it in the wanted list ?".format(input_name), section) - - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 300)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - logger.debug("Result: {0}".format(r.text), section) - - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif r.text == "OK": - logger.postprocess("SUCCESS: Post-Processing started for {0} in folder {1} ...".format(input_name, dir_name), section) - else: - logger.error("FAILED: Post-Processing has NOT started for {0} in folder {1}. exiting!".format(input_name, dir_name), section) - return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] - - # we will now wait for this album to be processed before returning to TorrentToMedia and unpausing. - timeout = time.time() + 60 * wait_for - while time.time() < timeout: - current_status = self.get_status(url, apikey, dir_name) - if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie. - logger.postprocess("SUCCESS: This release is now marked as status [{0}]".format(current_status), section) - return [0, "{0}: Successfully post-processed {1}".format(section, input_name)] - if not os.path.isdir(dir_name): - logger.postprocess("SUCCESS: The input directory {0} has been removed Processing must have finished.".format(dir_name), section) - return [0, "{0}: Successfully post-processed {1}".format(section, input_name)] - time.sleep(10 * wait_for) - # The status hasn't changed. - return [2, "no change"] - def process(self, section, dir_name, input_name=None, status=0, client_agent="manual", input_category=None): status = int(status) @@ -236,4 +155,85 @@ class Music(object): if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: logger.postprocess("Deleting failed files and folder {0}".format(dir_name), section) remove_dir(dir_name) - return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader. \ No newline at end of file + return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader. + + def command_complete(self, url, params, headers, section): + try: + r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL: {0}".format(url), section) + return None + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return None + else: + try: + return r.json()['state'] + except (ValueError, KeyError): + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error("{0} did not return expected json data.".format(section), section) + return None + + def get_status(self, url, apikey, dir_name): + logger.debug("Attempting to get current status for release:{0}".format(os.path.basename(dir_name))) + + params = { + 'apikey': apikey, + 'cmd': "getHistory" + } + + logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.RequestException: + logger.error("Unable to open URL") + return None + + try: + result = r.json() + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + return None + + for album in result: + if os.path.basename(dir_name) == album['FolderName']: + return album["Status"].lower() + + def force_process(self, params, url, apikey, input_name, dir_name, section, wait_for): + release_status = self.get_status(url, apikey, dir_name) + if not release_status: + logger.error("Could not find a status for {0}, is it in the wanted list ?".format(input_name), section) + + logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error("Unable to open URL {0}".format(url), section) + return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] + + logger.debug("Result: {0}".format(r.text), section) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] + elif r.text == "OK": + logger.postprocess("SUCCESS: Post-Processing started for {0} in folder {1} ...".format(input_name, dir_name), section) + else: + logger.error("FAILED: Post-Processing has NOT started for {0} in folder {1}. exiting!".format(input_name, dir_name), section) + return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] + + # we will now wait for this album to be processed before returning to TorrentToMedia and unpausing. + timeout = time.time() + 60 * wait_for + while time.time() < timeout: + current_status = self.get_status(url, apikey, dir_name) + if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie. + logger.postprocess("SUCCESS: This release is now marked as status [{0}]".format(current_status), section) + return [0, "{0}: Successfully post-processed {1}".format(section, input_name)] + if not os.path.isdir(dir_name): + logger.postprocess("SUCCESS: The input directory {0} has been removed Processing must have finished.".format(dir_name), section) + return [0, "{0}: Successfully post-processed {1}".format(section, input_name)] + time.sleep(10 * wait_for) + # The status hasn't changed. + return [2, "no change"] diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py index 988ebade..930823ab 100644 --- a/core/auto_process/tv.py +++ b/core/auto_process/tv.py @@ -18,39 +18,6 @@ requests.packages.urllib3.disable_warnings() class TV(object): - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def completed_download_handling(self, url2, headers, section="MAIN"): - try: - r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url2), section) - return False - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return False - else: - try: - return r.json().get("enableCompletedDownloadHandling", False) - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return False - def process(self, section, dir_name, input_name=None, failed=False, client_agent="manual", download_id=None, input_category=None, failure_link=None): cfg = dict(core.CFG[section][input_category]) @@ -372,3 +339,36 @@ class TV(object): return [1, "{0}: Failed to post-process {1}".format(section, input_name)] else: return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] # We did not receive Success confirmation. + + def command_complete(self, url, params, headers, section): + try: + r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL: {0}".format(url), section) + return None + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return None + else: + try: + return r.json()['state'] + except (ValueError, KeyError): + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error("{0} did not return expected json data.".format(section), section) + return None + + def completed_download_handling(self, url2, headers, section="MAIN"): + try: + r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error("Unable to open URL: {0}".format(url2), section) + return False + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error("Server returned status {0}".format(r.status_code), section) + return False + else: + try: + return r.json().get("enableCompletedDownloadHandling", False) + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + return False