github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-08-21 22:03:13 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'three' into deadsnakes

Browse source 
* three:
  Remove six.moves.urllib.request
  Remove six.string_types
  Remove six.iteritems
  Remove six.text_type usage
  Remove shutil_custom monkeypatch
  Remove PY2 checks
This commit is contained in:
Labrys of Knossos 2022-12-03 07:30:03 -05:00
commit 55e751c9a3
16 changed files with 65 additions and 196 deletions
Download patch file Download diff file Expand all files Collapse all files

20
core/__init__.py
View file

@ -37,9 +37,6 @@ CONFIG_TV_FILE = os.path.join(APP_ROOT, 'autoProcessTv.cfg')
TEST_FILE = os.path.join(APP_ROOT, 'tests', 'test.mp4')
MYAPP = None
import six
from six.moves import reload_module
from core import logger, main_db, version_check, databases, transcoder
from core.configuration import config
from core.plugins.downloaders.configuration import (
@ -305,23 +302,6 @@ def configure_locale():
if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
SYS_ENCODING = 'UTF-8'
if six.PY2:
if not hasattr(sys, 'setdefaultencoding'):
reload_module(sys)
try:
# pylint: disable=E1101
# On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError
sys.setdefaultencoding(SYS_ENCODING)
except Exception:
print('Sorry, you MUST add the nzbToMedia folder to the PYTHONPATH environment variable'
'\nor find another way to force Python to use {codec} for string encoding.'.format
(codec=SYS_ENCODING))
if 'NZBOP_SCRIPTDIR' in os.environ:
sys.exit(NZBGET_POSTPROCESS_ERROR)
else:
sys.exit(1)
def configure_migration():
global CONFIG_FILE

11
core/auto_process/managers/sickbeard.py
View file

@ -13,9 +13,6 @@ import requests
from requests_oauthlib import OAuth2Session
import six
from six import iteritems
class InitSickBeard:
"""Sickbeard init class.
@ -183,11 +180,7 @@ class InitSickBeard:
logger.debug('Response received: {}'.format(json_data))
raise
else:
if six.PY3:
str_type = (str)
else:
str_type = (str, unicode)
if isinstance(json_data, str_type):
if isinstance(json_data, str):
return rem_params, False
json_data = json_data.get('data', json_data)
@ -270,7 +263,7 @@ class InitSickBeard:
for param in rem_params:
params.pop(param)
for fork in sorted(iteritems(core.FORKS), reverse=False):
for fork in sorted(core.FORKS, reverse=False):
if params == fork[1]:
detected = True
break

7
core/configuration.py
View file

@ -4,7 +4,6 @@ import shutil
from itertools import chain
import configobj
from six import iteritems
import core
from core import logger
@ -161,7 +160,7 @@ class ConfigObj(configobj.ConfigObj, Section):
continue
def cleanup_values(values, section):
for option, value in iteritems(values):
for option, value in values.items():
if section in ['CouchPotato']:
if option == ['outputDirectory']:
CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0]
@ -242,7 +241,7 @@ class ConfigObj(configobj.ConfigObj, Section):
subsection = None
if section in list(chain.from_iterable(subsections.values())):
subsection = section
section = ''.join([k for k, v in iteritems(subsections) if subsection in v])
section = ''.join([k for k, v in subsections.items() if subsection in v])
process_section(section, subsection)
elif section in subsections.keys():
subsection = subsections[section]
@ -252,7 +251,7 @@ class ConfigObj(configobj.ConfigObj, Section):
# migrate SiCRKAGE settings from SickBeard section to new dedicated SiCRKAGE section
if CFG_OLD['SickBeard']['tv']['enabled'] and CFG_OLD['SickBeard']['tv']['fork'] == 'sickrage-api':
for option, value in iteritems(CFG_OLD['SickBeard']['tv']):
for option, value in CFG_OLD['SickBeard']['tv'].items():
if option in CFG_NEW['SiCKRAGE']['tv']:
CFG_NEW['SiCKRAGE']['tv'][option] = value

31
core/main_db.py
View file

@ -2,36 +2,9 @@ import re
import sqlite3
import time
from six import text_type, PY2
import core
from core import logger
if PY2:
class Row(sqlite3.Row, object):
"""
Row factory that uses Byte Strings for keys.
The sqlite3.Row in Python 2 does not support unicode keys.
This overrides __getitem__ to attempt to encode the key to bytes first.
"""
def __getitem__(self, item):
"""
Get an item from the row by index or key.
:param item: Index or Key of item to return.
:return: An item from the sqlite3.Row.
"""
try:
# sqlite3.Row column names should be Bytes in Python 2
item = item.encode()
except AttributeError:
pass # assume item is a numeric index
return super(Row, self).__getitem__(item)
else:
from sqlite3 import Row
def db_filename(filename='nzbtomedia.db', suffix=None):
"""
@ -53,7 +26,7 @@ class DBConnection:
self.filename = filename
self.connection = sqlite3.connect(db_filename(filename), 20)
self.connection.row_factory = Row
self.connection.row_factory = sqlite3.Row
def check_db_version(self):
result = None
@ -213,7 +186,7 @@ class DBConnection:
'INSERT OR IGNORE INTO {table} ({columns}) '
'VALUES ({values})'.format(
table=table_name,
columns=', '.join(map(text_type, value_dict.keys())),
columns=', '.join(map(str, value_dict.keys())),
values=', '.join(['?'] * len(value_dict.values())),
),
list(value_dict.values()),

11
core/processor/manual.py
View file

@ -9,11 +9,6 @@ from core.utils import (
get_download_info,
)
try:
text_type = unicode
except NameError:
text_type = str
def process():
# Perform Manual Post-Processing
@ -44,10 +39,8 @@ def process():
logger.info('Found download info for {0}, '
'setting variables now ...'.format
(os.path.basename(dir_name)))
client_agent = text_type(
core.DOWNLOAD_INFO[0]['client_agent']) or 'manual'
download_id = text_type(
core.DOWNLOAD_INFO[0]['input_id']) or ''
client_agent = core.DOWNLOAD_INFO[0]['client_agent'] or 'manual'
download_id = core.DOWNLOAD_INFO[0]['input_id'] or ''
else:
logger.info('Unable to locate download info for {0}, '
'continuing to try and process this release ...'.format

15
core/processor/nzb.py
View file

@ -15,11 +15,6 @@ from core.utils import (
update_download_info_status,
)
try:
text_type = unicode
except NameError:
text_type = str
def process(input_directory, input_name=None, status=0, client_agent='manual', download_id=None, input_category=None, failure_link=None):
if core.SAFE_MODE and input_directory == core.NZB_DEFAULT_DIRECTORY:
@ -48,12 +43,12 @@ def process(input_directory, input_name=None, status=0, client_agent='manual', d
except Exception:
pass
control_value_dict = {'input_directory': text_type(input_directory1)}
control_value_dict = {'input_directory': input_directory1}
new_value_dict = {
'input_name': text_type(input_name1),
'input_hash': text_type(download_id),
'input_id': text_type(download_id),
'client_agent': text_type(client_agent),
'input_name': input_name1,
'input_hash': download_id,
'input_id': download_id,
'client_agent': client_agent,
'status': 0,
'last_update': datetime.date.today().toordinal(),
}

20
core/transcoder.py
View file

@ -9,7 +9,6 @@ import shutil
import subprocess
from babelfish import Language
from six import iteritems, string_types, text_type
import core
from core import logger
@ -137,7 +136,7 @@ def check_vid_file(video_details, result):
def build_commands(file, new_dir, movie_name, bitbucket):
if isinstance(file, string_types):
if isinstance(file, str):
input_file = file
if 'concat:' in file:
file = file.split('|')[0].replace('concat:', '')
@ -155,7 +154,7 @@ def build_commands(file, new_dir, movie_name, bitbucket):
core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext'
new_file = file
else:
img, data = next(iteritems(file))
img, data = next(file.items())
name = data['name']
new_file = []
rem_vid = []
@ -511,7 +510,6 @@ def build_commands(file, new_dir, movie_name, bitbucket):
continue
command.extend(['-i', subfile])
lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0]
lan = text_type(lan)
metlan = None
try:
if len(lan) == 3:
@ -660,7 +658,7 @@ def process_list(it, new_dir, bitbucket):
if combine:
new_list.extend(combine_cd(combine))
for file in new_list:
if isinstance(file, string_types) and 'concat:' not in file and not os.path.isfile(file):
if isinstance(file, str) and 'concat:' not in file and not os.path.isfile(file):
success = False
break
if success and new_list:
@ -903,13 +901,13 @@ def transcode_directory(dir_name):
return 1, dir_name
for file in file_list:
if isinstance(file, string_types) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS:
if isinstance(file, str) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS:
continue
command, file = build_commands(file, new_dir, movie_name, bitbucket)
newfile_path = command[-1]
# transcoding files may remove the original file, so make sure to extract subtitles first
if core.SEXTRACT and isinstance(file, string_types):
if core.SEXTRACT and isinstance(file, str):
extract_subs(file, newfile_path, bitbucket)
try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason)
@ -924,10 +922,10 @@ def transcode_directory(dir_name):
print_cmd(command)
result = 1 # set result to failed in case call fails.
try:
if isinstance(file, string_types):
if isinstance(file, str):
proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE)
else:
img, data = next(iteritems(file))
img, data = next(file.items())
proc = subprocess.Popen(command, stdout=bitbucket, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
for vob in data['files']:
procin = zip_out(vob, img, bitbucket)
@ -942,7 +940,7 @@ def transcode_directory(dir_name):
except Exception:
logger.error('Transcoding of video {0} has failed'.format(newfile_path))
if core.SUBSDIR and result == 0 and isinstance(file, string_types):
if core.SUBSDIR and result == 0 and isinstance(file, str):
for sub in get_subs(file):
name = os.path.splitext(os.path.split(file)[1])[0]
subname = os.path.split(sub)[1]
@ -981,7 +979,7 @@ def transcode_directory(dir_name):
os.unlink(file)
except Exception:
pass
if not os.listdir(text_type(new_dir)): # this is an empty directory and we didn't transcode into it.
if not os.listdir(new_dir): # this is an empty directory and we didn't transcode into it.
os.rmdir(new_dir)
new_dir = dir_name
if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB

2
core/utils/__init__.py
View file

@ -1,6 +1,5 @@
import requests
from core.utils import shutil_custom
from core.utils.common import clean_dir, flatten, get_dirs, process_dir
from core.utils.download_info import get_download_info, update_download_info_status
from core.utils.encoding import char_replace, convert_to_ascii
@ -42,4 +41,3 @@ from core.utils.paths import (
from core.utils.processes import RunningProcess, restart
requests.packages.urllib3.disable_warnings()
shutil_custom.monkey_patch()

6
core/utils/common.py
View file

@ -1,7 +1,5 @@
import os.path
from six import text_type
import core
from core import logger
from core.utils.files import list_media_files, move_file
@ -27,7 +25,7 @@ def process_dir(path, link):
folders = []
logger.info('Searching {0} for mediafiles to post-process ...'.format(path))
dir_contents = os.listdir(text_type(path))
dir_contents = os.listdir(path)
# search for single files and move them into their own folder for post-processing
@ -63,7 +61,7 @@ def process_dir(path, link):
# Generate all path contents
path_contents = (
os.path.join(path, item)
for item in os.listdir(text_type(path))
for item in os.listdir(path)
)
# Generate all directories from path contents

6
core/utils/download_info.py
View file

@ -1,7 +1,5 @@
import datetime
from six import text_type
from core import logger, main_db
database = main_db.DBConnection()
@ -10,7 +8,7 @@ database = main_db.DBConnection()
def update_download_info_status(input_name, status):
msg = 'Updating DB download status of {0} to {1}'
action = 'UPDATE downloads SET status=?, last_update=? WHERE input_name=?'
args = [status, datetime.date.today().toordinal(), text_type(input_name)]
args = [status, datetime.date.today().toordinal(), input_name]
logger.db(msg.format(input_name, status))
database.action(action, args)
@ -18,6 +16,6 @@ def update_download_info_status(input_name, status):
def get_download_info(input_name, status):
msg = 'Getting download info for {0} from the DB'
action = 'SELECT * FROM downloads WHERE input_name=? AND status=?'
args = [text_type(input_name), status]
args = [input_name, status]
logger.db(msg.format(input_name))
return database.select(action, args)

94
core/utils/encoding.py
View file

@ -1,14 +1,9 @@
import os
from six import text_type
from six import PY2
from builtins import bytes
import core
from core import logger
if not PY2:
from builtins import bytes
def char_replace(name_in):
# Special character hex range:
@ -19,68 +14,39 @@ def char_replace(name_in):
# If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding
encoded = False
encoding = None
if isinstance(name_in, text_type):
if isinstance(name_in, str):
return encoded, name_in
if PY2:
name = name_in
for Idx in range(len(name)):
# print('Trying to intuit the encoding')
# /!\ detection is done 2char by 2char for UTF-8 special character
if (len(name) != 1) & (Idx < (len(name) - 1)):
# Detect UTF-8
if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & (
(name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')):
encoding = 'utf-8'
break
# Detect CP850
elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'):
encoding = 'iso-8859-15'
break
else:
# Detect CP850
if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'):
encoding = 'iso-8859-15'
break
else:
name = bytes(name_in)
for Idx in range(len(name)):
# print('Trying to intuit the encoding')
# /!\ detection is done 2char by 2char for UTF-8 special character
if (len(name) != 1) & (Idx < (len(name) - 1)):
# Detect UTF-8
if ((name[Idx] == 0xC2) | (name[Idx] == 0xC3)) & (
(name[Idx + 1] >= 0xA0) & (name[Idx + 1] <= 0xFF)):
encoding = 'utf-8'
break
# Detect CP850
elif (name[Idx] >= 0x80) & (name[Idx] <= 0xA5):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= 0xA6) & (name[Idx] <= 0xFF):
encoding = 'iso-8859-15'
break
else:
# Detect CP850
if (name[Idx] >= 0x80) & (name[Idx] <= 0xA5):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= 0xA6) & (name[Idx] <= 0xFF):
encoding = 'iso-8859-15'
break
name = bytes(name_in)
for Idx in range(len(name)):
# print('Trying to intuit the encoding')
# /!\ detection is done 2char by 2char for UTF-8 special character
if (len(name) != 1) & (Idx < (len(name) - 1)):
# Detect UTF-8
if ((name[Idx] == 0xC2) | (name[Idx] == 0xC3)) & (
(name[Idx + 1] >= 0xA0) & (name[Idx + 1] <= 0xFF)):
encoding = 'utf-8'
break
# Detect CP850
elif (name[Idx] >= 0x80) & (name[Idx] <= 0xA5):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= 0xA6) & (name[Idx] <= 0xFF):
encoding = 'iso-8859-15'
break
else:
# Detect CP850
if (name[Idx] >= 0x80) & (name[Idx] <= 0xA5):
encoding = 'cp850'
break
# Detect ISO-8859-15
elif (name[Idx] >= 0xA6) & (name[Idx] <= 0xFF):
encoding = 'iso-8859-15'
break
if encoding:
encoded = True
name = name.decode(encoding)
elif not PY2:
else:
name = name.decode()
return encoded, name

3
core/utils/files.py
View file

@ -6,7 +6,6 @@ import time
import beets.mediafile
import guessit
from six import text_type
import core
from core import extractor, logger
@ -141,7 +140,7 @@ def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True,
return files
for cur_file in os.listdir(text_type(path)):
for cur_file in os.listdir(path):
full_cur_file = os.path.join(path, cur_file)
# if it's a folder do it recursively

5
core/utils/identification.py
View file

@ -3,7 +3,6 @@ import re
import guessit
import requests
from six import text_type
from core import logger
from core.utils.naming import sanitize_name
@ -22,7 +21,7 @@ def find_imdbid(dir_name, input_name, omdb_api_key):
logger.info('Found imdbID [{0}]'.format(imdbid))
return imdbid
if os.path.isdir(dir_name):
for file in os.listdir(text_type(dir_name)):
for file in os.listdir(dir_name):
m = re.search(r'\b(tt\d{7,8})\b', file)
if m:
imdbid = m.group(1)
@ -138,7 +137,7 @@ def category_search(input_directory, input_name, input_category, root, categorie
logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory))
tordir = True
elif input_name and os.path.isdir(input_directory):
for file in os.listdir(text_type(input_directory)):
for file in os.listdir(input_directory):
if os.path.splitext(file)[0] in [input_name, sanitize_name(input_name)]:
logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(file, input_directory))
input_directory = os.path.join(input_directory, file)

10
core/utils/paths.py
View file

@ -4,8 +4,6 @@ import re
import shutil
import stat
from six import text_type
import core
from core import logger
@ -32,7 +30,7 @@ def onerror(func, path, exc_info):
def remove_dir(dir_name):
logger.info('Deleting {0}'.format(dir_name))
try:
shutil.rmtree(text_type(dir_name), onerror=onerror)
shutil.rmtree(dir_name, onerror=onerror)
except Exception:
logger.error('Unable to delete folder {0}'.format(dir_name))
@ -68,7 +66,7 @@ def get_dir_size(input_path):
prepend = partial(os.path.join, input_path)
return sum(
(os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f))
for f in map(prepend, os.listdir(text_type(input_path)))
for f in map(prepend, os.listdir(input_path))
)
@ -79,7 +77,7 @@ def remove_empty_folders(path, remove_root=True):
# remove empty subfolders
logger.debug('Checking for empty folders in:{0}'.format(path))
files = os.listdir(text_type(path))
files = os.listdir(path)
if len(files):
for f in files:
fullpath = os.path.join(path, f)
@ -87,7 +85,7 @@ def remove_empty_folders(path, remove_root=True):
remove_empty_folders(fullpath)
# if folder empty, delete it
files = os.listdir(text_type(path))
files = os.listdir(path)
if len(files) == 0 and remove_root:
logger.debug('Removing empty folder:{}'.format(path))
os.rmdir(path)

18
core/utils/shutil_custom.py
View file

@ -1,18 +0,0 @@
from __future__ import (
absolute_import,
division,
print_function,
unicode_literals,
)
from functools import partial
import shutil
from six import PY2
def monkey_patch(length=512 * 1024):
if PY2:
# On Python 2 monkey patch shutil.copyfileobj()
# to adjust the buffer length to 512KB rather than 4KB
original_copyfileobj = shutil.copyfileobj
shutil.copyfileobj = partial(original_copyfileobj, length=length)

2
core/version_check.py
View file

@ -10,7 +10,7 @@ import subprocess
import tarfile
import traceback
from six.moves.urllib.request import urlretrieve
from urllib.request import urlretrieve
import cleanup
import core