diff --git a/.github/README.md b/.github/README.md index 6e50347a..2428e348 100644 --- a/.github/README.md +++ b/.github/README.md @@ -2,7 +2,7 @@ nzbToMedia ========== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) -when using one of the popular NZB download clients like [SABnzbd](http://sabnzbd.org/ "SABnzbd") and [NZBGet](https://nzbget.com/ "NZBGet") on low performance systems like a NAS. +when using one of the popular NZB download clients like [SABnzbd](http://sabnzbd.org/ "SABnzbd") and [NZBGet](http://nzbget.sourceforge.net/ "NZBGet") on low performance systems like a NAS. This script is based on sabToSickBeard (written by Nic Wolfe and supplied with SickBeard), with the support for NZBGet being added by [thorli](https://github.com/thorli "thorli") and further contributions by [schumi2004](https://github.com/schumi2004 "schumi2004") and [hugbug](https://sourceforge.net/apps/phpbb/nzbget/memberlist.php?mode=viewprofile&u=67 "hugbug"). Torrent suport added by [jkaberg](https://github.com/jkaberg "jkaberg") and [berkona](https://github.com/berkona "berkona") Corrupt video checking, auto SickBeard fork determination and a whole lot of code improvement was done by [echel0n](https://github.com/echel0n "echel0n") diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3ed96ee..c7a04b35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,8 @@ jobs: vmImage: 'Ubuntu-latest' strategy: matrix: + Python38: + python.version: '3.8' Python39: python.version: '3.9' Python310: @@ -21,8 +23,6 @@ jobs: python.version: '3.11' Python312: python.version: '3.12' - Python313: - python.version: '3.13' maxParallel: 3 steps: diff --git a/libs/common/filetype/__init__.py b/libs/common/filetype/__init__.py deleted file mode 100644 index 3d0610ea..00000000 --- a/libs/common/filetype/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .filetype import * # noqa -from .helpers import * # noqa -from .match import * # noqa - -# Current package semver version -__version__ = version = '1.2.0' diff --git a/libs/common/filetype/__main__.py b/libs/common/filetype/__main__.py deleted file mode 100644 index b79aaa0a..00000000 --- a/libs/common/filetype/__main__.py +++ /dev/null @@ -1,41 +0,0 @@ -import glob -from itertools import chain -from os.path import isfile - -import filetype - - -def guess(path): - kind = filetype.guess(path) - if kind is None: - print('{}: File type determination failure.'.format(path)) - else: - print('{}: {} ({})'.format(path, kind.extension, kind.mime)) - - -def main(): - import argparse - - parser = argparse.ArgumentParser( - prog='filetype', description='Determine type of FILEs.' - ) - parser.add_argument( - 'file', nargs='+', - help='files, wildcard is supported' - ) - parser.add_argument( - '-v', '--version', action='version', - version=f'%(prog)s {filetype.version}', - help='output version information and exit' - ) - - args = parser.parse_args() - items = chain.from_iterable(map(glob.iglob, args.file)) - files = filter(isfile, items) - - for file in files: - guess(file) - - -if __name__ == '__main__': - main() diff --git a/libs/common/filetype/filetype.py b/libs/common/filetype/filetype.py deleted file mode 100644 index cc374add..00000000 --- a/libs/common/filetype/filetype.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .match import match -from .types import TYPES, Type - -# Expose supported matchers types -types = TYPES - - -def guess(obj): - """ - Infers the type of the given input. - - Function is overloaded to accept multiple types in input - and perform the needed type inference based on it. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - The matched type instance. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj) if obj else None - - -def guess_mime(obj): - """ - Infers the file type of the given input - and returns its MIME type. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - The matched MIME type as string. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - kind = guess(obj) - return kind.mime if kind else kind - - -def guess_extension(obj): - """ - Infers the file type of the given input - and returns its RFC file extension. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - The matched file extension as string. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - kind = guess(obj) - return kind.extension if kind else kind - - -def get_type(mime=None, ext=None): - """ - Returns the file type instance searching by - MIME type or file extension. - - Args: - ext: file extension string. E.g: jpg, png, mp4, mp3 - mime: MIME string. E.g: image/jpeg, video/mpeg - - Returns: - The matched file type instance. Otherwise None. - """ - for kind in types: - if kind.extension == ext or kind.mime == mime: - return kind - return None - - -def add_type(instance): - """ - Adds a new type matcher instance to the supported types. - - Args: - instance: Type inherited instance. - - Returns: - None - """ - if not isinstance(instance, Type): - raise TypeError('instance must inherit from filetype.types.Type') - - types.insert(0, instance) diff --git a/libs/common/filetype/helpers.py b/libs/common/filetype/helpers.py deleted file mode 100644 index afe4e684..00000000 --- a/libs/common/filetype/helpers.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import -from .types import TYPES -from .match import ( - image_match, font_match, document_match, - video_match, audio_match, archive_match -) - - -def is_extension_supported(ext): - """ - Checks if the given extension string is - one of the supported by the file matchers. - - Args: - ext (str): file extension string. E.g: jpg, png, mp4, mp3 - - Returns: - True if the file extension is supported. - Otherwise False. - """ - for kind in TYPES: - if kind.extension == ext: - return True - return False - - -def is_mime_supported(mime): - """ - Checks if the given MIME type string is - one of the supported by the file matchers. - - Args: - mime (str): MIME string. E.g: image/jpeg, video/mpeg - - Returns: - True if the MIME type is supported. - Otherwise False. - """ - for kind in TYPES: - if kind.mime == mime: - return True - return False - - -def is_image(obj): - """ - Checks if a given input is a supported type image. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid image. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return image_match(obj) is not None - - -def is_archive(obj): - """ - Checks if a given input is a supported type archive. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid archive. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return archive_match(obj) is not None - - -def is_audio(obj): - """ - Checks if a given input is a supported type audio. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid audio. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return audio_match(obj) is not None - - -def is_video(obj): - """ - Checks if a given input is a supported type video. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid video. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return video_match(obj) is not None - - -def is_font(obj): - """ - Checks if a given input is a supported type font. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid font. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return font_match(obj) is not None - - -def is_document(obj): - """ - Checks if a given input is a supported type document. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - True if obj is a valid document. Otherwise False. - - Raises: - TypeError: if obj is not a supported type. - """ - return document_match(obj) is not None diff --git a/libs/common/filetype/match.py b/libs/common/filetype/match.py deleted file mode 100644 index 8b6f11b7..00000000 --- a/libs/common/filetype/match.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .types import ARCHIVE as archive_matchers -from .types import AUDIO as audio_matchers -from .types import APPLICATION as application_matchers -from .types import DOCUMENT as document_matchers -from .types import FONT as font_matchers -from .types import IMAGE as image_matchers -from .types import VIDEO as video_matchers -from .types import TYPES -from .utils import get_bytes - - -def match(obj, matchers=TYPES): - """ - Matches the given input against the available - file type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if type matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - buf = get_bytes(obj) - - for matcher in matchers: - if matcher.match(buf): - return matcher - - return None - - -def image_match(obj): - """ - Matches the given input against the available - image type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, image_matchers) - - -def font_match(obj): - """ - Matches the given input against the available - font type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, font_matchers) - - -def video_match(obj): - """ - Matches the given input against the available - video type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, video_matchers) - - -def audio_match(obj): - """ - Matches the given input against the available - autio type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, audio_matchers) - - -def archive_match(obj): - """ - Matches the given input against the available - archive type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, archive_matchers) - - -def application_match(obj): - """ - Matches the given input against the available - application type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, application_matchers) - - -def document_match(obj): - """ - Matches the given input against the available - document type matchers. - - Args: - obj: path to file, bytes or bytearray. - - Returns: - Type instance if matches. Otherwise None. - - Raises: - TypeError: if obj is not a supported type. - """ - return match(obj, document_matchers) diff --git a/libs/common/filetype/types/__init__.py b/libs/common/filetype/types/__init__.py deleted file mode 100644 index e833ab42..00000000 --- a/libs/common/filetype/types/__init__.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from . import archive -from . import audio -from . import application -from . import document -from . import font -from . import image -from . import video -from .base import Type # noqa - -# Supported image types -IMAGE = ( - image.Dwg(), - image.Xcf(), - image.Jpeg(), - image.Jpx(), - image.Jxl(), - image.Apng(), - image.Png(), - image.Gif(), - image.Webp(), - image.Tiff(), - image.Cr2(), - image.Bmp(), - image.Jxr(), - image.Psd(), - image.Ico(), - image.Heic(), - image.Dcm(), - image.Avif(), - image.Qoi(), - image.Dds(), -) - -# Supported video types -VIDEO = ( - video.M3gp(), - video.Mp4(), - video.M4v(), - video.Mkv(), - video.Mov(), - video.Avi(), - video.Wmv(), - video.Mpeg(), - video.Webm(), - video.Flv(), -) - -# Supported audio types -AUDIO = ( - audio.Aac(), - audio.Midi(), - audio.Mp3(), - audio.M4a(), - audio.Ogg(), - audio.Flac(), - audio.Wav(), - audio.Amr(), - audio.Aiff(), -) - -# Supported font types -FONT = (font.Woff(), font.Woff2(), font.Ttf(), font.Otf()) - -# Supported archive container types -ARCHIVE = ( - archive.Br(), - archive.Rpm(), - archive.Dcm(), - archive.Epub(), - archive.Zip(), - archive.Tar(), - archive.Rar(), - archive.Gz(), - archive.Bz2(), - archive.SevenZ(), - archive.Pdf(), - archive.Exe(), - archive.Swf(), - archive.Rtf(), - archive.Nes(), - archive.Crx(), - archive.Cab(), - archive.Eot(), - archive.Ps(), - archive.Xz(), - archive.Sqlite(), - archive.Deb(), - archive.Ar(), - archive.Z(), - archive.Lzop(), - archive.Lz(), - archive.Elf(), - archive.Lz4(), - archive.Zstd(), -) - -# Supported archive container types -APPLICATION = ( - application.Wasm(), -) - -# Supported document types -DOCUMENT = ( - document.Doc(), - document.Docx(), - document.Odt(), - document.Xls(), - document.Xlsx(), - document.Ods(), - document.Ppt(), - document.Pptx(), - document.Odp(), -) - - -# Expose supported type matchers -TYPES = list(IMAGE + AUDIO + VIDEO + FONT + DOCUMENT + ARCHIVE + APPLICATION) diff --git a/libs/common/filetype/types/application.py b/libs/common/filetype/types/application.py deleted file mode 100644 index 6f02370c..00000000 --- a/libs/common/filetype/types/application.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type - - -class Wasm(Type): - """Implements the Wasm image type matcher.""" - - MIME = 'application/wasm' - EXTENSION = 'wasm' - - def __init__(self): - super(Wasm, self).__init__( - mime=Wasm.MIME, - extension=Wasm.EXTENSION - ) - - def match(self, buf): - return buf[:8] == bytearray([0x00, 0x61, 0x73, 0x6d, - 0x01, 0x00, 0x00, 0x00]) diff --git a/libs/common/filetype/types/archive.py b/libs/common/filetype/types/archive.py deleted file mode 100644 index 0ffc9548..00000000 --- a/libs/common/filetype/types/archive.py +++ /dev/null @@ -1,688 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import struct - -from .base import Type - - -class Epub(Type): - """ - Implements the EPUB archive type matcher. - """ - MIME = 'application/epub+zip' - EXTENSION = 'epub' - - def __init__(self): - super(Epub, self).__init__( - mime=Epub.MIME, - extension=Epub.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 57 and - buf[0] == 0x50 and buf[1] == 0x4B and - buf[2] == 0x3 and buf[3] == 0x4 and - buf[30] == 0x6D and buf[31] == 0x69 and - buf[32] == 0x6D and buf[33] == 0x65 and - buf[34] == 0x74 and buf[35] == 0x79 and - buf[36] == 0x70 and buf[37] == 0x65 and - buf[38] == 0x61 and buf[39] == 0x70 and - buf[40] == 0x70 and buf[41] == 0x6C and - buf[42] == 0x69 and buf[43] == 0x63 and - buf[44] == 0x61 and buf[45] == 0x74 and - buf[46] == 0x69 and buf[47] == 0x6F and - buf[48] == 0x6E and buf[49] == 0x2F and - buf[50] == 0x65 and buf[51] == 0x70 and - buf[52] == 0x75 and buf[53] == 0x62 and - buf[54] == 0x2B and buf[55] == 0x7A and - buf[56] == 0x69 and buf[57] == 0x70) - - -class Zip(Type): - """ - Implements the Zip archive type matcher. - """ - MIME = 'application/zip' - EXTENSION = 'zip' - - def __init__(self): - super(Zip, self).__init__( - mime=Zip.MIME, - extension=Zip.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x50 and buf[1] == 0x4B and - (buf[2] == 0x3 or buf[2] == 0x5 or - buf[2] == 0x7) and - (buf[3] == 0x4 or buf[3] == 0x6 or - buf[3] == 0x8)) - - -class Tar(Type): - """ - Implements the Tar archive type matcher. - """ - MIME = 'application/x-tar' - EXTENSION = 'tar' - - def __init__(self): - super(Tar, self).__init__( - mime=Tar.MIME, - extension=Tar.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 261 and - buf[257] == 0x75 and - buf[258] == 0x73 and - buf[259] == 0x74 and - buf[260] == 0x61 and - buf[261] == 0x72) - - -class Rar(Type): - """ - Implements the RAR archive type matcher. - """ - MIME = 'application/x-rar-compressed' - EXTENSION = 'rar' - - def __init__(self): - super(Rar, self).__init__( - mime=Rar.MIME, - extension=Rar.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 6 and - buf[0] == 0x52 and - buf[1] == 0x61 and - buf[2] == 0x72 and - buf[3] == 0x21 and - buf[4] == 0x1A and - buf[5] == 0x7 and - (buf[6] == 0x0 or - buf[6] == 0x1)) - - -class Gz(Type): - """ - Implements the GZ archive type matcher. - """ - MIME = 'application/gzip' - EXTENSION = 'gz' - - def __init__(self): - super(Gz, self).__init__( - mime=Gz.MIME, - extension=Gz.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 2 and - buf[0] == 0x1F and - buf[1] == 0x8B and - buf[2] == 0x8) - - -class Bz2(Type): - """ - Implements the BZ2 archive type matcher. - """ - MIME = 'application/x-bzip2' - EXTENSION = 'bz2' - - def __init__(self): - super(Bz2, self).__init__( - mime=Bz2.MIME, - extension=Bz2.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 2 and - buf[0] == 0x42 and - buf[1] == 0x5A and - buf[2] == 0x68) - - -class SevenZ(Type): - """ - Implements the SevenZ (7z) archive type matcher. - """ - MIME = 'application/x-7z-compressed' - EXTENSION = '7z' - - def __init__(self): - super(SevenZ, self).__init__( - mime=SevenZ.MIME, - extension=SevenZ.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 5 and - buf[0] == 0x37 and - buf[1] == 0x7A and - buf[2] == 0xBC and - buf[3] == 0xAF and - buf[4] == 0x27 and - buf[5] == 0x1C) - - -class Pdf(Type): - """ - Implements the PDF archive type matcher. - """ - MIME = 'application/pdf' - EXTENSION = 'pdf' - - def __init__(self): - super(Pdf, self).__init__( - mime=Pdf.MIME, - extension=Pdf.EXTENSION - ) - - def match(self, buf): - # Detect BOM and skip first 3 bytes - if (len(buf) > 3 and - buf[0] == 0xEF and - buf[1] == 0xBB and - buf[2] == 0xBF): # noqa E129 - buf = buf[3:] - - return (len(buf) > 3 and - buf[0] == 0x25 and - buf[1] == 0x50 and - buf[2] == 0x44 and - buf[3] == 0x46) - - -class Exe(Type): - """ - Implements the EXE archive type matcher. - """ - MIME = 'application/x-msdownload' - EXTENSION = 'exe' - - def __init__(self): - super(Exe, self).__init__( - mime=Exe.MIME, - extension=Exe.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 1 and - buf[0] == 0x4D and - buf[1] == 0x5A) - - -class Swf(Type): - """ - Implements the SWF archive type matcher. - """ - MIME = 'application/x-shockwave-flash' - EXTENSION = 'swf' - - def __init__(self): - super(Swf, self).__init__( - mime=Swf.MIME, - extension=Swf.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 2 and - (buf[0] == 0x46 or - buf[0] == 0x43 or - buf[0] == 0x5A) and - buf[1] == 0x57 and - buf[2] == 0x53) - - -class Rtf(Type): - """ - Implements the RTF archive type matcher. - """ - MIME = 'application/rtf' - EXTENSION = 'rtf' - - def __init__(self): - super(Rtf, self).__init__( - mime=Rtf.MIME, - extension=Rtf.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 4 and - buf[0] == 0x7B and - buf[1] == 0x5C and - buf[2] == 0x72 and - buf[3] == 0x74 and - buf[4] == 0x66) - - -class Nes(Type): - """ - Implements the NES archive type matcher. - """ - MIME = 'application/x-nintendo-nes-rom' - EXTENSION = 'nes' - - def __init__(self): - super(Nes, self).__init__( - mime=Nes.MIME, - extension=Nes.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x4E and - buf[1] == 0x45 and - buf[2] == 0x53 and - buf[3] == 0x1A) - - -class Crx(Type): - """ - Implements the CRX archive type matcher. - """ - MIME = 'application/x-google-chrome-extension' - EXTENSION = 'crx' - - def __init__(self): - super(Crx, self).__init__( - mime=Crx.MIME, - extension=Crx.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x43 and - buf[1] == 0x72 and - buf[2] == 0x32 and - buf[3] == 0x34) - - -class Cab(Type): - """ - Implements the CAB archive type matcher. - """ - MIME = 'application/vnd.ms-cab-compressed' - EXTENSION = 'cab' - - def __init__(self): - super(Cab, self).__init__( - mime=Cab.MIME, - extension=Cab.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - ((buf[0] == 0x4D and - buf[1] == 0x53 and - buf[2] == 0x43 and - buf[3] == 0x46) or - (buf[0] == 0x49 and - buf[1] == 0x53 and - buf[2] == 0x63 and - buf[3] == 0x28))) - - -class Eot(Type): - """ - Implements the EOT archive type matcher. - """ - MIME = 'application/octet-stream' - EXTENSION = 'eot' - - def __init__(self): - super(Eot, self).__init__( - mime=Eot.MIME, - extension=Eot.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 35 and - buf[34] == 0x4C and - buf[35] == 0x50 and - ((buf[8] == 0x02 and - buf[9] == 0x00 and - buf[10] == 0x01) or - (buf[8] == 0x01 and - buf[9] == 0x00 and - buf[10] == 0x00) or - (buf[8] == 0x02 and - buf[9] == 0x00 and - buf[10] == 0x02))) - - -class Ps(Type): - """ - Implements the PS archive type matcher. - """ - MIME = 'application/postscript' - EXTENSION = 'ps' - - def __init__(self): - super(Ps, self).__init__( - mime=Ps.MIME, - extension=Ps.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 1 and - buf[0] == 0x25 and - buf[1] == 0x21) - - -class Xz(Type): - """ - Implements the XS archive type matcher. - """ - MIME = 'application/x-xz' - EXTENSION = 'xz' - - def __init__(self): - super(Xz, self).__init__( - mime=Xz.MIME, - extension=Xz.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 5 and - buf[0] == 0xFD and - buf[1] == 0x37 and - buf[2] == 0x7A and - buf[3] == 0x58 and - buf[4] == 0x5A and - buf[5] == 0x00) - - -class Sqlite(Type): - """ - Implements the Sqlite DB archive type matcher. - """ - MIME = 'application/x-sqlite3' - EXTENSION = 'sqlite' - - def __init__(self): - super(Sqlite, self).__init__( - mime=Sqlite.MIME, - extension=Sqlite.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x53 and - buf[1] == 0x51 and - buf[2] == 0x4C and - buf[3] == 0x69) - - -class Deb(Type): - """ - Implements the DEB archive type matcher. - """ - MIME = 'application/x-deb' - EXTENSION = 'deb' - - def __init__(self): - super(Deb, self).__init__( - mime=Deb.MIME, - extension=Deb.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 20 and - buf[0] == 0x21 and - buf[1] == 0x3C and - buf[2] == 0x61 and - buf[3] == 0x72 and - buf[4] == 0x63 and - buf[5] == 0x68 and - buf[6] == 0x3E and - buf[7] == 0x0A and - buf[8] == 0x64 and - buf[9] == 0x65 and - buf[10] == 0x62 and - buf[11] == 0x69 and - buf[12] == 0x61 and - buf[13] == 0x6E and - buf[14] == 0x2D and - buf[15] == 0x62 and - buf[16] == 0x69 and - buf[17] == 0x6E and - buf[18] == 0x61 and - buf[19] == 0x72 and - buf[20] == 0x79) - - -class Ar(Type): - """ - Implements the AR archive type matcher. - """ - MIME = 'application/x-unix-archive' - EXTENSION = 'ar' - - def __init__(self): - super(Ar, self).__init__( - mime=Ar.MIME, - extension=Ar.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 6 and - buf[0] == 0x21 and - buf[1] == 0x3C and - buf[2] == 0x61 and - buf[3] == 0x72 and - buf[4] == 0x63 and - buf[5] == 0x68 and - buf[6] == 0x3E) - - -class Z(Type): - """ - Implements the Z archive type matcher. - """ - MIME = 'application/x-compress' - EXTENSION = 'Z' - - def __init__(self): - super(Z, self).__init__( - mime=Z.MIME, - extension=Z.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 1 and - ((buf[0] == 0x1F and - buf[1] == 0xA0) or - (buf[0] == 0x1F and - buf[1] == 0x9D))) - - -class Lzop(Type): - """ - Implements the Lzop archive type matcher. - """ - MIME = 'application/x-lzop' - EXTENSION = 'lzo' - - def __init__(self): - super(Lzop, self).__init__( - mime=Lzop.MIME, - extension=Lzop.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 7 and - buf[0] == 0x89 and - buf[1] == 0x4C and - buf[2] == 0x5A and - buf[3] == 0x4F and - buf[4] == 0x00 and - buf[5] == 0x0D and - buf[6] == 0x0A and - buf[7] == 0x1A) - - -class Lz(Type): - """ - Implements the Lz archive type matcher. - """ - MIME = 'application/x-lzip' - EXTENSION = 'lz' - - def __init__(self): - super(Lz, self).__init__( - mime=Lz.MIME, - extension=Lz.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x4C and - buf[1] == 0x5A and - buf[2] == 0x49 and - buf[3] == 0x50) - - -class Elf(Type): - """ - Implements the Elf archive type matcher - """ - MIME = 'application/x-executable' - EXTENSION = 'elf' - - def __init__(self): - super(Elf, self).__init__( - mime=Elf.MIME, - extension=Elf.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 52 and - buf[0] == 0x7F and - buf[1] == 0x45 and - buf[2] == 0x4C and - buf[3] == 0x46) - - -class Lz4(Type): - """ - Implements the Lz4 archive type matcher. - """ - MIME = 'application/x-lz4' - EXTENSION = 'lz4' - - def __init__(self): - super(Lz4, self).__init__( - mime=Lz4.MIME, - extension=Lz4.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x04 and - buf[1] == 0x22 and - buf[2] == 0x4D and - buf[3] == 0x18) - - -class Br(Type): - """Implements the Br image type matcher.""" - - MIME = 'application/x-brotli' - EXTENSION = 'br' - - def __init__(self): - super(Br, self).__init__( - mime=Br.MIME, - extension=Br.EXTENSION - ) - - def match(self, buf): - return buf[:4] == bytearray([0xce, 0xb2, 0xcf, 0x81]) - - -class Dcm(Type): - """Implements the Dcm image type matcher.""" - - MIME = 'application/dicom' - EXTENSION = 'dcm' - - def __init__(self): - super(Dcm, self).__init__( - mime=Dcm.MIME, - extension=Dcm.EXTENSION - ) - - def match(self, buf): - return buf[128:131] == bytearray([0x44, 0x49, 0x43, 0x4d]) - - -class Rpm(Type): - """Implements the Rpm image type matcher.""" - - MIME = 'application/x-rpm' - EXTENSION = 'rpm' - - def __init__(self): - super(Rpm, self).__init__( - mime=Rpm.MIME, - extension=Rpm.EXTENSION - ) - - def match(self, buf): - return buf[:4] == bytearray([0xed, 0xab, 0xee, 0xdb]) - - -class Zstd(Type): - """ - Implements the Zstd archive type matcher. - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md - """ - MIME = 'application/zstd' - EXTENSION = 'zst' - MAGIC_SKIPPABLE_START = 0x184D2A50 - MAGIC_SKIPPABLE_MASK = 0xFFFFFFF0 - - def __init__(self): - super(Zstd, self).__init__( - mime=Zstd.MIME, - extension=Zstd.EXTENSION - ) - - @staticmethod - def _to_little_endian_int(buf): - # return int.from_bytes(buf, byteorder='little') - return struct.unpack(' 3 and - buf[0] in (0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28) and - buf[1] == 0xb5 and - buf[2] == 0x2f and - buf[3] == 0xfd) - if is_zstd: - return True - # skippable frames - if len(buf) < 8: - return False - magic = self._to_little_endian_int(buf[:4]) & Zstd.MAGIC_SKIPPABLE_MASK - if magic == Zstd.MAGIC_SKIPPABLE_START: - user_data_len = self._to_little_endian_int(buf[4:8]) - if len(buf) < 8 + user_data_len: - return False - next_frame = buf[8 + user_data_len:] - return self.match(next_frame) - return False diff --git a/libs/common/filetype/types/audio.py b/libs/common/filetype/types/audio.py deleted file mode 100644 index 429d8053..00000000 --- a/libs/common/filetype/types/audio.py +++ /dev/null @@ -1,221 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type - - -class Midi(Type): - """ - Implements the Midi audio type matcher. - """ - MIME = 'audio/midi' - EXTENSION = 'midi' - - def __init__(self): - super(Midi, self).__init__( - mime=Midi.MIME, - extension=Midi.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x4D and - buf[1] == 0x54 and - buf[2] == 0x68 and - buf[3] == 0x64) - - -class Mp3(Type): - """ - Implements the MP3 audio type matcher. - """ - MIME = 'audio/mpeg' - EXTENSION = 'mp3' - - def __init__(self): - super(Mp3, self).__init__( - mime=Mp3.MIME, - extension=Mp3.EXTENSION - ) - - def match(self, buf): - if len(buf) > 2: - if ( - buf[0] == 0x49 and - buf[1] == 0x44 and - buf[2] == 0x33 - ): - return True - - if buf[0] == 0xFF: - if ( - buf[1] == 0xE2 or # MPEG 2.5 with error protection - buf[1] == 0xE3 or # MPEG 2.5 w/o error protection - buf[1] == 0xF2 or # MPEG 2 with error protection - buf[1] == 0xF3 or # MPEG 2 w/o error protection - buf[1] == 0xFA or # MPEG 1 with error protection - buf[1] == 0xFB # MPEG 1 w/o error protection - ): - return True - return False - - -class M4a(Type): - """ - Implements the M4A audio type matcher. - """ - MIME = 'audio/mp4' - EXTENSION = 'm4a' - - def __init__(self): - super(M4a, self).__init__( - mime=M4a.MIME, - extension=M4a.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 10 and - ((buf[4] == 0x66 and - buf[5] == 0x74 and - buf[6] == 0x79 and - buf[7] == 0x70 and - buf[8] == 0x4D and - buf[9] == 0x34 and - buf[10] == 0x41) or - (buf[0] == 0x4D and - buf[1] == 0x34 and - buf[2] == 0x41 and - buf[3] == 0x20))) - - -class Ogg(Type): - """ - Implements the OGG audio type matcher. - """ - MIME = 'audio/ogg' - EXTENSION = 'ogg' - - def __init__(self): - super(Ogg, self).__init__( - mime=Ogg.MIME, - extension=Ogg.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x4F and - buf[1] == 0x67 and - buf[2] == 0x67 and - buf[3] == 0x53) - - -class Flac(Type): - """ - Implements the FLAC audio type matcher. - """ - MIME = 'audio/x-flac' - EXTENSION = 'flac' - - def __init__(self): - super(Flac, self).__init__( - mime=Flac.MIME, - extension=Flac.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x66 and - buf[1] == 0x4C and - buf[2] == 0x61 and - buf[3] == 0x43) - - -class Wav(Type): - """ - Implements the WAV audio type matcher. - """ - MIME = 'audio/x-wav' - EXTENSION = 'wav' - - def __init__(self): - super(Wav, self).__init__( - mime=Wav.MIME, - extension=Wav.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 11 and - buf[0] == 0x52 and - buf[1] == 0x49 and - buf[2] == 0x46 and - buf[3] == 0x46 and - buf[8] == 0x57 and - buf[9] == 0x41 and - buf[10] == 0x56 and - buf[11] == 0x45) - - -class Amr(Type): - """ - Implements the AMR audio type matcher. - """ - MIME = 'audio/amr' - EXTENSION = 'amr' - - def __init__(self): - super(Amr, self).__init__( - mime=Amr.MIME, - extension=Amr.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 11 and - buf[0] == 0x23 and - buf[1] == 0x21 and - buf[2] == 0x41 and - buf[3] == 0x4D and - buf[4] == 0x52 and - buf[5] == 0x0A) - - -class Aac(Type): - """Implements the Aac audio type matcher.""" - - MIME = 'audio/aac' - EXTENSION = 'aac' - - def __init__(self): - super(Aac, self).__init__( - mime=Aac.MIME, - extension=Aac.EXTENSION - ) - - def match(self, buf): - return (buf[:2] == bytearray([0xff, 0xf1]) or - buf[:2] == bytearray([0xff, 0xf9])) - - -class Aiff(Type): - """ - Implements the AIFF audio type matcher. - """ - MIME = 'audio/x-aiff' - EXTENSION = 'aiff' - - def __init__(self): - super(Aiff, self).__init__( - mime=Aiff.MIME, - extension=Aiff.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 11 and - buf[0] == 0x46 and - buf[1] == 0x4F and - buf[2] == 0x52 and - buf[3] == 0x4D and - buf[8] == 0x41 and - buf[9] == 0x49 and - buf[10] == 0x46 and - buf[11] == 0x46) diff --git a/libs/common/filetype/types/base.py b/libs/common/filetype/types/base.py deleted file mode 100644 index 7c0c0d26..00000000 --- a/libs/common/filetype/types/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- - - -class Type(object): - """ - Represents the file type object inherited by - specific file type matchers. - Provides convenient accessor and helper methods. - """ - def __init__(self, mime, extension): - self.__mime = mime - self.__extension = extension - - @property - def mime(self): - return self.__mime - - @property - def extension(self): - return self.__extension - - def is_extension(self, extension): - return self.__extension is extension - - def is_mime(self, mime): - return self.__mime is mime - - def match(self, buf): - raise NotImplementedError diff --git a/libs/common/filetype/types/document.py b/libs/common/filetype/types/document.py deleted file mode 100644 index e5236735..00000000 --- a/libs/common/filetype/types/document.py +++ /dev/null @@ -1,265 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type - - -class ZippedDocumentBase(Type): - def match(self, buf): - # start by checking for ZIP local file header signature - idx = self.search_signature(buf, 0, 6000) - if idx != 0: - return - - return self.match_document(buf) - - def match_document(self, buf): - raise NotImplementedError - - def compare_bytes(self, buf, subslice, start_offset): - sl = len(subslice) - - if start_offset + sl > len(buf): - return False - - return buf[start_offset:start_offset + sl] == subslice - - def search_signature(self, buf, start, rangeNum): - signature = b"PK\x03\x04" - length = len(buf) - - end = start + rangeNum - end = length if end > length else end - - if start >= end: - return -1 - - try: - return buf.index(signature, start, end) - except ValueError: - return -1 - - -class OpenDocument(ZippedDocumentBase): - def match_document(self, buf): - # Check if first file in archive is the identifying file - if not self.compare_bytes(buf, b"mimetype", 0x1E): - return - - # Check content of mimetype file if it matches current mime - return self.compare_bytes(buf, bytes(self.mime, "ASCII"), 0x26) - - -class OfficeOpenXml(ZippedDocumentBase): - def match_document(self, buf): - # Check if first file in archive is the identifying file - ft = self.match_filename(buf, 0x1E) - if ft: - return ft - - # Otherwise check that the fist file is one of these - if ( - not self.compare_bytes(buf, b"[Content_Types].xml", 0x1E) - and not self.compare_bytes(buf, b"_rels/.rels", 0x1E) - and not self.compare_bytes(buf, b"docProps", 0x1E) - ): - return - - # Loop through next 3 files and check if they match - # NOTE: OpenOffice/Libreoffice orders ZIP entry differently, so check the 4th file - # https://github.com/h2non/filetype/blob/d730d98ad5c990883148485b6fd5adbdd378364a/matchers/document.go#L134 - idx = 0 - for i in range(4): - # Search for next file header - idx = self.search_signature(buf, idx + 4, 6000) - if idx == -1: - return - - # Filename is at file header + 30 - ft = self.match_filename(buf, idx + 30) - if ft: - return ft - - def match_filename(self, buf, offset): - if self.compare_bytes(buf, b"word/", offset): - return ( - self.mime - == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ) - if self.compare_bytes(buf, b"ppt/", offset): - return ( - self.mime - == "application/vnd.openxmlformats-officedocument.presentationml.presentation" - ) - if self.compare_bytes(buf, b"xl/", offset): - return ( - self.mime - == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - - -class Doc(Type): - """ - Implements the Microsoft Word (Office 97-2003) document type matcher. - """ - - MIME = "application/msword" - EXTENSION = "doc" - - def __init__(self): - super(Doc, self).__init__(mime=Doc.MIME, extension=Doc.EXTENSION) - - def match(self, buf): - if len(buf) > 515 and buf[0:8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1": - if buf[512:516] == b"\xEC\xA5\xC1\x00": - return True - if ( - len(buf) > 2142 - and ( - b"\x00\x0A\x00\x00\x00MSWordDoc\x00\x10\x00\x00\x00Word.Document.8\x00\xF49\xB2q" - in buf[2075:2142] - or b"W\0o\0r\0d\0D\0o\0c\0u\0m\0e\0n\0t\0" - in buf[0x580:0x598] - ) - ): - return True - if ( - len(buf) > 663 and buf[512:531] == b"R\x00o\x00o\x00t\x00 \x00E\x00n\x00t\x00r\x00y" - and buf[640:663] == b"W\x00o\x00r\x00d\x00D\x00o\x00c\x00u\x00m\x00e\x00n\x00t" - ): - return True - - return False - - -class Docx(OfficeOpenXml): - """ - Implements the Microsoft Word OOXML (Office 2007+) document type matcher. - """ - - MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - EXTENSION = "docx" - - def __init__(self): - super(Docx, self).__init__(mime=Docx.MIME, extension=Docx.EXTENSION) - - -class Odt(OpenDocument): - """ - Implements the OpenDocument Text document type matcher. - """ - - MIME = "application/vnd.oasis.opendocument.text" - EXTENSION = "odt" - - def __init__(self): - super(Odt, self).__init__(mime=Odt.MIME, extension=Odt.EXTENSION) - - -class Xls(Type): - """ - Implements the Microsoft Excel (Office 97-2003) document type matcher. - """ - - MIME = "application/vnd.ms-excel" - EXTENSION = "xls" - - def __init__(self): - super(Xls, self).__init__(mime=Xls.MIME, extension=Xls.EXTENSION) - - def match(self, buf): - if len(buf) > 520 and buf[0:8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1": - if buf[512:516] == b"\xFD\xFF\xFF\xFF" and ( - buf[518] == 0x00 or buf[518] == 0x02 - ): - return True - if buf[512:520] == b"\x09\x08\x10\x00\x00\x06\x05\x00": - return True - if ( - len(buf) > 2095 - and b"\xE2\x00\x00\x00\x5C\x00\x70\x00\x04\x00\x00Calc" - in buf[1568:2095] - ): - return True - - return False - - -class Xlsx(OfficeOpenXml): - """ - Implements the Microsoft Excel OOXML (Office 2007+) document type matcher. - """ - - MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - EXTENSION = "xlsx" - - def __init__(self): - super(Xlsx, self).__init__(mime=Xlsx.MIME, extension=Xlsx.EXTENSION) - - -class Ods(OpenDocument): - """ - Implements the OpenDocument Spreadsheet document type matcher. - """ - - MIME = "application/vnd.oasis.opendocument.spreadsheet" - EXTENSION = "ods" - - def __init__(self): - super(Ods, self).__init__(mime=Ods.MIME, extension=Ods.EXTENSION) - - -class Ppt(Type): - """ - Implements the Microsoft PowerPoint (Office 97-2003) document type matcher. - """ - - MIME = "application/vnd.ms-powerpoint" - EXTENSION = "ppt" - - def __init__(self): - super(Ppt, self).__init__(mime=Ppt.MIME, extension=Ppt.EXTENSION) - - def match(self, buf): - if len(buf) > 524 and buf[0:8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1": - if buf[512:516] == b"\xA0\x46\x1D\xF0": - return True - if buf[512:516] == b"\x00\x6E\x1E\xF0": - return True - if buf[512:516] == b"\x0F\x00\xE8\x03": - return True - if buf[512:516] == b"\xFD\xFF\xFF\xFF" and buf[522:524] == b"\x00\x00": - return True - if ( - len(buf) > 2096 - and buf[2072:2096] - == b"\x00\xB9\x29\xE8\x11\x00\x00\x00MS PowerPoint 97" - ): - return True - - return False - - -class Pptx(OfficeOpenXml): - """ - Implements the Microsoft PowerPoint OOXML (Office 2007+) document type matcher. - """ - - MIME = "application/vnd.openxmlformats-officedocument.presentationml.presentation" - EXTENSION = "pptx" - - def __init__(self): - super(Pptx, self).__init__(mime=Pptx.MIME, extension=Pptx.EXTENSION) - - -class Odp(OpenDocument): - """ - Implements the OpenDocument Presentation document type matcher. - """ - - MIME = "application/vnd.oasis.opendocument.presentation" - EXTENSION = "odp" - - def __init__(self): - super(Odp, self).__init__(mime=Odp.MIME, extension=Odp.EXTENSION) diff --git a/libs/common/filetype/types/font.py b/libs/common/filetype/types/font.py deleted file mode 100644 index 461f5c44..00000000 --- a/libs/common/filetype/types/font.py +++ /dev/null @@ -1,115 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type - - -class Woff(Type): - """ - Implements the WOFF font type matcher. - """ - MIME = 'application/font-woff' - EXTENSION = 'woff' - - def __init__(self): - super(Woff, self).__init__( - mime=Woff.MIME, - extension=Woff.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 7 and - buf[0] == 0x77 and - buf[1] == 0x4F and - buf[2] == 0x46 and - buf[3] == 0x46 and - ((buf[4] == 0x00 and - buf[5] == 0x01 and - buf[6] == 0x00 and - buf[7] == 0x00) or - (buf[4] == 0x4F and - buf[5] == 0x54 and - buf[6] == 0x54 and - buf[7] == 0x4F) or - (buf[4] == 0x74 and - buf[5] == 0x72 and - buf[6] == 0x75 and - buf[7] == 0x65))) - - -class Woff2(Type): - """ - Implements the WOFF2 font type matcher. - """ - MIME = 'application/font-woff' - EXTENSION = 'woff2' - - def __init__(self): - super(Woff2, self).__init__( - mime=Woff2.MIME, - extension=Woff2.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 7 and - buf[0] == 0x77 and - buf[1] == 0x4F and - buf[2] == 0x46 and - buf[3] == 0x32 and - ((buf[4] == 0x00 and - buf[5] == 0x01 and - buf[6] == 0x00 and - buf[7] == 0x00) or - (buf[4] == 0x4F and - buf[5] == 0x54 and - buf[6] == 0x54 and - buf[7] == 0x4F) or - (buf[4] == 0x74 and - buf[5] == 0x72 and - buf[6] == 0x75 and - buf[7] == 0x65))) - - -class Ttf(Type): - """ - Implements the TTF font type matcher. - """ - MIME = 'application/font-sfnt' - EXTENSION = 'ttf' - - def __init__(self): - super(Ttf, self).__init__( - mime=Ttf.MIME, - extension=Ttf.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 4 and - buf[0] == 0x00 and - buf[1] == 0x01 and - buf[2] == 0x00 and - buf[3] == 0x00 and - buf[4] == 0x00) - - -class Otf(Type): - """ - Implements the OTF font type matcher. - """ - MIME = 'application/font-sfnt' - EXTENSION = 'otf' - - def __init__(self): - super(Otf, self).__init__( - mime=Otf.MIME, - extension=Otf.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 4 and - buf[0] == 0x4F and - buf[1] == 0x54 and - buf[2] == 0x54 and - buf[3] == 0x4F and - buf[4] == 0x00) diff --git a/libs/common/filetype/types/image.py b/libs/common/filetype/types/image.py deleted file mode 100644 index 783aa7b1..00000000 --- a/libs/common/filetype/types/image.py +++ /dev/null @@ -1,453 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type -from .isobmff import IsoBmff - - -class Jpeg(Type): - """ - Implements the JPEG image type matcher. - """ - MIME = 'image/jpeg' - EXTENSION = 'jpg' - - def __init__(self): - super(Jpeg, self).__init__( - mime=Jpeg.MIME, - extension=Jpeg.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 2 and - buf[0] == 0xFF and - buf[1] == 0xD8 and - buf[2] == 0xFF) - - -class Jpx(Type): - """ - Implements the JPEG2000 image type matcher. - """ - - MIME = "image/jpx" - EXTENSION = "jpx" - - def __init__(self): - super(Jpx, self).__init__(mime=Jpx.MIME, extension=Jpx.EXTENSION) - - def match(self, buf): - return ( - len(buf) > 50 - and buf[0] == 0x00 - and buf[1] == 0x00 - and buf[2] == 0x00 - and buf[3] == 0x0C - and buf[16:24] == b"ftypjp2 " - ) - - -class Jxl(Type): - """ - Implements the JPEG XL image type matcher. - """ - - MIME = "image/jxl" - EXTENSION = "jxl" - - def __init__(self): - super(Jxl, self).__init__(mime=Jxl.MIME, extension=Jxl.EXTENSION) - - def match(self, buf): - return ( - (len(buf) > 1 and - buf[0] == 0xFF and - buf[1] == 0x0A) or - (len(buf) > 11 and - buf[0] == 0x00 and - buf[1] == 0x00 and - buf[2] == 0x00 and - buf[3] == 0x00 and - buf[4] == 0x0C and - buf[5] == 0x4A and - buf[6] == 0x58 and - buf[7] == 0x4C and - buf[8] == 0x20 and - buf[9] == 0x0D and - buf[10] == 0x87 and - buf[11] == 0x0A) - ) - - -class Apng(Type): - """ - Implements the APNG image type matcher. - """ - MIME = 'image/apng' - EXTENSION = 'apng' - - def __init__(self): - super(Apng, self).__init__( - mime=Apng.MIME, - extension=Apng.EXTENSION - ) - - def match(self, buf): - if (len(buf) > 8 and - buf[:8] == bytearray([0x89, 0x50, 0x4e, 0x47, - 0x0d, 0x0a, 0x1a, 0x0a])): - # cursor in buf, skip already readed 8 bytes - i = 8 - while len(buf) > i: - data_length = int.from_bytes(buf[i:i+4], byteorder="big") - i += 4 - - chunk_type = buf[i:i+4].decode("ascii", errors='ignore') - i += 4 - - # acTL chunk in APNG must appear before IDAT - # IEND is end of PNG - if (chunk_type == "IDAT" or chunk_type == "IEND"): - return False - if (chunk_type == "acTL"): - return True - - # move to the next chunk by skipping data and crc (4 bytes) - i += data_length + 4 - - return False - - -class Png(Type): - """ - Implements the PNG image type matcher. - """ - MIME = 'image/png' - EXTENSION = 'png' - - def __init__(self): - super(Png, self).__init__( - mime=Png.MIME, - extension=Png.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x89 and - buf[1] == 0x50 and - buf[2] == 0x4E and - buf[3] == 0x47) - - -class Gif(Type): - """ - Implements the GIF image type matcher. - """ - MIME = 'image/gif' - EXTENSION = 'gif' - - def __init__(self): - super(Gif, self).__init__( - mime=Gif.MIME, - extension=Gif.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 2 and - buf[0] == 0x47 and - buf[1] == 0x49 and - buf[2] == 0x46) - - -class Webp(Type): - """ - Implements the WEBP image type matcher. - """ - MIME = 'image/webp' - EXTENSION = 'webp' - - def __init__(self): - super(Webp, self).__init__( - mime=Webp.MIME, - extension=Webp.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 13 and - buf[0] == 0x52 and - buf[1] == 0x49 and - buf[2] == 0x46 and - buf[3] == 0x46 and - buf[8] == 0x57 and - buf[9] == 0x45 and - buf[10] == 0x42 and - buf[11] == 0x50 and - buf[12] == 0x56 and - buf[13] == 0x50) - - -class Cr2(Type): - """ - Implements the CR2 image type matcher. - """ - MIME = 'image/x-canon-cr2' - EXTENSION = 'cr2' - - def __init__(self): - super(Cr2, self).__init__( - mime=Cr2.MIME, - extension=Cr2.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 9 and - ((buf[0] == 0x49 and buf[1] == 0x49 and - buf[2] == 0x2A and buf[3] == 0x0) or - (buf[0] == 0x4D and buf[1] == 0x4D and - buf[2] == 0x0 and buf[3] == 0x2A)) and - buf[8] == 0x43 and buf[9] == 0x52) - - -class Tiff(Type): - """ - Implements the TIFF image type matcher. - """ - MIME = 'image/tiff' - EXTENSION = 'tif' - - def __init__(self): - super(Tiff, self).__init__( - mime=Tiff.MIME, - extension=Tiff.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 9 and - ((buf[0] == 0x49 and buf[1] == 0x49 and - buf[2] == 0x2A and buf[3] == 0x0) or - (buf[0] == 0x4D and buf[1] == 0x4D and - buf[2] == 0x0 and buf[3] == 0x2A)) - and not (buf[8] == 0x43 and buf[9] == 0x52)) - - -class Bmp(Type): - """ - Implements the BMP image type matcher. - """ - MIME = 'image/bmp' - EXTENSION = 'bmp' - - def __init__(self): - super(Bmp, self).__init__( - mime=Bmp.MIME, - extension=Bmp.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 1 and - buf[0] == 0x42 and - buf[1] == 0x4D) - - -class Jxr(Type): - """ - Implements the JXR image type matcher. - """ - MIME = 'image/vnd.ms-photo' - EXTENSION = 'jxr' - - def __init__(self): - super(Jxr, self).__init__( - mime=Jxr.MIME, - extension=Jxr.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 2 and - buf[0] == 0x49 and - buf[1] == 0x49 and - buf[2] == 0xBC) - - -class Psd(Type): - """ - Implements the PSD image type matcher. - """ - MIME = 'image/vnd.adobe.photoshop' - EXTENSION = 'psd' - - def __init__(self): - super(Psd, self).__init__( - mime=Psd.MIME, - extension=Psd.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x38 and - buf[1] == 0x42 and - buf[2] == 0x50 and - buf[3] == 0x53) - - -class Ico(Type): - """ - Implements the ICO image type matcher. - """ - MIME = 'image/x-icon' - EXTENSION = 'ico' - - def __init__(self): - super(Ico, self).__init__( - mime=Ico.MIME, - extension=Ico.EXTENSION, - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x00 and - buf[1] == 0x00 and - buf[2] == 0x01 and - buf[3] == 0x00) - - -class Heic(IsoBmff): - """ - Implements the HEIC image type matcher. - """ - MIME = 'image/heic' - EXTENSION = 'heic' - - def __init__(self): - super(Heic, self).__init__( - mime=Heic.MIME, - extension=Heic.EXTENSION - ) - - def match(self, buf): - if not self._is_isobmff(buf): - return False - - major_brand, minor_version, compatible_brands = self._get_ftyp(buf) - if major_brand == 'heic': - return True - if major_brand in ['mif1', 'msf1'] and 'heic' in compatible_brands: - return True - return False - - -class Dcm(Type): - - MIME = 'application/dicom' - EXTENSION = 'dcm' - OFFSET = 128 - - def __init__(self): - super(Dcm, self).__init__( - mime=Dcm.MIME, - extension=Dcm.EXTENSION - ) - - def match(self, buf): - return (len(buf) > Dcm.OFFSET + 4 and - buf[Dcm.OFFSET + 0] == 0x44 and - buf[Dcm.OFFSET + 1] == 0x49 and - buf[Dcm.OFFSET + 2] == 0x43 and - buf[Dcm.OFFSET + 3] == 0x4D) - - -class Dwg(Type): - """Implements the Dwg image type matcher.""" - - MIME = 'image/vnd.dwg' - EXTENSION = 'dwg' - - def __init__(self): - super(Dwg, self).__init__( - mime=Dwg.MIME, - extension=Dwg.EXTENSION - ) - - def match(self, buf): - return buf[:4] == bytearray([0x41, 0x43, 0x31, 0x30]) - - -class Xcf(Type): - """Implements the Xcf image type matcher.""" - - MIME = 'image/x-xcf' - EXTENSION = 'xcf' - - def __init__(self): - super(Xcf, self).__init__( - mime=Xcf.MIME, - extension=Xcf.EXTENSION - ) - - def match(self, buf): - return buf[:10] == bytearray([0x67, 0x69, 0x6d, 0x70, 0x20, - 0x78, 0x63, 0x66, 0x20, 0x76]) - - -class Avif(IsoBmff): - """ - Implements the AVIF image type matcher. - """ - MIME = 'image/avif' - EXTENSION = 'avif' - - def __init__(self): - super(Avif, self).__init__( - mime=Avif.MIME, - extension=Avif.EXTENSION - ) - - def match(self, buf): - if not self._is_isobmff(buf): - return False - - major_brand, minor_version, compatible_brands = self._get_ftyp(buf) - if major_brand in ['avif', 'avis']: - return True - if major_brand in ['mif1', 'msf1'] and 'avif' in compatible_brands: - return True - return False - - -class Qoi(Type): - """ - Implements the QOI image type matcher. - """ - MIME = 'image/qoi' - EXTENSION = 'qoi' - - def __init__(self): - super(Qoi, self).__init__( - mime=Qoi.MIME, - extension=Qoi.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x71 and - buf[1] == 0x6F and - buf[2] == 0x69 and - buf[3] == 0x66) - - -class Dds(Type): - """ - Implements the DDS image type matcher. - """ - MIME = 'image/dds' - EXTENSION = 'dds' - - def __init__(self): - super(Dds, self).__init__( - mime=Dds.MIME, - extension=Dds.EXTENSION - ) - - def match(self, buf): - return buf.startswith(b'\x44\x44\x53\x20') diff --git a/libs/common/filetype/types/isobmff.py b/libs/common/filetype/types/isobmff.py deleted file mode 100644 index 2ac0ffe8..00000000 --- a/libs/common/filetype/types/isobmff.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -import codecs - -from .base import Type - - -class IsoBmff(Type): - """ - Implements the ISO-BMFF base type. - """ - def __init__(self, mime, extension): - super(IsoBmff, self).__init__( - mime=mime, - extension=extension - ) - - def _is_isobmff(self, buf): - if len(buf) < 16 or buf[4:8] != b'ftyp': - return False - if len(buf) < int(codecs.encode(buf[0:4], 'hex'), 16): - return False - return True - - def _get_ftyp(self, buf): - ftyp_len = int(codecs.encode(buf[0:4], 'hex'), 16) - major_brand = buf[8:12].decode(errors='ignore') - minor_version = int(codecs.encode(buf[12:16], 'hex'), 16) - compatible_brands = [] - for i in range(16, ftyp_len, 4): - compatible_brands.append(buf[i:i+4].decode(errors='ignore')) - - return major_brand, minor_version, compatible_brands diff --git a/libs/common/filetype/types/video.py b/libs/common/filetype/types/video.py deleted file mode 100644 index 12672eda..00000000 --- a/libs/common/filetype/types/video.py +++ /dev/null @@ -1,230 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .base import Type -from .isobmff import IsoBmff - - -class Mp4(IsoBmff): - """ - Implements the MP4 video type matcher. - """ - MIME = 'video/mp4' - EXTENSION = 'mp4' - - def __init__(self): - super(Mp4, self).__init__( - mime=Mp4.MIME, - extension=Mp4.EXTENSION - ) - - def match(self, buf): - if not self._is_isobmff(buf): - return False - - major_brand, minor_version, compatible_brands = self._get_ftyp(buf) - for brand in compatible_brands: - if brand in ['mp41', 'mp42', 'isom']: - return True - return major_brand in ['mp41', 'mp42', 'isom'] - - -class M4v(Type): - """ - Implements the M4V video type matcher. - """ - MIME = 'video/x-m4v' - EXTENSION = 'm4v' - - def __init__(self): - super(M4v, self).__init__( - mime=M4v.MIME, - extension=M4v.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 10 and - buf[0] == 0x0 and buf[1] == 0x0 and - buf[2] == 0x0 and buf[3] == 0x1C and - buf[4] == 0x66 and buf[5] == 0x74 and - buf[6] == 0x79 and buf[7] == 0x70 and - buf[8] == 0x4D and buf[9] == 0x34 and - buf[10] == 0x56) - - -class Mkv(Type): - """ - Implements the MKV video type matcher. - """ - MIME = 'video/x-matroska' - EXTENSION = 'mkv' - - def __init__(self): - super(Mkv, self).__init__( - mime=Mkv.MIME, - extension=Mkv.EXTENSION - ) - - def match(self, buf): - contains_ebml_element = buf.startswith(b'\x1A\x45\xDF\xA3') - contains_doctype_element = buf.find(b'\x42\x82\x88matroska') > -1 - return contains_ebml_element and contains_doctype_element - - -class Webm(Type): - """ - Implements the WebM video type matcher. - """ - MIME = 'video/webm' - EXTENSION = 'webm' - - def __init__(self): - super(Webm, self).__init__( - mime=Webm.MIME, - extension=Webm.EXTENSION - ) - - def match(self, buf): - contains_ebml_element = buf.startswith(b'\x1A\x45\xDF\xA3') - contains_doctype_element = buf.find(b'\x42\x82\x84webm') > -1 - return contains_ebml_element and contains_doctype_element - - -class Mov(IsoBmff): - """ - Implements the MOV video type matcher. - """ - MIME = 'video/quicktime' - EXTENSION = 'mov' - - def __init__(self): - super(Mov, self).__init__( - mime=Mov.MIME, - extension=Mov.EXTENSION - ) - - def match(self, buf): - if not self._is_isobmff(buf): - return False - - major_brand, minor_version, compatible_brands = self._get_ftyp(buf) - return major_brand == 'qt ' - - -class Avi(Type): - """ - Implements the AVI video type matcher. - """ - MIME = 'video/x-msvideo' - EXTENSION = 'avi' - - def __init__(self): - super(Avi, self).__init__( - mime=Avi.MIME, - extension=Avi.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 11 and - buf[0] == 0x52 and - buf[1] == 0x49 and - buf[2] == 0x46 and - buf[3] == 0x46 and - buf[8] == 0x41 and - buf[9] == 0x56 and - buf[10] == 0x49 and - buf[11] == 0x20) - - -class Wmv(Type): - """ - Implements the WMV video type matcher. - """ - MIME = 'video/x-ms-wmv' - EXTENSION = 'wmv' - - def __init__(self): - super(Wmv, self).__init__( - mime=Wmv.MIME, - extension=Wmv.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 9 and - buf[0] == 0x30 and - buf[1] == 0x26 and - buf[2] == 0xB2 and - buf[3] == 0x75 and - buf[4] == 0x8E and - buf[5] == 0x66 and - buf[6] == 0xCF and - buf[7] == 0x11 and - buf[8] == 0xA6 and - buf[9] == 0xD9) - - -class Flv(Type): - """ - Implements the FLV video type matcher. - """ - MIME = 'video/x-flv' - EXTENSION = 'flv' - - def __init__(self): - super(Flv, self).__init__( - mime=Flv.MIME, - extension=Flv.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x46 and - buf[1] == 0x4C and - buf[2] == 0x56 and - buf[3] == 0x01) - - -class Mpeg(Type): - """ - Implements the MPEG video type matcher. - """ - MIME = 'video/mpeg' - EXTENSION = 'mpg' - - def __init__(self): - super(Mpeg, self).__init__( - mime=Mpeg.MIME, - extension=Mpeg.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 3 and - buf[0] == 0x0 and - buf[1] == 0x0 and - buf[2] == 0x1 and - buf[3] >= 0xb0 and - buf[3] <= 0xbf) - - -class M3gp(Type): - """Implements the 3gp video type matcher.""" - - MIME = 'video/3gpp' - EXTENSION = '3gp' - - def __init__(self): - super(M3gp, self).__init__( - mime=M3gp.MIME, - extension=M3gp.EXTENSION - ) - - def match(self, buf): - return (len(buf) > 10 and - buf[4] == 0x66 and - buf[5] == 0x74 and - buf[6] == 0x79 and - buf[7] == 0x70 and - buf[8] == 0x33 and - buf[9] == 0x67 and - buf[10] == 0x70) diff --git a/libs/common/filetype/utils.py b/libs/common/filetype/utils.py deleted file mode 100644 index c954876a..00000000 --- a/libs/common/filetype/utils.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding: utf-8 -*- - -# Python 2.7 workaround -try: - import pathlib -except ImportError: - pass - - -_NUM_SIGNATURE_BYTES = 8192 - - -def get_signature_bytes(path): - """ - Reads file from disk and returns the first 8192 bytes - of data representing the magic number header signature. - - Args: - path: path string to file. - - Returns: - First 8192 bytes of the file content as bytearray type. - """ - with open(path, 'rb') as fp: - return bytearray(fp.read(_NUM_SIGNATURE_BYTES)) - - -def signature(array): - """ - Returns the first 8192 bytes of the given bytearray - as part of the file header signature. - - Args: - array: bytearray to extract the header signature. - - Returns: - First 8192 bytes of the file content as bytearray type. - """ - length = len(array) - index = _NUM_SIGNATURE_BYTES if length > _NUM_SIGNATURE_BYTES else length - - return array[:index] - - -def get_bytes(obj): - """ - Infers the input type and reads the first 8192 bytes, - returning a sliced bytearray. - - Args: - obj: path to readable, file-like object(with read() method), bytes, - bytearray or memoryview - - Returns: - First 8192 bytes of the file content as bytearray type. - - Raises: - TypeError: if obj is not a supported type. - """ - if isinstance(obj, bytearray): - return signature(obj) - - if isinstance(obj, str): - return get_signature_bytes(obj) - - if isinstance(obj, bytes): - return signature(obj) - - if isinstance(obj, memoryview): - return bytearray(signature(obj).tolist()) - - if isinstance(obj, pathlib.PurePath): - return get_signature_bytes(obj) - - if hasattr(obj, 'read'): - if hasattr(obj, 'tell') and hasattr(obj, 'seek'): - start_pos = obj.tell() - obj.seek(0) - magic_bytes = obj.read(_NUM_SIGNATURE_BYTES) - obj.seek(start_pos) - return get_bytes(magic_bytes) - return get_bytes(obj.read(_NUM_SIGNATURE_BYTES)) - - raise TypeError('Unsupported type as file input: %s' % type(obj)) diff --git a/libs/common/mediafile.py b/libs/common/mediafile.py index 5600c496..ca70c944 100644 --- a/libs/common/mediafile.py +++ b/libs/common/mediafile.py @@ -33,6 +33,8 @@ Internally ``MediaFile`` uses ``MediaField`` descriptors to access the data from the tags. In turn ``MediaField`` uses a number of ``StorageStyle`` strategies to handle format specific logic. """ +from __future__ import division, absolute_import, print_function + import mutagen import mutagen.id3 import mutagen.mp3 @@ -46,17 +48,18 @@ import binascii import codecs import datetime import enum -import filetype import functools +import imghdr import logging import math import os import re +import six import struct import traceback -__version__ = '0.13.0' +__version__ = '0.10.1' __all__ = ['UnreadableFileError', 'FileTypeError', 'MediaFile'] log = logging.getLogger(__name__) @@ -78,6 +81,8 @@ TYPES = { 'wav': 'WAVE', } +PREFERRED_IMAGE_EXTENSIONS = {'jpeg': 'jpg'} + # Exceptions. @@ -131,8 +136,8 @@ def mutagen_call(action, filename, func, *args, **kwargs): try: return func(*args, **kwargs) except mutagen.MutagenError as exc: - log.debug(u'%s failed: %s', action, str(exc)) - raise UnreadableFileError(filename, str(exc)) + log.debug(u'%s failed: %s', action, six.text_type(exc)) + raise UnreadableFileError(filename, six.text_type(exc)) except UnreadableFileError: # Reraise our errors without changes. # Used in case of decorating functions (e.g. by `loadfile`). @@ -197,8 +202,8 @@ def _safe_cast(out_type, val): # Process any other type as a string. if isinstance(val, bytes): val = val.decode('utf-8', 'ignore') - elif not isinstance(val, str): - val = str(val) + elif not isinstance(val, six.string_types): + val = six.text_type(val) # Get a number from the front of the string. match = re.match(r'[\+-]?[0-9]+', val.strip()) return int(match.group(0)) if match else 0 @@ -210,13 +215,13 @@ def _safe_cast(out_type, val): except ValueError: return False - elif out_type == str: + elif out_type == six.text_type: if isinstance(val, bytes): return val.decode('utf-8', 'ignore') - elif isinstance(val, str): + elif isinstance(val, six.text_type): return val else: - return str(val) + return six.text_type(val) elif out_type == float: if isinstance(val, int) or isinstance(val, float): @@ -225,7 +230,7 @@ def _safe_cast(out_type, val): if isinstance(val, bytes): val = val.decode('utf-8', 'ignore') else: - val = str(val) + val = six.text_type(val) match = re.match(r'[\+-]?([0-9]+\.?[0-9]*|[0-9]*\.[0-9]+)', val.strip()) if match: @@ -284,7 +289,7 @@ def _sc_decode(soundcheck): """ # We decode binary data. If one of the formats gives us a text # string, interpret it as UTF-8. - if isinstance(soundcheck, str): + if isinstance(soundcheck, six.text_type): soundcheck = soundcheck.encode('utf-8') # SoundCheck tags consist of 10 numbers, each represented by 8 @@ -344,15 +349,52 @@ def _sc_encode(gain, peak): # Cover art and other images. +def _imghdr_what_wrapper(data): + """A wrapper around imghdr.what to account for jpeg files that can only be + identified as such using their magic bytes + See #1545 + See https://github.com/file/file/blob/master/magic/Magdir/jpeg#L12 + """ + # imghdr.what returns none for jpegs with only the magic bytes, so + # _wider_test_jpeg is run in that case. It still returns None if it didn't + # match such a jpeg file. + return imghdr.what(None, h=data) or _wider_test_jpeg(data) + + +def _wider_test_jpeg(data): + """Test for a jpeg file following the UNIX file implementation which + uses the magic bytes rather than just looking for the bytes that + represent 'JFIF' or 'EXIF' at a fixed position. + """ + if data[:2] == b'\xff\xd8': + return 'jpeg' + def image_mime_type(data): """Return the MIME type of the image data (a bytestring). """ - return filetype.guess_mime(data) + # This checks for a jpeg file with only the magic bytes (unrecognized by + # imghdr.what). imghdr.what returns none for that type of file, so + # _wider_test_jpeg is run in that case. It still returns None if it didn't + # match such a jpeg file. + kind = _imghdr_what_wrapper(data) + if kind in ['gif', 'jpeg', 'png', 'tiff', 'bmp']: + return 'image/{0}'.format(kind) + elif kind == 'pgm': + return 'image/x-portable-graymap' + elif kind == 'pbm': + return 'image/x-portable-bitmap' + elif kind == 'ppm': + return 'image/x-portable-pixmap' + elif kind == 'xbm': + return 'image/x-xbitmap' + else: + return 'image/x-{0}'.format(kind) def image_extension(data): - return filetype.guess_extension(data) + ext = _imghdr_what_wrapper(data) + return PREFERRED_IMAGE_EXTENSIONS.get(ext, ext) class ImageType(enum.Enum): @@ -395,7 +437,7 @@ class Image(object): def __init__(self, data, desc=None, type=None): assert isinstance(data, bytes) if desc is not None: - assert isinstance(desc, str) + assert isinstance(desc, six.text_type) self.data = data self.desc = desc if isinstance(type, int): @@ -453,7 +495,7 @@ class StorageStyle(object): """List of mutagen classes the StorageStyle can handle. """ - def __init__(self, key, as_type=str, suffix=None, + def __init__(self, key, as_type=six.text_type, suffix=None, float_places=2, read_only=False): """Create a basic storage strategy. Parameters: @@ -478,8 +520,8 @@ class StorageStyle(object): self.read_only = read_only # Convert suffix to correct string type. - if self.suffix and self.as_type is str \ - and not isinstance(self.suffix, str): + if self.suffix and self.as_type is six.text_type \ + and not isinstance(self.suffix, six.text_type): self.suffix = self.suffix.decode('utf-8') # Getter. @@ -502,7 +544,7 @@ class StorageStyle(object): """Given a raw value stored on a Mutagen object, decode and return the represented value. """ - if self.suffix and isinstance(mutagen_value, str) \ + if self.suffix and isinstance(mutagen_value, six.text_type) \ and mutagen_value.endswith(self.suffix): return mutagen_value[:-len(self.suffix)] else: @@ -524,17 +566,17 @@ class StorageStyle(object): """Convert the external Python value to a type that is suitable for storing in a Mutagen file object. """ - if isinstance(value, float) and self.as_type is str: + if isinstance(value, float) and self.as_type is six.text_type: value = u'{0:.{1}f}'.format(value, self.float_places) value = self.as_type(value) - elif self.as_type is str: + elif self.as_type is six.text_type: if isinstance(value, bool): # Store bools as 1/0 instead of True/False. - value = str(int(bool(value))) + value = six.text_type(int(bool(value))) elif isinstance(value, bytes): value = value.decode('utf-8', 'ignore') else: - value = str(value) + value = six.text_type(value) else: value = self.as_type(value) @@ -558,8 +600,8 @@ class ListStorageStyle(StorageStyle): object to each. Subclasses may overwrite ``fetch`` and ``store``. ``fetch`` must - return a (possibly empty) list or `None` if the tag does not exist. - ``store`` receives a serialized list of values as the second argument. + return a (possibly empty) list and ``store`` receives a serialized + list of values as the second argument. The `serialize` and `deserialize` methods (from the base `StorageStyle`) are still called with individual values. This class @@ -568,23 +610,15 @@ class ListStorageStyle(StorageStyle): def get(self, mutagen_file): """Get the first value in the field's value list. """ - values = self.get_list(mutagen_file) - if values is None: - return None - try: - return values[0] + return self.get_list(mutagen_file)[0] except IndexError: return None def get_list(self, mutagen_file): """Get a list of all values for the field using this style. """ - raw_values = self.fetch(mutagen_file) - if raw_values is None: - return None - - return [self.deserialize(item) for item in raw_values] + return [self.deserialize(item) for item in self.fetch(mutagen_file)] def fetch(self, mutagen_file): """Get the list of raw (serialized) values. @@ -592,27 +626,19 @@ class ListStorageStyle(StorageStyle): try: return mutagen_file[self.key] except KeyError: - return None + return [] def set(self, mutagen_file, value): """Set an individual value as the only value for the field using this style. """ - if value is None: - self.store(mutagen_file, None) - else: - self.set_list(mutagen_file, [value]) + self.set_list(mutagen_file, [value]) def set_list(self, mutagen_file, values): """Set all values for the field using this style. `values` should be an iterable. """ - if values is None: - self.delete(mutagen_file) - else: - self.store( - mutagen_file, [self.serialize(value) for value in values] - ) + self.store(mutagen_file, [self.serialize(value) for value in values]) def store(self, mutagen_file, values): """Set the list of all raw (serialized) values for this field. @@ -660,7 +686,7 @@ class MP4StorageStyle(StorageStyle): def serialize(self, value): value = super(MP4StorageStyle, self).serialize(value) - if self.key.startswith('----:') and isinstance(value, str): + if self.key.startswith('----:') and isinstance(value, six.text_type): value = value.encode('utf-8') return value @@ -839,7 +865,7 @@ class MP3UFIDStorageStyle(MP3StorageStyle): def store(self, mutagen_file, value): # This field type stores text data as encoded data. - assert isinstance(value, str) + assert isinstance(value, six.text_type) value = value.encode('utf-8') frames = mutagen_file.tags.getall(self.key) @@ -863,7 +889,7 @@ class MP3DescStorageStyle(MP3StorageStyle): """ def __init__(self, desc=u'', key='TXXX', attr='text', multispec=True, **kwargs): - assert isinstance(desc, str) + assert isinstance(desc, six.text_type) self.description = desc self.attr = attr self.multispec = multispec @@ -952,7 +978,7 @@ class MP3SlashPackStorageStyle(MP3StorageStyle): def _fetch_unpacked(self, mutagen_file): data = self.fetch(mutagen_file) if data: - items = str(data).split('/') + items = six.text_type(data).split('/') else: items = [] packing_length = 2 @@ -968,7 +994,7 @@ class MP3SlashPackStorageStyle(MP3StorageStyle): items[0] = '' if items[1] is None: items.pop() # Do not store last value - self.store(mutagen_file, '/'.join(map(str, items))) + self.store(mutagen_file, '/'.join(map(six.text_type, items))) def delete(self, mutagen_file): if self.pack_pos == 0: @@ -1235,7 +1261,7 @@ class MediaField(object): getting this property. """ - self.out_type = kwargs.get('out_type', str) + self.out_type = kwargs.get('out_type', six.text_type) self._styles = styles def styles(self, mutagen_file): @@ -1275,7 +1301,7 @@ class MediaField(object): return 0.0 elif self.out_type == bool: return False - elif self.out_type == str: + elif self.out_type == six.text_type: return u'' @@ -1291,7 +1317,7 @@ class ListMediaField(MediaField): values = style.get_list(mediafile.mgfile) if values: return [_safe_cast(self.out_type, value) for value in values] - return None + return [] def __set__(self, mediafile, values): for style in self.styles(mediafile.mgfile): @@ -1358,9 +1384,9 @@ class DateField(MediaField): """ # Get the underlying data and split on hyphens and slashes. datestring = super(DateField, self).__get__(mediafile, None) - if isinstance(datestring, str): - datestring = re.sub(r'[Tt ].*$', '', str(datestring)) - items = re.split('[-/]', str(datestring)) + if isinstance(datestring, six.string_types): + datestring = re.sub(r'[Tt ].*$', '', six.text_type(datestring)) + items = re.split('[-/]', six.text_type(datestring)) else: items = [] @@ -1397,7 +1423,7 @@ class DateField(MediaField): date.append(u'{0:02d}'.format(int(month))) if month and day: date.append(u'{0:02d}'.format(int(day))) - date = map(str, date) + date = map(six.text_type, date) super(DateField, self).__set__(mediafile, u'-'.join(date)) if hasattr(self, '_year_field'): @@ -2045,7 +2071,6 @@ class MediaFile(object): original_date = DateField( MP3StorageStyle('TDOR'), MP4StorageStyle('----:com.apple.iTunes:ORIGINAL YEAR'), - MP4StorageStyle('----:com.apple.iTunes:ORIGINALDATE'), StorageStyle('ORIGINALDATE'), ASFStorageStyle('WM/OriginalReleaseYear')) @@ -2060,36 +2085,12 @@ class MediaFile(object): StorageStyle('ARTIST_CREDIT'), ASFStorageStyle('beets/Artist Credit'), ) - artists_credit = ListMediaField( - MP3ListDescStorageStyle(desc=u'ARTISTS_CREDIT'), - MP4ListStorageStyle('----:com.apple.iTunes:ARTISTS_CREDIT'), - ListStorageStyle('ARTISTS_CREDIT'), - ASFStorageStyle('beets/ArtistsCredit'), - ) - artists_sort = ListMediaField( - MP3ListDescStorageStyle(desc=u'ARTISTS_SORT'), - MP4ListStorageStyle('----:com.apple.iTunes:ARTISTS_SORT'), - ListStorageStyle('ARTISTS_SORT'), - ASFStorageStyle('beets/ArtistsSort'), - ) albumartist_credit = MediaField( MP3DescStorageStyle(u'Album Artist Credit'), MP4StorageStyle('----:com.apple.iTunes:Album Artist Credit'), StorageStyle('ALBUMARTIST_CREDIT'), ASFStorageStyle('beets/Album Artist Credit'), ) - albumartists_credit = ListMediaField( - MP3ListDescStorageStyle(desc=u'ALBUMARTISTS_CREDIT'), - MP4ListStorageStyle('----:com.apple.iTunes:ALBUMARTISTS_CREDIT'), - ListStorageStyle('ALBUMARTISTS_CREDIT'), - ASFStorageStyle('beets/AlbumArtistsCredit'), - ) - albumartists_sort = ListMediaField( - MP3ListDescStorageStyle(desc=u'ALBUMARTISTS_SORT'), - MP4ListStorageStyle('----:com.apple.iTunes:ALBUMARTISTS_SORT'), - ListStorageStyle('ALBUMARTISTS_SORT'), - ASFStorageStyle('beets/AlbumArtistsSort'), - ) # Legacy album art field art = CoverArtField() diff --git a/libs/common/qbittorrent/client.py b/libs/common/qbittorrent/client.py index ecd4c1bd..9c734c2f 100644 --- a/libs/common/qbittorrent/client.py +++ b/libs/common/qbittorrent/client.py @@ -442,23 +442,13 @@ class Client(object): :param infohash: INFO HASH of torrent. """ - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/stop', data={'hashes': infohash.lower()}) - else: - return self._post('torrents/pause', data={'hashes': infohash.lower()}) + return self._post('torrents/pause', data={'hashes': infohash.lower()}) def pause_all(self): """ Pause all torrents. """ - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/stop', data={'hashes': infohash.lower()}) - else: - return self._post('torrents/pause', data={'hashes': infohash.lower()}) + return self._post('torrents/pause', data={'hashes': 'all'}) def pause_multiple(self, infohash_list): """ @@ -467,12 +457,7 @@ class Client(object): :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/stop', data=data) - else: - return self._post('torrents/pause', data=data) + return self._post('torrents/pause', data=data) def set_category(self, infohash_list, category): """ @@ -512,23 +497,13 @@ class Client(object): :param infohash: INFO HASH of torrent. """ - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/start', data={'hashes': infohash.lower()}) - else: - return self._post('torrents/resume', data={'hashes': infohash.lower()}) + return self._post('torrents/resume', data={'hashes': infohash.lower()}) def resume_all(self): """ Resume all torrents. """ - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/start', data={'hashes': 'all'}) - else: - return self._post('torrents/resume', data={'hashes': 'all'}) + return self._post('torrents/resume', data={'hashes': 'all'}) def resume_multiple(self, infohash_list): """ @@ -537,12 +512,7 @@ class Client(object): :param infohash_list: Single or list() of infohashes. """ data = self._process_infohash_list(infohash_list) - app_ver = self.qbittorrent_version - maj_ver = int(app_ver.replace('v','').split('.')[0]) - if maj_ver >= 5: - return self._post('torrents/start', data=data) - else: - return self._post('torrents/resume', data=data) + return self._post('torrents/resume', data=data) def delete(self, infohash_list): """ diff --git a/tox.ini b/tox.ini index c7438177..e5fd86e7 100644 --- a/tox.ini +++ b/tox.ini @@ -4,11 +4,12 @@ envlist = clean, check, - {py39, py310, py311, py312, py313}, + {py38, py39, py310, py311, py312, py313}, report [testenv] basepython = + py38: {env:TOXPYTHON:python3.8} py39: {env:TOXPYTHON:python3.9} py310: {env:TOXPYTHON:python3.10} py311: {env:TOXPYTHON:python3.11}