Move common libs to libs/common

libs/common/subliminal/providers/__init__.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+import logging
+
+from bs4 import BeautifulSoup, FeatureNotFound
+from six.moves.xmlrpc_client import SafeTransport
+
+from ..video import Episode, Movie
+
+logger = logging.getLogger(__name__)
+
+
+class TimeoutSafeTransport(SafeTransport):
+    """Timeout support for ``xmlrpc.client.SafeTransport``."""
+    def __init__(self, timeout, *args, **kwargs):
+        SafeTransport.__init__(self, *args, **kwargs)
+        self.timeout = timeout
+
+    def make_connection(self, host):
+        c = SafeTransport.make_connection(self, host)
+        c.timeout = self.timeout
+
+        return c
+
+
+class ParserBeautifulSoup(BeautifulSoup):
+    """A ``bs4.BeautifulSoup`` that picks the first parser available in `parsers`.
+
+    :param markup: markup for the ``bs4.BeautifulSoup``.
+    :param list parsers: parser names, in order of preference.
+
+    """
+    def __init__(self, markup, parsers, **kwargs):
+        # reject features
+        if set(parsers).intersection({'fast', 'permissive', 'strict', 'xml', 'html', 'html5'}):
+            raise ValueError('Features not allowed, only parser names')
+
+        # reject some kwargs
+        if 'features' in kwargs:
+            raise ValueError('Cannot use features kwarg')
+        if 'builder' in kwargs:
+            raise ValueError('Cannot use builder kwarg')
+
+        # pick the first parser available
+        for parser in parsers:
+            try:
+                super(ParserBeautifulSoup, self).__init__(markup, parser, **kwargs)
+                return
+            except FeatureNotFound:
+                pass
+
+        raise FeatureNotFound
+
+
+class Provider(object):
+    """Base class for providers.
+
+    If any configuration is possible for the provider, like credentials, it must take place during instantiation.
+
+    :raise: :class:`~subliminal.exceptions.ConfigurationError` if there is a configuration error
+
+    """
+    #: Supported set of :class:`~babelfish.language.Language`
+    languages = set()
+
+    #: Supported video types
+    video_types = (Episode, Movie)
+
+    #: Required hash, if any
+    required_hash = None
+
+    def __enter__(self):
+        self.initialize()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.terminate()
+
+    def initialize(self):
+        """Initialize the provider.
+
+        Must be called when starting to work with the provider. This is the place for network initialization
+        or login operations.
+
+        .. note::
+            This is called automatically when entering the `with` statement
+
+        """
+        raise NotImplementedError
+
+    def terminate(self):
+        """Terminate the provider.
+
+        Must be called when done with the provider. This is the place for network shutdown or logout operations.
+
+        .. note::
+            This is called automatically when exiting the `with` statement
+
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def check(cls, video):
+        """Check if the `video` can be processed.
+
+        The `video` is considered invalid if not an instance of :attr:`video_types` or if the :attr:`required_hash` is
+        not present in :attr:`~subliminal.video.Video.hashes` attribute of the `video`.
+
+        :param video: the video to check.
+        :type video: :class:`~subliminal.video.Video`
+        :return: `True` if the `video` is valid, `False` otherwise.
+        :rtype: bool
+
+        """
+        if not isinstance(video, cls.video_types):
+            return False
+        if cls.required_hash is not None and cls.required_hash not in video.hashes:
+            return False
+
+        return True
+
+    def query(self, *args, **kwargs):
+        """Query the provider for subtitles.
+
+        Arguments should match as much as possible the actual parameters for querying the provider
+
+        :return: found subtitles.
+        :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+        :raise: :class:`~subliminal.exceptions.ProviderError`
+
+        """
+        raise NotImplementedError
+
+    def list_subtitles(self, video, languages):
+        """List subtitles for the `video` with the given `languages`.
+
+        This will call the :meth:`query` method internally. The parameters passed to the :meth:`query` method may
+        vary depending on the amount of information available in the `video`.
+
+        :param video: video to list subtitles for.
+        :type video: :class:`~subliminal.video.Video`
+        :param languages: languages to search for.
+        :type languages: set of :class:`~babelfish.language.Language`
+        :return: found subtitles.
+        :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+        :raise: :class:`~subliminal.exceptions.ProviderError`
+
+        """
+        raise NotImplementedError
+
+    def download_subtitle(self, subtitle):
+        """Download `subtitle`'s :attr:`~subliminal.subtitle.Subtitle.content`.
+
+        :param subtitle: subtitle to download.
+        :type subtitle: :class:`~subliminal.subtitle.Subtitle`
+        :raise: :class:`~subliminal.exceptions.ProviderError`
+
+        """
+        raise NotImplementedError
+
+    def __repr__(self):
+        return '<%s [%r]>' % (self.__class__.__name__, self.video_types)

libs/common/subliminal/providers/addic7ed.py

@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+import logging
+import re
+
+from babelfish import Language, language_converters
+from guessit import guessit
+from requests import Session
+
+from . import ParserBeautifulSoup, Provider
+from .. import __short_version__
+from ..cache import SHOW_EXPIRATION_TIME, region
+from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, TooManyRequests
+from ..score import get_equivalent_release_groups
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
+from ..utils import sanitize, sanitize_release_group
+from ..video import Episode
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
+
+#: Series header parsing regex
+series_year_re = re.compile(r'^(?P<series>[ \w\'.:(),&!?-]+?)(?: \((?P<year>\d{4})\))?$')
+
+
+class Addic7edSubtitle(Subtitle):
+    """Addic7ed Subtitle."""
+    provider_name = 'addic7ed'
+
+    def __init__(self, language, hearing_impaired, page_link, series, season, episode, title, year, version,
+                 download_link):
+        super(Addic7edSubtitle, self).__init__(language, hearing_impaired, page_link)
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.title = title
+        self.year = year
+        self.version = version
+        self.download_link = download_link
+
+    @property
+    def id(self):
+        return self.download_link
+
+    def get_matches(self, video):
+        matches = set()
+
+        # series
+        if video.series and sanitize(self.series) == sanitize(video.series):
+            matches.add('series')
+        # season
+        if video.season and self.season == video.season:
+            matches.add('season')
+        # episode
+        if video.episode and self.episode == video.episode:
+            matches.add('episode')
+        # title
+        if video.title and sanitize(self.title) == sanitize(video.title):
+            matches.add('title')
+        # year
+        if video.original_series and self.year is None or video.year and video.year == self.year:
+            matches.add('year')
+        # release_group
+        if (video.release_group and self.version and
+                any(r in sanitize_release_group(self.version)
+                    for r in get_equivalent_release_groups(sanitize_release_group(video.release_group)))):
+            matches.add('release_group')
+        # resolution
+        if video.resolution and self.version and video.resolution in self.version.lower():
+            matches.add('resolution')
+        # format
+        if video.format and self.version and video.format.lower() in self.version.lower():
+            matches.add('format')
+        # other properties
+        matches |= guess_matches(video, guessit(self.version), partial=True)
+
+        return matches
+
+
+class Addic7edProvider(Provider):
+    """Addic7ed Provider."""
+    languages = {Language('por', 'BR')} | {Language(l) for l in [
+        'ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas', 'fin', 'fra', 'glg',
+        'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa', 'nld', 'nor', 'pol', 'por', 'ron', 'rus',
+        'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'zho'
+    ]}
+    video_types = (Episode,)
+    server_url = 'http://www.addic7ed.com/'
+
+    def __init__(self, username=None, password=None):
+        if username is not None and password is None or username is None and password is not None:
+            raise ConfigurationError('Username and password must be specified')
+
+        self.username = username
+        self.password = password
+        self.logged_in = False
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+        # login
+        if self.username is not None and self.password is not None:
+            logger.info('Logging in')
+            data = {'username': self.username, 'password': self.password, 'Submit': 'Log in'}
+            r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10)
+
+            if r.status_code != 302:
+                raise AuthenticationError(self.username)
+
+            logger.debug('Logged in')
+            self.logged_in = True
+
+    def terminate(self):
+        # logout
+        if self.logged_in:
+            logger.info('Logging out')
+            r = self.session.get(self.server_url + 'logout.php', timeout=10)
+            r.raise_for_status()
+            logger.debug('Logged out')
+            self.logged_in = False
+
+        self.session.close()
+
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+    def _get_show_ids(self):
+        """Get the ``dict`` of show ids per series by querying the `shows.php` page.
+
+        :return: show id per series, lower case and without quotes.
+        :rtype: dict
+
+        """
+        # get the show page
+        logger.info('Getting show ids')
+        r = self.session.get(self.server_url + 'shows.php', timeout=10)
+        r.raise_for_status()
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # populate the show ids
+        show_ids = {}
+        for show in soup.select('td.version > h3 > a[href^="/show/"]'):
+            show_ids[sanitize(show.text)] = int(show['href'][6:])
+        logger.debug('Found %d show ids', len(show_ids))
+
+        return show_ids
+
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+    def _search_show_id(self, series, year=None):
+        """Search the show id from the `series` and `year`.
+
+        :param str series: series of the episode.
+        :param year: year of the series, if any.
+        :type year: int
+        :return: the show id, if found.
+        :rtype: int
+
+        """
+        # addic7ed doesn't support search with quotes
+        series = series.replace('\'', ' ')
+
+        # build the params
+        series_year = '%s %d' % (series, year) if year is not None else series
+        params = {'search': series_year, 'Submit': 'Search'}
+
+        # make the search
+        logger.info('Searching show ids with %r', params)
+        r = self.session.get(self.server_url + 'search.php', params=params, timeout=10)
+        r.raise_for_status()
+        if r.status_code == 304:
+            raise TooManyRequests()
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # get the suggestion
+        suggestion = soup.select('span.titulo > a[href^="/show/"]')
+        if not suggestion:
+            logger.warning('Show id not found: no suggestion')
+            return None
+        if not sanitize(suggestion[0].i.text.replace('\'', ' ')) == sanitize(series_year):
+            logger.warning('Show id not found: suggestion does not match')
+            return None
+        show_id = int(suggestion[0]['href'][6:])
+        logger.debug('Found show id %d', show_id)
+
+        return show_id
+
+    def get_show_id(self, series, year=None, country_code=None):
+        """Get the best matching show id for `series`, `year` and `country_code`.
+
+        First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id`.
+
+        :param str series: series of the episode.
+        :param year: year of the series, if any.
+        :type year: int
+        :param country_code: country code of the series, if any.
+        :type country_code: str
+        :return: the show id, if found.
+        :rtype: int
+
+        """
+        series_sanitized = sanitize(series).lower()
+        show_ids = self._get_show_ids()
+        show_id = None
+
+        # attempt with country
+        if not show_id and country_code:
+            logger.debug('Getting show id with country')
+            show_id = show_ids.get('%s %s' % (series_sanitized, country_code.lower()))
+
+        # attempt with year
+        if not show_id and year:
+            logger.debug('Getting show id with year')
+            show_id = show_ids.get('%s %d' % (series_sanitized, year))
+
+        # attempt clean
+        if not show_id:
+            logger.debug('Getting show id')
+            show_id = show_ids.get(series_sanitized)
+
+        # search as last resort
+        if not show_id:
+            logger.warning('Series not found in show ids')
+            show_id = self._search_show_id(series)
+
+        return show_id
+
+    def query(self, series, season, year=None, country=None):
+        # get the show id
+        show_id = self.get_show_id(series, year, country)
+        if show_id is None:
+            logger.error('No show id found for %r (%r)', series, {'year': year, 'country': country})
+            return []
+
+        # get the page of the season of the show
+        logger.info('Getting the page of show id %d, season %d', show_id, season)
+        r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10)
+        r.raise_for_status()
+        if r.status_code == 304:
+            raise TooManyRequests()
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # loop over subtitle rows
+        match = series_year_re.match(soup.select('#header font')[0].text.strip()[:-10])
+        series = match.group('series')
+        year = int(match.group('year')) if match.group('year') else None
+        subtitles = []
+        for row in soup.select('tr.epeven'):
+            cells = row('td')
+
+            # ignore incomplete subtitles
+            status = cells[5].text
+            if status != 'Completed':
+                logger.debug('Ignoring subtitle with status %s', status)
+                continue
+
+            # read the item
+            language = Language.fromaddic7ed(cells[3].text)
+            hearing_impaired = bool(cells[6].text)
+            page_link = self.server_url + cells[2].a['href'][1:]
+            season = int(cells[0].text)
+            episode = int(cells[1].text)
+            title = cells[2].text
+            version = cells[4].text
+            download_link = cells[9].a['href'][1:]
+
+            subtitle = Addic7edSubtitle(language, hearing_impaired, page_link, series, season, episode, title, year,
+                                        version, download_link)
+            logger.debug('Found subtitle %r', subtitle)
+            subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return [s for s in self.query(video.series, video.season, video.year)
+                if s.language in languages and s.episode == video.episode]
+
+    def download_subtitle(self, subtitle):
+        # download the subtitle
+        logger.info('Downloading subtitle %r', subtitle)
+        r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link},
+                             timeout=10)
+        r.raise_for_status()
+
+        # detect download limit exceeded
+        if r.headers['Content-Type'] == 'text/html':
+            raise DownloadLimitExceeded
+
+        subtitle.content = fix_line_ending(r.content)

libs/common/subliminal/providers/legendastv.py

@@ -0,0 +1,448 @@
+# -*- coding: utf-8 -*-
+import io
+import json
+import logging
+import os
+import re
+
+from babelfish import Language, language_converters
+from datetime import datetime, timedelta
+from dogpile.cache.api import NO_VALUE
+from guessit import guessit
+import pytz
+import rarfile
+from rarfile import RarFile, is_rarfile
+from requests import Session
+from zipfile import ZipFile, is_zipfile
+
+from . import ParserBeautifulSoup, Provider
+from .. import __short_version__
+from ..cache import SHOW_EXPIRATION_TIME, region
+from ..exceptions import AuthenticationError, ConfigurationError, ProviderError
+from ..subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending, guess_matches, sanitize
+from ..video import Episode, Movie
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('legendastv = subliminal.converters.legendastv:LegendasTVConverter')
+
+# Configure :mod:`rarfile` to use the same path separator as :mod:`zipfile`
+rarfile.PATH_SEP = '/'
+
+#: Conversion map for types
+type_map = {'M': 'movie', 'S': 'episode', 'C': 'episode'}
+
+#: BR title season parsing regex
+season_re = re.compile(r' - (?P<season>\d+)(\xaa|a|st|nd|rd|th) (temporada|season)', re.IGNORECASE)
+
+#: Downloads parsing regex
+downloads_re = re.compile(r'(?P<downloads>\d+) downloads')
+
+#: Rating parsing regex
+rating_re = re.compile(r'nota (?P<rating>\d+)')
+
+#: Timestamp parsing regex
+timestamp_re = re.compile(r'(?P<day>\d+)/(?P<month>\d+)/(?P<year>\d+) - (?P<hour>\d+):(?P<minute>\d+)')
+
+#: Cache key for releases
+releases_key = __name__ + ':releases|{archive_id}'
+
+
+class LegendasTVArchive(object):
+    """LegendasTV Archive.
+
+    :param str id: identifier.
+    :param str name: name.
+    :param bool pack: contains subtitles for multiple episodes.
+    :param bool pack: featured.
+    :param str link: link.
+    :param int downloads: download count.
+    :param int rating: rating (0-10).
+    :param timestamp: timestamp.
+    :type timestamp: datetime.datetime
+
+    """
+    def __init__(self, id, name, pack, featured, link, downloads=0, rating=0, timestamp=None):
+        #: Identifier
+        self.id = id
+
+        #: Name
+        self.name = name
+
+        #: Pack
+        self.pack = pack
+
+        #: Featured
+        self.featured = featured
+
+        #: Link
+        self.link = link
+
+        #: Download count
+        self.downloads = downloads
+
+        #: Rating (0-10)
+        self.rating = rating
+
+        #: Timestamp
+        self.timestamp = timestamp
+
+        #: Compressed content as :class:`rarfile.RarFile` or :class:`zipfile.ZipFile`
+        self.content = None
+
+    def __repr__(self):
+        return '<%s [%s] %r>' % (self.__class__.__name__, self.id, self.name)
+
+
+class LegendasTVSubtitle(Subtitle):
+    """LegendasTV Subtitle."""
+    provider_name = 'legendastv'
+
+    def __init__(self, language, type, title, year, imdb_id, season, archive, name):
+        super(LegendasTVSubtitle, self).__init__(language, archive.link)
+        self.type = type
+        self.title = title
+        self.year = year
+        self.imdb_id = imdb_id
+        self.season = season
+        self.archive = archive
+        self.name = name
+
+    @property
+    def id(self):
+        return '%s-%s' % (self.archive.id, self.name.lower())
+
+    def get_matches(self, video, hearing_impaired=False):
+        matches = set()
+
+        # episode
+        if isinstance(video, Episode) and self.type == 'episode':
+            # series
+            if video.series and sanitize(self.title) == sanitize(video.series):
+                matches.add('series')
+
+            # year (year is based on season air date hence the adjustment)
+            if video.original_series and self.year is None or video.year and video.year == self.year - self.season + 1:
+                matches.add('year')
+
+            # imdb_id
+            if video.series_imdb_id and self.imdb_id == video.series_imdb_id:
+                matches.add('series_imdb_id')
+
+        # movie
+        elif isinstance(video, Movie) and self.type == 'movie':
+            # title
+            if video.title and sanitize(self.title) == sanitize(video.title):
+                matches.add('title')
+
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+
+            # imdb_id
+            if video.imdb_id and self.imdb_id == video.imdb_id:
+                matches.add('imdb_id')
+
+        # archive name
+        matches |= guess_matches(video, guessit(self.archive.name, {'type': self.type}))
+
+        # name
+        matches |= guess_matches(video, guessit(self.name, {'type': self.type}))
+
+        return matches
+
+
+class LegendasTVProvider(Provider):
+    """LegendasTV Provider.
+
+    :param str username: username.
+    :param str password: password.
+
+    """
+    languages = {Language.fromlegendastv(l) for l in language_converters['legendastv'].codes}
+    server_url = 'http://legendas.tv/'
+
+    def __init__(self, username=None, password=None):
+        if username and not password or not username and password:
+            raise ConfigurationError('Username and password must be specified')
+
+        self.username = username
+        self.password = password
+        self.logged_in = False
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+        # login
+        if self.username is not None and self.password is not None:
+            logger.info('Logging in')
+            data = {'_method': 'POST', 'data[User][username]': self.username, 'data[User][password]': self.password}
+            r = self.session.post(self.server_url + 'login', data, allow_redirects=False, timeout=10)
+            r.raise_for_status()
+
+            soup = ParserBeautifulSoup(r.content, ['html.parser'])
+            if soup.find('div', {'class': 'alert-error'}, string=re.compile(u'Usuário ou senha inválidos')):
+                raise AuthenticationError(self.username)
+
+            logger.debug('Logged in')
+            self.logged_in = True
+
+    def terminate(self):
+        # logout
+        if self.logged_in:
+            logger.info('Logging out')
+            r = self.session.get(self.server_url + 'users/logout', allow_redirects=False, timeout=10)
+            r.raise_for_status()
+            logger.debug('Logged out')
+            self.logged_in = False
+
+        self.session.close()
+
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+    def search_titles(self, title):
+        """Search for titles matching the `title`.
+
+        :param str title: the title to search for.
+        :return: found titles.
+        :rtype: dict
+
+        """
+        # make the query
+        logger.info('Searching title %r', title)
+        r = self.session.get(self.server_url + 'legenda/sugestao/{}'.format(title), timeout=10)
+        r.raise_for_status()
+        results = json.loads(r.text)
+
+        # loop over results
+        titles = {}
+        for result in results:
+            source = result['_source']
+
+            # extract id
+            title_id = int(source['id_filme'])
+
+            # extract type and title
+            title = {'type': type_map[source['tipo']], 'title': source['dsc_nome']}
+
+            # extract year
+            if source['dsc_data_lancamento'] and source['dsc_data_lancamento'].isdigit():
+                title['year'] = int(source['dsc_data_lancamento'])
+
+            # extract imdb_id
+            if source['id_imdb'] != '0':
+                if not source['id_imdb'].startswith('tt'):
+                    title['imdb_id'] = 'tt' + source['id_imdb'].zfill(7)
+                else:
+                    title['imdb_id'] = source['id_imdb']
+
+            # extract season
+            if title['type'] == 'episode':
+                if source['temporada'] and source['temporada'].isdigit():
+                    title['season'] = int(source['temporada'])
+                else:
+                    match = season_re.search(source['dsc_nome_br'])
+                    if match:
+                        title['season'] = int(match.group('season'))
+                    else:
+                        logger.warning('No season detected for title %d', title_id)
+
+            # add title
+            titles[title_id] = title
+
+        logger.debug('Found %d titles', len(titles))
+
+        return titles
+
+    @region.cache_on_arguments(expiration_time=timedelta(minutes=15).total_seconds())
+    def get_archives(self, title_id, language_code):
+        """Get the archive list from a given `title_id` and `language_code`.
+
+        :param int title_id: title id.
+        :param int language_code: language code.
+        :return: the archives.
+        :rtype: list of :class:`LegendasTVArchive`
+
+        """
+        logger.info('Getting archives for title %d and language %d', title_id, language_code)
+        archives = []
+        page = 1
+        while True:
+            # get the archive page
+            url = self.server_url + 'util/carrega_legendas_busca_filme/{title}/{language}/-/{page}'.format(
+                title=title_id, language=language_code, page=page)
+            r = self.session.get(url)
+            r.raise_for_status()
+
+            # parse the results
+            soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+            for archive_soup in soup.select('div.list_element > article > div'):
+                # create archive
+                archive = LegendasTVArchive(archive_soup.a['href'].split('/')[2], archive_soup.a.text,
+                                            'pack' in archive_soup['class'], 'destaque' in archive_soup['class'],
+                                            self.server_url + archive_soup.a['href'][1:])
+
+                # extract text containing downloads, rating and timestamp
+                data_text = archive_soup.find('p', class_='data').text
+
+                # match downloads
+                archive.downloads = int(downloads_re.search(data_text).group('downloads'))
+
+                # match rating
+                match = rating_re.search(data_text)
+                if match:
+                    archive.rating = int(match.group('rating'))
+
+                # match timestamp and validate it
+                time_data = {k: int(v) for k, v in timestamp_re.search(data_text).groupdict().items()}
+                archive.timestamp = pytz.timezone('America/Sao_Paulo').localize(datetime(**time_data))
+                if archive.timestamp > datetime.utcnow().replace(tzinfo=pytz.utc):
+                    raise ProviderError('Archive timestamp is in the future')
+
+                # add archive
+                archives.append(archive)
+
+            # stop on last page
+            if soup.find('a', attrs={'class': 'load_more'}, string='carregar mais') is None:
+                break
+
+            # increment page count
+            page += 1
+
+        logger.debug('Found %d archives', len(archives))
+
+        return archives
+
+    def download_archive(self, archive):
+        """Download an archive's :attr:`~LegendasTVArchive.content`.
+
+        :param archive: the archive to download :attr:`~LegendasTVArchive.content` of.
+        :type archive: :class:`LegendasTVArchive`
+
+        """
+        logger.info('Downloading archive %s', archive.id)
+        r = self.session.get(self.server_url + 'downloadarquivo/{}'.format(archive.id))
+        r.raise_for_status()
+
+        # open the archive
+        archive_stream = io.BytesIO(r.content)
+        if is_rarfile(archive_stream):
+            logger.debug('Identified rar archive')
+            archive.content = RarFile(archive_stream)
+        elif is_zipfile(archive_stream):
+            logger.debug('Identified zip archive')
+            archive.content = ZipFile(archive_stream)
+        else:
+            raise ValueError('Not a valid archive')
+
+    def query(self, language, title, season=None, episode=None, year=None):
+        # search for titles
+        titles = self.search_titles(sanitize(title))
+
+        # search for titles with the quote or dot character
+        ignore_characters = {'\'', '.'}
+        if any(c in title for c in ignore_characters):
+            titles.update(self.search_titles(sanitize(title, ignore_characters=ignore_characters)))
+
+        subtitles = []
+        # iterate over titles
+        for title_id, t in titles.items():
+            # discard mismatches on title
+            if sanitize(t['title']) != sanitize(title):
+                continue
+
+            # episode
+            if season and episode:
+                # discard mismatches on type
+                if t['type'] != 'episode':
+                    continue
+
+                # discard mismatches on season
+                if 'season' not in t or t['season'] != season:
+                    continue
+            # movie
+            else:
+                # discard mismatches on type
+                if t['type'] != 'movie':
+                    continue
+
+                # discard mismatches on year
+                if year is not None and 'year' in t and t['year'] != year:
+                    continue
+
+            # iterate over title's archives
+            for a in self.get_archives(title_id, language.legendastv):
+                # clean name of path separators and pack flags
+                clean_name = a.name.replace('/', '-')
+                if a.pack and clean_name.startswith('(p)'):
+                    clean_name = clean_name[3:]
+
+                # guess from name
+                guess = guessit(clean_name, {'type': t['type']})
+
+                # episode
+                if season and episode:
+                    # discard mismatches on episode in non-pack archives
+                    if not a.pack and 'episode' in guess and guess['episode'] != episode:
+                        continue
+
+                # compute an expiration time based on the archive timestamp
+                expiration_time = (datetime.utcnow().replace(tzinfo=pytz.utc) - a.timestamp).total_seconds()
+
+                # attempt to get the releases from the cache
+                releases = region.get(releases_key.format(archive_id=a.id), expiration_time=expiration_time)
+
+                # the releases are not in cache or cache is expired
+                if releases == NO_VALUE:
+                    logger.info('Releases not found in cache')
+
+                    # download archive
+                    self.download_archive(a)
+
+                    # extract the releases
+                    releases = []
+                    for name in a.content.namelist():
+                        # discard the legendastv file
+                        if name.startswith('Legendas.tv'):
+                            continue
+
+                        # discard hidden files
+                        if os.path.split(name)[-1].startswith('.'):
+                            continue
+
+                        # discard non-subtitle files
+                        if not name.lower().endswith(SUBTITLE_EXTENSIONS):
+                            continue
+
+                        releases.append(name)
+
+                    # cache the releases
+                    region.set(releases_key.format(archive_id=a.id), releases)
+
+                # iterate over releases
+                for r in releases:
+                    subtitle = LegendasTVSubtitle(language, t['type'], t['title'], t.get('year'), t.get('imdb_id'),
+                                                  t.get('season'), a, r)
+                    logger.debug('Found subtitle %r', subtitle)
+                    subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        season = episode = None
+        if isinstance(video, Episode):
+            title = video.series
+            season = video.season
+            episode = video.episode
+        else:
+            title = video.title
+
+        return [s for l in languages for s in self.query(l, title, season=season, episode=episode, year=video.year)]
+
+    def download_subtitle(self, subtitle):
+        # download archive in case we previously hit the releases cache and didn't download it
+        if subtitle.archive.content is None:
+            self.download_archive(subtitle.archive)
+
+        # extract subtitle's content
+        subtitle.content = fix_line_ending(subtitle.archive.content.read(subtitle.name))

libs/common/subliminal/providers/napiprojekt.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+import logging
+
+from babelfish import Language
+from requests import Session
+
+from . import Provider
+from .. import __short_version__
+from ..subtitle import Subtitle
+
+logger = logging.getLogger(__name__)
+
+
+def get_subhash(hash):
+    """Get a second hash based on napiprojekt's hash.
+
+    :param str hash: napiprojekt's hash.
+    :return: the subhash.
+    :rtype: str
+
+    """
+    idx = [0xe, 0x3, 0x6, 0x8, 0x2]
+    mul = [2, 2, 5, 4, 3]
+    add = [0, 0xd, 0x10, 0xb, 0x5]
+
+    b = []
+    for i in range(len(idx)):
+        a = add[i]
+        m = mul[i]
+        i = idx[i]
+        t = a + int(hash[i], 16)
+        v = int(hash[t:t + 2], 16)
+        b.append(('%x' % (v * m))[-1])
+
+    return ''.join(b)
+
+
+class NapiProjektSubtitle(Subtitle):
+    """NapiProjekt Subtitle."""
+    provider_name = 'napiprojekt'
+
+    def __init__(self, language, hash):
+        super(NapiProjektSubtitle, self).__init__(language)
+        self.hash = hash
+
+    @property
+    def id(self):
+        return self.hash
+
+    def get_matches(self, video):
+        matches = set()
+
+        # hash
+        if 'napiprojekt' in video.hashes and video.hashes['napiprojekt'] == self.hash:
+            matches.add('hash')
+
+        return matches
+
+
+class NapiProjektProvider(Provider):
+    """NapiProjekt Provider."""
+    languages = {Language.fromalpha2(l) for l in ['pl']}
+    required_hash = 'napiprojekt'
+    server_url = 'http://napiprojekt.pl/unit_napisy/dl.php'
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+    def terminate(self):
+        self.session.close()
+
+    def query(self, language, hash):
+        params = {
+            'v': 'dreambox',
+            'kolejka': 'false',
+            'nick': '',
+            'pass': '',
+            'napios': 'Linux',
+            'l': language.alpha2.upper(),
+            'f': hash,
+            't': get_subhash(hash)}
+        logger.info('Searching subtitle %r', params)
+        response = self.session.get(self.server_url, params=params, timeout=10)
+        response.raise_for_status()
+
+        # handle subtitles not found and errors
+        if response.content[:4] == b'NPc0':
+            logger.debug('No subtitles found')
+            return None
+
+        subtitle = NapiProjektSubtitle(language, hash)
+        subtitle.content = response.content
+        logger.debug('Found subtitle %r', subtitle)
+
+        return subtitle
+
+    def list_subtitles(self, video, languages):
+        return [s for s in [self.query(l, video.hashes['napiprojekt']) for l in languages] if s is not None]
+
+    def download_subtitle(self, subtitle):
+        # there is no download step, content is already filled from listing subtitles
+        pass

libs/common/subliminal/providers/opensubtitles.py

@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+import base64
+import logging
+import os
+import re
+import zlib
+
+from babelfish import Language, language_converters
+from guessit import guessit
+from six.moves.xmlrpc_client import ServerProxy
+
+from . import Provider, TimeoutSafeTransport
+from .. import __short_version__
+from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
+from ..utils import sanitize
+from ..video import Episode, Movie
+
+logger = logging.getLogger(__name__)
+
+
+class OpenSubtitlesSubtitle(Subtitle):
+    """OpenSubtitles Subtitle."""
+    provider_name = 'opensubtitles'
+    series_re = re.compile(r'^"(?P<series_name>.*)" (?P<series_title>.*)$')
+
+    def __init__(self, language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind, hash, movie_name,
+                 movie_release_name, movie_year, movie_imdb_id, series_season, series_episode, filename, encoding):
+        super(OpenSubtitlesSubtitle, self).__init__(language, hearing_impaired, page_link, encoding)
+        self.subtitle_id = subtitle_id
+        self.matched_by = matched_by
+        self.movie_kind = movie_kind
+        self.hash = hash
+        self.movie_name = movie_name
+        self.movie_release_name = movie_release_name
+        self.movie_year = movie_year
+        self.movie_imdb_id = movie_imdb_id
+        self.series_season = series_season
+        self.series_episode = series_episode
+        self.filename = filename
+
+    @property
+    def id(self):
+        return str(self.subtitle_id)
+
+    @property
+    def series_name(self):
+        return self.series_re.match(self.movie_name).group('series_name')
+
+    @property
+    def series_title(self):
+        return self.series_re.match(self.movie_name).group('series_title')
+
+    def get_matches(self, video):
+        matches = set()
+
+        # episode
+        if isinstance(video, Episode) and self.movie_kind == 'episode':
+            # tag match, assume series, year, season and episode matches
+            if self.matched_by == 'tag':
+                matches |= {'series', 'year', 'season', 'episode'}
+            # series
+            if video.series and sanitize(self.series_name) == sanitize(video.series):
+                matches.add('series')
+            # year
+            if video.original_series and self.movie_year is None or video.year and video.year == self.movie_year:
+                matches.add('year')
+            # season
+            if video.season and self.series_season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.series_episode == video.episode:
+                matches.add('episode')
+            # title
+            if video.title and sanitize(self.series_title) == sanitize(video.title):
+                matches.add('title')
+            # guess
+            matches |= guess_matches(video, guessit(self.movie_release_name, {'type': 'episode'}))
+            matches |= guess_matches(video, guessit(self.filename, {'type': 'episode'}))
+            # hash
+            if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']:
+                if 'series' in matches and 'season' in matches and 'episode' in matches:
+                    matches.add('hash')
+                else:
+                    logger.debug('Match on hash discarded')
+        # movie
+        elif isinstance(video, Movie) and self.movie_kind == 'movie':
+            # tag match, assume title and year matches
+            if self.matched_by == 'tag':
+                matches |= {'title', 'year'}
+            # title
+            if video.title and sanitize(self.movie_name) == sanitize(video.title):
+                matches.add('title')
+            # year
+            if video.year and self.movie_year == video.year:
+                matches.add('year')
+            # guess
+            matches |= guess_matches(video, guessit(self.movie_release_name, {'type': 'movie'}))
+            matches |= guess_matches(video, guessit(self.filename, {'type': 'movie'}))
+            # hash
+            if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']:
+                if 'title' in matches:
+                    matches.add('hash')
+                else:
+                    logger.debug('Match on hash discarded')
+        else:
+            logger.info('%r is not a valid movie_kind', self.movie_kind)
+            return matches
+
+        # imdb_id
+        if video.imdb_id and self.movie_imdb_id == video.imdb_id:
+            matches.add('imdb_id')
+
+        return matches
+
+
+class OpenSubtitlesProvider(Provider):
+    """OpenSubtitles Provider.
+
+    :param str username: username.
+    :param str password: password.
+
+    """
+    languages = {Language.fromopensubtitles(l) for l in language_converters['opensubtitles'].codes}
+
+    def __init__(self, username=None, password=None):
+        self.server = ServerProxy('https://api.opensubtitles.org/xml-rpc', TimeoutSafeTransport(10))
+        if username and not password or not username and password:
+            raise ConfigurationError('Username and password must be specified')
+        # None values not allowed for logging in, so replace it by ''
+        self.username = username or ''
+        self.password = password or ''
+        self.token = None
+
+    def initialize(self):
+        logger.info('Logging in')
+        response = checked(self.server.LogIn(self.username, self.password, 'eng',
+                                             'subliminal v%s' % __short_version__))
+        self.token = response['token']
+        logger.debug('Logged in with token %r', self.token)
+
+    def terminate(self):
+        logger.info('Logging out')
+        checked(self.server.LogOut(self.token))
+        self.server.close()
+        self.token = None
+        logger.debug('Logged out')
+
+    def no_operation(self):
+        logger.debug('No operation')
+        checked(self.server.NoOperation(self.token))
+
+    def query(self, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None, tag=None):
+        # fill the search criteria
+        criteria = []
+        if hash and size:
+            criteria.append({'moviehash': hash, 'moviebytesize': str(size)})
+        if imdb_id:
+            criteria.append({'imdbid': imdb_id[2:]})
+        if tag:
+            criteria.append({'tag': tag})
+        if query and season and episode:
+            criteria.append({'query': query.replace('\'', ''), 'season': season, 'episode': episode})
+        elif query:
+            criteria.append({'query': query.replace('\'', '')})
+        if not criteria:
+            raise ValueError('Not enough information')
+
+        # add the language
+        for criterion in criteria:
+            criterion['sublanguageid'] = ','.join(sorted(l.opensubtitles for l in languages))
+
+        # query the server
+        logger.info('Searching subtitles %r', criteria)
+        response = checked(self.server.SearchSubtitles(self.token, criteria))
+        subtitles = []
+
+        # exit if no data
+        if not response['data']:
+            logger.debug('No subtitles found')
+            return subtitles
+
+        # loop over subtitle items
+        for subtitle_item in response['data']:
+            # read the item
+            language = Language.fromopensubtitles(subtitle_item['SubLanguageID'])
+            hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
+            page_link = subtitle_item['SubtitlesLink']
+            subtitle_id = int(subtitle_item['IDSubtitleFile'])
+            matched_by = subtitle_item['MatchedBy']
+            movie_kind = subtitle_item['MovieKind']
+            hash = subtitle_item['MovieHash']
+            movie_name = subtitle_item['MovieName']
+            movie_release_name = subtitle_item['MovieReleaseName']
+            movie_year = int(subtitle_item['MovieYear']) if subtitle_item['MovieYear'] else None
+            movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb']
+            series_season = int(subtitle_item['SeriesSeason']) if subtitle_item['SeriesSeason'] else None
+            series_episode = int(subtitle_item['SeriesEpisode']) if subtitle_item['SeriesEpisode'] else None
+            filename = subtitle_item['SubFileName']
+            encoding = subtitle_item.get('SubEncoding') or None
+
+            subtitle = OpenSubtitlesSubtitle(language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind,
+                                             hash, movie_name, movie_release_name, movie_year, movie_imdb_id,
+                                             series_season, series_episode, filename, encoding)
+            logger.debug('Found subtitle %r by %s', subtitle, matched_by)
+            subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        season = episode = None
+        if isinstance(video, Episode):
+            query = video.series
+            season = video.season
+            episode = video.episode
+        else:
+            query = video.title
+
+        return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id,
+                          query=query, season=season, episode=episode, tag=os.path.basename(video.name))
+
+    def download_subtitle(self, subtitle):
+        logger.info('Downloading subtitle %r', subtitle)
+        response = checked(self.server.DownloadSubtitles(self.token, [str(subtitle.subtitle_id)]))
+        subtitle.content = fix_line_ending(zlib.decompress(base64.b64decode(response['data'][0]['data']), 47))
+
+
+class OpenSubtitlesError(ProviderError):
+    """Base class for non-generic :class:`OpenSubtitlesProvider` exceptions."""
+    pass
+
+
+class Unauthorized(OpenSubtitlesError, AuthenticationError):
+    """Exception raised when status is '401 Unauthorized'."""
+    pass
+
+
+class NoSession(OpenSubtitlesError, AuthenticationError):
+    """Exception raised when status is '406 No session'."""
+    pass
+
+
+class DownloadLimitReached(OpenSubtitlesError, DownloadLimitExceeded):
+    """Exception raised when status is '407 Download limit reached'."""
+    pass
+
+
+class InvalidImdbid(OpenSubtitlesError):
+    """Exception raised when status is '413 Invalid ImdbID'."""
+    pass
+
+
+class UnknownUserAgent(OpenSubtitlesError, AuthenticationError):
+    """Exception raised when status is '414 Unknown User Agent'."""
+    pass
+
+
+class DisabledUserAgent(OpenSubtitlesError, AuthenticationError):
+    """Exception raised when status is '415 Disabled user agent'."""
+    pass
+
+
+class ServiceUnavailable(OpenSubtitlesError):
+    """Exception raised when status is '503 Service Unavailable'."""
+    pass
+
+
+def checked(response):
+    """Check a response status before returning it.
+
+    :param response: a response from a XMLRPC call to OpenSubtitles.
+    :return: the response.
+    :raise: :class:`OpenSubtitlesError`
+
+    """
+    status_code = int(response['status'][:3])
+    if status_code == 401:
+        raise Unauthorized
+    if status_code == 406:
+        raise NoSession
+    if status_code == 407:
+        raise DownloadLimitReached
+    if status_code == 413:
+        raise InvalidImdbid
+    if status_code == 414:
+        raise UnknownUserAgent
+    if status_code == 415:
+        raise DisabledUserAgent
+    if status_code == 503:
+        raise ServiceUnavailable
+    if status_code != 200:
+        raise OpenSubtitlesError(response['status'])
+
+    return response

libs/common/subliminal/providers/podnapisi.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+import io
+import logging
+import re
+
+from babelfish import Language, language_converters
+from guessit import guessit
+try:
+    from lxml import etree
+except ImportError:
+    try:
+        import xml.etree.cElementTree as etree
+    except ImportError:
+        import xml.etree.ElementTree as etree
+from requests import Session
+from zipfile import ZipFile
+
+from . import Provider
+from .. import __short_version__
+from ..exceptions import ProviderError
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
+from ..utils import sanitize
+from ..video import Episode, Movie
+
+logger = logging.getLogger(__name__)
+
+
+class PodnapisiSubtitle(Subtitle):
+    """Podnapisi Subtitle."""
+    provider_name = 'podnapisi'
+
+    def __init__(self, language, hearing_impaired, page_link, pid, releases, title, season=None, episode=None,
+                 year=None):
+        super(PodnapisiSubtitle, self).__init__(language, hearing_impaired, page_link)
+        self.pid = pid
+        self.releases = releases
+        self.title = title
+        self.season = season
+        self.episode = episode
+        self.year = year
+
+    @property
+    def id(self):
+        return self.pid
+
+    def get_matches(self, video):
+        matches = set()
+
+        # episode
+        if isinstance(video, Episode):
+            # series
+            if video.series and sanitize(self.title) == sanitize(video.series):
+                matches.add('series')
+            # year
+            if video.original_series and self.year is None or video.year and video.year == self.year:
+                matches.add('year')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+            # guess
+            for release in self.releases:
+                matches |= guess_matches(video, guessit(release, {'type': 'episode'}))
+        # movie
+        elif isinstance(video, Movie):
+            # title
+            if video.title and sanitize(self.title) == sanitize(video.title):
+                matches.add('title')
+            # year
+            if video.year and self.year == video.year:
+                matches.add('year')
+            # guess
+            for release in self.releases:
+                matches |= guess_matches(video, guessit(release, {'type': 'movie'}))
+
+        return matches
+
+
+class PodnapisiProvider(Provider):
+    """Podnapisi Provider."""
+    languages = ({Language('por', 'BR'), Language('srp', script='Latn')} |
+                 {Language.fromalpha2(l) for l in language_converters['alpha2'].codes})
+    server_url = 'http://podnapisi.net/subtitles/'
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+    def terminate(self):
+        self.session.close()
+
+    def query(self, language, keyword, season=None, episode=None, year=None):
+        # set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
+        params = {'sXML': 1, 'sL': str(language), 'sK': keyword}
+        is_episode = False
+        if season and episode:
+            is_episode = True
+            params['sTS'] = season
+            params['sTE'] = episode
+        if year:
+            params['sY'] = year
+
+        # loop over paginated results
+        logger.info('Searching subtitles %r', params)
+        subtitles = []
+        pids = set()
+        while True:
+            # query the server
+            xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content)
+
+            # exit if no results
+            if not int(xml.find('pagination/results').text):
+                logger.debug('No subtitles found')
+                break
+
+            # loop over subtitles
+            for subtitle_xml in xml.findall('subtitle'):
+                # read xml elements
+                language = Language.fromietf(subtitle_xml.find('language').text)
+                hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
+                page_link = subtitle_xml.find('url').text
+                pid = subtitle_xml.find('pid').text
+                releases = []
+                if subtitle_xml.find('release').text:
+                    for release in subtitle_xml.find('release').text.split():
+                        release = re.sub(r'\.+$', '', release)  # remove trailing dots
+                        release = ''.join(filter(lambda x: ord(x) < 128, release))  # remove non-ascii characters
+                        releases.append(release)
+                title = subtitle_xml.find('title').text
+                season = int(subtitle_xml.find('tvSeason').text)
+                episode = int(subtitle_xml.find('tvEpisode').text)
+                year = int(subtitle_xml.find('year').text)
+
+                if is_episode:
+                    subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+                                                 season=season, episode=episode, year=year)
+                else:
+                    subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+                                                 year=year)
+
+                # ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
+                if pid in pids:
+                    continue
+
+                logger.debug('Found subtitle %r', subtitle)
+                subtitles.append(subtitle)
+                pids.add(pid)
+
+            # stop on last page
+            if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
+                break
+
+            # increment current page
+            params['page'] = int(xml.find('pagination/current').text) + 1
+            logger.debug('Getting page %d', params['page'])
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        if isinstance(video, Episode):
+            return [s for l in languages for s in self.query(l, video.series, season=video.season,
+                                                             episode=video.episode, year=video.year)]
+        elif isinstance(video, Movie):
+            return [s for l in languages for s in self.query(l, video.title, year=video.year)]
+
+    def download_subtitle(self, subtitle):
+        # download as a zip
+        logger.info('Downloading subtitle %r', subtitle)
+        r = self.session.get(self.server_url + subtitle.pid + '/download', params={'container': 'zip'}, timeout=10)
+        r.raise_for_status()
+
+        # open the zip
+        with ZipFile(io.BytesIO(r.content)) as zf:
+            if len(zf.namelist()) > 1:
+                raise ProviderError('More than one file to unzip')
+
+            subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))

libs/common/subliminal/providers/shooter.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+import json
+import logging
+import os
+
+from babelfish import Language, language_converters
+from requests import Session
+
+from . import Provider
+from .. import __short_version__
+from ..subtitle import Subtitle, fix_line_ending
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('shooter = subliminal.converters.shooter:ShooterConverter')
+
+
+class ShooterSubtitle(Subtitle):
+    """Shooter Subtitle."""
+    provider_name = 'shooter'
+
+    def __init__(self, language, hash, download_link):
+        super(ShooterSubtitle, self).__init__(language)
+        self.hash = hash
+        self.download_link = download_link
+
+    @property
+    def id(self):
+        return self.download_link
+
+    def get_matches(self, video):
+        matches = set()
+
+        # hash
+        if 'shooter' in video.hashes and video.hashes['shooter'] == self.hash:
+            matches.add('hash')
+
+        return matches
+
+
+class ShooterProvider(Provider):
+    """Shooter Provider."""
+    languages = {Language(l) for l in ['eng', 'zho']}
+    server_url = 'https://www.shooter.cn/api/subapi.php'
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+    def terminate(self):
+        self.session.close()
+
+    def query(self, language, filename, hash=None):
+        # query the server
+        params = {'filehash': hash, 'pathinfo': os.path.realpath(filename), 'format': 'json', 'lang': language.shooter}
+        logger.debug('Searching subtitles %r', params)
+        r = self.session.post(self.server_url, params=params, timeout=10)
+        r.raise_for_status()
+
+        # handle subtitles not found
+        if r.content == b'\xff':
+            logger.debug('No subtitles found')
+            return []
+
+        # parse the subtitles
+        results = json.loads(r.text)
+        subtitles = [ShooterSubtitle(language, hash, t['Link']) for s in results for t in s['Files']]
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return [s for l in languages for s in self.query(l, video.name, video.hashes.get('shooter'))]
+
+    def download_subtitle(self, subtitle):
+        logger.info('Downloading subtitle %r', subtitle)
+        r = self.session.get(subtitle.download_link, timeout=10)
+        r.raise_for_status()
+
+        subtitle.content = fix_line_ending(r.content)

libs/common/subliminal/providers/subscenter.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+import bisect
+from collections import defaultdict
+import io
+import json
+import logging
+import zipfile
+
+from babelfish import Language
+from guessit import guessit
+from requests import Session
+
+from . import ParserBeautifulSoup, Provider
+from .. import __short_version__
+from ..cache import SHOW_EXPIRATION_TIME, region
+from ..exceptions import AuthenticationError, ConfigurationError, ProviderError
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
+from ..utils import sanitize
+from ..video import Episode, Movie
+
+logger = logging.getLogger(__name__)
+
+
+class SubsCenterSubtitle(Subtitle):
+    """SubsCenter Subtitle."""
+    provider_name = 'subscenter'
+
+    def __init__(self, language, hearing_impaired, page_link, series, season, episode, title, subtitle_id, subtitle_key,
+                 downloaded, releases):
+        super(SubsCenterSubtitle, self).__init__(language, hearing_impaired, page_link)
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.title = title
+        self.subtitle_id = subtitle_id
+        self.subtitle_key = subtitle_key
+        self.downloaded = downloaded
+        self.releases = releases
+
+    @property
+    def id(self):
+        return str(self.subtitle_id)
+
+    def get_matches(self, video):
+        matches = set()
+
+        # episode
+        if isinstance(video, Episode):
+            # series
+            if video.series and sanitize(self.series) == sanitize(video.series):
+                matches.add('series')
+            # season
+            if video.season and self.season == video.season:
+                matches.add('season')
+            # episode
+            if video.episode and self.episode == video.episode:
+                matches.add('episode')
+            # guess
+            for release in self.releases:
+                matches |= guess_matches(video, guessit(release, {'type': 'episode'}))
+        # movie
+        elif isinstance(video, Movie):
+            # guess
+            for release in self.releases:
+                matches |= guess_matches(video, guessit(release, {'type': 'movie'}))
+
+        # title
+        if video.title and sanitize(self.title) == sanitize(video.title):
+            matches.add('title')
+
+        return matches
+
+
+class SubsCenterProvider(Provider):
+    """SubsCenter Provider."""
+    languages = {Language.fromalpha2(l) for l in ['he']}
+    server_url = 'http://www.subscenter.co/he/'
+
+    def __init__(self, username=None, password=None):
+        if username is not None and password is None or username is None and password is not None:
+            raise ConfigurationError('Username and password must be specified')
+
+        self.session = None
+        self.username = username
+        self.password = password
+        self.logged_in = False
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/{}'.format(__short_version__)
+
+        # login
+        if self.username is not None and self.password is not None:
+            logger.debug('Logging in')
+            url = self.server_url + 'subscenter/accounts/login/'
+
+            # retrieve CSRF token
+            self.session.get(url)
+            csrf_token = self.session.cookies['csrftoken']
+
+            # actual login
+            data = {'username': self.username, 'password': self.password, 'csrfmiddlewaretoken': csrf_token}
+            r = self.session.post(url, data, allow_redirects=False, timeout=10)
+
+            if r.status_code != 302:
+                raise AuthenticationError(self.username)
+
+            logger.info('Logged in')
+            self.logged_in = True
+
+    def terminate(self):
+        # logout
+        if self.logged_in:
+            logger.info('Logging out')
+            r = self.session.get(self.server_url + 'subscenter/accounts/logout/', timeout=10)
+            r.raise_for_status()
+            logger.info('Logged out')
+            self.logged_in = False
+
+        self.session.close()
+
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+    def _search_url_titles(self, title):
+        """Search the URL titles by kind for the given `title`.
+
+        :param str title: title to search for.
+        :return: the URL titles by kind.
+        :rtype: collections.defaultdict
+
+        """
+        # make the search
+        logger.info('Searching title name for %r', title)
+        r = self.session.get(self.server_url + 'subtitle/search/', params={'q': title}, timeout=10)
+        r.raise_for_status()
+
+        # check for redirections
+        if r.history and all([h.status_code == 302 for h in r.history]):
+            logger.debug('Redirected to the subtitles page')
+            links = [r.url]
+        else:
+            # get the suggestions (if needed)
+            soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+            links = [link.attrs['href'] for link in soup.select('#processes div.generalWindowTop a')]
+            logger.debug('Found %d suggestions', len(links))
+
+        url_titles = defaultdict(list)
+        for link in links:
+            parts = link.split('/')
+            url_titles[parts[-3]].append(parts[-2])
+
+        return url_titles
+
+    def query(self, title, season=None, episode=None):
+        # search for the url title
+        url_titles = self._search_url_titles(title)
+
+        # episode
+        if season and episode:
+            if 'series' not in url_titles:
+                logger.error('No URL title found for series %r', title)
+                return []
+            url_title = url_titles['series'][0]
+            logger.debug('Using series title %r', url_title)
+            url = self.server_url + 'cst/data/series/sb/{}/{}/{}/'.format(url_title, season, episode)
+            page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(url_title, season, episode)
+        else:
+            if 'movie' not in url_titles:
+                logger.error('No URL title found for movie %r', title)
+                return []
+            url_title = url_titles['movie'][0]
+            logger.debug('Using movie title %r', url_title)
+            url = self.server_url + 'cst/data/movie/sb/{}/'.format(url_title)
+            page_link = self.server_url + 'subtitle/movie/{}/'.format(url_title)
+
+        # get the list of subtitles
+        logger.debug('Getting the list of subtitles')
+        r = self.session.get(url)
+        r.raise_for_status()
+        results = json.loads(r.text)
+
+        # loop over results
+        subtitles = {}
+        for language_code, language_data in results.items():
+            for quality_data in language_data.values():
+                for quality, subtitles_data in quality_data.items():
+                    for subtitle_item in subtitles_data.values():
+                        # read the item
+                        language = Language.fromalpha2(language_code)
+                        hearing_impaired = bool(subtitle_item['hearing_impaired'])
+                        subtitle_id = subtitle_item['id']
+                        subtitle_key = subtitle_item['key']
+                        downloaded = subtitle_item['downloaded']
+                        release = subtitle_item['subtitle_version']
+
+                        # add the release and increment downloaded count if we already have the subtitle
+                        if subtitle_id in subtitles:
+                            logger.debug('Found additional release %r for subtitle %d', release, subtitle_id)
+                            bisect.insort_left(subtitles[subtitle_id].releases, release)  # deterministic order
+                            subtitles[subtitle_id].downloaded += downloaded
+                            continue
+
+                        # otherwise create it
+                        subtitle = SubsCenterSubtitle(language, hearing_impaired, page_link, title, season, episode,
+                                                      title, subtitle_id, subtitle_key, downloaded, [release])
+                        logger.debug('Found subtitle %r', subtitle)
+                        subtitles[subtitle_id] = subtitle
+
+        return subtitles.values()
+
+    def list_subtitles(self, video, languages):
+        season = episode = None
+        title = video.title
+
+        if isinstance(video, Episode):
+            title = video.series
+            season = video.season
+            episode = video.episode
+
+        return [s for s in self.query(title, season, episode) if s.language in languages]
+
+    def download_subtitle(self, subtitle):
+        # download
+        url = self.server_url + 'subtitle/download/{}/{}/'.format(subtitle.language.alpha2, subtitle.subtitle_id)
+        params = {'v': subtitle.releases[0], 'key': subtitle.subtitle_key}
+        r = self.session.get(url, params=params, headers={'Referer': subtitle.page_link}, timeout=10)
+        r.raise_for_status()
+
+        # open the zip
+        with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+            # remove some filenames from the namelist
+            namelist = [n for n in zf.namelist() if not n.endswith('.txt')]
+            if len(namelist) > 1:
+                raise ProviderError('More than one file to unzip')
+
+            subtitle.content = fix_line_ending(zf.read(namelist[0]))

libs/common/subliminal/providers/thesubdb.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+import logging
+
+from babelfish import Language, language_converters
+from requests import Session
+
+from . import Provider
+from .. import __short_version__
+from ..subtitle import Subtitle, fix_line_ending
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('thesubdb = subliminal.converters.thesubdb:TheSubDBConverter')
+
+
+class TheSubDBSubtitle(Subtitle):
+    """TheSubDB Subtitle."""
+    provider_name = 'thesubdb'
+
+    def __init__(self, language, hash):
+        super(TheSubDBSubtitle, self).__init__(language)
+        self.hash = hash
+
+    @property
+    def id(self):
+        return self.hash + '-' + str(self.language)
+
+    def get_matches(self, video):
+        matches = set()
+
+        # hash
+        if 'thesubdb' in video.hashes and video.hashes['thesubdb'] == self.hash:
+            matches.add('hash')
+
+        return matches
+
+
+class TheSubDBProvider(Provider):
+    """TheSubDB Provider."""
+    languages = {Language.fromthesubdb(l) for l in language_converters['thesubdb'].codes}
+    required_hash = 'thesubdb'
+    server_url = 'http://api.thesubdb.com/'
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = ('SubDB/1.0 (subliminal/%s; https://github.com/Diaoul/subliminal)' %
+                                              __short_version__)
+
+    def terminate(self):
+        self.session.close()
+
+    def query(self, hash):
+        # make the query
+        params = {'action': 'search', 'hash': hash}
+        logger.info('Searching subtitles %r', params)
+        r = self.session.get(self.server_url, params=params, timeout=10)
+
+        # handle subtitles not found and errors
+        if r.status_code == 404:
+            logger.debug('No subtitles found')
+            return []
+        r.raise_for_status()
+
+        # loop over languages
+        subtitles = []
+        for language_code in r.text.split(','):
+            language = Language.fromthesubdb(language_code)
+
+            subtitle = TheSubDBSubtitle(language, hash)
+            logger.debug('Found subtitle %r', subtitle)
+            subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return [s for s in self.query(video.hashes['thesubdb']) if s.language in languages]
+
+    def download_subtitle(self, subtitle):
+        logger.info('Downloading subtitle %r', subtitle)
+        params = {'action': 'download', 'hash': subtitle.hash, 'language': subtitle.language.alpha2}
+        r = self.session.get(self.server_url, params=params, timeout=10)
+        r.raise_for_status()
+
+        subtitle.content = fix_line_ending(r.content)

libs/common/subliminal/providers/tvsubtitles.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+import io
+import logging
+import re
+from zipfile import ZipFile
+
+from babelfish import Language, language_converters
+from guessit import guessit
+from requests import Session
+
+from . import ParserBeautifulSoup, Provider
+from .. import __short_version__
+from ..cache import EPISODE_EXPIRATION_TIME, SHOW_EXPIRATION_TIME, region
+from ..exceptions import ProviderError
+from ..score import get_equivalent_release_groups
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
+from ..utils import sanitize, sanitize_release_group
+from ..video import Episode
+
+logger = logging.getLogger(__name__)
+
+language_converters.register('tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter')
+
+link_re = re.compile(r'^(?P<series>.+?)(?: \(?\d{4}\)?| \((?:US|UK)\))? \((?P<first_year>\d{4})-\d{4}\)$')
+episode_id_re = re.compile(r'^episode-\d+\.html$')
+
+
+class TVsubtitlesSubtitle(Subtitle):
+    """TVsubtitles Subtitle."""
+    provider_name = 'tvsubtitles'
+
+    def __init__(self, language, page_link, subtitle_id, series, season, episode, year, rip, release):
+        super(TVsubtitlesSubtitle, self).__init__(language, page_link=page_link)
+        self.subtitle_id = subtitle_id
+        self.series = series
+        self.season = season
+        self.episode = episode
+        self.year = year
+        self.rip = rip
+        self.release = release
+
+    @property
+    def id(self):
+        return str(self.subtitle_id)
+
+    def get_matches(self, video):
+        matches = set()
+
+        # series
+        if video.series and sanitize(self.series) == sanitize(video.series):
+            matches.add('series')
+        # season
+        if video.season and self.season == video.season:
+            matches.add('season')
+        # episode
+        if video.episode and self.episode == video.episode:
+            matches.add('episode')
+        # year
+        if video.original_series and self.year is None or video.year and video.year == self.year:
+            matches.add('year')
+        # release_group
+        if (video.release_group and self.release and
+                any(r in sanitize_release_group(self.release)
+                    for r in get_equivalent_release_groups(sanitize_release_group(video.release_group)))):
+            matches.add('release_group')
+        # other properties
+        if self.release:
+            matches |= guess_matches(video, guessit(self.release, {'type': 'episode'}), partial=True)
+        if self.rip:
+            matches |= guess_matches(video, guessit(self.rip), partial=True)
+
+        return matches
+
+
+class TVsubtitlesProvider(Provider):
+    """TVsubtitles Provider."""
+    languages = {Language('por', 'BR')} | {Language(l) for l in [
+        'ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor', 'nld', 'pol', 'por',
+        'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho'
+    ]}
+    video_types = (Episode,)
+    server_url = 'http://www.tvsubtitles.net/'
+
+    def initialize(self):
+        self.session = Session()
+        self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
+
+    def terminate(self):
+        self.session.close()
+
+    @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+    def search_show_id(self, series, year=None):
+        """Search the show id from the `series` and `year`.
+
+        :param str series: series of the episode.
+        :param year: year of the series, if any.
+        :type year: int
+        :return: the show id, if any.
+        :rtype: int
+
+        """
+        # make the search
+        logger.info('Searching show id for %r', series)
+        r = self.session.post(self.server_url + 'search.php', data={'q': series}, timeout=10)
+        r.raise_for_status()
+
+        # get the series out of the suggestions
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+        show_id = None
+        for suggestion in soup.select('div.left li div a[href^="/tvshow-"]'):
+            match = link_re.match(suggestion.text)
+            if not match:
+                logger.error('Failed to match %s', suggestion.text)
+                continue
+
+            if match.group('series').lower() == series.lower():
+                if year is not None and int(match.group('first_year')) != year:
+                    logger.debug('Year does not match')
+                    continue
+                show_id = int(suggestion['href'][8:-5])
+                logger.debug('Found show id %d', show_id)
+                break
+
+        return show_id
+
+    @region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME)
+    def get_episode_ids(self, show_id, season):
+        """Get episode ids from the show id and the season.
+
+        :param int show_id: show id.
+        :param int season: season of the episode.
+        :return: episode ids per episode number.
+        :rtype: dict
+
+        """
+        # get the page of the season of the show
+        logger.info('Getting the page of show id %d, season %d', show_id, season)
+        r = self.session.get(self.server_url + 'tvshow-%d-%d.html' % (show_id, season), timeout=10)
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # loop over episode rows
+        episode_ids = {}
+        for row in soup.select('table#table5 tr'):
+            # skip rows that do not have a link to the episode page
+            if not row('a', href=episode_id_re):
+                continue
+
+            # extract data from the cells
+            cells = row('td')
+            episode = int(cells[0].text.split('x')[1])
+            episode_id = int(cells[1].a['href'][8:-5])
+            episode_ids[episode] = episode_id
+
+        if episode_ids:
+            logger.debug('Found episode ids %r', episode_ids)
+        else:
+            logger.warning('No episode ids found')
+
+        return episode_ids
+
+    def query(self, series, season, episode, year=None):
+        # search the show id
+        show_id = self.search_show_id(series, year)
+        if show_id is None:
+            logger.error('No show id found for %r (%r)', series, {'year': year})
+            return []
+
+        # get the episode ids
+        episode_ids = self.get_episode_ids(show_id, season)
+        if episode not in episode_ids:
+            logger.error('Episode %d not found', episode)
+            return []
+
+        # get the episode page
+        logger.info('Getting the page for episode %d', episode_ids[episode])
+        r = self.session.get(self.server_url + 'episode-%d.html' % episode_ids[episode], timeout=10)
+        soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+        # loop over subtitles rows
+        subtitles = []
+        for row in soup.select('.subtitlen'):
+            # read the item
+            language = Language.fromtvsubtitles(row.h5.img['src'][13:-4])
+            subtitle_id = int(row.parent['href'][10:-5])
+            page_link = self.server_url + 'subtitle-%d.html' % subtitle_id
+            rip = row.find('p', title='rip').text.strip() or None
+            release = row.find('p', title='release').text.strip() or None
+
+            subtitle = TVsubtitlesSubtitle(language, page_link, subtitle_id, series, season, episode, year, rip,
+                                           release)
+            logger.debug('Found subtitle %s', subtitle)
+            subtitles.append(subtitle)
+
+        return subtitles
+
+    def list_subtitles(self, video, languages):
+        return [s for s in self.query(video.series, video.season, video.episode, video.year) if s.language in languages]
+
+    def download_subtitle(self, subtitle):
+        # download as a zip
+        logger.info('Downloading subtitle %r', subtitle)
+        r = self.session.get(self.server_url + 'download-%d.html' % subtitle.subtitle_id, timeout=10)
+        r.raise_for_status()
+
+        # open the zip
+        with ZipFile(io.BytesIO(r.content)) as zf:
+            if len(zf.namelist()) > 1:
+                raise ProviderError('More than one file to unzip')
+
+            subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))