Merge pull request #1915 from clinton-hall/vendor

Update vendored libraries
2025-08-21 05:43:16 -07:00 · 2022-12-02 22:14:22 -05:00 · 2022-12-02 22:14:22 -05:00 · 1adc1fab80
commit 1adc1fab80
parent ba96e40653 4f109fc898
586 changed files with 98455 additions and 39275 deletions
--- a/libs/common/_distutils_hack/init.py
+++ b/libs/common/_distutils_hack/init.py
@ -0,0 +1,222 @@
+# don't import any costly modules
+import sys
+import os
+
+
+is_pypy = '__pypy__' in sys.builtin_module_names
+
+
+def warn_distutils_present():
+    if 'distutils' not in sys.modules:
+        return
+    if is_pypy and sys.version_info < (3, 7):
+        # PyPy for 3.6 unconditionally imports distutils, so bypass the warning
+        # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250
+        return
+    import warnings
+
+    warnings.warn(
+        "Distutils was imported before Setuptools, but importing Setuptools "
+        "also replaces the `distutils` module in `sys.modules`. This may lead "
+        "to undesirable behaviors or errors. To avoid these issues, avoid "
+        "using distutils directly, ensure that setuptools is installed in the "
+        "traditional way (e.g. not an editable install), and/or make sure "
+        "that setuptools is always imported before distutils."
+    )
+
+
+def clear_distutils():
+    if 'distutils' not in sys.modules:
+        return
+    import warnings
+
+    warnings.warn("Setuptools is replacing distutils.")
+    mods = [
+        name
+        for name in sys.modules
+        if name == "distutils" or name.startswith("distutils.")
+    ]
+    for name in mods:
+        del sys.modules[name]
+
+
+def enabled():
+    """
+    Allow selection of distutils by environment variable.
+    """
+    which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
+    return which == 'local'
+
+
+def ensure_local_distutils():
+    import importlib
+
+    clear_distutils()
+
+    # With the DistutilsMetaFinder in place,
+    # perform an import to cause distutils to be
+    # loaded from setuptools._distutils. Ref #2906.
+    with shim():
+        importlib.import_module('distutils')
+
+    # check that submodules load as expected
+    core = importlib.import_module('distutils.core')
+    assert '_distutils' in core.__file__, core.__file__
+    assert 'setuptools._distutils.log' not in sys.modules
+
+
+def do_override():
+    """
+    Ensure that the local copy of distutils is preferred over stdlib.
+
+    See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
+    for more motivation.
+    """
+    if enabled():
+        warn_distutils_present()
+        ensure_local_distutils()
+
+
+class _TrivialRe:
+    def __init__(self, *patterns):
+        self._patterns = patterns
+
+    def match(self, string):
+        return all(pat in string for pat in self._patterns)
+
+
+class DistutilsMetaFinder:
+    def find_spec(self, fullname, path, target=None):
+        # optimization: only consider top level modules and those
+        # found in the CPython test suite.
+        if path is not None and not fullname.startswith('test.'):
+            return
+
+        method_name = 'spec_for_{fullname}'.format(**locals())
+        method = getattr(self, method_name, lambda: None)
+        return method()
+
+    def spec_for_distutils(self):
+        if self.is_cpython():
+            return
+
+        import importlib
+        import importlib.abc
+        import importlib.util
+
+        try:
+            mod = importlib.import_module('setuptools._distutils')
+        except Exception:
+            # There are a couple of cases where setuptools._distutils
+            # may not be present:
+            # - An older Setuptools without a local distutils is
+            #   taking precedence. Ref #2957.
+            # - Path manipulation during sitecustomize removes
+            #   setuptools from the path but only after the hook
+            #   has been loaded. Ref #2980.
+            # In either case, fall back to stdlib behavior.
+            return
+
+        class DistutilsLoader(importlib.abc.Loader):
+            def create_module(self, spec):
+                mod.__name__ = 'distutils'
+                return mod
+
+            def exec_module(self, module):
+                pass
+
+        return importlib.util.spec_from_loader(
+            'distutils', DistutilsLoader(), origin=mod.__file__
+        )
+
+    @staticmethod
+    def is_cpython():
+        """
+        Suppress supplying distutils for CPython (build and tests).
+        Ref #2965 and #3007.
+        """
+        return os.path.isfile('pybuilddir.txt')
+
+    def spec_for_pip(self):
+        """
+        Ensure stdlib distutils when running under pip.
+        See pypa/pip#8761 for rationale.
+        """
+        if self.pip_imported_during_build():
+            return
+        clear_distutils()
+        self.spec_for_distutils = lambda: None
+
+    @classmethod
+    def pip_imported_during_build(cls):
+        """
+        Detect if pip is being imported in a build script. Ref #2355.
+        """
+        import traceback
+
+        return any(
+            cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
+        )
+
+    @staticmethod
+    def frame_file_is_setup(frame):
+        """
+        Return True if the indicated frame suggests a setup.py file.
+        """
+        # some frames may not have __file__ (#2940)
+        return frame.f_globals.get('__file__', '').endswith('setup.py')
+
+    def spec_for_sensitive_tests(self):
+        """
+        Ensure stdlib distutils when running select tests under CPython.
+
+        python/cpython#91169
+        """
+        clear_distutils()
+        self.spec_for_distutils = lambda: None
+
+    sensitive_tests = (
+        [
+            'test.test_distutils',
+            'test.test_peg_generator',
+            'test.test_importlib',
+        ]
+        if sys.version_info < (3, 10)
+        else [
+            'test.test_distutils',
+        ]
+    )
+
+
+for name in DistutilsMetaFinder.sensitive_tests:
+    setattr(
+        DistutilsMetaFinder,
+        f'spec_for_{name}',
+        DistutilsMetaFinder.spec_for_sensitive_tests,
+    )
+
+
+DISTUTILS_FINDER = DistutilsMetaFinder()
+
+
+def add_shim():
+    DISTUTILS_FINDER in sys.meta_path or insert_shim()
+
+
+class shim:
+    def __enter__(self):
+        insert_shim()
+
+    def __exit__(self, exc, value, tb):
+        remove_shim()
+
+
+def insert_shim():
+    sys.meta_path.insert(0, DISTUTILS_FINDER)
+
+
+def remove_shim():
+    try:
+        sys.meta_path.remove(DISTUTILS_FINDER)
+    except ValueError:
+        pass
--- a/libs/common/_distutils_hack/override.py
+++ b/libs/common/_distutils_hack/override.py
@ -0,0 +1 @@
+__import__('_distutils_hack').do_override()
--- a/libs/common/bin/beet.exe
+++ b/libs/common/bin/beet.exe
--- a/libs/common/bin/chardetect.exe
+++ b/libs/common/bin/chardetect.exe
--- a/libs/common/bin/easy_install.exe
+++ b/libs/common/bin/easy_install.exe
--- a/libs/common/bin/guessit.exe
+++ b/libs/common/bin/guessit.exe
--- a/libs/common/bin/mid3cp.exe
+++ b/libs/common/bin/mid3cp.exe
--- a/libs/common/bin/mid3iconv.exe
+++ b/libs/common/bin/mid3iconv.exe
--- a/libs/common/bin/mid3v2.exe
+++ b/libs/common/bin/mid3v2.exe
--- a/libs/common/bin/moggsplit.exe
+++ b/libs/common/bin/moggsplit.exe
--- a/libs/common/bin/mutagen-inspect.exe
+++ b/libs/common/bin/mutagen-inspect.exe
--- a/libs/common/bin/mutagen-pony.exe
+++ b/libs/common/bin/mutagen-pony.exe
--- a/libs/common/bin/easy_install-3.7.exe
+++ b/libs/common/bin/easy_install-3.7.exe
--- a/libs/common/bin/pbr.exe
+++ b/libs/common/bin/pbr.exe
--- a/libs/common/bin/srt.exe
+++ b/libs/common/bin/srt.exe
--- a/libs/common/bin/subliminal.exe
+++ b/libs/common/bin/subliminal.exe
--- a/libs/common/bin/unidecode.exe
+++ b/libs/common/bin/unidecode.exe
--- a/libs/common/certifi/init.py
+++ b/libs/common/certifi/init.py
@ -1,3 +1,4 @@
 from .core import contents, where

-__version__ = "2021.05.30"
+__all__ = ["contents", "where"]
+__version__ = "2022.09.24"
--- a/libs/common/certifi/cacert.pem
+++ b/libs/common/certifi/cacert.pem
--- a/libs/common/certifi/core.py
+++ b/libs/common/certifi/core.py
@ -1,20 +1,20 @@
-# -*- coding: utf-8 -*-
-
 """
 certifi.py
 ~~~~~~~~~~

 This module returns the installation location of cacert.pem or its contents.
 """
-import os
+import sys

-try:
-    from importlib.resources import path as get_path, read_text
+
+if sys.version_info >= (3, 11):
+
+    from importlib.resources import as_file, files

    _CACERT_CTX = None
    _CACERT_PATH = None

-    def where():
+    def where() -> str:
        # This is slightly terrible, but we want to delay extracting the file
        # in cases where we're inside of a zipimport situation until someone
        # actually calls where(), but we don't want to re-extract the file
@ -33,28 +33,76 @@ try:
            # We also have to hold onto the actual context manager, because
            # it will do the cleanup whenever it gets garbage collected, so
            # we will also store that at the global level as well.
+            _CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem"))
+            _CACERT_PATH = str(_CACERT_CTX.__enter__())
+
+        return _CACERT_PATH
+
+    def contents() -> str:
+        return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii")
+
+elif sys.version_info >= (3, 7):
+
+    from importlib.resources import path as get_path, read_text
+
+    _CACERT_CTX = None
+    _CACERT_PATH = None
+
+    def where() -> str:
+        # This is slightly terrible, but we want to delay extracting the
+        # file in cases where we're inside of a zipimport situation until
+        # someone actually calls where(), but we don't want to re-extract
+        # the file on every call of where(), so we'll do it once then store
+        # it in a global variable.
+        global _CACERT_CTX
+        global _CACERT_PATH
+        if _CACERT_PATH is None:
+            # This is slightly janky, the importlib.resources API wants you
+            # to manage the cleanup of this file, so it doesn't actually
+            # return a path, it returns a context manager that will give
+            # you the path when you enter it and will do any cleanup when
+            # you leave it. In the common case of not needing a temporary
+            # file, it will just return the file system location and the
+            # __exit__() is a no-op.
+            #
+            # We also have to hold onto the actual context manager, because
+            # it will do the cleanup whenever it gets garbage collected, so
+            # we will also store that at the global level as well.
            _CACERT_CTX = get_path("certifi", "cacert.pem")
            _CACERT_PATH = str(_CACERT_CTX.__enter__())

        return _CACERT_PATH

+    def contents() -> str:
+        return read_text("certifi", "cacert.pem", encoding="ascii")
+
+else:
+    import os
+    import types
+    from typing import Union
+
+    Package = Union[types.ModuleType, str]
+    Resource = Union[str, "os.PathLike"]

-except ImportError:
    # This fallback will work for Python versions prior to 3.7 that lack the
    # importlib.resources module but relies on the existing `where` function
    # so won't address issues with environments like PyOxidizer that don't set
    # __file__ on modules.
-    def read_text(_module, _path, encoding="ascii"):
-        with open(where(), "r", encoding=encoding) as data:
+    def read_text(
+        package: Package,
+        resource: Resource,
+        encoding: str = 'utf-8',
+        errors: str = 'strict'
+    ) -> str:
+        with open(where(), encoding=encoding) as data:
            return data.read()

    # If we don't have importlib.resources, then we will just do the old logic
    # of assuming we're on the filesystem and munge the path directly.
-    def where():
+    def where() -> str:
        f = os.path.dirname(__file__)

        return os.path.join(f, "cacert.pem")

-
-def contents():
-    return read_text("certifi", "cacert.pem", encoding="ascii")
+    def contents() -> str:
+        return read_text("certifi", "cacert.pem", encoding="ascii")
--- a/libs/common/more_itertools/tests/init.py
+++ b/libs/common/more_itertools/tests/init.py
--- a/libs/common/chardet/init.py
+++ b/libs/common/chardet/init.py
@ -15,69 +15,101 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import List, Union

-from .universaldetector import UniversalDetector
+from .charsetgroupprober import CharSetGroupProber
+from .charsetprober import CharSetProber
 from .enums import InputState
-from .version import __version__, VERSION
+from .resultdict import ResultDict
+from .universaldetector import UniversalDetector
+from .version import VERSION, __version__
+
+__all__ = ["UniversalDetector", "detect", "detect_all", "__version__", "VERSION"]


-__all__ = ['UniversalDetector', 'detect', 'detect_all', '__version__', 'VERSION']
-
-
-def detect(byte_str):
+def detect(
+    byte_str: Union[bytes, bytearray], should_rename_legacy: bool = False
+) -> ResultDict:
    """
    Detect the encoding of the given byte string.

    :param byte_str:     The byte sequence to examine.
    :type byte_str:      ``bytes`` or ``bytearray``
+    :param should_rename_legacy:  Should we rename legacy encodings
+                                  to their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
    """
    if not isinstance(byte_str, bytearray):
        if not isinstance(byte_str, bytes):
-            raise TypeError('Expected object of type bytes or bytearray, got: '
-                            '{}'.format(type(byte_str)))
-        else:
-            byte_str = bytearray(byte_str)
-    detector = UniversalDetector()
+            raise TypeError(
+                f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
+            )
+        byte_str = bytearray(byte_str)
+    detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
    detector.feed(byte_str)
    return detector.close()


-def detect_all(byte_str):
+def detect_all(
+    byte_str: Union[bytes, bytearray],
+    ignore_threshold: bool = False,
+    should_rename_legacy: bool = False,
+) -> List[ResultDict]:
    """
    Detect all the possible encodings of the given byte string.

-    :param byte_str:     The byte sequence to examine.
-    :type byte_str:      ``bytes`` or ``bytearray``
+    :param byte_str:          The byte sequence to examine.
+    :type byte_str:           ``bytes`` or ``bytearray``
+    :param ignore_threshold:  Include encodings that are below
+                              ``UniversalDetector.MINIMUM_THRESHOLD``
+                              in results.
+    :type ignore_threshold:   ``bool``
+    :param should_rename_legacy:  Should we rename legacy encodings
+                                  to their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
    """
    if not isinstance(byte_str, bytearray):
        if not isinstance(byte_str, bytes):
-            raise TypeError('Expected object of type bytes or bytearray, got: '
-                            '{}'.format(type(byte_str)))
-        else:
-            byte_str = bytearray(byte_str)
+            raise TypeError(
+                f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
+            )
+        byte_str = bytearray(byte_str)

-    detector = UniversalDetector()
+    detector = UniversalDetector(should_rename_legacy=should_rename_legacy)
    detector.feed(byte_str)
    detector.close()

-    if detector._input_state == InputState.HIGH_BYTE:
-        results = []
-        for prober in detector._charset_probers:
-            if prober.get_confidence() > detector.MINIMUM_THRESHOLD:
-                charset_name = prober.charset_name
-                lower_charset_name = prober.charset_name.lower()
+    if detector.input_state == InputState.HIGH_BYTE:
+        results: List[ResultDict] = []
+        probers: List[CharSetProber] = []
+        for prober in detector.charset_probers:
+            if isinstance(prober, CharSetGroupProber):
+                probers.extend(p for p in prober.probers)
+            else:
+                probers.append(prober)
+        for prober in probers:
+            if ignore_threshold or prober.get_confidence() > detector.MINIMUM_THRESHOLD:
+                charset_name = prober.charset_name or ""
+                lower_charset_name = charset_name.lower()
                # Use Windows encoding name instead of ISO-8859 if we saw any
                # extra Windows-specific bytes
-                if lower_charset_name.startswith('iso-8859'):
-                    if detector._has_win_bytes:
-                        charset_name = detector.ISO_WIN_MAP.get(lower_charset_name,
-                                                            charset_name)
-                results.append({
-                    'encoding': charset_name,
-                    'confidence': prober.get_confidence(),
-                    'language': prober.language,
-                })
+                if lower_charset_name.startswith("iso-8859") and detector.has_win_bytes:
+                    charset_name = detector.ISO_WIN_MAP.get(
+                        lower_charset_name, charset_name
+                    )
+                # Rename legacy encodings with superset encodings if asked
+                if should_rename_legacy:
+                    charset_name = detector.LEGACY_MAP.get(
+                        charset_name.lower(), charset_name
+                    )
+                results.append(
+                    {
+                        "encoding": charset_name,
+                        "confidence": prober.get_confidence(),
+                        "language": prober.language,
+                    }
+                )
        if len(results) > 0:
-            return sorted(results, key=lambda result: -result['confidence'])
+            return sorted(results, key=lambda result: -result["confidence"])

    return [detector.result]
--- a/libs/common/chardet/big5freq.py
+++ b/libs/common/chardet/big5freq.py
@ -42,9 +42,9 @@

 BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75

-#Char to FreqOrder table
+# Char to FreqOrder table
 BIG5_TABLE_SIZE = 5376
-
+# fmt: off
 BIG5_CHAR_TO_FREQ_ORDER = (
   1,1801,1506, 255,1431, 198,   9,  82,   6,5008, 177, 202,3681,1256,2821, 110, #   16
 3814,  33,3274, 261,  76,  44,2114,  16,2946,2187,1176, 659,3971,  26,3451,2653, #   32
@ -383,4 +383,4 @@ BIG5_CHAR_TO_FREQ_ORDER = (
 890,3669,3943,5791,1878,3798,3439,5792,2186,2358,3440,1652,5793,5794,5795, 941, # 5360
 2299, 208,3546,4161,2020, 330,4438,3944,2906,2499,3799,4439,4811,5796,5797,5798, # 5376
 )
-
+# fmt: on
--- a/libs/common/chardet/big5prober.py
+++ b/libs/common/chardet/big5prober.py
@ -25,23 +25,23 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
 from .chardistribution import Big5DistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import BIG5_SM_MODEL


 class Big5Prober(MultiByteCharSetProber):
-    def __init__(self):
-        super(Big5Prober, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(BIG5_SM_MODEL)
        self.distribution_analyzer = Big5DistributionAnalysis()
        self.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "Big5"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Chinese"
--- a/libs/common/chardet/chardistribution.py
+++ b/libs/common/chardet/chardistribution.py
@ -25,40 +25,58 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .euctwfreq import (EUCTW_CHAR_TO_FREQ_ORDER, EUCTW_TABLE_SIZE,
-                        EUCTW_TYPICAL_DISTRIBUTION_RATIO)
-from .euckrfreq import (EUCKR_CHAR_TO_FREQ_ORDER, EUCKR_TABLE_SIZE,
-                        EUCKR_TYPICAL_DISTRIBUTION_RATIO)
-from .gb2312freq import (GB2312_CHAR_TO_FREQ_ORDER, GB2312_TABLE_SIZE,
-                         GB2312_TYPICAL_DISTRIBUTION_RATIO)
-from .big5freq import (BIG5_CHAR_TO_FREQ_ORDER, BIG5_TABLE_SIZE,
-                       BIG5_TYPICAL_DISTRIBUTION_RATIO)
-from .jisfreq import (JIS_CHAR_TO_FREQ_ORDER, JIS_TABLE_SIZE,
-                      JIS_TYPICAL_DISTRIBUTION_RATIO)
+from typing import Tuple, Union
+
+from .big5freq import (
+    BIG5_CHAR_TO_FREQ_ORDER,
+    BIG5_TABLE_SIZE,
+    BIG5_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .euckrfreq import (
+    EUCKR_CHAR_TO_FREQ_ORDER,
+    EUCKR_TABLE_SIZE,
+    EUCKR_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .euctwfreq import (
+    EUCTW_CHAR_TO_FREQ_ORDER,
+    EUCTW_TABLE_SIZE,
+    EUCTW_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .gb2312freq import (
+    GB2312_CHAR_TO_FREQ_ORDER,
+    GB2312_TABLE_SIZE,
+    GB2312_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .jisfreq import (
+    JIS_CHAR_TO_FREQ_ORDER,
+    JIS_TABLE_SIZE,
+    JIS_TYPICAL_DISTRIBUTION_RATIO,
+)
+from .johabfreq import JOHAB_TO_EUCKR_ORDER_TABLE


-class CharDistributionAnalysis(object):
+class CharDistributionAnalysis:
    ENOUGH_DATA_THRESHOLD = 1024
    SURE_YES = 0.99
    SURE_NO = 0.01
    MINIMUM_DATA_THRESHOLD = 3

-    def __init__(self):
+    def __init__(self) -> None:
        # Mapping table to get frequency order from char order (get from
        # GetOrder())
-        self._char_to_freq_order = None
-        self._table_size = None  # Size of above table
+        self._char_to_freq_order: Tuple[int, ...] = tuple()
+        self._table_size = 0  # Size of above table
        # This is a constant value which varies from language to language,
        # used in calculating confidence.  See
        # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
        # for further detail.
-        self.typical_distribution_ratio = None
-        self._done = None
-        self._total_chars = None
-        self._freq_chars = None
+        self.typical_distribution_ratio = 0.0
+        self._done = False
+        self._total_chars = 0
+        self._freq_chars = 0
        self.reset()

-    def reset(self):
+    def reset(self) -> None:
        """reset analyser, clear any state"""
        # If this flag is set to True, detection is done and conclusion has
        # been made
@ -67,7 +85,7 @@ class CharDistributionAnalysis(object):
        # The number of characters whose frequency order is less than 512
        self._freq_chars = 0

-    def feed(self, char, char_len):
+    def feed(self, char: Union[bytes, bytearray], char_len: int) -> None:
        """feed a character with known length"""
        if char_len == 2:
            # we only care about 2-bytes character in our distribution analysis
@ -81,7 +99,7 @@ class CharDistributionAnalysis(object):
                if 512 > self._char_to_freq_order[order]:
                    self._freq_chars += 1

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        """return confidence based on existing data"""
        # if we didn't receive any character in our consideration range,
        # return negative answer
@ -89,20 +107,21 @@ class CharDistributionAnalysis(object):
            return self.SURE_NO

        if self._total_chars != self._freq_chars:
-            r = (self._freq_chars / ((self._total_chars - self._freq_chars)
-                 * self.typical_distribution_ratio))
+            r = self._freq_chars / (
+                (self._total_chars - self._freq_chars) * self.typical_distribution_ratio
+            )
            if r < self.SURE_YES:
                return r

        # normalize confidence (we don't want to be 100% sure)
        return self.SURE_YES

-    def got_enough_data(self):
+    def got_enough_data(self) -> bool:
        # It is not necessary to receive all data to draw conclusion.
        # For charset detection, certain amount of data is enough
        return self._total_chars > self.ENOUGH_DATA_THRESHOLD

-    def get_order(self, byte_str):
+    def get_order(self, _: Union[bytes, bytearray]) -> int:
        # We do not handle characters based on the original encoding string,
        # but convert this encoding string to a number, here called order.
        # This allows multiple encodings of a language to share one frequency
@ -111,13 +130,13 @@ class CharDistributionAnalysis(object):


 class EUCTWDistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(EUCTWDistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = EUCTW_CHAR_TO_FREQ_ORDER
        self._table_size = EUCTW_TABLE_SIZE
        self.typical_distribution_ratio = EUCTW_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for euc-TW encoding, we are interested
        #   first  byte range: 0xc4 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
@ -125,18 +144,17 @@ class EUCTWDistributionAnalysis(CharDistributionAnalysis):
        first_char = byte_str[0]
        if first_char >= 0xC4:
            return 94 * (first_char - 0xC4) + byte_str[1] - 0xA1
-        else:
-            return -1
+        return -1


 class EUCKRDistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(EUCKRDistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
        self._table_size = EUCKR_TABLE_SIZE
        self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for euc-KR encoding, we are interested
        #   first  byte range: 0xb0 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
@ -144,18 +162,32 @@ class EUCKRDistributionAnalysis(CharDistributionAnalysis):
        first_char = byte_str[0]
        if first_char >= 0xB0:
            return 94 * (first_char - 0xB0) + byte_str[1] - 0xA1
-        else:
-            return -1
+        return -1
+
+
+class JOHABDistributionAnalysis(CharDistributionAnalysis):
+    def __init__(self) -> None:
+        super().__init__()
+        self._char_to_freq_order = EUCKR_CHAR_TO_FREQ_ORDER
+        self._table_size = EUCKR_TABLE_SIZE
+        self.typical_distribution_ratio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
+        first_char = byte_str[0]
+        if 0x88 <= first_char < 0xD4:
+            code = first_char * 256 + byte_str[1]
+            return JOHAB_TO_EUCKR_ORDER_TABLE.get(code, -1)
+        return -1


 class GB2312DistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(GB2312DistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = GB2312_CHAR_TO_FREQ_ORDER
        self._table_size = GB2312_TABLE_SIZE
        self.typical_distribution_ratio = GB2312_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for GB2312 encoding, we are interested
        #  first  byte range: 0xb0 -- 0xfe
        #  second byte range: 0xa1 -- 0xfe
@ -163,18 +195,17 @@ class GB2312DistributionAnalysis(CharDistributionAnalysis):
        first_char, second_char = byte_str[0], byte_str[1]
        if (first_char >= 0xB0) and (second_char >= 0xA1):
            return 94 * (first_char - 0xB0) + second_char - 0xA1
-        else:
-            return -1
+        return -1


 class Big5DistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(Big5DistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = BIG5_CHAR_TO_FREQ_ORDER
        self._table_size = BIG5_TABLE_SIZE
        self.typical_distribution_ratio = BIG5_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for big5 encoding, we are interested
        #   first  byte range: 0xa4 -- 0xfe
        #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
@ -183,28 +214,26 @@ class Big5DistributionAnalysis(CharDistributionAnalysis):
        if first_char >= 0xA4:
            if second_char >= 0xA1:
                return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
-            else:
-                return 157 * (first_char - 0xA4) + second_char - 0x40
-        else:
-            return -1
+            return 157 * (first_char - 0xA4) + second_char - 0x40
+        return -1


 class SJISDistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(SJISDistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
        self._table_size = JIS_TABLE_SIZE
        self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for sjis encoding, we are interested
        #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
        #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
        # no validation needed here. State machine has done that
        first_char, second_char = byte_str[0], byte_str[1]
-        if (first_char >= 0x81) and (first_char <= 0x9F):
+        if 0x81 <= first_char <= 0x9F:
            order = 188 * (first_char - 0x81)
-        elif (first_char >= 0xE0) and (first_char <= 0xEF):
+        elif 0xE0 <= first_char <= 0xEF:
            order = 188 * (first_char - 0xE0 + 31)
        else:
            return -1
@ -215,19 +244,18 @@ class SJISDistributionAnalysis(CharDistributionAnalysis):


 class EUCJPDistributionAnalysis(CharDistributionAnalysis):
-    def __init__(self):
-        super(EUCJPDistributionAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._char_to_freq_order = JIS_CHAR_TO_FREQ_ORDER
        self._table_size = JIS_TABLE_SIZE
        self.typical_distribution_ratio = JIS_TYPICAL_DISTRIBUTION_RATIO

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> int:
        # for euc-JP encoding, we are interested
        #   first  byte range: 0xa0 -- 0xfe
        #   second byte range: 0xa1 -- 0xfe
        # no validation needed here. State machine has done that
        char = byte_str[0]
        if char >= 0xA0:
-            return 94 * (char - 0xA1) + byte_str[1] - 0xa1
-        else:
-            return -1
+            return 94 * (char - 0xA1) + byte_str[1] - 0xA1
+        return -1
--- a/libs/common/chardet/charsetgroupprober.py
+++ b/libs/common/chardet/charsetgroupprober.py
@ -25,29 +25,30 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .enums import ProbingState
+from typing import List, Optional, Union
+
 from .charsetprober import CharSetProber
+from .enums import LanguageFilter, ProbingState


 class CharSetGroupProber(CharSetProber):
-    def __init__(self, lang_filter=None):
-        super(CharSetGroupProber, self).__init__(lang_filter=lang_filter)
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
        self._active_num = 0
-        self.probers = []
-        self._best_guess_prober = None
+        self.probers: List[CharSetProber] = []
+        self._best_guess_prober: Optional[CharSetProber] = None

-    def reset(self):
-        super(CharSetGroupProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        self._active_num = 0
        for prober in self.probers:
-            if prober:
-                prober.reset()
-                prober.active = True
-                self._active_num += 1
+            prober.reset()
+            prober.active = True
+            self._active_num += 1
        self._best_guess_prober = None

    @property
-    def charset_name(self):
+    def charset_name(self) -> Optional[str]:
        if not self._best_guess_prober:
            self.get_confidence()
            if not self._best_guess_prober:
@ -55,17 +56,15 @@ class CharSetGroupProber(CharSetProber):
        return self._best_guess_prober.charset_name

    @property
-    def language(self):
+    def language(self) -> Optional[str]:
        if not self._best_guess_prober:
            self.get_confidence()
            if not self._best_guess_prober:
                return None
        return self._best_guess_prober.language

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        for prober in self.probers:
-            if not prober:
-                continue
            if not prober.active:
                continue
            state = prober.feed(byte_str)
@ -75,7 +74,7 @@ class CharSetGroupProber(CharSetProber):
                self._best_guess_prober = prober
                self._state = ProbingState.FOUND_IT
                return self.state
-            elif state == ProbingState.NOT_ME:
+            if state == ProbingState.NOT_ME:
                prober.active = False
                self._active_num -= 1
                if self._active_num <= 0:
@ -83,22 +82,22 @@ class CharSetGroupProber(CharSetProber):
                    return self.state
        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        state = self.state
        if state == ProbingState.FOUND_IT:
            return 0.99
-        elif state == ProbingState.NOT_ME:
+        if state == ProbingState.NOT_ME:
            return 0.01
        best_conf = 0.0
        self._best_guess_prober = None
        for prober in self.probers:
-            if not prober:
-                continue
            if not prober.active:
-                self.logger.debug('%s not active', prober.charset_name)
+                self.logger.debug("%s not active", prober.charset_name)
                continue
            conf = prober.get_confidence()
-            self.logger.debug('%s %s confidence = %s', prober.charset_name, prober.language, conf)
+            self.logger.debug(
+                "%s %s confidence = %s", prober.charset_name, prober.language, conf
+            )
            if best_conf < conf:
                best_conf = conf
                self._best_guess_prober = prober
--- a/libs/common/chardet/charsetprober.py
+++ b/libs/common/chardet/charsetprober.py
@ -28,54 +28,62 @@

 import logging
 import re
+from typing import Optional, Union

-from .enums import ProbingState
+from .enums import LanguageFilter, ProbingState
+
+INTERNATIONAL_WORDS_PATTERN = re.compile(
+    b"[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?"
+)


-class CharSetProber(object):
+class CharSetProber:

    SHORTCUT_THRESHOLD = 0.95

-    def __init__(self, lang_filter=None):
-        self._state = None
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        self._state = ProbingState.DETECTING
+        self.active = True
        self.lang_filter = lang_filter
        self.logger = logging.getLogger(__name__)

-    def reset(self):
+    def reset(self) -> None:
        self._state = ProbingState.DETECTING

    @property
-    def charset_name(self):
+    def charset_name(self) -> Optional[str]:
        return None

-    def feed(self, buf):
-        pass
+    @property
+    def language(self) -> Optional[str]:
+        raise NotImplementedError
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        raise NotImplementedError

    @property
-    def state(self):
+    def state(self) -> ProbingState:
        return self._state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        return 0.0

    @staticmethod
-    def filter_high_byte_only(buf):
-        buf = re.sub(b'([\x00-\x7F])+', b' ', buf)
+    def filter_high_byte_only(buf: Union[bytes, bytearray]) -> bytes:
+        buf = re.sub(b"([\x00-\x7F])+", b" ", buf)
        return buf

    @staticmethod
-    def filter_international_words(buf):
+    def filter_international_words(buf: Union[bytes, bytearray]) -> bytearray:
        """
        We define three types of bytes:
        alphabet: english alphabets [a-zA-Z]
        international: international characters [\x80-\xFF]
        marker: everything else [^a-zA-Z\x80-\xFF]
-
        The input buffer can be thought to contain a series of words delimited
        by markers. This function works to filter all words that contain at
        least one international character. All contiguous sequences of markers
        are replaced by a single space ascii character.
-
        This filter applies to all scripts which do not use English characters.
        """
        filtered = bytearray()
@ -83,8 +91,7 @@ class CharSetProber(object):
        # This regex expression filters out only words that have at-least one
        # international character. The word may include one marker character at
        # the end.
-        words = re.findall(b'[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?',
-                           buf)
+        words = INTERNATIONAL_WORDS_PATTERN.findall(buf)

        for word in words:
            filtered.extend(word[:-1])
@ -94,20 +101,17 @@ class CharSetProber(object):
            # similarly across all languages and may thus have similar
            # frequencies).
            last_char = word[-1:]
-            if not last_char.isalpha() and last_char < b'\x80':
-                last_char = b' '
+            if not last_char.isalpha() and last_char < b"\x80":
+                last_char = b" "
            filtered.extend(last_char)

        return filtered

    @staticmethod
-    def filter_with_english_letters(buf):
+    def remove_xml_tags(buf: Union[bytes, bytearray]) -> bytes:
        """
        Returns a copy of ``buf`` that retains only the sequences of English
        alphabet and high byte characters that are not between <> characters.
-        Also retains English alphabet and high byte characters immediately
-        before occurrences of >.
-
        This filter can be applied to all scripts which contain both English
        characters and extended ASCII characters, but is currently only used by
        ``Latin1Prober``.
@ -115,26 +119,24 @@ class CharSetProber(object):
        filtered = bytearray()
        in_tag = False
        prev = 0
+        buf = memoryview(buf).cast("c")

-        for curr in range(len(buf)):
-            # Slice here to get bytes instead of an int with Python 3
-            buf_char = buf[curr:curr + 1]
-            # Check if we're coming out of or entering an HTML tag
-            if buf_char == b'>':
+        for curr, buf_char in enumerate(buf):
+            # Check if we're coming out of or entering an XML tag
+
+            # https://github.com/python/typeshed/issues/8182
+            if buf_char == b">":  # type: ignore[comparison-overlap]
+                prev = curr + 1
                in_tag = False
-            elif buf_char == b'<':
-                in_tag = True
-
-            # If current character is not extended-ASCII and not alphabetic...
-            if buf_char < b'\x80' and not buf_char.isalpha():
-                # ...and we're not in a tag
+            # https://github.com/python/typeshed/issues/8182
+            elif buf_char == b"<":  # type: ignore[comparison-overlap]
                if curr > prev and not in_tag:
                    # Keep everything after last non-extended-ASCII,
                    # non-alphabetic character
                    filtered.extend(buf[prev:curr])
                    # Output a space to delimit stretch we kept
-                    filtered.extend(b' ')
-                prev = curr + 1
+                    filtered.extend(b" ")
+                in_tag = True

        # If we're not in a tag...
        if not in_tag:
--- a/libs/common/chardet/cli/init.py
+++ b/libs/common/chardet/cli/init.py
@ -1 +0,0 @@
-
--- a/libs/common/chardet/cli/chardetect.py
+++ b/libs/common/chardet/cli/chardetect.py
@ -12,17 +12,21 @@ If no paths are provided, it takes its input from stdin.

 """

-from __future__ import absolute_import, print_function, unicode_literals

 import argparse
 import sys
+from typing import Iterable, List, Optional

-from chardet import __version__
-from chardet.compat import PY2
-from chardet.universaldetector import UniversalDetector
+from .. import __version__
+from ..universaldetector import UniversalDetector


-def description_of(lines, name='stdin'):
+def description_of(
+    lines: Iterable[bytes],
+    name: str = "stdin",
+    minimal: bool = False,
+    should_rename_legacy: bool = False,
+) -> Optional[str]:
    """
    Return a string describing the probable encoding of a file or
    list of strings.
@ -31,8 +35,11 @@ def description_of(lines, name='stdin'):
    :type lines: Iterable of bytes
    :param name: Name of file or collection of lines
    :type name: str
+    :param should_rename_legacy:  Should we rename legacy encodings to
+                                  their more modern equivalents?
+    :type should_rename_legacy:   ``bool``
    """
-    u = UniversalDetector()
+    u = UniversalDetector(should_rename_legacy=should_rename_legacy)
    for line in lines:
        line = bytearray(line)
        u.feed(line)
@ -41,16 +48,14 @@ def description_of(lines, name='stdin'):
            break
    u.close()
    result = u.result
-    if PY2:
-        name = name.decode(sys.getfilesystemencoding(), 'ignore')
-    if result['encoding']:
-        return '{}: {} with confidence {}'.format(name, result['encoding'],
-                                                     result['confidence'])
-    else:
-        return '{}: no result'.format(name)
+    if minimal:
+        return result["encoding"]
+    if result["encoding"]:
+        return f'{name}: {result["encoding"]} with confidence {result["confidence"]}'
+    return f"{name}: no result"


-def main(argv=None):
+def main(argv: Optional[List[str]] = None) -> None:
    """
    Handles command line arguments and gets things started.

@ -60,25 +65,48 @@ def main(argv=None):
    """
    # Get command line arguments
    parser = argparse.ArgumentParser(
-        description="Takes one or more file paths and reports their detected \
-                     encodings")
-    parser.add_argument('input',
-                        help='File whose encoding we would like to determine. \
-                              (default: stdin)',
-                        type=argparse.FileType('rb'), nargs='*',
-                        default=[sys.stdin if PY2 else sys.stdin.buffer])
-    parser.add_argument('--version', action='version',
-                        version='%(prog)s {}'.format(__version__))
+        description=(
+            "Takes one or more file paths and reports their detected encodings"
+        )
+    )
+    parser.add_argument(
+        "input",
+        help="File whose encoding we would like to determine. (default: stdin)",
+        type=argparse.FileType("rb"),
+        nargs="*",
+        default=[sys.stdin.buffer],
+    )
+    parser.add_argument(
+        "--minimal",
+        help="Print only the encoding to standard output",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-l",
+        "--legacy",
+        help="Rename legacy encodings to more modern ones.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
    args = parser.parse_args(argv)

    for f in args.input:
        if f.isatty():
-            print("You are running chardetect interactively. Press " +
-                  "CTRL-D twice at the start of a blank line to signal the " +
-                  "end of your input. If you want help, run chardetect " +
-                  "--help\n", file=sys.stderr)
-        print(description_of(f, f.name))
+            print(
+                "You are running chardetect interactively. Press "
+                "CTRL-D twice at the start of a blank line to signal the "
+                "end of your input. If you want help, run chardetect "
+                "--help\n",
+                file=sys.stderr,
+            )
+        print(
+            description_of(
+                f, f.name, minimal=args.minimal, should_rename_legacy=args.legacy
+            )
+        )


-if __name__ == '__main__':
+if __name__ == "__main__":
    main()
--- a/libs/common/chardet/codingstatemachine.py
+++ b/libs/common/chardet/codingstatemachine.py
@ -27,10 +27,11 @@

 import logging

+from .codingstatemachinedict import CodingStateMachineDict
 from .enums import MachineState


-class CodingStateMachine(object):
+class CodingStateMachine:
    """
    A state machine to verify a byte sequence for a particular encoding. For
    each byte the detector receives, it will feed that byte to every active
@ -52,37 +53,38 @@ class CodingStateMachine(object):
                 negative answer for this encoding. Detector will exclude this
                 encoding from consideration from here on.
    """
-    def __init__(self, sm):
+
+    def __init__(self, sm: CodingStateMachineDict) -> None:
        self._model = sm
        self._curr_byte_pos = 0
        self._curr_char_len = 0
-        self._curr_state = None
+        self._curr_state = MachineState.START
+        self.active = True
        self.logger = logging.getLogger(__name__)
        self.reset()

-    def reset(self):
+    def reset(self) -> None:
        self._curr_state = MachineState.START

-    def next_state(self, c):
+    def next_state(self, c: int) -> int:
        # for each byte we get its class
        # if it is first byte, we also get byte length
-        byte_class = self._model['class_table'][c]
+        byte_class = self._model["class_table"][c]
        if self._curr_state == MachineState.START:
            self._curr_byte_pos = 0
-            self._curr_char_len = self._model['char_len_table'][byte_class]
+            self._curr_char_len = self._model["char_len_table"][byte_class]
        # from byte's class and state_table, we get its next state
-        curr_state = (self._curr_state * self._model['class_factor']
-                      + byte_class)
-        self._curr_state = self._model['state_table'][curr_state]
+        curr_state = self._curr_state * self._model["class_factor"] + byte_class
+        self._curr_state = self._model["state_table"][curr_state]
        self._curr_byte_pos += 1
        return self._curr_state

-    def get_current_charlen(self):
+    def get_current_charlen(self) -> int:
        return self._curr_char_len

-    def get_coding_state_machine(self):
-        return self._model['name']
+    def get_coding_state_machine(self) -> str:
+        return self._model["name"]

    @property
-    def language(self):
-        return self._model['language']
+    def language(self) -> str:
+        return self._model["language"]
--- a/libs/common/chardet/codingstatemachinedict.py
+++ b/libs/common/chardet/codingstatemachinedict.py
@ -0,0 +1,19 @@
+from typing import TYPE_CHECKING, Tuple
+
+if TYPE_CHECKING:
+    # TypedDict was introduced in Python 3.8.
+    #
+    # TODO: Remove the else block and TYPE_CHECKING check when dropping support
+    # for Python 3.7.
+    from typing import TypedDict
+
+    class CodingStateMachineDict(TypedDict, total=False):
+        class_table: Tuple[int, ...]
+        class_factor: int
+        state_table: Tuple[int, ...]
+        char_len_table: Tuple[int, ...]
+        name: str
+        language: str  # Optional key
+
+else:
+    CodingStateMachineDict = dict
--- a/libs/common/chardet/cp949prober.py
+++ b/libs/common/chardet/cp949prober.py
@ -32,8 +32,8 @@ from .mbcssm import CP949_SM_MODEL


 class CP949Prober(MultiByteCharSetProber):
-    def __init__(self):
-        super(CP949Prober, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(CP949_SM_MODEL)
        # NOTE: CP949 is a superset of EUC-KR, so the distribution should be
        #       not different.
@ -41,9 +41,9 @@ class CP949Prober(MultiByteCharSetProber):
        self.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "CP949"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Korean"
--- a/libs/common/chardet/enums.py
+++ b/libs/common/chardet/enums.py
@ -4,21 +4,26 @@ All of the Enums that are used throughout the chardet package.
 :author: Dan Blanchard (dan.blanchard@gmail.com)
 """

+from enum import Enum, Flag

-class InputState(object):
+
+class InputState:
    """
    This enum represents the different states a universal detector can be in.
    """
+
    PURE_ASCII = 0
    ESC_ASCII = 1
    HIGH_BYTE = 2


-class LanguageFilter(object):
+class LanguageFilter(Flag):
    """
    This enum represents the different language filters we can apply to a
    ``UniversalDetector``.
    """
+
+    NONE = 0x00
    CHINESE_SIMPLIFIED = 0x01
    CHINESE_TRADITIONAL = 0x02
    JAPANESE = 0x04
@ -29,46 +34,50 @@ class LanguageFilter(object):
    CJK = CHINESE | JAPANESE | KOREAN


-class ProbingState(object):
+class ProbingState(Enum):
    """
    This enum represents the different states a prober can be in.
    """
+
    DETECTING = 0
    FOUND_IT = 1
    NOT_ME = 2


-class MachineState(object):
+class MachineState:
    """
    This enum represents the different states a state machine can be in.
    """
+
    START = 0
    ERROR = 1
    ITS_ME = 2


-class SequenceLikelihood(object):
+class SequenceLikelihood:
    """
    This enum represents the likelihood of a character following the previous one.
    """
+
    NEGATIVE = 0
    UNLIKELY = 1
    LIKELY = 2
    POSITIVE = 3

    @classmethod
-    def get_num_categories(cls):
+    def get_num_categories(cls) -> int:
        """:returns: The number of likelihood categories in the enum."""
        return 4


-class CharacterCategory(object):
+class CharacterCategory:
    """
    This enum represents the different categories language models for
    ``SingleByteCharsetProber`` put characters into.

    Anything less than CONTROL is considered a letter.
    """
+
    UNDEFINED = 255
    LINE_BREAK = 254
    SYMBOL = 253
--- a/libs/common/chardet/escprober.py
+++ b/libs/common/chardet/escprober.py
@ -25,11 +25,17 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import Optional, Union
+
 from .charsetprober import CharSetProber
 from .codingstatemachine import CodingStateMachine
-from .enums import LanguageFilter, ProbingState, MachineState
-from .escsm import (HZ_SM_MODEL, ISO2022CN_SM_MODEL, ISO2022JP_SM_MODEL,
-                    ISO2022KR_SM_MODEL)
+from .enums import LanguageFilter, MachineState, ProbingState
+from .escsm import (
+    HZ_SM_MODEL,
+    ISO2022CN_SM_MODEL,
+    ISO2022JP_SM_MODEL,
+    ISO2022KR_SM_MODEL,
+)


 class EscCharSetProber(CharSetProber):
@ -39,8 +45,8 @@ class EscCharSetProber(CharSetProber):
    identify these encodings.
    """

-    def __init__(self, lang_filter=None):
-        super(EscCharSetProber, self).__init__(lang_filter=lang_filter)
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
        self.coding_sm = []
        if self.lang_filter & LanguageFilter.CHINESE_SIMPLIFIED:
            self.coding_sm.append(CodingStateMachine(HZ_SM_MODEL))
@ -49,17 +55,15 @@ class EscCharSetProber(CharSetProber):
            self.coding_sm.append(CodingStateMachine(ISO2022JP_SM_MODEL))
        if self.lang_filter & LanguageFilter.KOREAN:
            self.coding_sm.append(CodingStateMachine(ISO2022KR_SM_MODEL))
-        self.active_sm_count = None
-        self._detected_charset = None
-        self._detected_language = None
-        self._state = None
+        self.active_sm_count = 0
+        self._detected_charset: Optional[str] = None
+        self._detected_language: Optional[str] = None
+        self._state = ProbingState.DETECTING
        self.reset()

-    def reset(self):
-        super(EscCharSetProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        for coding_sm in self.coding_sm:
-            if not coding_sm:
-                continue
            coding_sm.active = True
            coding_sm.reset()
        self.active_sm_count = len(self.coding_sm)
@ -67,23 +71,20 @@ class EscCharSetProber(CharSetProber):
        self._detected_language = None

    @property
-    def charset_name(self):
+    def charset_name(self) -> Optional[str]:
        return self._detected_charset

    @property
-    def language(self):
+    def language(self) -> Optional[str]:
        return self._detected_language

-    def get_confidence(self):
-        if self._detected_charset:
-            return 0.99
-        else:
-            return 0.00
+    def get_confidence(self) -> float:
+        return 0.99 if self._detected_charset else 0.00

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        for c in byte_str:
            for coding_sm in self.coding_sm:
-                if not coding_sm or not coding_sm.active:
+                if not coding_sm.active:
                    continue
                coding_state = coding_sm.next_state(c)
                if coding_state == MachineState.ERROR:
--- a/libs/common/chardet/escsm.py
+++ b/libs/common/chardet/escsm.py
@ -12,7 +12,7 @@
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
+# version 2.1 of the License,  or (at your option) any later version.
 #
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -20,227 +20,242 @@
 # Lesser General Public License for more details.
 #
 # You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# License along with this library; if not,  write to the Free Software
+# Foundation,  Inc.,  51 Franklin St,  Fifth Floor,  Boston,  MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from .codingstatemachinedict import CodingStateMachineDict
 from .enums import MachineState

+# fmt: off
 HZ_CLS = (
-1,0,0,0,0,0,0,0,  # 00 - 07
-0,0,0,0,0,0,0,0,  # 08 - 0f
-0,0,0,0,0,0,0,0,  # 10 - 17
-0,0,0,1,0,0,0,0,  # 18 - 1f
-0,0,0,0,0,0,0,0,  # 20 - 27
-0,0,0,0,0,0,0,0,  # 28 - 2f
-0,0,0,0,0,0,0,0,  # 30 - 37
-0,0,0,0,0,0,0,0,  # 38 - 3f
-0,0,0,0,0,0,0,0,  # 40 - 47
-0,0,0,0,0,0,0,0,  # 48 - 4f
-0,0,0,0,0,0,0,0,  # 50 - 57
-0,0,0,0,0,0,0,0,  # 58 - 5f
-0,0,0,0,0,0,0,0,  # 60 - 67
-0,0,0,0,0,0,0,0,  # 68 - 6f
-0,0,0,0,0,0,0,0,  # 70 - 77
-0,0,0,4,0,5,2,0,  # 78 - 7f
-1,1,1,1,1,1,1,1,  # 80 - 87
-1,1,1,1,1,1,1,1,  # 88 - 8f
-1,1,1,1,1,1,1,1,  # 90 - 97
-1,1,1,1,1,1,1,1,  # 98 - 9f
-1,1,1,1,1,1,1,1,  # a0 - a7
-1,1,1,1,1,1,1,1,  # a8 - af
-1,1,1,1,1,1,1,1,  # b0 - b7
-1,1,1,1,1,1,1,1,  # b8 - bf
-1,1,1,1,1,1,1,1,  # c0 - c7
-1,1,1,1,1,1,1,1,  # c8 - cf
-1,1,1,1,1,1,1,1,  # d0 - d7
-1,1,1,1,1,1,1,1,  # d8 - df
-1,1,1,1,1,1,1,1,  # e0 - e7
-1,1,1,1,1,1,1,1,  # e8 - ef
-1,1,1,1,1,1,1,1,  # f0 - f7
-1,1,1,1,1,1,1,1,  # f8 - ff
+    1, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 0, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 4, 0, 5, 2, 0,  # 78 - 7f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 80 - 87
+    1, 1, 1, 1, 1, 1, 1, 1,  # 88 - 8f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 90 - 97
+    1, 1, 1, 1, 1, 1, 1, 1,  # 98 - 9f
+    1, 1, 1, 1, 1, 1, 1, 1,  # a0 - a7
+    1, 1, 1, 1, 1, 1, 1, 1,  # a8 - af
+    1, 1, 1, 1, 1, 1, 1, 1,  # b0 - b7
+    1, 1, 1, 1, 1, 1, 1, 1,  # b8 - bf
+    1, 1, 1, 1, 1, 1, 1, 1,  # c0 - c7
+    1, 1, 1, 1, 1, 1, 1, 1,  # c8 - cf
+    1, 1, 1, 1, 1, 1, 1, 1,  # d0 - d7
+    1, 1, 1, 1, 1, 1, 1, 1,  # d8 - df
+    1, 1, 1, 1, 1, 1, 1, 1,  # e0 - e7
+    1, 1, 1, 1, 1, 1, 1, 1,  # e8 - ef
+    1, 1, 1, 1, 1, 1, 1, 1,  # f0 - f7
+    1, 1, 1, 1, 1, 1, 1, 1,  # f8 - ff
 )

 HZ_ST = (
-MachineState.START,MachineState.ERROR,     3,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,# 00-07
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 08-0f
-MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,     4,MachineState.ERROR,# 10-17
-     5,MachineState.ERROR,     6,MachineState.ERROR,     5,     5,     4,MachineState.ERROR,# 18-1f
-     4,MachineState.ERROR,     4,     4,     4,MachineState.ERROR,     4,MachineState.ERROR,# 20-27
-     4,MachineState.ITS_ME,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 28-2f
+MachineState.START, MachineState.ERROR,      3, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
+MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
+MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.START, MachineState.START,      4, MachineState.ERROR, # 10-17
+     5, MachineState.ERROR,      6, MachineState.ERROR,      5,      5,      4, MachineState.ERROR, # 18-1f
+     4, MachineState.ERROR,      4,      4,      4, MachineState.ERROR,      4, MachineState.ERROR, # 20-27
+     4, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 28-2f
 )
+# fmt: on

 HZ_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)

-HZ_SM_MODEL = {'class_table': HZ_CLS,
-               'class_factor': 6,
-               'state_table': HZ_ST,
-               'char_len_table': HZ_CHAR_LEN_TABLE,
-               'name': "HZ-GB-2312",
-               'language': 'Chinese'}
+HZ_SM_MODEL: CodingStateMachineDict = {
+    "class_table": HZ_CLS,
+    "class_factor": 6,
+    "state_table": HZ_ST,
+    "char_len_table": HZ_CHAR_LEN_TABLE,
+    "name": "HZ-GB-2312",
+    "language": "Chinese",
+}

+# fmt: off
 ISO2022CN_CLS = (
-2,0,0,0,0,0,0,0,  # 00 - 07
-0,0,0,0,0,0,0,0,  # 08 - 0f
-0,0,0,0,0,0,0,0,  # 10 - 17
-0,0,0,1,0,0,0,0,  # 18 - 1f
-0,0,0,0,0,0,0,0,  # 20 - 27
-0,3,0,0,0,0,0,0,  # 28 - 2f
-0,0,0,0,0,0,0,0,  # 30 - 37
-0,0,0,0,0,0,0,0,  # 38 - 3f
-0,0,0,4,0,0,0,0,  # 40 - 47
-0,0,0,0,0,0,0,0,  # 48 - 4f
-0,0,0,0,0,0,0,0,  # 50 - 57
-0,0,0,0,0,0,0,0,  # 58 - 5f
-0,0,0,0,0,0,0,0,  # 60 - 67
-0,0,0,0,0,0,0,0,  # 68 - 6f
-0,0,0,0,0,0,0,0,  # 70 - 77
-0,0,0,0,0,0,0,0,  # 78 - 7f
-2,2,2,2,2,2,2,2,  # 80 - 87
-2,2,2,2,2,2,2,2,  # 88 - 8f
-2,2,2,2,2,2,2,2,  # 90 - 97
-2,2,2,2,2,2,2,2,  # 98 - 9f
-2,2,2,2,2,2,2,2,  # a0 - a7
-2,2,2,2,2,2,2,2,  # a8 - af
-2,2,2,2,2,2,2,2,  # b0 - b7
-2,2,2,2,2,2,2,2,  # b8 - bf
-2,2,2,2,2,2,2,2,  # c0 - c7
-2,2,2,2,2,2,2,2,  # c8 - cf
-2,2,2,2,2,2,2,2,  # d0 - d7
-2,2,2,2,2,2,2,2,  # d8 - df
-2,2,2,2,2,2,2,2,  # e0 - e7
-2,2,2,2,2,2,2,2,  # e8 - ef
-2,2,2,2,2,2,2,2,  # f0 - f7
-2,2,2,2,2,2,2,2,  # f8 - ff
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 4, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
 )

 ISO2022CN_ST = (
-MachineState.START,     3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 00-07
-MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 08-0f
-MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 10-17
-MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     4,MachineState.ERROR,# 18-1f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 20-27
-     5,     6,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 28-2f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 30-37
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.START,# 38-3f
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
+    MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, # 18-1f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 20-27
+        5,      6, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 28-2f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 30-37
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, # 38-3f
 )
+# fmt: on

 ISO2022CN_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0)

-ISO2022CN_SM_MODEL = {'class_table': ISO2022CN_CLS,
-                      'class_factor': 9,
-                      'state_table': ISO2022CN_ST,
-                      'char_len_table': ISO2022CN_CHAR_LEN_TABLE,
-                      'name': "ISO-2022-CN",
-                      'language': 'Chinese'}
+ISO2022CN_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022CN_CLS,
+    "class_factor": 9,
+    "state_table": ISO2022CN_ST,
+    "char_len_table": ISO2022CN_CHAR_LEN_TABLE,
+    "name": "ISO-2022-CN",
+    "language": "Chinese",
+}

+# fmt: off
 ISO2022JP_CLS = (
-2,0,0,0,0,0,0,0,  # 00 - 07
-0,0,0,0,0,0,2,2,  # 08 - 0f
-0,0,0,0,0,0,0,0,  # 10 - 17
-0,0,0,1,0,0,0,0,  # 18 - 1f
-0,0,0,0,7,0,0,0,  # 20 - 27
-3,0,0,0,0,0,0,0,  # 28 - 2f
-0,0,0,0,0,0,0,0,  # 30 - 37
-0,0,0,0,0,0,0,0,  # 38 - 3f
-6,0,4,0,8,0,0,0,  # 40 - 47
-0,9,5,0,0,0,0,0,  # 48 - 4f
-0,0,0,0,0,0,0,0,  # 50 - 57
-0,0,0,0,0,0,0,0,  # 58 - 5f
-0,0,0,0,0,0,0,0,  # 60 - 67
-0,0,0,0,0,0,0,0,  # 68 - 6f
-0,0,0,0,0,0,0,0,  # 70 - 77
-0,0,0,0,0,0,0,0,  # 78 - 7f
-2,2,2,2,2,2,2,2,  # 80 - 87
-2,2,2,2,2,2,2,2,  # 88 - 8f
-2,2,2,2,2,2,2,2,  # 90 - 97
-2,2,2,2,2,2,2,2,  # 98 - 9f
-2,2,2,2,2,2,2,2,  # a0 - a7
-2,2,2,2,2,2,2,2,  # a8 - af
-2,2,2,2,2,2,2,2,  # b0 - b7
-2,2,2,2,2,2,2,2,  # b8 - bf
-2,2,2,2,2,2,2,2,  # c0 - c7
-2,2,2,2,2,2,2,2,  # c8 - cf
-2,2,2,2,2,2,2,2,  # d0 - d7
-2,2,2,2,2,2,2,2,  # d8 - df
-2,2,2,2,2,2,2,2,  # e0 - e7
-2,2,2,2,2,2,2,2,  # e8 - ef
-2,2,2,2,2,2,2,2,  # f0 - f7
-2,2,2,2,2,2,2,2,  # f8 - ff
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 2, 2,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 7, 0, 0, 0,  # 20 - 27
+    3, 0, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    6, 0, 4, 0, 8, 0, 0, 0,  # 40 - 47
+    0, 9, 5, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
 )

 ISO2022JP_ST = (
-MachineState.START,     3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 00-07
-MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 08-0f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 10-17
-MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,# 18-1f
-MachineState.ERROR,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     4,MachineState.ERROR,MachineState.ERROR,# 20-27
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     6,MachineState.ITS_ME,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,# 28-2f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,# 30-37
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 38-3f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,MachineState.START,MachineState.START,# 40-47
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 00-07
+    MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 08-0f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 10-17
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, # 18-1f
+    MachineState.ERROR,      5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, MachineState.ERROR, # 20-27
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      6, MachineState.ITS_ME, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, # 28-2f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, # 30-37
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 38-3f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ERROR, MachineState.START, MachineState.START, # 40-47
 )
+# fmt: on

 ISO2022JP_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)

-ISO2022JP_SM_MODEL = {'class_table': ISO2022JP_CLS,
-                      'class_factor': 10,
-                      'state_table': ISO2022JP_ST,
-                      'char_len_table': ISO2022JP_CHAR_LEN_TABLE,
-                      'name': "ISO-2022-JP",
-                      'language': 'Japanese'}
+ISO2022JP_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022JP_CLS,
+    "class_factor": 10,
+    "state_table": ISO2022JP_ST,
+    "char_len_table": ISO2022JP_CHAR_LEN_TABLE,
+    "name": "ISO-2022-JP",
+    "language": "Japanese",
+}

+# fmt: off
 ISO2022KR_CLS = (
-2,0,0,0,0,0,0,0,  # 00 - 07
-0,0,0,0,0,0,0,0,  # 08 - 0f
-0,0,0,0,0,0,0,0,  # 10 - 17
-0,0,0,1,0,0,0,0,  # 18 - 1f
-0,0,0,0,3,0,0,0,  # 20 - 27
-0,4,0,0,0,0,0,0,  # 28 - 2f
-0,0,0,0,0,0,0,0,  # 30 - 37
-0,0,0,0,0,0,0,0,  # 38 - 3f
-0,0,0,5,0,0,0,0,  # 40 - 47
-0,0,0,0,0,0,0,0,  # 48 - 4f
-0,0,0,0,0,0,0,0,  # 50 - 57
-0,0,0,0,0,0,0,0,  # 58 - 5f
-0,0,0,0,0,0,0,0,  # 60 - 67
-0,0,0,0,0,0,0,0,  # 68 - 6f
-0,0,0,0,0,0,0,0,  # 70 - 77
-0,0,0,0,0,0,0,0,  # 78 - 7f
-2,2,2,2,2,2,2,2,  # 80 - 87
-2,2,2,2,2,2,2,2,  # 88 - 8f
-2,2,2,2,2,2,2,2,  # 90 - 97
-2,2,2,2,2,2,2,2,  # 98 - 9f
-2,2,2,2,2,2,2,2,  # a0 - a7
-2,2,2,2,2,2,2,2,  # a8 - af
-2,2,2,2,2,2,2,2,  # b0 - b7
-2,2,2,2,2,2,2,2,  # b8 - bf
-2,2,2,2,2,2,2,2,  # c0 - c7
-2,2,2,2,2,2,2,2,  # c8 - cf
-2,2,2,2,2,2,2,2,  # d0 - d7
-2,2,2,2,2,2,2,2,  # d8 - df
-2,2,2,2,2,2,2,2,  # e0 - e7
-2,2,2,2,2,2,2,2,  # e8 - ef
-2,2,2,2,2,2,2,2,  # f0 - f7
-2,2,2,2,2,2,2,2,  # f8 - ff
+    2, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 0, 0, 0, 0, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 1, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 3, 0, 0, 0,  # 20 - 27
+    0, 4, 0, 0, 0, 0, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 5, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 80 - 87
+    2, 2, 2, 2, 2, 2, 2, 2,  # 88 - 8f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 90 - 97
+    2, 2, 2, 2, 2, 2, 2, 2,  # 98 - 9f
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 2,  # f8 - ff
 )

 ISO2022KR_ST = (
-MachineState.START,     3,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,# 00-07
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,# 08-0f
-MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     4,MachineState.ERROR,MachineState.ERROR,# 10-17
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,# 18-1f
-MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.START,MachineState.START,MachineState.START,MachineState.START,# 20-27
+    MachineState.START,      3, MachineState.ERROR, MachineState.START, MachineState.START, MachineState.START, MachineState.ERROR, MachineState.ERROR, # 00-07
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ITS_ME, # 08-0f
+    MachineState.ITS_ME, MachineState.ITS_ME, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      4, MachineState.ERROR, MachineState.ERROR, # 10-17
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR,      5, MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, # 18-1f
+    MachineState.ERROR, MachineState.ERROR, MachineState.ERROR, MachineState.ITS_ME, MachineState.START, MachineState.START, MachineState.START, MachineState.START, # 20-27
 )
+# fmt: on

 ISO2022KR_CHAR_LEN_TABLE = (0, 0, 0, 0, 0, 0)

-ISO2022KR_SM_MODEL = {'class_table': ISO2022KR_CLS,
-                      'class_factor': 6,
-                      'state_table': ISO2022KR_ST,
-                      'char_len_table': ISO2022KR_CHAR_LEN_TABLE,
-                      'name': "ISO-2022-KR",
-                      'language': 'Korean'}
-
-
+ISO2022KR_SM_MODEL: CodingStateMachineDict = {
+    "class_table": ISO2022KR_CLS,
+    "class_factor": 6,
+    "state_table": ISO2022KR_ST,
+    "char_len_table": ISO2022KR_CHAR_LEN_TABLE,
+    "name": "ISO-2022-KR",
+    "language": "Korean",
+}
--- a/libs/common/chardet/eucjpprober.py
+++ b/libs/common/chardet/eucjpprober.py
@ -25,68 +25,78 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .enums import ProbingState, MachineState
-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
+from typing import Union
+
 from .chardistribution import EUCJPDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
 from .jpcntx import EUCJPContextAnalysis
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import EUCJP_SM_MODEL


 class EUCJPProber(MultiByteCharSetProber):
-    def __init__(self):
-        super(EUCJPProber, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(EUCJP_SM_MODEL)
        self.distribution_analyzer = EUCJPDistributionAnalysis()
        self.context_analyzer = EUCJPContextAnalysis()
        self.reset()

-    def reset(self):
-        super(EUCJPProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        self.context_analyzer.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "EUC-JP"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Japanese"

-    def feed(self, byte_str):
-        for i in range(len(byte_str)):
-            # PY3K: byte_str is a byte array, so byte_str[i] is an int, not a byte
-            coding_state = self.coding_sm.next_state(byte_str[i])
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None
+
+        for i, byte in enumerate(byte_str):
+            # PY3K: byte_str is a byte array, so byte is an int, not a byte
+            coding_state = self.coding_sm.next_state(byte)
            if coding_state == MachineState.ERROR:
-                self.logger.debug('%s %s prober hit error at byte %s',
-                                  self.charset_name, self.language, i)
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
                self._state = ProbingState.NOT_ME
                break
-            elif coding_state == MachineState.ITS_ME:
+            if coding_state == MachineState.ITS_ME:
                self._state = ProbingState.FOUND_IT
                break
-            elif coding_state == MachineState.START:
+            if coding_state == MachineState.START:
                char_len = self.coding_sm.get_current_charlen()
                if i == 0:
-                    self._last_char[1] = byte_str[0]
+                    self._last_char[1] = byte
                    self.context_analyzer.feed(self._last_char, char_len)
                    self.distribution_analyzer.feed(self._last_char, char_len)
                else:
-                    self.context_analyzer.feed(byte_str[i - 1:i + 1],
-                                                char_len)
-                    self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
-                                                     char_len)
+                    self.context_analyzer.feed(byte_str[i - 1 : i + 1], char_len)
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)

        self._last_char[0] = byte_str[-1]

        if self.state == ProbingState.DETECTING:
-            if (self.context_analyzer.got_enough_data() and
-               (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
+            if self.context_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
                self._state = ProbingState.FOUND_IT

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
+
        context_conf = self.context_analyzer.get_confidence()
        distrib_conf = self.distribution_analyzer.get_confidence()
        return max(context_conf, distrib_conf)
--- a/libs/common/chardet/euckrfreq.py
+++ b/libs/common/chardet/euckrfreq.py
@ -43,6 +43,7 @@ EUCKR_TYPICAL_DISTRIBUTION_RATIO = 6.0
 EUCKR_TABLE_SIZE = 2352

 # Char to FreqOrder table ,
+# fmt: off
 EUCKR_CHAR_TO_FREQ_ORDER = (
  13, 130, 120,1396, 481,1719,1720, 328, 609, 212,1721, 707, 400, 299,1722,  87,
 1397,1723, 104, 536,1117,1203,1724,1267, 685,1268, 508,1725,1726,1727,1728,1398,
@ -192,4 +193,4 @@ EUCKR_CHAR_TO_FREQ_ORDER = (
 2629,2630,2631, 924, 648, 863, 603,2632,2633, 934,1540, 864, 865,2634, 642,1042,
 670,1190,2635,2636,2637,2638, 168,2639, 652, 873, 542,1054,1541,2640,2641,2642,  # 512, 256
 )
-
+# fmt: on
--- a/libs/common/chardet/euckrprober.py
+++ b/libs/common/chardet/euckrprober.py
@ -25,23 +25,23 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
 from .chardistribution import EUCKRDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import EUCKR_SM_MODEL


 class EUCKRProber(MultiByteCharSetProber):
-    def __init__(self):
-        super(EUCKRProber, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(EUCKR_SM_MODEL)
        self.distribution_analyzer = EUCKRDistributionAnalysis()
        self.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "EUC-KR"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Korean"
--- a/libs/common/chardet/euctwfreq.py
+++ b/libs/common/chardet/euctwfreq.py
@ -43,345 +43,346 @@

 EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75

-# Char to FreqOrder table ,
+# Char to FreqOrder table
 EUCTW_TABLE_SIZE = 5376

+# fmt: off
 EUCTW_CHAR_TO_FREQ_ORDER = (
-   1,1800,1506, 255,1431, 198,   9,  82,   6,7310, 177, 202,3615,1256,2808, 110,  # 2742
-3735,  33,3241, 261,  76,  44,2113,  16,2931,2184,1176, 659,3868,  26,3404,2643,  # 2758
-1198,3869,3313,4060, 410,2211, 302, 590, 361,1963,   8, 204,  58,4296,7311,1931,  # 2774
-  63,7312,7313, 317,1614,  75, 222, 159,4061,2412,1480,7314,3500,3068, 224,2809,  # 2790
-3616,   3,  10,3870,1471,  29,2774,1135,2852,1939, 873, 130,3242,1123, 312,7315,  # 2806
-4297,2051, 507, 252, 682,7316, 142,1914, 124, 206,2932,  34,3501,3173,  64, 604,  # 2822
-7317,2494,1976,1977, 155,1990, 645, 641,1606,7318,3405, 337,  72, 406,7319,  80,  # 2838
- 630, 238,3174,1509, 263, 939,1092,2644, 756,1440,1094,3406, 449,  69,2969, 591,  # 2854
- 179,2095, 471, 115,2034,1843,  60,  50,2970, 134, 806,1868, 734,2035,3407, 180,  # 2870
- 995,1607, 156, 537,2893, 688,7320, 319,1305, 779,2144, 514,2374, 298,4298, 359,  # 2886
-2495,  90,2707,1338, 663,  11, 906,1099,2545,  20,2436, 182, 532,1716,7321, 732,  # 2902
-1376,4062,1311,1420,3175,  25,2312,1056, 113, 399, 382,1949, 242,3408,2467, 529,  # 2918
-3243, 475,1447,3617,7322, 117,  21, 656, 810,1297,2295,2329,3502,7323, 126,4063,  # 2934
- 706, 456, 150, 613,4299,  71,1118,2036,4064, 145,3069,  85, 835, 486,2114,1246,  # 2950
-1426, 428, 727,1285,1015, 800, 106, 623, 303,1281,7324,2127,2354, 347,3736, 221,  # 2966
-3503,3110,7325,1955,1153,4065,  83, 296,1199,3070, 192, 624,  93,7326, 822,1897,  # 2982
-2810,3111, 795,2064, 991,1554,1542,1592,  27,  43,2853, 859, 139,1456, 860,4300,  # 2998
- 437, 712,3871, 164,2392,3112, 695, 211,3017,2096, 195,3872,1608,3504,3505,3618,  # 3014
-3873, 234, 811,2971,2097,3874,2229,1441,3506,1615,2375, 668,2076,1638, 305, 228,  # 3030
-1664,4301, 467, 415,7327, 262,2098,1593, 239, 108, 300, 200,1033, 512,1247,2077,  # 3046
-7328,7329,2173,3176,3619,2673, 593, 845,1062,3244,  88,1723,2037,3875,1950, 212,  # 3062
- 266, 152, 149, 468,1898,4066,4302,  77, 187,7330,3018,  37,   5,2972,7331,3876,  # 3078
-7332,7333,  39,2517,4303,2894,3177,2078,  55, 148,  74,4304, 545, 483,1474,1029,  # 3094
-1665, 217,1869,1531,3113,1104,2645,4067,  24, 172,3507, 900,3877,3508,3509,4305,  # 3110
-  32,1408,2811,1312, 329, 487,2355,2247,2708, 784,2674,   4,3019,3314,1427,1788,  # 3126
- 188, 109, 499,7334,3620,1717,1789, 888,1217,3020,4306,7335,3510,7336,3315,1520,  # 3142
-3621,3878, 196,1034, 775,7337,7338, 929,1815, 249, 439,  38,7339,1063,7340, 794,  # 3158
-3879,1435,2296,  46, 178,3245,2065,7341,2376,7342, 214,1709,4307, 804,  35, 707,  # 3174
- 324,3622,1601,2546, 140, 459,4068,7343,7344,1365, 839, 272, 978,2257,2572,3409,  # 3190
-2128,1363,3623,1423, 697, 100,3071,  48,  70,1231, 495,3114,2193,7345,1294,7346,  # 3206
-2079, 462, 586,1042,3246, 853, 256, 988, 185,2377,3410,1698, 434,1084,7347,3411,  # 3222
- 314,2615,2775,4308,2330,2331, 569,2280, 637,1816,2518, 757,1162,1878,1616,3412,  # 3238
- 287,1577,2115, 768,4309,1671,2854,3511,2519,1321,3737, 909,2413,7348,4069, 933,  # 3254
-3738,7349,2052,2356,1222,4310, 765,2414,1322, 786,4311,7350,1919,1462,1677,2895,  # 3270
-1699,7351,4312,1424,2437,3115,3624,2590,3316,1774,1940,3413,3880,4070, 309,1369,  # 3286
-1130,2812, 364,2230,1653,1299,3881,3512,3882,3883,2646, 525,1085,3021, 902,2000,  # 3302
-1475, 964,4313, 421,1844,1415,1057,2281, 940,1364,3116, 376,4314,4315,1381,   7,  # 3318
-2520, 983,2378, 336,1710,2675,1845, 321,3414, 559,1131,3022,2742,1808,1132,1313,  # 3334
- 265,1481,1857,7352, 352,1203,2813,3247, 167,1089, 420,2814, 776, 792,1724,3513,  # 3350
-4071,2438,3248,7353,4072,7354, 446, 229, 333,2743, 901,3739,1200,1557,4316,2647,  # 3366
-1920, 395,2744,2676,3740,4073,1835, 125, 916,3178,2616,4317,7355,7356,3741,7357,  # 3382
-7358,7359,4318,3117,3625,1133,2547,1757,3415,1510,2313,1409,3514,7360,2145, 438,  # 3398
-2591,2896,2379,3317,1068, 958,3023, 461, 311,2855,2677,4074,1915,3179,4075,1978,  # 3414
- 383, 750,2745,2617,4076, 274, 539, 385,1278,1442,7361,1154,1964, 384, 561, 210,  # 3430
-  98,1295,2548,3515,7362,1711,2415,1482,3416,3884,2897,1257, 129,7363,3742, 642,  # 3446
- 523,2776,2777,2648,7364, 141,2231,1333,  68, 176, 441, 876, 907,4077, 603,2592,  # 3462
- 710, 171,3417, 404, 549,  18,3118,2393,1410,3626,1666,7365,3516,4319,2898,4320,  # 3478
-7366,2973, 368,7367, 146, 366,  99, 871,3627,1543, 748, 807,1586,1185,  22,2258,  # 3494
- 379,3743,3180,7368,3181, 505,1941,2618,1991,1382,2314,7369, 380,2357, 218, 702,  # 3510
-1817,1248,3418,3024,3517,3318,3249,7370,2974,3628, 930,3250,3744,7371,  59,7372,  # 3526
- 585, 601,4078, 497,3419,1112,1314,4321,1801,7373,1223,1472,2174,7374, 749,1836,  # 3542
- 690,1899,3745,1772,3885,1476, 429,1043,1790,2232,2116, 917,4079, 447,1086,1629,  # 3558
-7375, 556,7376,7377,2020,1654, 844,1090, 105, 550, 966,1758,2815,1008,1782, 686,  # 3574
-1095,7378,2282, 793,1602,7379,3518,2593,4322,4080,2933,2297,4323,3746, 980,2496,  # 3590
- 544, 353, 527,4324, 908,2678,2899,7380, 381,2619,1942,1348,7381,1341,1252, 560,  # 3606
-3072,7382,3420,2856,7383,2053, 973, 886,2080, 143,4325,7384,7385, 157,3886, 496,  # 3622
-4081,  57, 840, 540,2038,4326,4327,3421,2117,1445, 970,2259,1748,1965,2081,4082,  # 3638
-3119,1234,1775,3251,2816,3629, 773,1206,2129,1066,2039,1326,3887,1738,1725,4083,  # 3654
- 279,3120,  51,1544,2594, 423,1578,2130,2066, 173,4328,1879,7386,7387,1583, 264,  # 3670
- 610,3630,4329,2439, 280, 154,7388,7389,7390,1739, 338,1282,3073, 693,2857,1411,  # 3686
-1074,3747,2440,7391,4330,7392,7393,1240, 952,2394,7394,2900,1538,2679, 685,1483,  # 3702
-4084,2468,1436, 953,4085,2054,4331, 671,2395,  79,4086,2441,3252, 608, 567,2680,  # 3718
-3422,4087,4088,1691, 393,1261,1791,2396,7395,4332,7396,7397,7398,7399,1383,1672,  # 3734
-3748,3182,1464, 522,1119, 661,1150, 216, 675,4333,3888,1432,3519, 609,4334,2681,  # 3750
-2397,7400,7401,7402,4089,3025,   0,7403,2469, 315, 231,2442, 301,3319,4335,2380,  # 3766
-7404, 233,4090,3631,1818,4336,4337,7405,  96,1776,1315,2082,7406, 257,7407,1809,  # 3782
-3632,2709,1139,1819,4091,2021,1124,2163,2778,1777,2649,7408,3074, 363,1655,3183,  # 3798
-7409,2975,7410,7411,7412,3889,1567,3890, 718, 103,3184, 849,1443, 341,3320,2934,  # 3814
-1484,7413,1712, 127,  67, 339,4092,2398, 679,1412, 821,7414,7415, 834, 738, 351,  # 3830
-2976,2146, 846, 235,1497,1880, 418,1992,3749,2710, 186,1100,2147,2746,3520,1545,  # 3846
-1355,2935,2858,1377, 583,3891,4093,2573,2977,7416,1298,3633,1078,2549,3634,2358,  # 3862
-  78,3750,3751, 267,1289,2099,2001,1594,4094, 348, 369,1274,2194,2175,1837,4338,  # 3878
-1820,2817,3635,2747,2283,2002,4339,2936,2748, 144,3321, 882,4340,3892,2749,3423,  # 3894
-4341,2901,7417,4095,1726, 320,7418,3893,3026, 788,2978,7419,2818,1773,1327,2859,  # 3910
-3894,2819,7420,1306,4342,2003,1700,3752,3521,2359,2650, 787,2022, 506, 824,3636,  # 3926
- 534, 323,4343,1044,3322,2023,1900, 946,3424,7421,1778,1500,1678,7422,1881,4344,  # 3942
- 165, 243,4345,3637,2521, 123, 683,4096, 764,4346,  36,3895,1792, 589,2902, 816,  # 3958
- 626,1667,3027,2233,1639,1555,1622,3753,3896,7423,3897,2860,1370,1228,1932, 891,  # 3974
-2083,2903, 304,4097,7424, 292,2979,2711,3522, 691,2100,4098,1115,4347, 118, 662,  # 3990
-7425, 611,1156, 854,2381,1316,2861,   2, 386, 515,2904,7426,7427,3253, 868,2234,  # 4006
-1486, 855,2651, 785,2212,3028,7428,1040,3185,3523,7429,3121, 448,7430,1525,7431,  # 4022
-2164,4348,7432,3754,7433,4099,2820,3524,3122, 503, 818,3898,3123,1568, 814, 676,  # 4038
-1444, 306,1749,7434,3755,1416,1030, 197,1428, 805,2821,1501,4349,7435,7436,7437,  # 4054
-1993,7438,4350,7439,7440,2195,  13,2779,3638,2980,3124,1229,1916,7441,3756,2131,  # 4070
-7442,4100,4351,2399,3525,7443,2213,1511,1727,1120,7444,7445, 646,3757,2443, 307,  # 4086
-7446,7447,1595,3186,7448,7449,7450,3639,1113,1356,3899,1465,2522,2523,7451, 519,  # 4102
-7452, 128,2132,  92,2284,1979,7453,3900,1512, 342,3125,2196,7454,2780,2214,1980,  # 4118
-3323,7455, 290,1656,1317, 789, 827,2360,7456,3758,4352, 562, 581,3901,7457, 401,  # 4134
-4353,2248,  94,4354,1399,2781,7458,1463,2024,4355,3187,1943,7459, 828,1105,4101,  # 4150
-1262,1394,7460,4102, 605,4356,7461,1783,2862,7462,2822, 819,2101, 578,2197,2937,  # 4166
-7463,1502, 436,3254,4103,3255,2823,3902,2905,3425,3426,7464,2712,2315,7465,7466,  # 4182
-2332,2067,  23,4357, 193, 826,3759,2102, 699,1630,4104,3075, 390,1793,1064,3526,  # 4198
-7467,1579,3076,3077,1400,7468,4105,1838,1640,2863,7469,4358,4359, 137,4106, 598,  # 4214
-3078,1966, 780, 104, 974,2938,7470, 278, 899, 253, 402, 572, 504, 493,1339,7471,  # 4230
-3903,1275,4360,2574,2550,7472,3640,3029,3079,2249, 565,1334,2713, 863,  41,7473,  # 4246
-7474,4361,7475,1657,2333,  19, 463,2750,4107, 606,7476,2981,3256,1087,2084,1323,  # 4262
-2652,2982,7477,1631,1623,1750,4108,2682,7478,2864, 791,2714,2653,2334, 232,2416,  # 4278
-7479,2983,1498,7480,2654,2620, 755,1366,3641,3257,3126,2025,1609, 119,1917,3427,  # 4294
- 862,1026,4109,7481,3904,3760,4362,3905,4363,2260,1951,2470,7482,1125, 817,4110,  # 4310
-4111,3906,1513,1766,2040,1487,4112,3030,3258,2824,3761,3127,7483,7484,1507,7485,  # 4326
-2683, 733,  40,1632,1106,2865, 345,4113, 841,2524, 230,4364,2984,1846,3259,3428,  # 4342
-7486,1263, 986,3429,7487, 735, 879, 254,1137, 857, 622,1300,1180,1388,1562,3907,  # 4358
-3908,2939, 967,2751,2655,1349, 592,2133,1692,3324,2985,1994,4114,1679,3909,1901,  # 4374
-2185,7488, 739,3642,2715,1296,1290,7489,4115,2198,2199,1921,1563,2595,2551,1870,  # 4390
-2752,2986,7490, 435,7491, 343,1108, 596,  17,1751,4365,2235,3430,3643,7492,4366,  # 4406
- 294,3527,2940,1693, 477, 979, 281,2041,3528, 643,2042,3644,2621,2782,2261,1031,  # 4422
-2335,2134,2298,3529,4367, 367,1249,2552,7493,3530,7494,4368,1283,3325,2004, 240,  # 4438
-1762,3326,4369,4370, 836,1069,3128, 474,7495,2148,2525, 268,3531,7496,3188,1521,  # 4454
-1284,7497,1658,1546,4116,7498,3532,3533,7499,4117,3327,2684,1685,4118, 961,1673,  # 4470
-2622, 190,2005,2200,3762,4371,4372,7500, 570,2497,3645,1490,7501,4373,2623,3260,  # 4486
-1956,4374, 584,1514, 396,1045,1944,7502,4375,1967,2444,7503,7504,4376,3910, 619,  # 4502
-7505,3129,3261, 215,2006,2783,2553,3189,4377,3190,4378, 763,4119,3763,4379,7506,  # 4518
-7507,1957,1767,2941,3328,3646,1174, 452,1477,4380,3329,3130,7508,2825,1253,2382,  # 4534
-2186,1091,2285,4120, 492,7509, 638,1169,1824,2135,1752,3911, 648, 926,1021,1324,  # 4550
-4381, 520,4382, 997, 847,1007, 892,4383,3764,2262,1871,3647,7510,2400,1784,4384,  # 4566
-1952,2942,3080,3191,1728,4121,2043,3648,4385,2007,1701,3131,1551,  30,2263,4122,  # 4582
-7511,2026,4386,3534,7512, 501,7513,4123, 594,3431,2165,1821,3535,3432,3536,3192,  # 4598
- 829,2826,4124,7514,1680,3132,1225,4125,7515,3262,4387,4126,3133,2336,7516,4388,  # 4614
-4127,7517,3912,3913,7518,1847,2383,2596,3330,7519,4389, 374,3914, 652,4128,4129,  # 4630
- 375,1140, 798,7520,7521,7522,2361,4390,2264, 546,1659, 138,3031,2445,4391,7523,  # 4646
-2250, 612,1848, 910, 796,3765,1740,1371, 825,3766,3767,7524,2906,2554,7525, 692,  # 4662
- 444,3032,2624, 801,4392,4130,7526,1491, 244,1053,3033,4131,4132, 340,7527,3915,  # 4678
-1041,2987, 293,1168,  87,1357,7528,1539, 959,7529,2236, 721, 694,4133,3768, 219,  # 4694
-1478, 644,1417,3331,2656,1413,1401,1335,1389,3916,7530,7531,2988,2362,3134,1825,  # 4710
- 730,1515, 184,2827,  66,4393,7532,1660,2943, 246,3332, 378,1457, 226,3433, 975,  # 4726
-3917,2944,1264,3537, 674, 696,7533, 163,7534,1141,2417,2166, 713,3538,3333,4394,  # 4742
-3918,7535,7536,1186,  15,7537,1079,1070,7538,1522,3193,3539, 276,1050,2716, 758,  # 4758
-1126, 653,2945,3263,7539,2337, 889,3540,3919,3081,2989, 903,1250,4395,3920,3434,  # 4774
-3541,1342,1681,1718, 766,3264, 286,  89,2946,3649,7540,1713,7541,2597,3334,2990,  # 4790
-7542,2947,2215,3194,2866,7543,4396,2498,2526, 181, 387,1075,3921, 731,2187,3335,  # 4806
-7544,3265, 310, 313,3435,2299, 770,4134,  54,3034, 189,4397,3082,3769,3922,7545,  # 4822
-1230,1617,1849, 355,3542,4135,4398,3336, 111,4136,3650,1350,3135,3436,3035,4137,  # 4838
-2149,3266,3543,7546,2784,3923,3924,2991, 722,2008,7547,1071, 247,1207,2338,2471,  # 4854
-1378,4399,2009, 864,1437,1214,4400, 373,3770,1142,2216, 667,4401, 442,2753,2555,  # 4870
-3771,3925,1968,4138,3267,1839, 837, 170,1107, 934,1336,1882,7548,7549,2118,4139,  # 4886
-2828, 743,1569,7550,4402,4140, 582,2384,1418,3437,7551,1802,7552, 357,1395,1729,  # 4902
-3651,3268,2418,1564,2237,7553,3083,3772,1633,4403,1114,2085,4141,1532,7554, 482,  # 4918
-2446,4404,7555,7556,1492, 833,1466,7557,2717,3544,1641,2829,7558,1526,1272,3652,  # 4934
-4142,1686,1794, 416,2556,1902,1953,1803,7559,3773,2785,3774,1159,2316,7560,2867,  # 4950
-4405,1610,1584,3036,2419,2754, 443,3269,1163,3136,7561,7562,3926,7563,4143,2499,  # 4966
-3037,4406,3927,3137,2103,1647,3545,2010,1872,4144,7564,4145, 431,3438,7565, 250,  # 4982
-  97,  81,4146,7566,1648,1850,1558, 160, 848,7567, 866, 740,1694,7568,2201,2830,  # 4998
-3195,4147,4407,3653,1687, 950,2472, 426, 469,3196,3654,3655,3928,7569,7570,1188,  # 5014
- 424,1995, 861,3546,4148,3775,2202,2685, 168,1235,3547,4149,7571,2086,1674,4408,  # 5030
-3337,3270, 220,2557,1009,7572,3776, 670,2992, 332,1208, 717,7573,7574,3548,2447,  # 5046
-3929,3338,7575, 513,7576,1209,2868,3339,3138,4409,1080,7577,7578,7579,7580,2527,  # 5062
-3656,3549, 815,1587,3930,3931,7581,3550,3439,3777,1254,4410,1328,3038,1390,3932,  # 5078
-1741,3933,3778,3934,7582, 236,3779,2448,3271,7583,7584,3657,3780,1273,3781,4411,  # 5094
-7585, 308,7586,4412, 245,4413,1851,2473,1307,2575, 430, 715,2136,2449,7587, 270,  # 5110
- 199,2869,3935,7588,3551,2718,1753, 761,1754, 725,1661,1840,4414,3440,3658,7589,  # 5126
-7590, 587,  14,3272, 227,2598, 326, 480,2265, 943,2755,3552, 291, 650,1883,7591,  # 5142
-1702,1226, 102,1547,  62,3441, 904,4415,3442,1164,4150,7592,7593,1224,1548,2756,  # 5158
- 391, 498,1493,7594,1386,1419,7595,2055,1177,4416, 813, 880,1081,2363, 566,1145,  # 5174
-4417,2286,1001,1035,2558,2599,2238, 394,1286,7596,7597,2068,7598,  86,1494,1730,  # 5190
-3936, 491,1588, 745, 897,2948, 843,3340,3937,2757,2870,3273,1768, 998,2217,2069,  # 5206
- 397,1826,1195,1969,3659,2993,3341, 284,7599,3782,2500,2137,2119,1903,7600,3938,  # 5222
-2150,3939,4151,1036,3443,1904, 114,2559,4152, 209,1527,7601,7602,2949,2831,2625,  # 5238
-2385,2719,3139, 812,2560,7603,3274,7604,1559, 737,1884,3660,1210, 885,  28,2686,  # 5254
-3553,3783,7605,4153,1004,1779,4418,7606, 346,1981,2218,2687,4419,3784,1742, 797,  # 5270
-1642,3940,1933,1072,1384,2151, 896,3941,3275,3661,3197,2871,3554,7607,2561,1958,  # 5286
-4420,2450,1785,7608,7609,7610,3942,4154,1005,1308,3662,4155,2720,4421,4422,1528,  # 5302
-2600, 161,1178,4156,1982, 987,4423,1101,4157, 631,3943,1157,3198,2420,1343,1241,  # 5318
-1016,2239,2562, 372, 877,2339,2501,1160, 555,1934, 911,3944,7611, 466,1170, 169,  # 5334
-1051,2907,2688,3663,2474,2994,1182,2011,2563,1251,2626,7612, 992,2340,3444,1540,  # 5350
-2721,1201,2070,2401,1996,2475,7613,4424, 528,1922,2188,1503,1873,1570,2364,3342,  # 5366
-3276,7614, 557,1073,7615,1827,3445,2087,2266,3140,3039,3084, 767,3085,2786,4425,  # 5382
-1006,4158,4426,2341,1267,2176,3664,3199, 778,3945,3200,2722,1597,2657,7616,4427,  # 5398
-7617,3446,7618,7619,7620,3277,2689,1433,3278, 131,  95,1504,3946, 723,4159,3141,  # 5414
-1841,3555,2758,2189,3947,2027,2104,3665,7621,2995,3948,1218,7622,3343,3201,3949,  # 5430
-4160,2576, 248,1634,3785, 912,7623,2832,3666,3040,3786, 654,  53,7624,2996,7625,  # 5446
-1688,4428, 777,3447,1032,3950,1425,7626, 191, 820,2120,2833, 971,4429, 931,3202,  # 5462
- 135, 664, 783,3787,1997, 772,2908,1935,3951,3788,4430,2909,3203, 282,2723, 640,  # 5478
-1372,3448,1127, 922, 325,3344,7627,7628, 711,2044,7629,7630,3952,2219,2787,1936,  # 5494
-3953,3345,2220,2251,3789,2300,7631,4431,3790,1258,3279,3954,3204,2138,2950,3955,  # 5510
-3956,7632,2221, 258,3205,4432, 101,1227,7633,3280,1755,7634,1391,3281,7635,2910,  # 5526
-2056, 893,7636,7637,7638,1402,4161,2342,7639,7640,3206,3556,7641,7642, 878,1325,  # 5542
-1780,2788,4433, 259,1385,2577, 744,1183,2267,4434,7643,3957,2502,7644, 684,1024,  # 5558
-4162,7645, 472,3557,3449,1165,3282,3958,3959, 322,2152, 881, 455,1695,1152,1340,  # 5574
- 660, 554,2153,4435,1058,4436,4163, 830,1065,3346,3960,4437,1923,7646,1703,1918,  # 5590
-7647, 932,2268, 122,7648,4438, 947, 677,7649,3791,2627, 297,1905,1924,2269,4439,  # 5606
-2317,3283,7650,7651,4164,7652,4165,  84,4166, 112, 989,7653, 547,1059,3961, 701,  # 5622
-3558,1019,7654,4167,7655,3450, 942, 639, 457,2301,2451, 993,2951, 407, 851, 494,  # 5638
-4440,3347, 927,7656,1237,7657,2421,3348, 573,4168, 680, 921,2911,1279,1874, 285,  # 5654
- 790,1448,1983, 719,2167,7658,7659,4441,3962,3963,1649,7660,1541, 563,7661,1077,  # 5670
-7662,3349,3041,3451, 511,2997,3964,3965,3667,3966,1268,2564,3350,3207,4442,4443,  # 5686
-7663, 535,1048,1276,1189,2912,2028,3142,1438,1373,2834,2952,1134,2012,7664,4169,  # 5702
-1238,2578,3086,1259,7665, 700,7666,2953,3143,3668,4170,7667,4171,1146,1875,1906,  # 5718
-4444,2601,3967, 781,2422, 132,1589, 203, 147, 273,2789,2402, 898,1786,2154,3968,  # 5734
-3969,7668,3792,2790,7669,7670,4445,4446,7671,3208,7672,1635,3793, 965,7673,1804,  # 5750
-2690,1516,3559,1121,1082,1329,3284,3970,1449,3794,  65,1128,2835,2913,2759,1590,  # 5766
-3795,7674,7675,  12,2658,  45, 976,2579,3144,4447, 517,2528,1013,1037,3209,7676,  # 5782
-3796,2836,7677,3797,7678,3452,7679,2602, 614,1998,2318,3798,3087,2724,2628,7680,  # 5798
-2580,4172, 599,1269,7681,1810,3669,7682,2691,3088, 759,1060, 489,1805,3351,3285,  # 5814
-1358,7683,7684,2386,1387,1215,2629,2252, 490,7685,7686,4173,1759,2387,2343,7687,  # 5830
-4448,3799,1907,3971,2630,1806,3210,4449,3453,3286,2760,2344, 874,7688,7689,3454,  # 5846
-3670,1858,  91,2914,3671,3042,3800,4450,7690,3145,3972,2659,7691,3455,1202,1403,  # 5862
-3801,2954,2529,1517,2503,4451,3456,2504,7692,4452,7693,2692,1885,1495,1731,3973,  # 5878
-2365,4453,7694,2029,7695,7696,3974,2693,1216, 237,2581,4174,2319,3975,3802,4454,  # 5894
-4455,2694,3560,3457, 445,4456,7697,7698,7699,7700,2761,  61,3976,3672,1822,3977,  # 5910
-7701, 687,2045, 935, 925, 405,2660, 703,1096,1859,2725,4457,3978,1876,1367,2695,  # 5926
-3352, 918,2105,1781,2476, 334,3287,1611,1093,4458, 564,3146,3458,3673,3353, 945,  # 5942
-2631,2057,4459,7702,1925, 872,4175,7703,3459,2696,3089, 349,4176,3674,3979,4460,  # 5958
-3803,4177,3675,2155,3980,4461,4462,4178,4463,2403,2046, 782,3981, 400, 251,4179,  # 5974
-1624,7704,7705, 277,3676, 299,1265, 476,1191,3804,2121,4180,4181,1109, 205,7706,  # 5990
-2582,1000,2156,3561,1860,7707,7708,7709,4464,7710,4465,2565, 107,2477,2157,3982,  # 6006
-3460,3147,7711,1533, 541,1301, 158, 753,4182,2872,3562,7712,1696, 370,1088,4183,  # 6022
-4466,3563, 579, 327, 440, 162,2240, 269,1937,1374,3461, 968,3043,  56,1396,3090,  # 6038
-2106,3288,3354,7713,1926,2158,4467,2998,7714,3564,7715,7716,3677,4468,2478,7717,  # 6054
-2791,7718,1650,4469,7719,2603,7720,7721,3983,2661,3355,1149,3356,3984,3805,3985,  # 6070
-7722,1076,  49,7723, 951,3211,3289,3290, 450,2837, 920,7724,1811,2792,2366,4184,  # 6086
-1908,1138,2367,3806,3462,7725,3212,4470,1909,1147,1518,2423,4471,3807,7726,4472,  # 6102
-2388,2604, 260,1795,3213,7727,7728,3808,3291, 708,7729,3565,1704,7730,3566,1351,  # 6118
-1618,3357,2999,1886, 944,4185,3358,4186,3044,3359,4187,7731,3678, 422, 413,1714,  # 6134
-3292, 500,2058,2345,4188,2479,7732,1344,1910, 954,7733,1668,7734,7735,3986,2404,  # 6150
-4189,3567,3809,4190,7736,2302,1318,2505,3091, 133,3092,2873,4473, 629,  31,2838,  # 6166
-2697,3810,4474, 850, 949,4475,3987,2955,1732,2088,4191,1496,1852,7737,3988, 620,  # 6182
-3214, 981,1242,3679,3360,1619,3680,1643,3293,2139,2452,1970,1719,3463,2168,7738,  # 6198
-3215,7739,7740,3361,1828,7741,1277,4476,1565,2047,7742,1636,3568,3093,7743, 869,  # 6214
-2839, 655,3811,3812,3094,3989,3000,3813,1310,3569,4477,7744,7745,7746,1733, 558,  # 6230
-4478,3681, 335,1549,3045,1756,4192,3682,1945,3464,1829,1291,1192, 470,2726,2107,  # 6246
-2793, 913,1054,3990,7747,1027,7748,3046,3991,4479, 982,2662,3362,3148,3465,3216,  # 6262
-3217,1946,2794,7749, 571,4480,7750,1830,7751,3570,2583,1523,2424,7752,2089, 984,  # 6278
-4481,3683,1959,7753,3684, 852, 923,2795,3466,3685, 969,1519, 999,2048,2320,1705,  # 6294
-7754,3095, 615,1662, 151, 597,3992,2405,2321,1049, 275,4482,3686,4193, 568,3687,  # 6310
-3571,2480,4194,3688,7755,2425,2270, 409,3218,7756,1566,2874,3467,1002, 769,2840,  # 6326
- 194,2090,3149,3689,2222,3294,4195, 628,1505,7757,7758,1763,2177,3001,3993, 521,  # 6342
-1161,2584,1787,2203,2406,4483,3994,1625,4196,4197, 412,  42,3096, 464,7759,2632,  # 6358
-4484,3363,1760,1571,2875,3468,2530,1219,2204,3814,2633,2140,2368,4485,4486,3295,  # 6374
-1651,3364,3572,7760,7761,3573,2481,3469,7762,3690,7763,7764,2271,2091, 460,7765,  # 6390
-4487,7766,3002, 962, 588,3574, 289,3219,2634,1116,  52,7767,3047,1796,7768,7769,  # 6406
-7770,1467,7771,1598,1143,3691,4198,1984,1734,1067,4488,1280,3365, 465,4489,1572,  # 6422
- 510,7772,1927,2241,1812,1644,3575,7773,4490,3692,7774,7775,2663,1573,1534,7776,  # 6438
-7777,4199, 536,1807,1761,3470,3815,3150,2635,7778,7779,7780,4491,3471,2915,1911,  # 6454
-2796,7781,3296,1122, 377,3220,7782, 360,7783,7784,4200,1529, 551,7785,2059,3693,  # 6470
-1769,2426,7786,2916,4201,3297,3097,2322,2108,2030,4492,1404, 136,1468,1479, 672,  # 6486
-1171,3221,2303, 271,3151,7787,2762,7788,2049, 678,2727, 865,1947,4493,7789,2013,  # 6502
-3995,2956,7790,2728,2223,1397,3048,3694,4494,4495,1735,2917,3366,3576,7791,3816,  # 6518
- 509,2841,2453,2876,3817,7792,7793,3152,3153,4496,4202,2531,4497,2304,1166,1010,  # 6534
- 552, 681,1887,7794,7795,2957,2958,3996,1287,1596,1861,3154, 358, 453, 736, 175,  # 6550
- 478,1117, 905,1167,1097,7796,1853,1530,7797,1706,7798,2178,3472,2287,3695,3473,  # 6566
-3577,4203,2092,4204,7799,3367,1193,2482,4205,1458,2190,2205,1862,1888,1421,3298,  # 6582
-2918,3049,2179,3474, 595,2122,7800,3997,7801,7802,4206,1707,2636, 223,3696,1359,  # 6598
- 751,3098, 183,3475,7803,2797,3003, 419,2369, 633, 704,3818,2389, 241,7804,7805,  # 6614
-7806, 838,3004,3697,2272,2763,2454,3819,1938,2050,3998,1309,3099,2242,1181,7807,  # 6630
-1136,2206,3820,2370,1446,4207,2305,4498,7808,7809,4208,1055,2605, 484,3698,7810,  # 6646
-3999, 625,4209,2273,3368,1499,4210,4000,7811,4001,4211,3222,2274,2275,3476,7812,  # 6662
-7813,2764, 808,2606,3699,3369,4002,4212,3100,2532, 526,3370,3821,4213, 955,7814,  # 6678
-1620,4214,2637,2427,7815,1429,3700,1669,1831, 994, 928,7816,3578,1260,7817,7818,  # 6694
-7819,1948,2288, 741,2919,1626,4215,2729,2455, 867,1184, 362,3371,1392,7820,7821,  # 6710
-4003,4216,1770,1736,3223,2920,4499,4500,1928,2698,1459,1158,7822,3050,3372,2877,  # 6726
-1292,1929,2506,2842,3701,1985,1187,2071,2014,2607,4217,7823,2566,2507,2169,3702,  # 6742
-2483,3299,7824,3703,4501,7825,7826, 666,1003,3005,1022,3579,4218,7827,4502,1813,  # 6758
-2253, 574,3822,1603, 295,1535, 705,3823,4219, 283, 858, 417,7828,7829,3224,4503,  # 6774
-4504,3051,1220,1889,1046,2276,2456,4004,1393,1599, 689,2567, 388,4220,7830,2484,  # 6790
- 802,7831,2798,3824,2060,1405,2254,7832,4505,3825,2109,1052,1345,3225,1585,7833,  # 6806
- 809,7834,7835,7836, 575,2730,3477, 956,1552,1469,1144,2323,7837,2324,1560,2457,  # 6822
-3580,3226,4005, 616,2207,3155,2180,2289,7838,1832,7839,3478,4506,7840,1319,3704,  # 6838
-3705,1211,3581,1023,3227,1293,2799,7841,7842,7843,3826, 607,2306,3827, 762,2878,  # 6854
-1439,4221,1360,7844,1485,3052,7845,4507,1038,4222,1450,2061,2638,4223,1379,4508,  # 6870
-2585,7846,7847,4224,1352,1414,2325,2921,1172,7848,7849,3828,3829,7850,1797,1451,  # 6886
-7851,7852,7853,7854,2922,4006,4007,2485,2346, 411,4008,4009,3582,3300,3101,4509,  # 6902
-1561,2664,1452,4010,1375,7855,7856,  47,2959, 316,7857,1406,1591,2923,3156,7858,  # 6918
-1025,2141,3102,3157, 354,2731, 884,2224,4225,2407, 508,3706, 726,3583, 996,2428,  # 6934
-3584, 729,7859, 392,2191,1453,4011,4510,3707,7860,7861,2458,3585,2608,1675,2800,  # 6950
- 919,2347,2960,2348,1270,4511,4012,  73,7862,7863, 647,7864,3228,2843,2255,1550,  # 6966
-1346,3006,7865,1332, 883,3479,7866,7867,7868,7869,3301,2765,7870,1212, 831,1347,  # 6982
-4226,4512,2326,3830,1863,3053, 720,3831,4513,4514,3832,7871,4227,7872,7873,4515,  # 6998
-7874,7875,1798,4516,3708,2609,4517,3586,1645,2371,7876,7877,2924, 669,2208,2665,  # 7014
-2429,7878,2879,7879,7880,1028,3229,7881,4228,2408,7882,2256,1353,7883,7884,4518,  # 7030
-3158, 518,7885,4013,7886,4229,1960,7887,2142,4230,7888,7889,3007,2349,2350,3833,  # 7046
- 516,1833,1454,4014,2699,4231,4519,2225,2610,1971,1129,3587,7890,2766,7891,2961,  # 7062
-1422, 577,1470,3008,1524,3373,7892,7893, 432,4232,3054,3480,7894,2586,1455,2508,  # 7078
-2226,1972,1175,7895,1020,2732,4015,3481,4520,7896,2733,7897,1743,1361,3055,3482,  # 7094
-2639,4016,4233,4521,2290, 895, 924,4234,2170, 331,2243,3056, 166,1627,3057,1098,  # 7110
-7898,1232,2880,2227,3374,4522, 657, 403,1196,2372, 542,3709,3375,1600,4235,3483,  # 7126
-7899,4523,2767,3230, 576, 530,1362,7900,4524,2533,2666,3710,4017,7901, 842,3834,  # 7142
-7902,2801,2031,1014,4018, 213,2700,3376, 665, 621,4236,7903,3711,2925,2430,7904,  # 7158
-2431,3302,3588,3377,7905,4237,2534,4238,4525,3589,1682,4239,3484,1380,7906, 724,  # 7174
-2277, 600,1670,7907,1337,1233,4526,3103,2244,7908,1621,4527,7909, 651,4240,7910,  # 7190
-1612,4241,2611,7911,2844,7912,2734,2307,3058,7913, 716,2459,3059, 174,1255,2701,  # 7206
-4019,3590, 548,1320,1398, 728,4020,1574,7914,1890,1197,3060,4021,7915,3061,3062,  # 7222
-3712,3591,3713, 747,7916, 635,4242,4528,7917,7918,7919,4243,7920,7921,4529,7922,  # 7238
-3378,4530,2432, 451,7923,3714,2535,2072,4244,2735,4245,4022,7924,1764,4531,7925,  # 7254
-4246, 350,7926,2278,2390,2486,7927,4247,4023,2245,1434,4024, 488,4532, 458,4248,  # 7270
-4025,3715, 771,1330,2391,3835,2568,3159,2159,2409,1553,2667,3160,4249,7928,2487,  # 7286
-2881,2612,1720,2702,4250,3379,4533,7929,2536,4251,7930,3231,4252,2768,7931,2015,  # 7302
-2736,7932,1155,1017,3716,3836,7933,3303,2308, 201,1864,4253,1430,7934,4026,7935,  # 7318
-7936,7937,7938,7939,4254,1604,7940, 414,1865, 371,2587,4534,4535,3485,2016,3104,  # 7334
-4536,1708, 960,4255, 887, 389,2171,1536,1663,1721,7941,2228,4027,2351,2926,1580,  # 7350
-7942,7943,7944,1744,7945,2537,4537,4538,7946,4539,7947,2073,7948,7949,3592,3380,  # 7366
-2882,4256,7950,4257,2640,3381,2802, 673,2703,2460, 709,3486,4028,3593,4258,7951,  # 7382
-1148, 502, 634,7952,7953,1204,4540,3594,1575,4541,2613,3717,7954,3718,3105, 948,  # 7398
-3232, 121,1745,3837,1110,7955,4259,3063,2509,3009,4029,3719,1151,1771,3838,1488,  # 7414
-4030,1986,7956,2433,3487,7957,7958,2093,7959,4260,3839,1213,1407,2803, 531,2737,  # 7430
-2538,3233,1011,1537,7960,2769,4261,3106,1061,7961,3720,3721,1866,2883,7962,2017,  # 7446
- 120,4262,4263,2062,3595,3234,2309,3840,2668,3382,1954,4542,7963,7964,3488,1047,  # 7462
-2704,1266,7965,1368,4543,2845, 649,3383,3841,2539,2738,1102,2846,2669,7966,7967,  # 7478
-1999,7968,1111,3596,2962,7969,2488,3842,3597,2804,1854,3384,3722,7970,7971,3385,  # 7494
-2410,2884,3304,3235,3598,7972,2569,7973,3599,2805,4031,1460, 856,7974,3600,7975,  # 7510
-2885,2963,7976,2886,3843,7977,4264, 632,2510, 875,3844,1697,3845,2291,7978,7979,  # 7526
-4544,3010,1239, 580,4545,4265,7980, 914, 936,2074,1190,4032,1039,2123,7981,7982,  # 7542
-7983,3386,1473,7984,1354,4266,3846,7985,2172,3064,4033, 915,3305,4267,4268,3306,  # 7558
-1605,1834,7986,2739, 398,3601,4269,3847,4034, 328,1912,2847,4035,3848,1331,4270,  # 7574
-3011, 937,4271,7987,3602,4036,4037,3387,2160,4546,3388, 524, 742, 538,3065,1012,  # 7590
-7988,7989,3849,2461,7990, 658,1103, 225,3850,7991,7992,4547,7993,4548,7994,3236,  # 7606
-1243,7995,4038, 963,2246,4549,7996,2705,3603,3161,7997,7998,2588,2327,7999,4550,  # 7622
-8000,8001,8002,3489,3307, 957,3389,2540,2032,1930,2927,2462, 870,2018,3604,1746,  # 7638
-2770,2771,2434,2463,8003,3851,8004,3723,3107,3724,3490,3390,3725,8005,1179,3066,  # 7654
-8006,3162,2373,4272,3726,2541,3163,3108,2740,4039,8007,3391,1556,2542,2292, 977,  # 7670
-2887,2033,4040,1205,3392,8008,1765,3393,3164,2124,1271,1689, 714,4551,3491,8009,  # 7686
-2328,3852, 533,4273,3605,2181, 617,8010,2464,3308,3492,2310,8011,8012,3165,8013,  # 7702
-8014,3853,1987, 618, 427,2641,3493,3394,8015,8016,1244,1690,8017,2806,4274,4552,  # 7718
-8018,3494,8019,8020,2279,1576, 473,3606,4275,3395, 972,8021,3607,8022,3067,8023,  # 7734
-8024,4553,4554,8025,3727,4041,4042,8026, 153,4555, 356,8027,1891,2888,4276,2143,  # 7750
- 408, 803,2352,8028,3854,8029,4277,1646,2570,2511,4556,4557,3855,8030,3856,4278,  # 7766
-8031,2411,3396, 752,8032,8033,1961,2964,8034, 746,3012,2465,8035,4279,3728, 698,  # 7782
-4558,1892,4280,3608,2543,4559,3609,3857,8036,3166,3397,8037,1823,1302,4043,2706,  # 7798
-3858,1973,4281,8038,4282,3167, 823,1303,1288,1236,2848,3495,4044,3398, 774,3859,  # 7814
-8039,1581,4560,1304,2849,3860,4561,8040,2435,2161,1083,3237,4283,4045,4284, 344,  # 7830
-1173, 288,2311, 454,1683,8041,8042,1461,4562,4046,2589,8043,8044,4563, 985, 894,  # 7846
-8045,3399,3168,8046,1913,2928,3729,1988,8047,2110,1974,8048,4047,8049,2571,1194,  # 7862
- 425,8050,4564,3169,1245,3730,4285,8051,8052,2850,8053, 636,4565,1855,3861, 760,  # 7878
-1799,8054,4286,2209,1508,4566,4048,1893,1684,2293,8055,8056,8057,4287,4288,2210,  # 7894
- 479,8058,8059, 832,8060,4049,2489,8061,2965,2490,3731, 990,3109, 627,1814,2642,  # 7910
-4289,1582,4290,2125,2111,3496,4567,8062, 799,4291,3170,8063,4568,2112,1737,3013,  # 7926
-1018, 543, 754,4292,3309,1676,4569,4570,4050,8064,1489,8065,3497,8066,2614,2889,  # 7942
-4051,8067,8068,2966,8069,8070,8071,8072,3171,4571,4572,2182,1722,8073,3238,3239,  # 7958
-1842,3610,1715, 481, 365,1975,1856,8074,8075,1962,2491,4573,8076,2126,3611,3240,  # 7974
- 433,1894,2063,2075,8077, 602,2741,8078,8079,8080,8081,8082,3014,1628,3400,8083,  # 7990
-3172,4574,4052,2890,4575,2512,8084,2544,2772,8085,8086,8087,3310,4576,2891,8088,  # 8006
-4577,8089,2851,4578,4579,1221,2967,4053,2513,8090,8091,8092,1867,1989,8093,8094,  # 8022
-8095,1895,8096,8097,4580,1896,4054, 318,8098,2094,4055,4293,8099,8100, 485,8101,  # 8038
- 938,3862, 553,2670, 116,8102,3863,3612,8103,3498,2671,2773,3401,3311,2807,8104,  # 8054
-3613,2929,4056,1747,2930,2968,8105,8106, 207,8107,8108,2672,4581,2514,8109,3015,  # 8070
- 890,3614,3864,8110,1877,3732,3402,8111,2183,2353,3403,1652,8112,8113,8114, 941,  # 8086
-2294, 208,3499,4057,2019, 330,4294,3865,2892,2492,3733,4295,8115,8116,8117,8118,  # 8102
+    1, 1800, 1506, 255, 1431, 198, 9, 82, 6, 7310, 177, 202, 3615, 1256, 2808, 110,  # 2742
+    3735, 33, 3241, 261, 76, 44, 2113, 16, 2931, 2184, 1176, 659, 3868, 26, 3404, 2643,  # 2758
+    1198, 3869, 3313, 4060, 410, 2211, 302, 590, 361, 1963, 8, 204, 58, 4296, 7311, 1931,  # 2774
+    63, 7312, 7313, 317, 1614, 75, 222, 159, 4061, 2412, 1480, 7314, 3500, 3068, 224, 2809,  # 2790
+    3616, 3, 10, 3870, 1471, 29, 2774, 1135, 2852, 1939, 873, 130, 3242, 1123, 312, 7315,  # 2806
+    4297, 2051, 507, 252, 682, 7316, 142, 1914, 124, 206, 2932, 34, 3501, 3173, 64, 604,  # 2822
+    7317, 2494, 1976, 1977, 155, 1990, 645, 641, 1606, 7318, 3405, 337, 72, 406, 7319, 80,  # 2838
+    630, 238, 3174, 1509, 263, 939, 1092, 2644, 756, 1440, 1094, 3406, 449, 69, 2969, 591,  # 2854
+    179, 2095, 471, 115, 2034, 1843, 60, 50, 2970, 134, 806, 1868, 734, 2035, 3407, 180,  # 2870
+    995, 1607, 156, 537, 2893, 688, 7320, 319, 1305, 779, 2144, 514, 2374, 298, 4298, 359,  # 2886
+    2495, 90, 2707, 1338, 663, 11, 906, 1099, 2545, 20, 2436, 182, 532, 1716, 7321, 732,  # 2902
+    1376, 4062, 1311, 1420, 3175, 25, 2312, 1056, 113, 399, 382, 1949, 242, 3408, 2467, 529,  # 2918
+    3243, 475, 1447, 3617, 7322, 117, 21, 656, 810, 1297, 2295, 2329, 3502, 7323, 126, 4063,  # 2934
+    706, 456, 150, 613, 4299, 71, 1118, 2036, 4064, 145, 3069, 85, 835, 486, 2114, 1246,  # 2950
+    1426, 428, 727, 1285, 1015, 800, 106, 623, 303, 1281, 7324, 2127, 2354, 347, 3736, 221,  # 2966
+    3503, 3110, 7325, 1955, 1153, 4065, 83, 296, 1199, 3070, 192, 624, 93, 7326, 822, 1897,  # 2982
+    2810, 3111, 795, 2064, 991, 1554, 1542, 1592, 27, 43, 2853, 859, 139, 1456, 860, 4300,  # 2998
+    437, 712, 3871, 164, 2392, 3112, 695, 211, 3017, 2096, 195, 3872, 1608, 3504, 3505, 3618,  # 3014
+    3873, 234, 811, 2971, 2097, 3874, 2229, 1441, 3506, 1615, 2375, 668, 2076, 1638, 305, 228,  # 3030
+    1664, 4301, 467, 415, 7327, 262, 2098, 1593, 239, 108, 300, 200, 1033, 512, 1247, 2077,  # 3046
+    7328, 7329, 2173, 3176, 3619, 2673, 593, 845, 1062, 3244, 88, 1723, 2037, 3875, 1950, 212,  # 3062
+    266, 152, 149, 468, 1898, 4066, 4302, 77, 187, 7330, 3018, 37, 5, 2972, 7331, 3876,  # 3078
+    7332, 7333, 39, 2517, 4303, 2894, 3177, 2078, 55, 148, 74, 4304, 545, 483, 1474, 1029,  # 3094
+    1665, 217, 1869, 1531, 3113, 1104, 2645, 4067, 24, 172, 3507, 900, 3877, 3508, 3509, 4305,  # 3110
+    32, 1408, 2811, 1312, 329, 487, 2355, 2247, 2708, 784, 2674, 4, 3019, 3314, 1427, 1788,  # 3126
+    188, 109, 499, 7334, 3620, 1717, 1789, 888, 1217, 3020, 4306, 7335, 3510, 7336, 3315, 1520,  # 3142
+    3621, 3878, 196, 1034, 775, 7337, 7338, 929, 1815, 249, 439, 38, 7339, 1063, 7340, 794,  # 3158
+    3879, 1435, 2296, 46, 178, 3245, 2065, 7341, 2376, 7342, 214, 1709, 4307, 804, 35, 707,  # 3174
+    324, 3622, 1601, 2546, 140, 459, 4068, 7343, 7344, 1365, 839, 272, 978, 2257, 2572, 3409,  # 3190
+    2128, 1363, 3623, 1423, 697, 100, 3071, 48, 70, 1231, 495, 3114, 2193, 7345, 1294, 7346,  # 3206
+    2079, 462, 586, 1042, 3246, 853, 256, 988, 185, 2377, 3410, 1698, 434, 1084, 7347, 3411,  # 3222
+    314, 2615, 2775, 4308, 2330, 2331, 569, 2280, 637, 1816, 2518, 757, 1162, 1878, 1616, 3412,  # 3238
+    287, 1577, 2115, 768, 4309, 1671, 2854, 3511, 2519, 1321, 3737, 909, 2413, 7348, 4069, 933,  # 3254
+    3738, 7349, 2052, 2356, 1222, 4310, 765, 2414, 1322, 786, 4311, 7350, 1919, 1462, 1677, 2895,  # 3270
+    1699, 7351, 4312, 1424, 2437, 3115, 3624, 2590, 3316, 1774, 1940, 3413, 3880, 4070, 309, 1369,  # 3286
+    1130, 2812, 364, 2230, 1653, 1299, 3881, 3512, 3882, 3883, 2646, 525, 1085, 3021, 902, 2000,  # 3302
+    1475, 964, 4313, 421, 1844, 1415, 1057, 2281, 940, 1364, 3116, 376, 4314, 4315, 1381, 7,  # 3318
+    2520, 983, 2378, 336, 1710, 2675, 1845, 321, 3414, 559, 1131, 3022, 2742, 1808, 1132, 1313,  # 3334
+    265, 1481, 1857, 7352, 352, 1203, 2813, 3247, 167, 1089, 420, 2814, 776, 792, 1724, 3513,  # 3350
+    4071, 2438, 3248, 7353, 4072, 7354, 446, 229, 333, 2743, 901, 3739, 1200, 1557, 4316, 2647,  # 3366
+    1920, 395, 2744, 2676, 3740, 4073, 1835, 125, 916, 3178, 2616, 4317, 7355, 7356, 3741, 7357,  # 3382
+    7358, 7359, 4318, 3117, 3625, 1133, 2547, 1757, 3415, 1510, 2313, 1409, 3514, 7360, 2145, 438,  # 3398
+    2591, 2896, 2379, 3317, 1068, 958, 3023, 461, 311, 2855, 2677, 4074, 1915, 3179, 4075, 1978,  # 3414
+    383, 750, 2745, 2617, 4076, 274, 539, 385, 1278, 1442, 7361, 1154, 1964, 384, 561, 210,  # 3430
+    98, 1295, 2548, 3515, 7362, 1711, 2415, 1482, 3416, 3884, 2897, 1257, 129, 7363, 3742, 642,  # 3446
+    523, 2776, 2777, 2648, 7364, 141, 2231, 1333, 68, 176, 441, 876, 907, 4077, 603, 2592,  # 3462
+    710, 171, 3417, 404, 549, 18, 3118, 2393, 1410, 3626, 1666, 7365, 3516, 4319, 2898, 4320,  # 3478
+    7366, 2973, 368, 7367, 146, 366, 99, 871, 3627, 1543, 748, 807, 1586, 1185, 22, 2258,  # 3494
+    379, 3743, 3180, 7368, 3181, 505, 1941, 2618, 1991, 1382, 2314, 7369, 380, 2357, 218, 702,  # 3510
+    1817, 1248, 3418, 3024, 3517, 3318, 3249, 7370, 2974, 3628, 930, 3250, 3744, 7371, 59, 7372,  # 3526
+    585, 601, 4078, 497, 3419, 1112, 1314, 4321, 1801, 7373, 1223, 1472, 2174, 7374, 749, 1836,  # 3542
+    690, 1899, 3745, 1772, 3885, 1476, 429, 1043, 1790, 2232, 2116, 917, 4079, 447, 1086, 1629,  # 3558
+    7375, 556, 7376, 7377, 2020, 1654, 844, 1090, 105, 550, 966, 1758, 2815, 1008, 1782, 686,  # 3574
+    1095, 7378, 2282, 793, 1602, 7379, 3518, 2593, 4322, 4080, 2933, 2297, 4323, 3746, 980, 2496,  # 3590
+    544, 353, 527, 4324, 908, 2678, 2899, 7380, 381, 2619, 1942, 1348, 7381, 1341, 1252, 560,  # 3606
+    3072, 7382, 3420, 2856, 7383, 2053, 973, 886, 2080, 143, 4325, 7384, 7385, 157, 3886, 496,  # 3622
+    4081, 57, 840, 540, 2038, 4326, 4327, 3421, 2117, 1445, 970, 2259, 1748, 1965, 2081, 4082,  # 3638
+    3119, 1234, 1775, 3251, 2816, 3629, 773, 1206, 2129, 1066, 2039, 1326, 3887, 1738, 1725, 4083,  # 3654
+    279, 3120, 51, 1544, 2594, 423, 1578, 2130, 2066, 173, 4328, 1879, 7386, 7387, 1583, 264,  # 3670
+    610, 3630, 4329, 2439, 280, 154, 7388, 7389, 7390, 1739, 338, 1282, 3073, 693, 2857, 1411,  # 3686
+    1074, 3747, 2440, 7391, 4330, 7392, 7393, 1240, 952, 2394, 7394, 2900, 1538, 2679, 685, 1483,  # 3702
+    4084, 2468, 1436, 953, 4085, 2054, 4331, 671, 2395, 79, 4086, 2441, 3252, 608, 567, 2680,  # 3718
+    3422, 4087, 4088, 1691, 393, 1261, 1791, 2396, 7395, 4332, 7396, 7397, 7398, 7399, 1383, 1672,  # 3734
+    3748, 3182, 1464, 522, 1119, 661, 1150, 216, 675, 4333, 3888, 1432, 3519, 609, 4334, 2681,  # 3750
+    2397, 7400, 7401, 7402, 4089, 3025, 0, 7403, 2469, 315, 231, 2442, 301, 3319, 4335, 2380,  # 3766
+    7404, 233, 4090, 3631, 1818, 4336, 4337, 7405, 96, 1776, 1315, 2082, 7406, 257, 7407, 1809,  # 3782
+    3632, 2709, 1139, 1819, 4091, 2021, 1124, 2163, 2778, 1777, 2649, 7408, 3074, 363, 1655, 3183,  # 3798
+    7409, 2975, 7410, 7411, 7412, 3889, 1567, 3890, 718, 103, 3184, 849, 1443, 341, 3320, 2934,  # 3814
+    1484, 7413, 1712, 127, 67, 339, 4092, 2398, 679, 1412, 821, 7414, 7415, 834, 738, 351,  # 3830
+    2976, 2146, 846, 235, 1497, 1880, 418, 1992, 3749, 2710, 186, 1100, 2147, 2746, 3520, 1545,  # 3846
+    1355, 2935, 2858, 1377, 583, 3891, 4093, 2573, 2977, 7416, 1298, 3633, 1078, 2549, 3634, 2358,  # 3862
+    78, 3750, 3751, 267, 1289, 2099, 2001, 1594, 4094, 348, 369, 1274, 2194, 2175, 1837, 4338,  # 3878
+    1820, 2817, 3635, 2747, 2283, 2002, 4339, 2936, 2748, 144, 3321, 882, 4340, 3892, 2749, 3423,  # 3894
+    4341, 2901, 7417, 4095, 1726, 320, 7418, 3893, 3026, 788, 2978, 7419, 2818, 1773, 1327, 2859,  # 3910
+    3894, 2819, 7420, 1306, 4342, 2003, 1700, 3752, 3521, 2359, 2650, 787, 2022, 506, 824, 3636,  # 3926
+    534, 323, 4343, 1044, 3322, 2023, 1900, 946, 3424, 7421, 1778, 1500, 1678, 7422, 1881, 4344,  # 3942
+    165, 243, 4345, 3637, 2521, 123, 683, 4096, 764, 4346, 36, 3895, 1792, 589, 2902, 816,  # 3958
+    626, 1667, 3027, 2233, 1639, 1555, 1622, 3753, 3896, 7423, 3897, 2860, 1370, 1228, 1932, 891,  # 3974
+    2083, 2903, 304, 4097, 7424, 292, 2979, 2711, 3522, 691, 2100, 4098, 1115, 4347, 118, 662,  # 3990
+    7425, 611, 1156, 854, 2381, 1316, 2861, 2, 386, 515, 2904, 7426, 7427, 3253, 868, 2234,  # 4006
+    1486, 855, 2651, 785, 2212, 3028, 7428, 1040, 3185, 3523, 7429, 3121, 448, 7430, 1525, 7431,  # 4022
+    2164, 4348, 7432, 3754, 7433, 4099, 2820, 3524, 3122, 503, 818, 3898, 3123, 1568, 814, 676,  # 4038
+    1444, 306, 1749, 7434, 3755, 1416, 1030, 197, 1428, 805, 2821, 1501, 4349, 7435, 7436, 7437,  # 4054
+    1993, 7438, 4350, 7439, 7440, 2195, 13, 2779, 3638, 2980, 3124, 1229, 1916, 7441, 3756, 2131,  # 4070
+    7442, 4100, 4351, 2399, 3525, 7443, 2213, 1511, 1727, 1120, 7444, 7445, 646, 3757, 2443, 307,  # 4086
+    7446, 7447, 1595, 3186, 7448, 7449, 7450, 3639, 1113, 1356, 3899, 1465, 2522, 2523, 7451, 519,  # 4102
+    7452, 128, 2132, 92, 2284, 1979, 7453, 3900, 1512, 342, 3125, 2196, 7454, 2780, 2214, 1980,  # 4118
+    3323, 7455, 290, 1656, 1317, 789, 827, 2360, 7456, 3758, 4352, 562, 581, 3901, 7457, 401,  # 4134
+    4353, 2248, 94, 4354, 1399, 2781, 7458, 1463, 2024, 4355, 3187, 1943, 7459, 828, 1105, 4101,  # 4150
+    1262, 1394, 7460, 4102, 605, 4356, 7461, 1783, 2862, 7462, 2822, 819, 2101, 578, 2197, 2937,  # 4166
+    7463, 1502, 436, 3254, 4103, 3255, 2823, 3902, 2905, 3425, 3426, 7464, 2712, 2315, 7465, 7466,  # 4182
+    2332, 2067, 23, 4357, 193, 826, 3759, 2102, 699, 1630, 4104, 3075, 390, 1793, 1064, 3526,  # 4198
+    7467, 1579, 3076, 3077, 1400, 7468, 4105, 1838, 1640, 2863, 7469, 4358, 4359, 137, 4106, 598,  # 4214
+    3078, 1966, 780, 104, 974, 2938, 7470, 278, 899, 253, 402, 572, 504, 493, 1339, 7471,  # 4230
+    3903, 1275, 4360, 2574, 2550, 7472, 3640, 3029, 3079, 2249, 565, 1334, 2713, 863, 41, 7473,  # 4246
+    7474, 4361, 7475, 1657, 2333, 19, 463, 2750, 4107, 606, 7476, 2981, 3256, 1087, 2084, 1323,  # 4262
+    2652, 2982, 7477, 1631, 1623, 1750, 4108, 2682, 7478, 2864, 791, 2714, 2653, 2334, 232, 2416,  # 4278
+    7479, 2983, 1498, 7480, 2654, 2620, 755, 1366, 3641, 3257, 3126, 2025, 1609, 119, 1917, 3427,  # 4294
+    862, 1026, 4109, 7481, 3904, 3760, 4362, 3905, 4363, 2260, 1951, 2470, 7482, 1125, 817, 4110,  # 4310
+    4111, 3906, 1513, 1766, 2040, 1487, 4112, 3030, 3258, 2824, 3761, 3127, 7483, 7484, 1507, 7485,  # 4326
+    2683, 733, 40, 1632, 1106, 2865, 345, 4113, 841, 2524, 230, 4364, 2984, 1846, 3259, 3428,  # 4342
+    7486, 1263, 986, 3429, 7487, 735, 879, 254, 1137, 857, 622, 1300, 1180, 1388, 1562, 3907,  # 4358
+    3908, 2939, 967, 2751, 2655, 1349, 592, 2133, 1692, 3324, 2985, 1994, 4114, 1679, 3909, 1901,  # 4374
+    2185, 7488, 739, 3642, 2715, 1296, 1290, 7489, 4115, 2198, 2199, 1921, 1563, 2595, 2551, 1870,  # 4390
+    2752, 2986, 7490, 435, 7491, 343, 1108, 596, 17, 1751, 4365, 2235, 3430, 3643, 7492, 4366,  # 4406
+    294, 3527, 2940, 1693, 477, 979, 281, 2041, 3528, 643, 2042, 3644, 2621, 2782, 2261, 1031,  # 4422
+    2335, 2134, 2298, 3529, 4367, 367, 1249, 2552, 7493, 3530, 7494, 4368, 1283, 3325, 2004, 240,  # 4438
+    1762, 3326, 4369, 4370, 836, 1069, 3128, 474, 7495, 2148, 2525, 268, 3531, 7496, 3188, 1521,  # 4454
+    1284, 7497, 1658, 1546, 4116, 7498, 3532, 3533, 7499, 4117, 3327, 2684, 1685, 4118, 961, 1673,  # 4470
+    2622, 190, 2005, 2200, 3762, 4371, 4372, 7500, 570, 2497, 3645, 1490, 7501, 4373, 2623, 3260,  # 4486
+    1956, 4374, 584, 1514, 396, 1045, 1944, 7502, 4375, 1967, 2444, 7503, 7504, 4376, 3910, 619,  # 4502
+    7505, 3129, 3261, 215, 2006, 2783, 2553, 3189, 4377, 3190, 4378, 763, 4119, 3763, 4379, 7506,  # 4518
+    7507, 1957, 1767, 2941, 3328, 3646, 1174, 452, 1477, 4380, 3329, 3130, 7508, 2825, 1253, 2382,  # 4534
+    2186, 1091, 2285, 4120, 492, 7509, 638, 1169, 1824, 2135, 1752, 3911, 648, 926, 1021, 1324,  # 4550
+    4381, 520, 4382, 997, 847, 1007, 892, 4383, 3764, 2262, 1871, 3647, 7510, 2400, 1784, 4384,  # 4566
+    1952, 2942, 3080, 3191, 1728, 4121, 2043, 3648, 4385, 2007, 1701, 3131, 1551, 30, 2263, 4122,  # 4582
+    7511, 2026, 4386, 3534, 7512, 501, 7513, 4123, 594, 3431, 2165, 1821, 3535, 3432, 3536, 3192,  # 4598
+    829, 2826, 4124, 7514, 1680, 3132, 1225, 4125, 7515, 3262, 4387, 4126, 3133, 2336, 7516, 4388,  # 4614
+    4127, 7517, 3912, 3913, 7518, 1847, 2383, 2596, 3330, 7519, 4389, 374, 3914, 652, 4128, 4129,  # 4630
+    375, 1140, 798, 7520, 7521, 7522, 2361, 4390, 2264, 546, 1659, 138, 3031, 2445, 4391, 7523,  # 4646
+    2250, 612, 1848, 910, 796, 3765, 1740, 1371, 825, 3766, 3767, 7524, 2906, 2554, 7525, 692,  # 4662
+    444, 3032, 2624, 801, 4392, 4130, 7526, 1491, 244, 1053, 3033, 4131, 4132, 340, 7527, 3915,  # 4678
+    1041, 2987, 293, 1168, 87, 1357, 7528, 1539, 959, 7529, 2236, 721, 694, 4133, 3768, 219,  # 4694
+    1478, 644, 1417, 3331, 2656, 1413, 1401, 1335, 1389, 3916, 7530, 7531, 2988, 2362, 3134, 1825,  # 4710
+    730, 1515, 184, 2827, 66, 4393, 7532, 1660, 2943, 246, 3332, 378, 1457, 226, 3433, 975,  # 4726
+    3917, 2944, 1264, 3537, 674, 696, 7533, 163, 7534, 1141, 2417, 2166, 713, 3538, 3333, 4394,  # 4742
+    3918, 7535, 7536, 1186, 15, 7537, 1079, 1070, 7538, 1522, 3193, 3539, 276, 1050, 2716, 758,  # 4758
+    1126, 653, 2945, 3263, 7539, 2337, 889, 3540, 3919, 3081, 2989, 903, 1250, 4395, 3920, 3434,  # 4774
+    3541, 1342, 1681, 1718, 766, 3264, 286, 89, 2946, 3649, 7540, 1713, 7541, 2597, 3334, 2990,  # 4790
+    7542, 2947, 2215, 3194, 2866, 7543, 4396, 2498, 2526, 181, 387, 1075, 3921, 731, 2187, 3335,  # 4806
+    7544, 3265, 310, 313, 3435, 2299, 770, 4134, 54, 3034, 189, 4397, 3082, 3769, 3922, 7545,  # 4822
+    1230, 1617, 1849, 355, 3542, 4135, 4398, 3336, 111, 4136, 3650, 1350, 3135, 3436, 3035, 4137,  # 4838
+    2149, 3266, 3543, 7546, 2784, 3923, 3924, 2991, 722, 2008, 7547, 1071, 247, 1207, 2338, 2471,  # 4854
+    1378, 4399, 2009, 864, 1437, 1214, 4400, 373, 3770, 1142, 2216, 667, 4401, 442, 2753, 2555,  # 4870
+    3771, 3925, 1968, 4138, 3267, 1839, 837, 170, 1107, 934, 1336, 1882, 7548, 7549, 2118, 4139,  # 4886
+    2828, 743, 1569, 7550, 4402, 4140, 582, 2384, 1418, 3437, 7551, 1802, 7552, 357, 1395, 1729,  # 4902
+    3651, 3268, 2418, 1564, 2237, 7553, 3083, 3772, 1633, 4403, 1114, 2085, 4141, 1532, 7554, 482,  # 4918
+    2446, 4404, 7555, 7556, 1492, 833, 1466, 7557, 2717, 3544, 1641, 2829, 7558, 1526, 1272, 3652,  # 4934
+    4142, 1686, 1794, 416, 2556, 1902, 1953, 1803, 7559, 3773, 2785, 3774, 1159, 2316, 7560, 2867,  # 4950
+    4405, 1610, 1584, 3036, 2419, 2754, 443, 3269, 1163, 3136, 7561, 7562, 3926, 7563, 4143, 2499,  # 4966
+    3037, 4406, 3927, 3137, 2103, 1647, 3545, 2010, 1872, 4144, 7564, 4145, 431, 3438, 7565, 250,  # 4982
+    97, 81, 4146, 7566, 1648, 1850, 1558, 160, 848, 7567, 866, 740, 1694, 7568, 2201, 2830,  # 4998
+    3195, 4147, 4407, 3653, 1687, 950, 2472, 426, 469, 3196, 3654, 3655, 3928, 7569, 7570, 1188,  # 5014
+    424, 1995, 861, 3546, 4148, 3775, 2202, 2685, 168, 1235, 3547, 4149, 7571, 2086, 1674, 4408,  # 5030
+    3337, 3270, 220, 2557, 1009, 7572, 3776, 670, 2992, 332, 1208, 717, 7573, 7574, 3548, 2447,  # 5046
+    3929, 3338, 7575, 513, 7576, 1209, 2868, 3339, 3138, 4409, 1080, 7577, 7578, 7579, 7580, 2527,  # 5062
+    3656, 3549, 815, 1587, 3930, 3931, 7581, 3550, 3439, 3777, 1254, 4410, 1328, 3038, 1390, 3932,  # 5078
+    1741, 3933, 3778, 3934, 7582, 236, 3779, 2448, 3271, 7583, 7584, 3657, 3780, 1273, 3781, 4411,  # 5094
+    7585, 308, 7586, 4412, 245, 4413, 1851, 2473, 1307, 2575, 430, 715, 2136, 2449, 7587, 270,  # 5110
+    199, 2869, 3935, 7588, 3551, 2718, 1753, 761, 1754, 725, 1661, 1840, 4414, 3440, 3658, 7589,  # 5126
+    7590, 587, 14, 3272, 227, 2598, 326, 480, 2265, 943, 2755, 3552, 291, 650, 1883, 7591,  # 5142
+    1702, 1226, 102, 1547, 62, 3441, 904, 4415, 3442, 1164, 4150, 7592, 7593, 1224, 1548, 2756,  # 5158
+    391, 498, 1493, 7594, 1386, 1419, 7595, 2055, 1177, 4416, 813, 880, 1081, 2363, 566, 1145,  # 5174
+    4417, 2286, 1001, 1035, 2558, 2599, 2238, 394, 1286, 7596, 7597, 2068, 7598, 86, 1494, 1730,  # 5190
+    3936, 491, 1588, 745, 897, 2948, 843, 3340, 3937, 2757, 2870, 3273, 1768, 998, 2217, 2069,  # 5206
+    397, 1826, 1195, 1969, 3659, 2993, 3341, 284, 7599, 3782, 2500, 2137, 2119, 1903, 7600, 3938,  # 5222
+    2150, 3939, 4151, 1036, 3443, 1904, 114, 2559, 4152, 209, 1527, 7601, 7602, 2949, 2831, 2625,  # 5238
+    2385, 2719, 3139, 812, 2560, 7603, 3274, 7604, 1559, 737, 1884, 3660, 1210, 885, 28, 2686,  # 5254
+    3553, 3783, 7605, 4153, 1004, 1779, 4418, 7606, 346, 1981, 2218, 2687, 4419, 3784, 1742, 797,  # 5270
+    1642, 3940, 1933, 1072, 1384, 2151, 896, 3941, 3275, 3661, 3197, 2871, 3554, 7607, 2561, 1958,  # 5286
+    4420, 2450, 1785, 7608, 7609, 7610, 3942, 4154, 1005, 1308, 3662, 4155, 2720, 4421, 4422, 1528,  # 5302
+    2600, 161, 1178, 4156, 1982, 987, 4423, 1101, 4157, 631, 3943, 1157, 3198, 2420, 1343, 1241,  # 5318
+    1016, 2239, 2562, 372, 877, 2339, 2501, 1160, 555, 1934, 911, 3944, 7611, 466, 1170, 169,  # 5334
+    1051, 2907, 2688, 3663, 2474, 2994, 1182, 2011, 2563, 1251, 2626, 7612, 992, 2340, 3444, 1540,  # 5350
+    2721, 1201, 2070, 2401, 1996, 2475, 7613, 4424, 528, 1922, 2188, 1503, 1873, 1570, 2364, 3342,  # 5366
+    3276, 7614, 557, 1073, 7615, 1827, 3445, 2087, 2266, 3140, 3039, 3084, 767, 3085, 2786, 4425,  # 5382
+    1006, 4158, 4426, 2341, 1267, 2176, 3664, 3199, 778, 3945, 3200, 2722, 1597, 2657, 7616, 4427,  # 5398
+    7617, 3446, 7618, 7619, 7620, 3277, 2689, 1433, 3278, 131, 95, 1504, 3946, 723, 4159, 3141,  # 5414
+    1841, 3555, 2758, 2189, 3947, 2027, 2104, 3665, 7621, 2995, 3948, 1218, 7622, 3343, 3201, 3949,  # 5430
+    4160, 2576, 248, 1634, 3785, 912, 7623, 2832, 3666, 3040, 3786, 654, 53, 7624, 2996, 7625,  # 5446
+    1688, 4428, 777, 3447, 1032, 3950, 1425, 7626, 191, 820, 2120, 2833, 971, 4429, 931, 3202,  # 5462
+    135, 664, 783, 3787, 1997, 772, 2908, 1935, 3951, 3788, 4430, 2909, 3203, 282, 2723, 640,  # 5478
+    1372, 3448, 1127, 922, 325, 3344, 7627, 7628, 711, 2044, 7629, 7630, 3952, 2219, 2787, 1936,  # 5494
+    3953, 3345, 2220, 2251, 3789, 2300, 7631, 4431, 3790, 1258, 3279, 3954, 3204, 2138, 2950, 3955,  # 5510
+    3956, 7632, 2221, 258, 3205, 4432, 101, 1227, 7633, 3280, 1755, 7634, 1391, 3281, 7635, 2910,  # 5526
+    2056, 893, 7636, 7637, 7638, 1402, 4161, 2342, 7639, 7640, 3206, 3556, 7641, 7642, 878, 1325,  # 5542
+    1780, 2788, 4433, 259, 1385, 2577, 744, 1183, 2267, 4434, 7643, 3957, 2502, 7644, 684, 1024,  # 5558
+    4162, 7645, 472, 3557, 3449, 1165, 3282, 3958, 3959, 322, 2152, 881, 455, 1695, 1152, 1340,  # 5574
+    660, 554, 2153, 4435, 1058, 4436, 4163, 830, 1065, 3346, 3960, 4437, 1923, 7646, 1703, 1918,  # 5590
+    7647, 932, 2268, 122, 7648, 4438, 947, 677, 7649, 3791, 2627, 297, 1905, 1924, 2269, 4439,  # 5606
+    2317, 3283, 7650, 7651, 4164, 7652, 4165, 84, 4166, 112, 989, 7653, 547, 1059, 3961, 701,  # 5622
+    3558, 1019, 7654, 4167, 7655, 3450, 942, 639, 457, 2301, 2451, 993, 2951, 407, 851, 494,  # 5638
+    4440, 3347, 927, 7656, 1237, 7657, 2421, 3348, 573, 4168, 680, 921, 2911, 1279, 1874, 285,  # 5654
+    790, 1448, 1983, 719, 2167, 7658, 7659, 4441, 3962, 3963, 1649, 7660, 1541, 563, 7661, 1077,  # 5670
+    7662, 3349, 3041, 3451, 511, 2997, 3964, 3965, 3667, 3966, 1268, 2564, 3350, 3207, 4442, 4443,  # 5686
+    7663, 535, 1048, 1276, 1189, 2912, 2028, 3142, 1438, 1373, 2834, 2952, 1134, 2012, 7664, 4169,  # 5702
+    1238, 2578, 3086, 1259, 7665, 700, 7666, 2953, 3143, 3668, 4170, 7667, 4171, 1146, 1875, 1906,  # 5718
+    4444, 2601, 3967, 781, 2422, 132, 1589, 203, 147, 273, 2789, 2402, 898, 1786, 2154, 3968,  # 5734
+    3969, 7668, 3792, 2790, 7669, 7670, 4445, 4446, 7671, 3208, 7672, 1635, 3793, 965, 7673, 1804,  # 5750
+    2690, 1516, 3559, 1121, 1082, 1329, 3284, 3970, 1449, 3794, 65, 1128, 2835, 2913, 2759, 1590,  # 5766
+    3795, 7674, 7675, 12, 2658, 45, 976, 2579, 3144, 4447, 517, 2528, 1013, 1037, 3209, 7676,  # 5782
+    3796, 2836, 7677, 3797, 7678, 3452, 7679, 2602, 614, 1998, 2318, 3798, 3087, 2724, 2628, 7680,  # 5798
+    2580, 4172, 599, 1269, 7681, 1810, 3669, 7682, 2691, 3088, 759, 1060, 489, 1805, 3351, 3285,  # 5814
+    1358, 7683, 7684, 2386, 1387, 1215, 2629, 2252, 490, 7685, 7686, 4173, 1759, 2387, 2343, 7687,  # 5830
+    4448, 3799, 1907, 3971, 2630, 1806, 3210, 4449, 3453, 3286, 2760, 2344, 874, 7688, 7689, 3454,  # 5846
+    3670, 1858, 91, 2914, 3671, 3042, 3800, 4450, 7690, 3145, 3972, 2659, 7691, 3455, 1202, 1403,  # 5862
+    3801, 2954, 2529, 1517, 2503, 4451, 3456, 2504, 7692, 4452, 7693, 2692, 1885, 1495, 1731, 3973,  # 5878
+    2365, 4453, 7694, 2029, 7695, 7696, 3974, 2693, 1216, 237, 2581, 4174, 2319, 3975, 3802, 4454,  # 5894
+    4455, 2694, 3560, 3457, 445, 4456, 7697, 7698, 7699, 7700, 2761, 61, 3976, 3672, 1822, 3977,  # 5910
+    7701, 687, 2045, 935, 925, 405, 2660, 703, 1096, 1859, 2725, 4457, 3978, 1876, 1367, 2695,  # 5926
+    3352, 918, 2105, 1781, 2476, 334, 3287, 1611, 1093, 4458, 564, 3146, 3458, 3673, 3353, 945,  # 5942
+    2631, 2057, 4459, 7702, 1925, 872, 4175, 7703, 3459, 2696, 3089, 349, 4176, 3674, 3979, 4460,  # 5958
+    3803, 4177, 3675, 2155, 3980, 4461, 4462, 4178, 4463, 2403, 2046, 782, 3981, 400, 251, 4179,  # 5974
+    1624, 7704, 7705, 277, 3676, 299, 1265, 476, 1191, 3804, 2121, 4180, 4181, 1109, 205, 7706,  # 5990
+    2582, 1000, 2156, 3561, 1860, 7707, 7708, 7709, 4464, 7710, 4465, 2565, 107, 2477, 2157, 3982,  # 6006
+    3460, 3147, 7711, 1533, 541, 1301, 158, 753, 4182, 2872, 3562, 7712, 1696, 370, 1088, 4183,  # 6022
+    4466, 3563, 579, 327, 440, 162, 2240, 269, 1937, 1374, 3461, 968, 3043, 56, 1396, 3090,  # 6038
+    2106, 3288, 3354, 7713, 1926, 2158, 4467, 2998, 7714, 3564, 7715, 7716, 3677, 4468, 2478, 7717,  # 6054
+    2791, 7718, 1650, 4469, 7719, 2603, 7720, 7721, 3983, 2661, 3355, 1149, 3356, 3984, 3805, 3985,  # 6070
+    7722, 1076, 49, 7723, 951, 3211, 3289, 3290, 450, 2837, 920, 7724, 1811, 2792, 2366, 4184,  # 6086
+    1908, 1138, 2367, 3806, 3462, 7725, 3212, 4470, 1909, 1147, 1518, 2423, 4471, 3807, 7726, 4472,  # 6102
+    2388, 2604, 260, 1795, 3213, 7727, 7728, 3808, 3291, 708, 7729, 3565, 1704, 7730, 3566, 1351,  # 6118
+    1618, 3357, 2999, 1886, 944, 4185, 3358, 4186, 3044, 3359, 4187, 7731, 3678, 422, 413, 1714,  # 6134
+    3292, 500, 2058, 2345, 4188, 2479, 7732, 1344, 1910, 954, 7733, 1668, 7734, 7735, 3986, 2404,  # 6150
+    4189, 3567, 3809, 4190, 7736, 2302, 1318, 2505, 3091, 133, 3092, 2873, 4473, 629, 31, 2838,  # 6166
+    2697, 3810, 4474, 850, 949, 4475, 3987, 2955, 1732, 2088, 4191, 1496, 1852, 7737, 3988, 620,  # 6182
+    3214, 981, 1242, 3679, 3360, 1619, 3680, 1643, 3293, 2139, 2452, 1970, 1719, 3463, 2168, 7738,  # 6198
+    3215, 7739, 7740, 3361, 1828, 7741, 1277, 4476, 1565, 2047, 7742, 1636, 3568, 3093, 7743, 869,  # 6214
+    2839, 655, 3811, 3812, 3094, 3989, 3000, 3813, 1310, 3569, 4477, 7744, 7745, 7746, 1733, 558,  # 6230
+    4478, 3681, 335, 1549, 3045, 1756, 4192, 3682, 1945, 3464, 1829, 1291, 1192, 470, 2726, 2107,  # 6246
+    2793, 913, 1054, 3990, 7747, 1027, 7748, 3046, 3991, 4479, 982, 2662, 3362, 3148, 3465, 3216,  # 6262
+    3217, 1946, 2794, 7749, 571, 4480, 7750, 1830, 7751, 3570, 2583, 1523, 2424, 7752, 2089, 984,  # 6278
+    4481, 3683, 1959, 7753, 3684, 852, 923, 2795, 3466, 3685, 969, 1519, 999, 2048, 2320, 1705,  # 6294
+    7754, 3095, 615, 1662, 151, 597, 3992, 2405, 2321, 1049, 275, 4482, 3686, 4193, 568, 3687,  # 6310
+    3571, 2480, 4194, 3688, 7755, 2425, 2270, 409, 3218, 7756, 1566, 2874, 3467, 1002, 769, 2840,  # 6326
+    194, 2090, 3149, 3689, 2222, 3294, 4195, 628, 1505, 7757, 7758, 1763, 2177, 3001, 3993, 521,  # 6342
+    1161, 2584, 1787, 2203, 2406, 4483, 3994, 1625, 4196, 4197, 412, 42, 3096, 464, 7759, 2632,  # 6358
+    4484, 3363, 1760, 1571, 2875, 3468, 2530, 1219, 2204, 3814, 2633, 2140, 2368, 4485, 4486, 3295,  # 6374
+    1651, 3364, 3572, 7760, 7761, 3573, 2481, 3469, 7762, 3690, 7763, 7764, 2271, 2091, 460, 7765,  # 6390
+    4487, 7766, 3002, 962, 588, 3574, 289, 3219, 2634, 1116, 52, 7767, 3047, 1796, 7768, 7769,  # 6406
+    7770, 1467, 7771, 1598, 1143, 3691, 4198, 1984, 1734, 1067, 4488, 1280, 3365, 465, 4489, 1572,  # 6422
+    510, 7772, 1927, 2241, 1812, 1644, 3575, 7773, 4490, 3692, 7774, 7775, 2663, 1573, 1534, 7776,  # 6438
+    7777, 4199, 536, 1807, 1761, 3470, 3815, 3150, 2635, 7778, 7779, 7780, 4491, 3471, 2915, 1911,  # 6454
+    2796, 7781, 3296, 1122, 377, 3220, 7782, 360, 7783, 7784, 4200, 1529, 551, 7785, 2059, 3693,  # 6470
+    1769, 2426, 7786, 2916, 4201, 3297, 3097, 2322, 2108, 2030, 4492, 1404, 136, 1468, 1479, 672,  # 6486
+    1171, 3221, 2303, 271, 3151, 7787, 2762, 7788, 2049, 678, 2727, 865, 1947, 4493, 7789, 2013,  # 6502
+    3995, 2956, 7790, 2728, 2223, 1397, 3048, 3694, 4494, 4495, 1735, 2917, 3366, 3576, 7791, 3816,  # 6518
+    509, 2841, 2453, 2876, 3817, 7792, 7793, 3152, 3153, 4496, 4202, 2531, 4497, 2304, 1166, 1010,  # 6534
+    552, 681, 1887, 7794, 7795, 2957, 2958, 3996, 1287, 1596, 1861, 3154, 358, 453, 736, 175,  # 6550
+    478, 1117, 905, 1167, 1097, 7796, 1853, 1530, 7797, 1706, 7798, 2178, 3472, 2287, 3695, 3473,  # 6566
+    3577, 4203, 2092, 4204, 7799, 3367, 1193, 2482, 4205, 1458, 2190, 2205, 1862, 1888, 1421, 3298,  # 6582
+    2918, 3049, 2179, 3474, 595, 2122, 7800, 3997, 7801, 7802, 4206, 1707, 2636, 223, 3696, 1359,  # 6598
+    751, 3098, 183, 3475, 7803, 2797, 3003, 419, 2369, 633, 704, 3818, 2389, 241, 7804, 7805,  # 6614
+    7806, 838, 3004, 3697, 2272, 2763, 2454, 3819, 1938, 2050, 3998, 1309, 3099, 2242, 1181, 7807,  # 6630
+    1136, 2206, 3820, 2370, 1446, 4207, 2305, 4498, 7808, 7809, 4208, 1055, 2605, 484, 3698, 7810,  # 6646
+    3999, 625, 4209, 2273, 3368, 1499, 4210, 4000, 7811, 4001, 4211, 3222, 2274, 2275, 3476, 7812,  # 6662
+    7813, 2764, 808, 2606, 3699, 3369, 4002, 4212, 3100, 2532, 526, 3370, 3821, 4213, 955, 7814,  # 6678
+    1620, 4214, 2637, 2427, 7815, 1429, 3700, 1669, 1831, 994, 928, 7816, 3578, 1260, 7817, 7818,  # 6694
+    7819, 1948, 2288, 741, 2919, 1626, 4215, 2729, 2455, 867, 1184, 362, 3371, 1392, 7820, 7821,  # 6710
+    4003, 4216, 1770, 1736, 3223, 2920, 4499, 4500, 1928, 2698, 1459, 1158, 7822, 3050, 3372, 2877,  # 6726
+    1292, 1929, 2506, 2842, 3701, 1985, 1187, 2071, 2014, 2607, 4217, 7823, 2566, 2507, 2169, 3702,  # 6742
+    2483, 3299, 7824, 3703, 4501, 7825, 7826, 666, 1003, 3005, 1022, 3579, 4218, 7827, 4502, 1813,  # 6758
+    2253, 574, 3822, 1603, 295, 1535, 705, 3823, 4219, 283, 858, 417, 7828, 7829, 3224, 4503,  # 6774
+    4504, 3051, 1220, 1889, 1046, 2276, 2456, 4004, 1393, 1599, 689, 2567, 388, 4220, 7830, 2484,  # 6790
+    802, 7831, 2798, 3824, 2060, 1405, 2254, 7832, 4505, 3825, 2109, 1052, 1345, 3225, 1585, 7833,  # 6806
+    809, 7834, 7835, 7836, 575, 2730, 3477, 956, 1552, 1469, 1144, 2323, 7837, 2324, 1560, 2457,  # 6822
+    3580, 3226, 4005, 616, 2207, 3155, 2180, 2289, 7838, 1832, 7839, 3478, 4506, 7840, 1319, 3704,  # 6838
+    3705, 1211, 3581, 1023, 3227, 1293, 2799, 7841, 7842, 7843, 3826, 607, 2306, 3827, 762, 2878,  # 6854
+    1439, 4221, 1360, 7844, 1485, 3052, 7845, 4507, 1038, 4222, 1450, 2061, 2638, 4223, 1379, 4508,  # 6870
+    2585, 7846, 7847, 4224, 1352, 1414, 2325, 2921, 1172, 7848, 7849, 3828, 3829, 7850, 1797, 1451,  # 6886
+    7851, 7852, 7853, 7854, 2922, 4006, 4007, 2485, 2346, 411, 4008, 4009, 3582, 3300, 3101, 4509,  # 6902
+    1561, 2664, 1452, 4010, 1375, 7855, 7856, 47, 2959, 316, 7857, 1406, 1591, 2923, 3156, 7858,  # 6918
+    1025, 2141, 3102, 3157, 354, 2731, 884, 2224, 4225, 2407, 508, 3706, 726, 3583, 996, 2428,  # 6934
+    3584, 729, 7859, 392, 2191, 1453, 4011, 4510, 3707, 7860, 7861, 2458, 3585, 2608, 1675, 2800,  # 6950
+    919, 2347, 2960, 2348, 1270, 4511, 4012, 73, 7862, 7863, 647, 7864, 3228, 2843, 2255, 1550,  # 6966
+    1346, 3006, 7865, 1332, 883, 3479, 7866, 7867, 7868, 7869, 3301, 2765, 7870, 1212, 831, 1347,  # 6982
+    4226, 4512, 2326, 3830, 1863, 3053, 720, 3831, 4513, 4514, 3832, 7871, 4227, 7872, 7873, 4515,  # 6998
+    7874, 7875, 1798, 4516, 3708, 2609, 4517, 3586, 1645, 2371, 7876, 7877, 2924, 669, 2208, 2665,  # 7014
+    2429, 7878, 2879, 7879, 7880, 1028, 3229, 7881, 4228, 2408, 7882, 2256, 1353, 7883, 7884, 4518,  # 7030
+    3158, 518, 7885, 4013, 7886, 4229, 1960, 7887, 2142, 4230, 7888, 7889, 3007, 2349, 2350, 3833,  # 7046
+    516, 1833, 1454, 4014, 2699, 4231, 4519, 2225, 2610, 1971, 1129, 3587, 7890, 2766, 7891, 2961,  # 7062
+    1422, 577, 1470, 3008, 1524, 3373, 7892, 7893, 432, 4232, 3054, 3480, 7894, 2586, 1455, 2508,  # 7078
+    2226, 1972, 1175, 7895, 1020, 2732, 4015, 3481, 4520, 7896, 2733, 7897, 1743, 1361, 3055, 3482,  # 7094
+    2639, 4016, 4233, 4521, 2290, 895, 924, 4234, 2170, 331, 2243, 3056, 166, 1627, 3057, 1098,  # 7110
+    7898, 1232, 2880, 2227, 3374, 4522, 657, 403, 1196, 2372, 542, 3709, 3375, 1600, 4235, 3483,  # 7126
+    7899, 4523, 2767, 3230, 576, 530, 1362, 7900, 4524, 2533, 2666, 3710, 4017, 7901, 842, 3834,  # 7142
+    7902, 2801, 2031, 1014, 4018, 213, 2700, 3376, 665, 621, 4236, 7903, 3711, 2925, 2430, 7904,  # 7158
+    2431, 3302, 3588, 3377, 7905, 4237, 2534, 4238, 4525, 3589, 1682, 4239, 3484, 1380, 7906, 724,  # 7174
+    2277, 600, 1670, 7907, 1337, 1233, 4526, 3103, 2244, 7908, 1621, 4527, 7909, 651, 4240, 7910,  # 7190
+    1612, 4241, 2611, 7911, 2844, 7912, 2734, 2307, 3058, 7913, 716, 2459, 3059, 174, 1255, 2701,  # 7206
+    4019, 3590, 548, 1320, 1398, 728, 4020, 1574, 7914, 1890, 1197, 3060, 4021, 7915, 3061, 3062,  # 7222
+    3712, 3591, 3713, 747, 7916, 635, 4242, 4528, 7917, 7918, 7919, 4243, 7920, 7921, 4529, 7922,  # 7238
+    3378, 4530, 2432, 451, 7923, 3714, 2535, 2072, 4244, 2735, 4245, 4022, 7924, 1764, 4531, 7925,  # 7254
+    4246, 350, 7926, 2278, 2390, 2486, 7927, 4247, 4023, 2245, 1434, 4024, 488, 4532, 458, 4248,  # 7270
+    4025, 3715, 771, 1330, 2391, 3835, 2568, 3159, 2159, 2409, 1553, 2667, 3160, 4249, 7928, 2487,  # 7286
+    2881, 2612, 1720, 2702, 4250, 3379, 4533, 7929, 2536, 4251, 7930, 3231, 4252, 2768, 7931, 2015,  # 7302
+    2736, 7932, 1155, 1017, 3716, 3836, 7933, 3303, 2308, 201, 1864, 4253, 1430, 7934, 4026, 7935,  # 7318
+    7936, 7937, 7938, 7939, 4254, 1604, 7940, 414, 1865, 371, 2587, 4534, 4535, 3485, 2016, 3104,  # 7334
+    4536, 1708, 960, 4255, 887, 389, 2171, 1536, 1663, 1721, 7941, 2228, 4027, 2351, 2926, 1580,  # 7350
+    7942, 7943, 7944, 1744, 7945, 2537, 4537, 4538, 7946, 4539, 7947, 2073, 7948, 7949, 3592, 3380,  # 7366
+    2882, 4256, 7950, 4257, 2640, 3381, 2802, 673, 2703, 2460, 709, 3486, 4028, 3593, 4258, 7951,  # 7382
+    1148, 502, 634, 7952, 7953, 1204, 4540, 3594, 1575, 4541, 2613, 3717, 7954, 3718, 3105, 948,  # 7398
+    3232, 121, 1745, 3837, 1110, 7955, 4259, 3063, 2509, 3009, 4029, 3719, 1151, 1771, 3838, 1488,  # 7414
+    4030, 1986, 7956, 2433, 3487, 7957, 7958, 2093, 7959, 4260, 3839, 1213, 1407, 2803, 531, 2737,  # 7430
+    2538, 3233, 1011, 1537, 7960, 2769, 4261, 3106, 1061, 7961, 3720, 3721, 1866, 2883, 7962, 2017,  # 7446
+    120, 4262, 4263, 2062, 3595, 3234, 2309, 3840, 2668, 3382, 1954, 4542, 7963, 7964, 3488, 1047,  # 7462
+    2704, 1266, 7965, 1368, 4543, 2845, 649, 3383, 3841, 2539, 2738, 1102, 2846, 2669, 7966, 7967,  # 7478
+    1999, 7968, 1111, 3596, 2962, 7969, 2488, 3842, 3597, 2804, 1854, 3384, 3722, 7970, 7971, 3385,  # 7494
+    2410, 2884, 3304, 3235, 3598, 7972, 2569, 7973, 3599, 2805, 4031, 1460, 856, 7974, 3600, 7975,  # 7510
+    2885, 2963, 7976, 2886, 3843, 7977, 4264, 632, 2510, 875, 3844, 1697, 3845, 2291, 7978, 7979,  # 7526
+    4544, 3010, 1239, 580, 4545, 4265, 7980, 914, 936, 2074, 1190, 4032, 1039, 2123, 7981, 7982,  # 7542
+    7983, 3386, 1473, 7984, 1354, 4266, 3846, 7985, 2172, 3064, 4033, 915, 3305, 4267, 4268, 3306,  # 7558
+    1605, 1834, 7986, 2739, 398, 3601, 4269, 3847, 4034, 328, 1912, 2847, 4035, 3848, 1331, 4270,  # 7574
+    3011, 937, 4271, 7987, 3602, 4036, 4037, 3387, 2160, 4546, 3388, 524, 742, 538, 3065, 1012,  # 7590
+    7988, 7989, 3849, 2461, 7990, 658, 1103, 225, 3850, 7991, 7992, 4547, 7993, 4548, 7994, 3236,  # 7606
+    1243, 7995, 4038, 963, 2246, 4549, 7996, 2705, 3603, 3161, 7997, 7998, 2588, 2327, 7999, 4550,  # 7622
+    8000, 8001, 8002, 3489, 3307, 957, 3389, 2540, 2032, 1930, 2927, 2462, 870, 2018, 3604, 1746,  # 7638
+    2770, 2771, 2434, 2463, 8003, 3851, 8004, 3723, 3107, 3724, 3490, 3390, 3725, 8005, 1179, 3066,  # 7654
+    8006, 3162, 2373, 4272, 3726, 2541, 3163, 3108, 2740, 4039, 8007, 3391, 1556, 2542, 2292, 977,  # 7670
+    2887, 2033, 4040, 1205, 3392, 8008, 1765, 3393, 3164, 2124, 1271, 1689, 714, 4551, 3491, 8009,  # 7686
+    2328, 3852, 533, 4273, 3605, 2181, 617, 8010, 2464, 3308, 3492, 2310, 8011, 8012, 3165, 8013,  # 7702
+    8014, 3853, 1987, 618, 427, 2641, 3493, 3394, 8015, 8016, 1244, 1690, 8017, 2806, 4274, 4552,  # 7718
+    8018, 3494, 8019, 8020, 2279, 1576, 473, 3606, 4275, 3395, 972, 8021, 3607, 8022, 3067, 8023,  # 7734
+    8024, 4553, 4554, 8025, 3727, 4041, 4042, 8026, 153, 4555, 356, 8027, 1891, 2888, 4276, 2143,  # 7750
+    408, 803, 2352, 8028, 3854, 8029, 4277, 1646, 2570, 2511, 4556, 4557, 3855, 8030, 3856, 4278,  # 7766
+    8031, 2411, 3396, 752, 8032, 8033, 1961, 2964, 8034, 746, 3012, 2465, 8035, 4279, 3728, 698,  # 7782
+    4558, 1892, 4280, 3608, 2543, 4559, 3609, 3857, 8036, 3166, 3397, 8037, 1823, 1302, 4043, 2706,  # 7798
+    3858, 1973, 4281, 8038, 4282, 3167, 823, 1303, 1288, 1236, 2848, 3495, 4044, 3398, 774, 3859,  # 7814
+    8039, 1581, 4560, 1304, 2849, 3860, 4561, 8040, 2435, 2161, 1083, 3237, 4283, 4045, 4284, 344,  # 7830
+    1173, 288, 2311, 454, 1683, 8041, 8042, 1461, 4562, 4046, 2589, 8043, 8044, 4563, 985, 894,  # 7846
+    8045, 3399, 3168, 8046, 1913, 2928, 3729, 1988, 8047, 2110, 1974, 8048, 4047, 8049, 2571, 1194,  # 7862
+    425, 8050, 4564, 3169, 1245, 3730, 4285, 8051, 8052, 2850, 8053, 636, 4565, 1855, 3861, 760,  # 7878
+    1799, 8054, 4286, 2209, 1508, 4566, 4048, 1893, 1684, 2293, 8055, 8056, 8057, 4287, 4288, 2210,  # 7894
+    479, 8058, 8059, 832, 8060, 4049, 2489, 8061, 2965, 2490, 3731, 990, 3109, 627, 1814, 2642,  # 7910
+    4289, 1582, 4290, 2125, 2111, 3496, 4567, 8062, 799, 4291, 3170, 8063, 4568, 2112, 1737, 3013,  # 7926
+    1018, 543, 754, 4292, 3309, 1676, 4569, 4570, 4050, 8064, 1489, 8065, 3497, 8066, 2614, 2889,  # 7942
+    4051, 8067, 8068, 2966, 8069, 8070, 8071, 8072, 3171, 4571, 4572, 2182, 1722, 8073, 3238, 3239,  # 7958
+    1842, 3610, 1715, 481, 365, 1975, 1856, 8074, 8075, 1962, 2491, 4573, 8076, 2126, 3611, 3240,  # 7974
+    433, 1894, 2063, 2075, 8077, 602, 2741, 8078, 8079, 8080, 8081, 8082, 3014, 1628, 3400, 8083,  # 7990
+    3172, 4574, 4052, 2890, 4575, 2512, 8084, 2544, 2772, 8085, 8086, 8087, 3310, 4576, 2891, 8088,  # 8006
+    4577, 8089, 2851, 4578, 4579, 1221, 2967, 4053, 2513, 8090, 8091, 8092, 1867, 1989, 8093, 8094,  # 8022
+    8095, 1895, 8096, 8097, 4580, 1896, 4054, 318, 8098, 2094, 4055, 4293, 8099, 8100, 485, 8101,  # 8038
+    938, 3862, 553, 2670, 116, 8102, 3863, 3612, 8103, 3498, 2671, 2773, 3401, 3311, 2807, 8104,  # 8054
+    3613, 2929, 4056, 1747, 2930, 2968, 8105, 8106, 207, 8107, 8108, 2672, 4581, 2514, 8109, 3015,  # 8070
+    890, 3614, 3864, 8110, 1877, 3732, 3402, 8111, 2183, 2353, 3403, 1652, 8112, 8113, 8114, 941,  # 8086
+    2294, 208, 3499, 4057, 2019, 330, 4294, 3865, 2892, 2492, 3733, 4295, 8115, 8116, 8117, 8118,  # 8102
 )
-
+# fmt: on
--- a/libs/common/chardet/euctwprober.py
+++ b/libs/common/chardet/euctwprober.py
@ -25,22 +25,23 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
 from .chardistribution import EUCTWDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import EUCTW_SM_MODEL

+
 class EUCTWProber(MultiByteCharSetProber):
-    def __init__(self):
-        super(EUCTWProber, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(EUCTW_SM_MODEL)
        self.distribution_analyzer = EUCTWDistributionAnalysis()
        self.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "EUC-TW"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Taiwan"
--- a/libs/common/chardet/gb2312freq.py
+++ b/libs/common/chardet/gb2312freq.py
@ -43,6 +43,7 @@ GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9

 GB2312_TABLE_SIZE = 3760

+# fmt: off
 GB2312_CHAR_TO_FREQ_ORDER = (
 1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205,
 2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842,
@ -280,4 +281,4 @@ GB2312_CHAR_TO_FREQ_ORDER = (
 381,1638,4592,1020, 516,3214, 458, 947,4575,1432, 211,1514,2926,1865,2142, 189,
 852,1221,1400,1486, 882,2299,4036, 351,  28,1122, 700,6479,6480,6481,6482,6483,  #last 512
 )
-
+# fmt: on
--- a/libs/common/chardet/gb2312prober.py
+++ b/libs/common/chardet/gb2312prober.py
@ -25,22 +25,23 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
 from .chardistribution import GB2312DistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import GB2312_SM_MODEL

+
 class GB2312Prober(MultiByteCharSetProber):
-    def __init__(self):
-        super(GB2312Prober, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(GB2312_SM_MODEL)
        self.distribution_analyzer = GB2312DistributionAnalysis()
        self.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "GB2312"

    @property
-    def language(self):
+    def language(self) -> str:
        return "Chinese"
--- a/libs/common/chardet/hebrewprober.py
+++ b/libs/common/chardet/hebrewprober.py
@ -25,8 +25,11 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import Optional, Union
+
 from .charsetprober import CharSetProber
 from .enums import ProbingState
+from .sbcharsetprober import SingleByteCharSetProber

 # This prober doesn't actually recognize a language or a charset.
 # It is a helper prober for the use of the Hebrew model probers
@ -125,18 +128,20 @@ from .enums import ProbingState
 # model probers scores. The answer is returned in the form of the name of the
 # charset identified, either "windows-1255" or "ISO-8859-8".

+
 class HebrewProber(CharSetProber):
+    SPACE = 0x20
    # windows-1255 / ISO-8859-8 code points of interest
-    FINAL_KAF = 0xea
-    NORMAL_KAF = 0xeb
-    FINAL_MEM = 0xed
-    NORMAL_MEM = 0xee
-    FINAL_NUN = 0xef
-    NORMAL_NUN = 0xf0
-    FINAL_PE = 0xf3
-    NORMAL_PE = 0xf4
-    FINAL_TSADI = 0xf5
-    NORMAL_TSADI = 0xf6
+    FINAL_KAF = 0xEA
+    NORMAL_KAF = 0xEB
+    FINAL_MEM = 0xED
+    NORMAL_MEM = 0xEE
+    FINAL_NUN = 0xEF
+    NORMAL_NUN = 0xF0
+    FINAL_PE = 0xF3
+    NORMAL_PE = 0xF4
+    FINAL_TSADI = 0xF5
+    NORMAL_TSADI = 0xF6

    # Minimum Visual vs Logical final letter score difference.
    # If the difference is below this, don't rely solely on the final letter score
@ -151,35 +156,44 @@ class HebrewProber(CharSetProber):
    VISUAL_HEBREW_NAME = "ISO-8859-8"
    LOGICAL_HEBREW_NAME = "windows-1255"

-    def __init__(self):
-        super(HebrewProber, self).__init__()
-        self._final_char_logical_score = None
-        self._final_char_visual_score = None
-        self._prev = None
-        self._before_prev = None
-        self._logical_prober = None
-        self._visual_prober = None
+    def __init__(self) -> None:
+        super().__init__()
+        self._final_char_logical_score = 0
+        self._final_char_visual_score = 0
+        self._prev = self.SPACE
+        self._before_prev = self.SPACE
+        self._logical_prober: Optional[SingleByteCharSetProber] = None
+        self._visual_prober: Optional[SingleByteCharSetProber] = None
        self.reset()

-    def reset(self):
+    def reset(self) -> None:
        self._final_char_logical_score = 0
        self._final_char_visual_score = 0
        # The two last characters seen in the previous buffer,
        # mPrev and mBeforePrev are initialized to space in order to simulate
        # a word delimiter at the beginning of the data
-        self._prev = ' '
-        self._before_prev = ' '
+        self._prev = self.SPACE
+        self._before_prev = self.SPACE
        # These probers are owned by the group prober.

-    def set_model_probers(self, logicalProber, visualProber):
-        self._logical_prober = logicalProber
-        self._visual_prober = visualProber
+    def set_model_probers(
+        self,
+        logical_prober: SingleByteCharSetProber,
+        visual_prober: SingleByteCharSetProber,
+    ) -> None:
+        self._logical_prober = logical_prober
+        self._visual_prober = visual_prober

-    def is_final(self, c):
-        return c in [self.FINAL_KAF, self.FINAL_MEM, self.FINAL_NUN,
-                     self.FINAL_PE, self.FINAL_TSADI]
+    def is_final(self, c: int) -> bool:
+        return c in [
+            self.FINAL_KAF,
+            self.FINAL_MEM,
+            self.FINAL_NUN,
+            self.FINAL_PE,
+            self.FINAL_TSADI,
+        ]

-    def is_non_final(self, c):
+    def is_non_final(self, c: int) -> bool:
        # The normal Tsadi is not a good Non-Final letter due to words like
        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
        # apostrophe is converted to a space in FilterWithoutEnglishLetters
@ -190,10 +204,9 @@ class HebrewProber(CharSetProber):
        # for example legally end with a Non-Final Pe or Kaf. However, the
        # benefit of these letters as Non-Final letters outweighs the damage
        # since these words are quite rare.
-        return c in [self.NORMAL_KAF, self.NORMAL_MEM,
-                     self.NORMAL_NUN, self.NORMAL_PE]
+        return c in [self.NORMAL_KAF, self.NORMAL_MEM, self.NORMAL_NUN, self.NORMAL_PE]

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        # Final letter analysis for logical-visual decision.
        # Look for evidence that the received buffer is either logical Hebrew
        # or visual Hebrew.
@ -227,9 +240,9 @@ class HebrewProber(CharSetProber):
        byte_str = self.filter_high_byte_only(byte_str)

        for cur in byte_str:
-            if cur == ' ':
+            if cur == self.SPACE:
                # We stand on a space - a word just ended
-                if self._before_prev != ' ':
+                if self._before_prev != self.SPACE:
                    # next-to-last char was not a space so self._prev is not a
                    # 1 letter word
                    if self.is_final(self._prev):
@ -241,8 +254,11 @@ class HebrewProber(CharSetProber):
                        self._final_char_visual_score += 1
            else:
                # Not standing on a space
-                if ((self._before_prev == ' ') and
-                        (self.is_final(self._prev)) and (cur != ' ')):
+                if (
+                    (self._before_prev == self.SPACE)
+                    and (self.is_final(self._prev))
+                    and (cur != self.SPACE)
+                ):
                    # case (3) [-2:space][-1:final letter][cur:not space]
                    self._final_char_visual_score += 1
            self._before_prev = self._prev
@ -253,7 +269,10 @@ class HebrewProber(CharSetProber):
        return ProbingState.DETECTING

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
+        assert self._logical_prober is not None
+        assert self._visual_prober is not None
+
        # Make the decision: is it Logical or Visual?
        # If the final letter score distance is dominant enough, rely on it.
        finalsub = self._final_char_logical_score - self._final_char_visual_score
@ -263,8 +282,9 @@ class HebrewProber(CharSetProber):
            return self.VISUAL_HEBREW_NAME

        # It's not dominant enough, try to rely on the model scores instead.
-        modelsub = (self._logical_prober.get_confidence()
-                    - self._visual_prober.get_confidence())
+        modelsub = (
+            self._logical_prober.get_confidence() - self._visual_prober.get_confidence()
+        )
        if modelsub > self.MIN_MODEL_DISTANCE:
            return self.LOGICAL_HEBREW_NAME
        if modelsub < -self.MIN_MODEL_DISTANCE:
@ -280,13 +300,17 @@ class HebrewProber(CharSetProber):
        return self.LOGICAL_HEBREW_NAME

    @property
-    def language(self):
-        return 'Hebrew'
+    def language(self) -> str:
+        return "Hebrew"

    @property
-    def state(self):
+    def state(self) -> ProbingState:
+        assert self._logical_prober is not None
+        assert self._visual_prober is not None
+
        # Remain active as long as any of the model probers are active.
-        if (self._logical_prober.state == ProbingState.NOT_ME) and \
-           (self._visual_prober.state == ProbingState.NOT_ME):
+        if (self._logical_prober.state == ProbingState.NOT_ME) and (
+            self._visual_prober.state == ProbingState.NOT_ME
+        ):
            return ProbingState.NOT_ME
        return ProbingState.DETECTING
--- a/libs/common/chardet/jisfreq.py
+++ b/libs/common/chardet/jisfreq.py
@ -46,6 +46,7 @@ JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0
 # Char to FreqOrder table ,
 JIS_TABLE_SIZE = 4368

+# fmt: off
 JIS_CHAR_TO_FREQ_ORDER = (
  40,   1,   6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, #   16
 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247,  18, 179,5071, 856,1661, #   32
@ -321,5 +322,4 @@ JIS_CHAR_TO_FREQ_ORDER = (
 1444,1698,2385,2251,3729,1365,2281,2235,1717,6188, 864,3841,2515, 444, 527,2767, # 4352
 2922,3625, 544, 461,6189, 566, 209,2437,3398,2098,1065,2068,3331,3626,3257,2137, # 4368  #last 512
 )
-
-
+# fmt: on
--- a/libs/common/chardet/johabfreq.py
+++ b/libs/common/chardet/johabfreq.py
--- a/libs/common/chardet/johabprober.py
+++ b/libs/common/chardet/johabprober.py
@ -1,7 +1,13 @@
 ######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
 # Contributor(s):
-#   Dan Blanchard
-#   Ian Cordasco
+#   Mark Pilgrim - port to Python
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@ -19,18 +25,23 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-import sys
+from .chardistribution import JOHABDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .mbcharsetprober import MultiByteCharSetProber
+from .mbcssm import JOHAB_SM_MODEL


-if sys.version_info < (3, 0):
-    PY2 = True
-    PY3 = False
-    string_types = (str, unicode)
-    text_type = unicode
-    iteritems = dict.iteritems
-else:
-    PY2 = False
-    PY3 = True
-    string_types = (bytes, str)
-    text_type = str
-    iteritems = dict.items
+class JOHABProber(MultiByteCharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self.coding_sm = CodingStateMachine(JOHAB_SM_MODEL)
+        self.distribution_analyzer = JOHABDistributionAnalysis()
+        self.reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "Johab"
+
+    @property
+    def language(self) -> str:
+        return "Korean"
--- a/libs/common/chardet/jpcntx.py
+++ b/libs/common/chardet/jpcntx.py
@ -25,110 +25,114 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import List, Tuple, Union

 # This is hiragana 2-char sequence table, the number in each cell represents its frequency category
-jp2CharContext = (
-(0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1),
-(2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4),
-(0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2),
-(0,4,0,5,0,5,0,4,0,4,5,4,4,3,5,3,5,1,5,3,4,3,4,4,3,4,3,3,4,3,5,4,4,3,5,5,3,5,5,5,3,5,5,3,4,5,5,3,1,3,2,0,3,4,0,4,2,0,4,2,1,5,3,2,3,5,0,4,0,2,0,5,4,4,5,4,5,0,4,0,0,4,4),
-(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),
-(0,3,0,4,0,3,0,3,0,4,5,4,3,3,3,3,4,3,5,4,4,3,5,4,4,3,4,3,4,4,4,4,5,3,4,4,3,4,5,5,4,5,5,1,4,5,4,3,0,3,3,1,3,3,0,4,4,0,3,3,1,5,3,3,3,5,0,4,0,3,0,4,4,3,4,3,3,0,4,1,1,3,4),
-(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),
-(0,4,0,3,0,3,0,4,0,3,4,4,3,2,2,1,2,1,3,1,3,3,3,3,3,4,3,1,3,3,5,3,3,0,4,3,0,5,4,3,3,5,4,4,3,4,4,5,0,1,2,0,1,2,0,2,2,0,1,0,0,5,2,2,1,4,0,3,0,1,0,4,4,3,5,4,3,0,2,1,0,4,3),
-(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),
-(0,3,0,5,0,4,0,2,1,4,4,2,4,1,4,2,4,2,4,3,3,3,4,3,3,3,3,1,4,2,3,3,3,1,4,4,1,1,1,4,3,3,2,0,2,4,3,2,0,3,3,0,3,1,1,0,0,0,3,3,0,4,2,2,3,4,0,4,0,3,0,4,4,5,3,4,4,0,3,0,0,1,4),
-(1,4,0,4,0,4,0,4,0,3,5,4,4,3,4,3,5,4,3,3,4,3,5,4,4,4,4,3,4,2,4,3,3,1,5,4,3,2,4,5,4,5,5,4,4,5,4,4,0,3,2,2,3,3,0,4,3,1,3,2,1,4,3,3,4,5,0,3,0,2,0,4,5,5,4,5,4,0,4,0,0,5,4),
-(0,5,0,5,0,4,0,3,0,4,4,3,4,3,3,3,4,0,4,4,4,3,4,3,4,3,3,1,4,2,4,3,4,0,5,4,1,4,5,4,4,5,3,2,4,3,4,3,2,4,1,3,3,3,2,3,2,0,4,3,3,4,3,3,3,4,0,4,0,3,0,4,5,4,4,4,3,0,4,1,0,1,3),
-(0,3,1,4,0,3,0,2,0,3,4,4,3,1,4,2,3,3,4,3,4,3,4,3,4,4,3,2,3,1,5,4,4,1,4,4,3,5,4,4,3,5,5,4,3,4,4,3,1,2,3,1,2,2,0,3,2,0,3,1,0,5,3,3,3,4,3,3,3,3,4,4,4,4,5,4,2,0,3,3,2,4,3),
-(0,2,0,3,0,1,0,1,0,0,3,2,0,0,2,0,1,0,2,1,3,3,3,1,2,3,1,0,1,0,4,2,1,1,3,3,0,4,3,3,1,4,3,3,0,3,3,2,0,0,0,0,1,0,0,2,0,0,0,0,0,4,1,0,2,3,2,2,2,1,3,3,3,4,4,3,2,0,3,1,0,3,3),
-(0,4,0,4,0,3,0,3,0,4,4,4,3,3,3,3,3,3,4,3,4,2,4,3,4,3,3,2,4,3,4,5,4,1,4,5,3,5,4,5,3,5,4,0,3,5,5,3,1,3,3,2,2,3,0,3,4,1,3,3,2,4,3,3,3,4,0,4,0,3,0,4,5,4,4,5,3,0,4,1,0,3,4),
-(0,2,0,3,0,3,0,0,0,2,2,2,1,0,1,0,0,0,3,0,3,0,3,0,1,3,1,0,3,1,3,3,3,1,3,3,3,0,1,3,1,3,4,0,0,3,1,1,0,3,2,0,0,0,0,1,3,0,1,0,0,3,3,2,0,3,0,0,0,0,0,3,4,3,4,3,3,0,3,0,0,2,3),
-(2,3,0,3,0,2,0,1,0,3,3,4,3,1,3,1,1,1,3,1,4,3,4,3,3,3,0,0,3,1,5,4,3,1,4,3,2,5,5,4,4,4,4,3,3,4,4,4,0,2,1,1,3,2,0,1,2,0,0,1,0,4,1,3,3,3,0,3,0,1,0,4,4,4,5,5,3,0,2,0,0,4,4),
-(0,2,0,1,0,3,1,3,0,2,3,3,3,0,3,1,0,0,3,0,3,2,3,1,3,2,1,1,0,0,4,2,1,0,2,3,1,4,3,2,0,4,4,3,1,3,1,3,0,1,0,0,1,0,0,0,1,0,0,0,0,4,1,1,1,2,0,3,0,0,0,3,4,2,4,3,2,0,1,0,0,3,3),
-(0,1,0,4,0,5,0,4,0,2,4,4,2,3,3,2,3,3,5,3,3,3,4,3,4,2,3,0,4,3,3,3,4,1,4,3,2,1,5,5,3,4,5,1,3,5,4,2,0,3,3,0,1,3,0,4,2,0,1,3,1,4,3,3,3,3,0,3,0,1,0,3,4,4,4,5,5,0,3,0,1,4,5),
-(0,2,0,3,0,3,0,0,0,2,3,1,3,0,4,0,1,1,3,0,3,4,3,2,3,1,0,3,3,2,3,1,3,0,2,3,0,2,1,4,1,2,2,0,0,3,3,0,0,2,0,0,0,1,0,0,0,0,2,2,0,3,2,1,3,3,0,2,0,2,0,0,3,3,1,2,4,0,3,0,2,2,3),
-(2,4,0,5,0,4,0,4,0,2,4,4,4,3,4,3,3,3,1,2,4,3,4,3,4,4,5,0,3,3,3,3,2,0,4,3,1,4,3,4,1,4,4,3,3,4,4,3,1,2,3,0,4,2,0,4,1,0,3,3,0,4,3,3,3,4,0,4,0,2,0,3,5,3,4,5,2,0,3,0,0,4,5),
-(0,3,0,4,0,1,0,1,0,1,3,2,2,1,3,0,3,0,2,0,2,0,3,0,2,0,0,0,1,0,1,1,0,0,3,1,0,0,0,4,0,3,1,0,2,1,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,4,2,2,3,1,0,3,0,0,0,1,4,4,4,3,0,0,4,0,0,1,4),
-(1,4,1,5,0,3,0,3,0,4,5,4,4,3,5,3,3,4,4,3,4,1,3,3,3,3,2,1,4,1,5,4,3,1,4,4,3,5,4,4,3,5,4,3,3,4,4,4,0,3,3,1,2,3,0,3,1,0,3,3,0,5,4,4,4,4,4,4,3,3,5,4,4,3,3,5,4,0,3,2,0,4,4),
-(0,2,0,3,0,1,0,0,0,1,3,3,3,2,4,1,3,0,3,1,3,0,2,2,1,1,0,0,2,0,4,3,1,0,4,3,0,4,4,4,1,4,3,1,1,3,3,1,0,2,0,0,1,3,0,0,0,0,2,0,0,4,3,2,4,3,5,4,3,3,3,4,3,3,4,3,3,0,2,1,0,3,3),
-(0,2,0,4,0,3,0,2,0,2,5,5,3,4,4,4,4,1,4,3,3,0,4,3,4,3,1,3,3,2,4,3,0,3,4,3,0,3,4,4,2,4,4,0,4,5,3,3,2,2,1,1,1,2,0,1,5,0,3,3,2,4,3,3,3,4,0,3,0,2,0,4,4,3,5,5,0,0,3,0,2,3,3),
-(0,3,0,4,0,3,0,1,0,3,4,3,3,1,3,3,3,0,3,1,3,0,4,3,3,1,1,0,3,0,3,3,0,0,4,4,0,1,5,4,3,3,5,0,3,3,4,3,0,2,0,1,1,1,0,1,3,0,1,2,1,3,3,2,3,3,0,3,0,1,0,1,3,3,4,4,1,0,1,2,2,1,3),
-(0,1,0,4,0,4,0,3,0,1,3,3,3,2,3,1,1,0,3,0,3,3,4,3,2,4,2,0,1,0,4,3,2,0,4,3,0,5,3,3,2,4,4,4,3,3,3,4,0,1,3,0,0,1,0,0,1,0,0,0,0,4,2,3,3,3,0,3,0,0,0,4,4,4,5,3,2,0,3,3,0,3,5),
-(0,2,0,3,0,0,0,3,0,1,3,0,2,0,0,0,1,0,3,1,1,3,3,0,0,3,0,0,3,0,2,3,1,0,3,1,0,3,3,2,0,4,2,2,0,2,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,2,1,2,0,1,0,1,0,0,0,1,3,1,2,0,0,0,1,0,0,1,4),
-(0,3,0,3,0,5,0,1,0,2,4,3,1,3,3,2,1,1,5,2,1,0,5,1,2,0,0,0,3,3,2,2,3,2,4,3,0,0,3,3,1,3,3,0,2,5,3,4,0,3,3,0,1,2,0,2,2,0,3,2,0,2,2,3,3,3,0,2,0,1,0,3,4,4,2,5,4,0,3,0,0,3,5),
-(0,3,0,3,0,3,0,1,0,3,3,3,3,0,3,0,2,0,2,1,1,0,2,0,1,0,0,0,2,1,0,0,1,0,3,2,0,0,3,3,1,2,3,1,0,3,3,0,0,1,0,0,0,0,0,2,0,0,0,0,0,2,3,1,2,3,0,3,0,1,0,3,2,1,0,4,3,0,1,1,0,3,3),
-(0,4,0,5,0,3,0,3,0,4,5,5,4,3,5,3,4,3,5,3,3,2,5,3,4,4,4,3,4,3,4,5,5,3,4,4,3,4,4,5,4,4,4,3,4,5,5,4,2,3,4,2,3,4,0,3,3,1,4,3,2,4,3,3,5,5,0,3,0,3,0,5,5,5,5,4,4,0,4,0,1,4,4),
-(0,4,0,4,0,3,0,3,0,3,5,4,4,2,3,2,5,1,3,2,5,1,4,2,3,2,3,3,4,3,3,3,3,2,5,4,1,3,3,5,3,4,4,0,4,4,3,1,1,3,1,0,2,3,0,2,3,0,3,0,0,4,3,1,3,4,0,3,0,2,0,4,4,4,3,4,5,0,4,0,0,3,4),
-(0,3,0,3,0,3,1,2,0,3,4,4,3,3,3,0,2,2,4,3,3,1,3,3,3,1,1,0,3,1,4,3,2,3,4,4,2,4,4,4,3,4,4,3,2,4,4,3,1,3,3,1,3,3,0,4,1,0,2,2,1,4,3,2,3,3,5,4,3,3,5,4,4,3,3,0,4,0,3,2,2,4,4),
-(0,2,0,1,0,0,0,0,0,1,2,1,3,0,0,0,0,0,2,0,1,2,1,0,0,1,0,0,0,0,3,0,0,1,0,1,1,3,1,0,0,0,1,1,0,1,1,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,1,2,2,0,3,4,0,0,0,1,1,0,0,1,0,0,0,0,0,1,1),
-(0,1,0,0,0,1,0,0,0,0,4,0,4,1,4,0,3,0,4,0,3,0,4,0,3,0,3,0,4,1,5,1,4,0,0,3,0,5,0,5,2,0,1,0,0,0,2,1,4,0,1,3,0,0,3,0,0,3,1,1,4,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0),
-(1,4,0,5,0,3,0,2,0,3,5,4,4,3,4,3,5,3,4,3,3,0,4,3,3,3,3,3,3,2,4,4,3,1,3,4,4,5,4,4,3,4,4,1,3,5,4,3,3,3,1,2,2,3,3,1,3,1,3,3,3,5,3,3,4,5,0,3,0,3,0,3,4,3,4,4,3,0,3,0,2,4,3),
-(0,1,0,4,0,0,0,0,0,1,4,0,4,1,4,2,4,0,3,0,1,0,1,0,0,0,0,0,2,0,3,1,1,1,0,3,0,0,0,1,2,1,0,0,1,1,1,1,0,1,0,0,0,1,0,0,3,0,0,0,0,3,2,0,2,2,0,1,0,0,0,2,3,2,3,3,0,0,0,0,2,1,0),
-(0,5,1,5,0,3,0,3,0,5,4,4,5,1,5,3,3,0,4,3,4,3,5,3,4,3,3,2,4,3,4,3,3,0,3,3,1,4,4,3,4,4,4,3,4,5,5,3,2,3,1,1,3,3,1,3,1,1,3,3,2,4,5,3,3,5,0,4,0,3,0,4,4,3,5,3,3,0,3,4,0,4,3),
-(0,5,0,5,0,3,0,2,0,4,4,3,5,2,4,3,3,3,4,4,4,3,5,3,5,3,3,1,4,0,4,3,3,0,3,3,0,4,4,4,4,5,4,3,3,5,5,3,2,3,1,2,3,2,0,1,0,0,3,2,2,4,4,3,1,5,0,4,0,3,0,4,3,1,3,2,1,0,3,3,0,3,3),
-(0,4,0,5,0,5,0,4,0,4,5,5,5,3,4,3,3,2,5,4,4,3,5,3,5,3,4,0,4,3,4,4,3,2,4,4,3,4,5,4,4,5,5,0,3,5,5,4,1,3,3,2,3,3,1,3,1,0,4,3,1,4,4,3,4,5,0,4,0,2,0,4,3,4,4,3,3,0,4,0,0,5,5),
-(0,4,0,4,0,5,0,1,1,3,3,4,4,3,4,1,3,0,5,1,3,0,3,1,3,1,1,0,3,0,3,3,4,0,4,3,0,4,4,4,3,4,4,0,3,5,4,1,0,3,0,0,2,3,0,3,1,0,3,1,0,3,2,1,3,5,0,3,0,1,0,3,2,3,3,4,4,0,2,2,0,4,4),
-(2,4,0,5,0,4,0,3,0,4,5,5,4,3,5,3,5,3,5,3,5,2,5,3,4,3,3,4,3,4,5,3,2,1,5,4,3,2,3,4,5,3,4,1,2,5,4,3,0,3,3,0,3,2,0,2,3,0,4,1,0,3,4,3,3,5,0,3,0,1,0,4,5,5,5,4,3,0,4,2,0,3,5),
-(0,5,0,4,0,4,0,2,0,5,4,3,4,3,4,3,3,3,4,3,4,2,5,3,5,3,4,1,4,3,4,4,4,0,3,5,0,4,4,4,4,5,3,1,3,4,5,3,3,3,3,3,3,3,0,2,2,0,3,3,2,4,3,3,3,5,3,4,1,3,3,5,3,2,0,0,0,0,4,3,1,3,3),
-(0,1,0,3,0,3,0,1,0,1,3,3,3,2,3,3,3,0,3,0,0,0,3,1,3,0,0,0,2,2,2,3,0,0,3,2,0,1,2,4,1,3,3,0,0,3,3,3,0,1,0,0,2,1,0,0,3,0,3,1,0,3,0,0,1,3,0,2,0,1,0,3,3,1,3,3,0,0,1,1,0,3,3),
-(0,2,0,3,0,2,1,4,0,2,2,3,1,1,3,1,1,0,2,0,3,1,2,3,1,3,0,0,1,0,4,3,2,3,3,3,1,4,2,3,3,3,3,1,0,3,1,4,0,1,1,0,1,2,0,1,1,0,1,1,0,3,1,3,2,2,0,1,0,0,0,2,3,3,3,1,0,0,0,0,0,2,3),
-(0,5,0,4,0,5,0,2,0,4,5,5,3,3,4,3,3,1,5,4,4,2,4,4,4,3,4,2,4,3,5,5,4,3,3,4,3,3,5,5,4,5,5,1,3,4,5,3,1,4,3,1,3,3,0,3,3,1,4,3,1,4,5,3,3,5,0,4,0,3,0,5,3,3,1,4,3,0,4,0,1,5,3),
-(0,5,0,5,0,4,0,2,0,4,4,3,4,3,3,3,3,3,5,4,4,4,4,4,4,5,3,3,5,2,4,4,4,3,4,4,3,3,4,4,5,5,3,3,4,3,4,3,3,4,3,3,3,3,1,2,2,1,4,3,3,5,4,4,3,4,0,4,0,3,0,4,4,4,4,4,1,0,4,2,0,2,4),
-(0,4,0,4,0,3,0,1,0,3,5,2,3,0,3,0,2,1,4,2,3,3,4,1,4,3,3,2,4,1,3,3,3,0,3,3,0,0,3,3,3,5,3,3,3,3,3,2,0,2,0,0,2,0,0,2,0,0,1,0,0,3,1,2,2,3,0,3,0,2,0,4,4,3,3,4,1,0,3,0,0,2,4),
-(0,0,0,4,0,0,0,0,0,0,1,0,1,0,2,0,0,0,0,0,1,0,2,0,1,0,0,0,0,0,3,1,3,0,3,2,0,0,0,1,0,3,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,4,0,2,0,0,0,0,0,0,2),
-(0,2,1,3,0,2,0,2,0,3,3,3,3,1,3,1,3,3,3,3,3,3,4,2,2,1,2,1,4,0,4,3,1,3,3,3,2,4,3,5,4,3,3,3,3,3,3,3,0,1,3,0,2,0,0,1,0,0,1,0,0,4,2,0,2,3,0,3,3,0,3,3,4,2,3,1,4,0,1,2,0,2,3),
-(0,3,0,3,0,1,0,3,0,2,3,3,3,0,3,1,2,0,3,3,2,3,3,2,3,2,3,1,3,0,4,3,2,0,3,3,1,4,3,3,2,3,4,3,1,3,3,1,1,0,1,1,0,1,0,1,0,1,0,0,0,4,1,1,0,3,0,3,1,0,2,3,3,3,3,3,1,0,0,2,0,3,3),
-(0,0,0,0,0,0,0,0,0,0,3,0,2,0,3,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,3,0,3,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,2,0,2,3,0,0,0,0,0,0,0,0,3),
-(0,2,0,3,1,3,0,3,0,2,3,3,3,1,3,1,3,1,3,1,3,3,3,1,3,0,2,3,1,1,4,3,3,2,3,3,1,2,2,4,1,3,3,0,1,4,2,3,0,1,3,0,3,0,0,1,3,0,2,0,0,3,3,2,1,3,0,3,0,2,0,3,4,4,4,3,1,0,3,0,0,3,3),
-(0,2,0,1,0,2,0,0,0,1,3,2,2,1,3,0,1,1,3,0,3,2,3,1,2,0,2,0,1,1,3,3,3,0,3,3,1,1,2,3,2,3,3,1,2,3,2,0,0,1,0,0,0,0,0,0,3,0,1,0,0,2,1,2,1,3,0,3,0,0,0,3,4,4,4,3,2,0,2,0,0,2,4),
-(0,0,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,2,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,3,1,0,0,0,0,0,0,0,3),
-(0,3,0,3,0,2,0,3,0,3,3,3,2,3,2,2,2,0,3,1,3,3,3,2,3,3,0,0,3,0,3,2,2,0,2,3,1,4,3,4,3,3,2,3,1,5,4,4,0,3,1,2,1,3,0,3,1,1,2,0,2,3,1,3,1,3,0,3,0,1,0,3,3,4,4,2,1,0,2,1,0,2,4),
-(0,1,0,3,0,1,0,2,0,1,4,2,5,1,4,0,2,0,2,1,3,1,4,0,2,1,0,0,2,1,4,1,1,0,3,3,0,5,1,3,2,3,3,1,0,3,2,3,0,1,0,0,0,0,0,0,1,0,0,0,0,4,0,1,0,3,0,2,0,1,0,3,3,3,4,3,3,0,0,0,0,2,3),
-(0,0,0,1,0,0,0,0,0,0,2,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,1,0,0,0,0,0,3),
-(0,1,0,3,0,4,0,3,0,2,4,3,1,0,3,2,2,1,3,1,2,2,3,1,1,1,2,1,3,0,1,2,0,1,3,2,1,3,0,5,5,1,0,0,1,3,2,1,0,3,0,0,1,0,0,0,0,0,3,4,0,1,1,1,3,2,0,2,0,1,0,2,3,3,1,2,3,0,1,0,1,0,4),
-(0,0,0,1,0,3,0,3,0,2,2,1,0,0,4,0,3,0,3,1,3,0,3,0,3,0,1,0,3,0,3,1,3,0,3,3,0,0,1,2,1,1,1,0,1,2,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,2,2,1,2,0,0,2,0,0,0,0,2,3,3,3,3,0,0,0,0,1,4),
-(0,0,0,3,0,3,0,0,0,0,3,1,1,0,3,0,1,0,2,0,1,0,0,0,0,0,0,0,1,0,3,0,2,0,2,3,0,0,2,2,3,1,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,2,3),
-(2,4,0,5,0,5,0,4,0,3,4,3,3,3,4,3,3,3,4,3,4,4,5,4,5,5,5,2,3,0,5,5,4,1,5,4,3,1,5,4,3,4,4,3,3,4,3,3,0,3,2,0,2,3,0,3,0,0,3,3,0,5,3,2,3,3,0,3,0,3,0,3,4,5,4,5,3,0,4,3,0,3,4),
-(0,3,0,3,0,3,0,3,0,3,3,4,3,2,3,2,3,0,4,3,3,3,3,3,3,3,3,0,3,2,4,3,3,1,3,4,3,4,4,4,3,4,4,3,2,4,4,1,0,2,0,0,1,1,0,2,0,0,3,1,0,5,3,2,1,3,0,3,0,1,2,4,3,2,4,3,3,0,3,2,0,4,4),
-(0,3,0,3,0,1,0,0,0,1,4,3,3,2,3,1,3,1,4,2,3,2,4,2,3,4,3,0,2,2,3,3,3,0,3,3,3,0,3,4,1,3,3,0,3,4,3,3,0,1,1,0,1,0,0,0,4,0,3,0,0,3,1,2,1,3,0,4,0,1,0,4,3,3,4,3,3,0,2,0,0,3,3),
-(0,3,0,4,0,1,0,3,0,3,4,3,3,0,3,3,3,1,3,1,3,3,4,3,3,3,0,0,3,1,5,3,3,1,3,3,2,5,4,3,3,4,5,3,2,5,3,4,0,1,0,0,0,0,0,2,0,0,1,1,0,4,2,2,1,3,0,3,0,2,0,4,4,3,5,3,2,0,1,1,0,3,4),
-(0,5,0,4,0,5,0,2,0,4,4,3,3,2,3,3,3,1,4,3,4,1,5,3,4,3,4,0,4,2,4,3,4,1,5,4,0,4,4,4,4,5,4,1,3,5,4,2,1,4,1,1,3,2,0,3,1,0,3,2,1,4,3,3,3,4,0,4,0,3,0,4,4,4,3,3,3,0,4,2,0,3,4),
-(1,4,0,4,0,3,0,1,0,3,3,3,1,1,3,3,2,2,3,3,1,0,3,2,2,1,2,0,3,1,2,1,2,0,3,2,0,2,2,3,3,4,3,0,3,3,1,2,0,1,1,3,1,2,0,0,3,0,1,1,0,3,2,2,3,3,0,3,0,0,0,2,3,3,4,3,3,0,1,0,0,1,4),
-(0,4,0,4,0,4,0,0,0,3,4,4,3,1,4,2,3,2,3,3,3,1,4,3,4,0,3,0,4,2,3,3,2,2,5,4,2,1,3,4,3,4,3,1,3,3,4,2,0,2,1,0,3,3,0,0,2,0,3,1,0,4,4,3,4,3,0,4,0,1,0,2,4,4,4,4,4,0,3,2,0,3,3),
-(0,0,0,1,0,4,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,3,2,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2),
-(0,2,0,3,0,4,0,4,0,1,3,3,3,0,4,0,2,1,2,1,1,1,2,0,3,1,1,0,1,0,3,1,0,0,3,3,2,0,1,1,0,0,0,0,0,1,0,2,0,2,2,0,3,1,0,0,1,0,1,1,0,1,2,0,3,0,0,0,0,1,0,0,3,3,4,3,1,0,1,0,3,0,2),
-(0,0,0,3,0,5,0,0,0,0,1,0,2,0,3,1,0,1,3,0,0,0,2,0,0,0,1,0,0,0,1,1,0,0,4,0,0,0,2,3,0,1,4,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,3,0,0,0,0,0,3),
-(0,2,0,5,0,5,0,1,0,2,4,3,3,2,5,1,3,2,3,3,3,0,4,1,2,0,3,0,4,0,2,2,1,1,5,3,0,0,1,4,2,3,2,0,3,3,3,2,0,2,4,1,1,2,0,1,1,0,3,1,0,1,3,1,2,3,0,2,0,0,0,1,3,5,4,4,4,0,3,0,0,1,3),
-(0,4,0,5,0,4,0,4,0,4,5,4,3,3,4,3,3,3,4,3,4,4,5,3,4,5,4,2,4,2,3,4,3,1,4,4,1,3,5,4,4,5,5,4,4,5,5,5,2,3,3,1,4,3,1,3,3,0,3,3,1,4,3,4,4,4,0,3,0,4,0,3,3,4,4,5,0,0,4,3,0,4,5),
-(0,4,0,4,0,3,0,3,0,3,4,4,4,3,3,2,4,3,4,3,4,3,5,3,4,3,2,1,4,2,4,4,3,1,3,4,2,4,5,5,3,4,5,4,1,5,4,3,0,3,2,2,3,2,1,3,1,0,3,3,3,5,3,3,3,5,4,4,2,3,3,4,3,3,3,2,1,0,3,2,1,4,3),
-(0,4,0,5,0,4,0,3,0,3,5,5,3,2,4,3,4,0,5,4,4,1,4,4,4,3,3,3,4,3,5,5,2,3,3,4,1,2,5,5,3,5,5,2,3,5,5,4,0,3,2,0,3,3,1,1,5,1,4,1,0,4,3,2,3,5,0,4,0,3,0,5,4,3,4,3,0,0,4,1,0,4,4),
-(1,3,0,4,0,2,0,2,0,2,5,5,3,3,3,3,3,0,4,2,3,4,4,4,3,4,0,0,3,4,5,4,3,3,3,3,2,5,5,4,5,5,5,4,3,5,5,5,1,3,1,0,1,0,0,3,2,0,4,2,0,5,2,3,2,4,1,3,0,3,0,4,5,4,5,4,3,0,4,2,0,5,4),
-(0,3,0,4,0,5,0,3,0,3,4,4,3,2,3,2,3,3,3,3,3,2,4,3,3,2,2,0,3,3,3,3,3,1,3,3,3,0,4,4,3,4,4,1,1,4,4,2,0,3,1,0,1,1,0,4,1,0,2,3,1,3,3,1,3,4,0,3,0,1,0,3,1,3,0,0,1,0,2,0,0,4,4),
-(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),
-(0,3,0,3,0,2,0,3,0,1,5,4,3,3,3,1,4,2,1,2,3,4,4,2,4,4,5,0,3,1,4,3,4,0,4,3,3,3,2,3,2,5,3,4,3,2,2,3,0,0,3,0,2,1,0,1,2,0,0,0,0,2,1,1,3,1,0,2,0,4,0,3,4,4,4,5,2,0,2,0,0,1,3),
-(0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,1,0,0,1,1,0,0,0,4,2,1,1,0,1,0,3,2,0,0,3,1,1,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,1,0,0,0,2,0,0,0,1,4,0,4,2,1,0,0,0,0,0,1),
-(0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,3,1,0,0,0,2,0,2,1,0,0,1,2,1,0,1,1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,1,3,1,0,0,0,0,0,1,0,0,2,1,0,0,0,0,0,0,0,0,2),
-(0,4,0,4,0,4,0,3,0,4,4,3,4,2,4,3,2,0,4,4,4,3,5,3,5,3,3,2,4,2,4,3,4,3,1,4,0,2,3,4,4,4,3,3,3,4,4,4,3,4,1,3,4,3,2,1,2,1,3,3,3,4,4,3,3,5,0,4,0,3,0,4,3,3,3,2,1,0,3,0,0,3,3),
-(0,4,0,3,0,3,0,3,0,3,5,5,3,3,3,3,4,3,4,3,3,3,4,4,4,3,3,3,3,4,3,5,3,3,1,3,2,4,5,5,5,5,4,3,4,5,5,3,2,2,3,3,3,3,2,3,3,1,2,3,2,4,3,3,3,4,0,4,0,2,0,4,3,2,2,1,2,0,3,0,0,4,1),
+# fmt: off
+jp2_char_context = (
+    (0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1),
+    (2, 4, 0, 4, 0, 3, 0, 4, 0, 3, 4, 4, 4, 2, 4, 3, 3, 4, 3, 2, 3, 3, 4, 2, 3, 3, 3, 2, 4, 1, 4, 3, 3, 1, 5, 4, 3, 4, 3, 4, 3, 5, 3, 0, 3, 5, 4, 2, 0, 3, 1, 0, 3, 3, 0, 3, 3, 0, 1, 1, 0, 4, 3, 0, 3, 3, 0, 4, 0, 2, 0, 3, 5, 5, 5, 5, 4, 0, 4, 1, 0, 3, 4),
+    (0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2),
+    (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 4, 4, 3, 5, 3, 5, 1, 5, 3, 4, 3, 4, 4, 3, 4, 3, 3, 4, 3, 5, 4, 4, 3, 5, 5, 3, 5, 5, 5, 3, 5, 5, 3, 4, 5, 5, 3, 1, 3, 2, 0, 3, 4, 0, 4, 2, 0, 4, 2, 1, 5, 3, 2, 3, 5, 0, 4, 0, 2, 0, 5, 4, 4, 5, 4, 5, 0, 4, 0, 0, 4, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 5, 4, 3, 3, 3, 3, 4, 3, 5, 4, 4, 3, 5, 4, 4, 3, 4, 3, 4, 4, 4, 4, 5, 3, 4, 4, 3, 4, 5, 5, 4, 5, 5, 1, 4, 5, 4, 3, 0, 3, 3, 1, 3, 3, 0, 4, 4, 0, 3, 3, 1, 5, 3, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 0, 4, 1, 1, 3, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 4, 0, 3, 0, 3, 0, 4, 0, 3, 4, 4, 3, 2, 2, 1, 2, 1, 3, 1, 3, 3, 3, 3, 3, 4, 3, 1, 3, 3, 5, 3, 3, 0, 4, 3, 0, 5, 4, 3, 3, 5, 4, 4, 3, 4, 4, 5, 0, 1, 2, 0, 1, 2, 0, 2, 2, 0, 1, 0, 0, 5, 2, 2, 1, 4, 0, 3, 0, 1, 0, 4, 4, 3, 5, 4, 3, 0, 2, 1, 0, 4, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 5, 0, 4, 0, 2, 1, 4, 4, 2, 4, 1, 4, 2, 4, 2, 4, 3, 3, 3, 4, 3, 3, 3, 3, 1, 4, 2, 3, 3, 3, 1, 4, 4, 1, 1, 1, 4, 3, 3, 2, 0, 2, 4, 3, 2, 0, 3, 3, 0, 3, 1, 1, 0, 0, 0, 3, 3, 0, 4, 2, 2, 3, 4, 0, 4, 0, 3, 0, 4, 4, 5, 3, 4, 4, 0, 3, 0, 0, 1, 4),
+    (1, 4, 0, 4, 0, 4, 0, 4, 0, 3, 5, 4, 4, 3, 4, 3, 5, 4, 3, 3, 4, 3, 5, 4, 4, 4, 4, 3, 4, 2, 4, 3, 3, 1, 5, 4, 3, 2, 4, 5, 4, 5, 5, 4, 4, 5, 4, 4, 0, 3, 2, 2, 3, 3, 0, 4, 3, 1, 3, 2, 1, 4, 3, 3, 4, 5, 0, 3, 0, 2, 0, 4, 5, 5, 4, 5, 4, 0, 4, 0, 0, 5, 4),
+    (0, 5, 0, 5, 0, 4, 0, 3, 0, 4, 4, 3, 4, 3, 3, 3, 4, 0, 4, 4, 4, 3, 4, 3, 4, 3, 3, 1, 4, 2, 4, 3, 4, 0, 5, 4, 1, 4, 5, 4, 4, 5, 3, 2, 4, 3, 4, 3, 2, 4, 1, 3, 3, 3, 2, 3, 2, 0, 4, 3, 3, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 4, 3, 0, 4, 1, 0, 1, 3),
+    (0, 3, 1, 4, 0, 3, 0, 2, 0, 3, 4, 4, 3, 1, 4, 2, 3, 3, 4, 3, 4, 3, 4, 3, 4, 4, 3, 2, 3, 1, 5, 4, 4, 1, 4, 4, 3, 5, 4, 4, 3, 5, 5, 4, 3, 4, 4, 3, 1, 2, 3, 1, 2, 2, 0, 3, 2, 0, 3, 1, 0, 5, 3, 3, 3, 4, 3, 3, 3, 3, 4, 4, 4, 4, 5, 4, 2, 0, 3, 3, 2, 4, 3),
+    (0, 2, 0, 3, 0, 1, 0, 1, 0, 0, 3, 2, 0, 0, 2, 0, 1, 0, 2, 1, 3, 3, 3, 1, 2, 3, 1, 0, 1, 0, 4, 2, 1, 1, 3, 3, 0, 4, 3, 3, 1, 4, 3, 3, 0, 3, 3, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 4, 1, 0, 2, 3, 2, 2, 2, 1, 3, 3, 3, 4, 4, 3, 2, 0, 3, 1, 0, 3, 3),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 3, 4, 2, 4, 3, 4, 3, 3, 2, 4, 3, 4, 5, 4, 1, 4, 5, 3, 5, 4, 5, 3, 5, 4, 0, 3, 5, 5, 3, 1, 3, 3, 2, 2, 3, 0, 3, 4, 1, 3, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 5, 4, 4, 5, 3, 0, 4, 1, 0, 3, 4),
+    (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 2, 2, 1, 0, 1, 0, 0, 0, 3, 0, 3, 0, 3, 0, 1, 3, 1, 0, 3, 1, 3, 3, 3, 1, 3, 3, 3, 0, 1, 3, 1, 3, 4, 0, 0, 3, 1, 1, 0, 3, 2, 0, 0, 0, 0, 1, 3, 0, 1, 0, 0, 3, 3, 2, 0, 3, 0, 0, 0, 0, 0, 3, 4, 3, 4, 3, 3, 0, 3, 0, 0, 2, 3),
+    (2, 3, 0, 3, 0, 2, 0, 1, 0, 3, 3, 4, 3, 1, 3, 1, 1, 1, 3, 1, 4, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 4, 3, 1, 4, 3, 2, 5, 5, 4, 4, 4, 4, 3, 3, 4, 4, 4, 0, 2, 1, 1, 3, 2, 0, 1, 2, 0, 0, 1, 0, 4, 1, 3, 3, 3, 0, 3, 0, 1, 0, 4, 4, 4, 5, 5, 3, 0, 2, 0, 0, 4, 4),
+    (0, 2, 0, 1, 0, 3, 1, 3, 0, 2, 3, 3, 3, 0, 3, 1, 0, 0, 3, 0, 3, 2, 3, 1, 3, 2, 1, 1, 0, 0, 4, 2, 1, 0, 2, 3, 1, 4, 3, 2, 0, 4, 4, 3, 1, 3, 1, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 1, 1, 2, 0, 3, 0, 0, 0, 3, 4, 2, 4, 3, 2, 0, 1, 0, 0, 3, 3),
+    (0, 1, 0, 4, 0, 5, 0, 4, 0, 2, 4, 4, 2, 3, 3, 2, 3, 3, 5, 3, 3, 3, 4, 3, 4, 2, 3, 0, 4, 3, 3, 3, 4, 1, 4, 3, 2, 1, 5, 5, 3, 4, 5, 1, 3, 5, 4, 2, 0, 3, 3, 0, 1, 3, 0, 4, 2, 0, 1, 3, 1, 4, 3, 3, 3, 3, 0, 3, 0, 1, 0, 3, 4, 4, 4, 5, 5, 0, 3, 0, 1, 4, 5),
+    (0, 2, 0, 3, 0, 3, 0, 0, 0, 2, 3, 1, 3, 0, 4, 0, 1, 1, 3, 0, 3, 4, 3, 2, 3, 1, 0, 3, 3, 2, 3, 1, 3, 0, 2, 3, 0, 2, 1, 4, 1, 2, 2, 0, 0, 3, 3, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 3, 2, 1, 3, 3, 0, 2, 0, 2, 0, 0, 3, 3, 1, 2, 4, 0, 3, 0, 2, 2, 3),
+    (2, 4, 0, 5, 0, 4, 0, 4, 0, 2, 4, 4, 4, 3, 4, 3, 3, 3, 1, 2, 4, 3, 4, 3, 4, 4, 5, 0, 3, 3, 3, 3, 2, 0, 4, 3, 1, 4, 3, 4, 1, 4, 4, 3, 3, 4, 4, 3, 1, 2, 3, 0, 4, 2, 0, 4, 1, 0, 3, 3, 0, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 3, 5, 3, 4, 5, 2, 0, 3, 0, 0, 4, 5),
+    (0, 3, 0, 4, 0, 1, 0, 1, 0, 1, 3, 2, 2, 1, 3, 0, 3, 0, 2, 0, 2, 0, 3, 0, 2, 0, 0, 0, 1, 0, 1, 1, 0, 0, 3, 1, 0, 0, 0, 4, 0, 3, 1, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 2, 2, 3, 1, 0, 3, 0, 0, 0, 1, 4, 4, 4, 3, 0, 0, 4, 0, 0, 1, 4),
+    (1, 4, 1, 5, 0, 3, 0, 3, 0, 4, 5, 4, 4, 3, 5, 3, 3, 4, 4, 3, 4, 1, 3, 3, 3, 3, 2, 1, 4, 1, 5, 4, 3, 1, 4, 4, 3, 5, 4, 4, 3, 5, 4, 3, 3, 4, 4, 4, 0, 3, 3, 1, 2, 3, 0, 3, 1, 0, 3, 3, 0, 5, 4, 4, 4, 4, 4, 4, 3, 3, 5, 4, 4, 3, 3, 5, 4, 0, 3, 2, 0, 4, 4),
+    (0, 2, 0, 3, 0, 1, 0, 0, 0, 1, 3, 3, 3, 2, 4, 1, 3, 0, 3, 1, 3, 0, 2, 2, 1, 1, 0, 0, 2, 0, 4, 3, 1, 0, 4, 3, 0, 4, 4, 4, 1, 4, 3, 1, 1, 3, 3, 1, 0, 2, 0, 0, 1, 3, 0, 0, 0, 0, 2, 0, 0, 4, 3, 2, 4, 3, 5, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 0, 2, 1, 0, 3, 3),
+    (0, 2, 0, 4, 0, 3, 0, 2, 0, 2, 5, 5, 3, 4, 4, 4, 4, 1, 4, 3, 3, 0, 4, 3, 4, 3, 1, 3, 3, 2, 4, 3, 0, 3, 4, 3, 0, 3, 4, 4, 2, 4, 4, 0, 4, 5, 3, 3, 2, 2, 1, 1, 1, 2, 0, 1, 5, 0, 3, 3, 2, 4, 3, 3, 3, 4, 0, 3, 0, 2, 0, 4, 4, 3, 5, 5, 0, 0, 3, 0, 2, 3, 3),
+    (0, 3, 0, 4, 0, 3, 0, 1, 0, 3, 4, 3, 3, 1, 3, 3, 3, 0, 3, 1, 3, 0, 4, 3, 3, 1, 1, 0, 3, 0, 3, 3, 0, 0, 4, 4, 0, 1, 5, 4, 3, 3, 5, 0, 3, 3, 4, 3, 0, 2, 0, 1, 1, 1, 0, 1, 3, 0, 1, 2, 1, 3, 3, 2, 3, 3, 0, 3, 0, 1, 0, 1, 3, 3, 4, 4, 1, 0, 1, 2, 2, 1, 3),
+    (0, 1, 0, 4, 0, 4, 0, 3, 0, 1, 3, 3, 3, 2, 3, 1, 1, 0, 3, 0, 3, 3, 4, 3, 2, 4, 2, 0, 1, 0, 4, 3, 2, 0, 4, 3, 0, 5, 3, 3, 2, 4, 4, 4, 3, 3, 3, 4, 0, 1, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4, 2, 3, 3, 3, 0, 3, 0, 0, 0, 4, 4, 4, 5, 3, 2, 0, 3, 3, 0, 3, 5),
+    (0, 2, 0, 3, 0, 0, 0, 3, 0, 1, 3, 0, 2, 0, 0, 0, 1, 0, 3, 1, 1, 3, 3, 0, 0, 3, 0, 0, 3, 0, 2, 3, 1, 0, 3, 1, 0, 3, 3, 2, 0, 4, 2, 2, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 1, 0, 1, 0, 0, 0, 1, 3, 1, 2, 0, 0, 0, 1, 0, 0, 1, 4),
+    (0, 3, 0, 3, 0, 5, 0, 1, 0, 2, 4, 3, 1, 3, 3, 2, 1, 1, 5, 2, 1, 0, 5, 1, 2, 0, 0, 0, 3, 3, 2, 2, 3, 2, 4, 3, 0, 0, 3, 3, 1, 3, 3, 0, 2, 5, 3, 4, 0, 3, 3, 0, 1, 2, 0, 2, 2, 0, 3, 2, 0, 2, 2, 3, 3, 3, 0, 2, 0, 1, 0, 3, 4, 4, 2, 5, 4, 0, 3, 0, 0, 3, 5),
+    (0, 3, 0, 3, 0, 3, 0, 1, 0, 3, 3, 3, 3, 0, 3, 0, 2, 0, 2, 1, 1, 0, 2, 0, 1, 0, 0, 0, 2, 1, 0, 0, 1, 0, 3, 2, 0, 0, 3, 3, 1, 2, 3, 1, 0, 3, 3, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 3, 1, 2, 3, 0, 3, 0, 1, 0, 3, 2, 1, 0, 4, 3, 0, 1, 1, 0, 3, 3),
+    (0, 4, 0, 5, 0, 3, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 4, 3, 5, 3, 3, 2, 5, 3, 4, 4, 4, 3, 4, 3, 4, 5, 5, 3, 4, 4, 3, 4, 4, 5, 4, 4, 4, 3, 4, 5, 5, 4, 2, 3, 4, 2, 3, 4, 0, 3, 3, 1, 4, 3, 2, 4, 3, 3, 5, 5, 0, 3, 0, 3, 0, 5, 5, 5, 5, 4, 4, 0, 4, 0, 1, 4, 4),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 5, 4, 4, 2, 3, 2, 5, 1, 3, 2, 5, 1, 4, 2, 3, 2, 3, 3, 4, 3, 3, 3, 3, 2, 5, 4, 1, 3, 3, 5, 3, 4, 4, 0, 4, 4, 3, 1, 1, 3, 1, 0, 2, 3, 0, 2, 3, 0, 3, 0, 0, 4, 3, 1, 3, 4, 0, 3, 0, 2, 0, 4, 4, 4, 3, 4, 5, 0, 4, 0, 0, 3, 4),
+    (0, 3, 0, 3, 0, 3, 1, 2, 0, 3, 4, 4, 3, 3, 3, 0, 2, 2, 4, 3, 3, 1, 3, 3, 3, 1, 1, 0, 3, 1, 4, 3, 2, 3, 4, 4, 2, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 3, 1, 3, 3, 1, 3, 3, 0, 4, 1, 0, 2, 2, 1, 4, 3, 2, 3, 3, 5, 4, 3, 3, 5, 4, 4, 3, 3, 0, 4, 0, 3, 2, 2, 4, 4),
+    (0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 2, 1, 3, 0, 0, 0, 0, 0, 2, 0, 1, 2, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 1, 3, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 0, 3, 4, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1),
+    (0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 4, 1, 4, 0, 3, 0, 4, 0, 3, 0, 4, 0, 3, 0, 3, 0, 4, 1, 5, 1, 4, 0, 0, 3, 0, 5, 0, 5, 2, 0, 1, 0, 0, 0, 2, 1, 4, 0, 1, 3, 0, 0, 3, 0, 0, 3, 1, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0),
+    (1, 4, 0, 5, 0, 3, 0, 2, 0, 3, 5, 4, 4, 3, 4, 3, 5, 3, 4, 3, 3, 0, 4, 3, 3, 3, 3, 3, 3, 2, 4, 4, 3, 1, 3, 4, 4, 5, 4, 4, 3, 4, 4, 1, 3, 5, 4, 3, 3, 3, 1, 2, 2, 3, 3, 1, 3, 1, 3, 3, 3, 5, 3, 3, 4, 5, 0, 3, 0, 3, 0, 3, 4, 3, 4, 4, 3, 0, 3, 0, 2, 4, 3),
+    (0, 1, 0, 4, 0, 0, 0, 0, 0, 1, 4, 0, 4, 1, 4, 2, 4, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0, 3, 1, 1, 1, 0, 3, 0, 0, 0, 1, 2, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 3, 2, 0, 2, 2, 0, 1, 0, 0, 0, 2, 3, 2, 3, 3, 0, 0, 0, 0, 2, 1, 0),
+    (0, 5, 1, 5, 0, 3, 0, 3, 0, 5, 4, 4, 5, 1, 5, 3, 3, 0, 4, 3, 4, 3, 5, 3, 4, 3, 3, 2, 4, 3, 4, 3, 3, 0, 3, 3, 1, 4, 4, 3, 4, 4, 4, 3, 4, 5, 5, 3, 2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 3, 2, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 4, 4, 3, 5, 3, 3, 0, 3, 4, 0, 4, 3),
+    (0, 5, 0, 5, 0, 3, 0, 2, 0, 4, 4, 3, 5, 2, 4, 3, 3, 3, 4, 4, 4, 3, 5, 3, 5, 3, 3, 1, 4, 0, 4, 3, 3, 0, 3, 3, 0, 4, 4, 4, 4, 5, 4, 3, 3, 5, 5, 3, 2, 3, 1, 2, 3, 2, 0, 1, 0, 0, 3, 2, 2, 4, 4, 3, 1, 5, 0, 4, 0, 3, 0, 4, 3, 1, 3, 2, 1, 0, 3, 3, 0, 3, 3),
+    (0, 4, 0, 5, 0, 5, 0, 4, 0, 4, 5, 5, 5, 3, 4, 3, 3, 2, 5, 4, 4, 3, 5, 3, 5, 3, 4, 0, 4, 3, 4, 4, 3, 2, 4, 4, 3, 4, 5, 4, 4, 5, 5, 0, 3, 5, 5, 4, 1, 3, 3, 2, 3, 3, 1, 3, 1, 0, 4, 3, 1, 4, 4, 3, 4, 5, 0, 4, 0, 2, 0, 4, 3, 4, 4, 3, 3, 0, 4, 0, 0, 5, 5),
+    (0, 4, 0, 4, 0, 5, 0, 1, 1, 3, 3, 4, 4, 3, 4, 1, 3, 0, 5, 1, 3, 0, 3, 1, 3, 1, 1, 0, 3, 0, 3, 3, 4, 0, 4, 3, 0, 4, 4, 4, 3, 4, 4, 0, 3, 5, 4, 1, 0, 3, 0, 0, 2, 3, 0, 3, 1, 0, 3, 1, 0, 3, 2, 1, 3, 5, 0, 3, 0, 1, 0, 3, 2, 3, 3, 4, 4, 0, 2, 2, 0, 4, 4),
+    (2, 4, 0, 5, 0, 4, 0, 3, 0, 4, 5, 5, 4, 3, 5, 3, 5, 3, 5, 3, 5, 2, 5, 3, 4, 3, 3, 4, 3, 4, 5, 3, 2, 1, 5, 4, 3, 2, 3, 4, 5, 3, 4, 1, 2, 5, 4, 3, 0, 3, 3, 0, 3, 2, 0, 2, 3, 0, 4, 1, 0, 3, 4, 3, 3, 5, 0, 3, 0, 1, 0, 4, 5, 5, 5, 4, 3, 0, 4, 2, 0, 3, 5),
+    (0, 5, 0, 4, 0, 4, 0, 2, 0, 5, 4, 3, 4, 3, 4, 3, 3, 3, 4, 3, 4, 2, 5, 3, 5, 3, 4, 1, 4, 3, 4, 4, 4, 0, 3, 5, 0, 4, 4, 4, 4, 5, 3, 1, 3, 4, 5, 3, 3, 3, 3, 3, 3, 3, 0, 2, 2, 0, 3, 3, 2, 4, 3, 3, 3, 5, 3, 4, 1, 3, 3, 5, 3, 2, 0, 0, 0, 0, 4, 3, 1, 3, 3),
+    (0, 1, 0, 3, 0, 3, 0, 1, 0, 1, 3, 3, 3, 2, 3, 3, 3, 0, 3, 0, 0, 0, 3, 1, 3, 0, 0, 0, 2, 2, 2, 3, 0, 0, 3, 2, 0, 1, 2, 4, 1, 3, 3, 0, 0, 3, 3, 3, 0, 1, 0, 0, 2, 1, 0, 0, 3, 0, 3, 1, 0, 3, 0, 0, 1, 3, 0, 2, 0, 1, 0, 3, 3, 1, 3, 3, 0, 0, 1, 1, 0, 3, 3),
+    (0, 2, 0, 3, 0, 2, 1, 4, 0, 2, 2, 3, 1, 1, 3, 1, 1, 0, 2, 0, 3, 1, 2, 3, 1, 3, 0, 0, 1, 0, 4, 3, 2, 3, 3, 3, 1, 4, 2, 3, 3, 3, 3, 1, 0, 3, 1, 4, 0, 1, 1, 0, 1, 2, 0, 1, 1, 0, 1, 1, 0, 3, 1, 3, 2, 2, 0, 1, 0, 0, 0, 2, 3, 3, 3, 1, 0, 0, 0, 0, 0, 2, 3),
+    (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 5, 5, 3, 3, 4, 3, 3, 1, 5, 4, 4, 2, 4, 4, 4, 3, 4, 2, 4, 3, 5, 5, 4, 3, 3, 4, 3, 3, 5, 5, 4, 5, 5, 1, 3, 4, 5, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 1, 4, 5, 3, 3, 5, 0, 4, 0, 3, 0, 5, 3, 3, 1, 4, 3, 0, 4, 0, 1, 5, 3),
+    (0, 5, 0, 5, 0, 4, 0, 2, 0, 4, 4, 3, 4, 3, 3, 3, 3, 3, 5, 4, 4, 4, 4, 4, 4, 5, 3, 3, 5, 2, 4, 4, 4, 3, 4, 4, 3, 3, 4, 4, 5, 5, 3, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 3, 1, 2, 2, 1, 4, 3, 3, 5, 4, 4, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 4, 4, 1, 0, 4, 2, 0, 2, 4),
+    (0, 4, 0, 4, 0, 3, 0, 1, 0, 3, 5, 2, 3, 0, 3, 0, 2, 1, 4, 2, 3, 3, 4, 1, 4, 3, 3, 2, 4, 1, 3, 3, 3, 0, 3, 3, 0, 0, 3, 3, 3, 5, 3, 3, 3, 3, 3, 2, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0, 1, 0, 0, 3, 1, 2, 2, 3, 0, 3, 0, 2, 0, 4, 4, 3, 3, 4, 1, 0, 3, 0, 0, 2, 4),
+    (0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 3, 1, 3, 0, 3, 2, 0, 0, 0, 1, 0, 3, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 0, 0, 0, 0, 2),
+    (0, 2, 1, 3, 0, 2, 0, 2, 0, 3, 3, 3, 3, 1, 3, 1, 3, 3, 3, 3, 3, 3, 4, 2, 2, 1, 2, 1, 4, 0, 4, 3, 1, 3, 3, 3, 2, 4, 3, 5, 4, 3, 3, 3, 3, 3, 3, 3, 0, 1, 3, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 4, 2, 0, 2, 3, 0, 3, 3, 0, 3, 3, 4, 2, 3, 1, 4, 0, 1, 2, 0, 2, 3),
+    (0, 3, 0, 3, 0, 1, 0, 3, 0, 2, 3, 3, 3, 0, 3, 1, 2, 0, 3, 3, 2, 3, 3, 2, 3, 2, 3, 1, 3, 0, 4, 3, 2, 0, 3, 3, 1, 4, 3, 3, 2, 3, 4, 3, 1, 3, 3, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 4, 1, 1, 0, 3, 0, 3, 1, 0, 2, 3, 3, 3, 3, 3, 1, 0, 0, 2, 0, 3, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3),
+    (0, 2, 0, 3, 1, 3, 0, 3, 0, 2, 3, 3, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 1, 3, 0, 2, 3, 1, 1, 4, 3, 3, 2, 3, 3, 1, 2, 2, 4, 1, 3, 3, 0, 1, 4, 2, 3, 0, 1, 3, 0, 3, 0, 0, 1, 3, 0, 2, 0, 0, 3, 3, 2, 1, 3, 0, 3, 0, 2, 0, 3, 4, 4, 4, 3, 1, 0, 3, 0, 0, 3, 3),
+    (0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 3, 2, 2, 1, 3, 0, 1, 1, 3, 0, 3, 2, 3, 1, 2, 0, 2, 0, 1, 1, 3, 3, 3, 0, 3, 3, 1, 1, 2, 3, 2, 3, 3, 1, 2, 3, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2, 1, 2, 1, 3, 0, 3, 0, 0, 0, 3, 4, 4, 4, 3, 2, 0, 2, 0, 0, 2, 4),
+    (0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3),
+    (0, 3, 0, 3, 0, 2, 0, 3, 0, 3, 3, 3, 2, 3, 2, 2, 2, 0, 3, 1, 3, 3, 3, 2, 3, 3, 0, 0, 3, 0, 3, 2, 2, 0, 2, 3, 1, 4, 3, 4, 3, 3, 2, 3, 1, 5, 4, 4, 0, 3, 1, 2, 1, 3, 0, 3, 1, 1, 2, 0, 2, 3, 1, 3, 1, 3, 0, 3, 0, 1, 0, 3, 3, 4, 4, 2, 1, 0, 2, 1, 0, 2, 4),
+    (0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 4, 2, 5, 1, 4, 0, 2, 0, 2, 1, 3, 1, 4, 0, 2, 1, 0, 0, 2, 1, 4, 1, 1, 0, 3, 3, 0, 5, 1, 3, 2, 3, 3, 1, 0, 3, 2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 1, 0, 3, 0, 2, 0, 1, 0, 3, 3, 3, 4, 3, 3, 0, 0, 0, 0, 2, 3),
+    (0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0, 3),
+    (0, 1, 0, 3, 0, 4, 0, 3, 0, 2, 4, 3, 1, 0, 3, 2, 2, 1, 3, 1, 2, 2, 3, 1, 1, 1, 2, 1, 3, 0, 1, 2, 0, 1, 3, 2, 1, 3, 0, 5, 5, 1, 0, 0, 1, 3, 2, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4, 0, 1, 1, 1, 3, 2, 0, 2, 0, 1, 0, 2, 3, 3, 1, 2, 3, 0, 1, 0, 1, 0, 4),
+    (0, 0, 0, 1, 0, 3, 0, 3, 0, 2, 2, 1, 0, 0, 4, 0, 3, 0, 3, 1, 3, 0, 3, 0, 3, 0, 1, 0, 3, 0, 3, 1, 3, 0, 3, 3, 0, 0, 1, 2, 1, 1, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 2, 0, 0, 2, 0, 0, 0, 0, 2, 3, 3, 3, 3, 0, 0, 0, 0, 1, 4),
+    (0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 3, 1, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2, 0, 2, 3, 0, 0, 2, 2, 3, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 2, 3),
+    (2, 4, 0, 5, 0, 5, 0, 4, 0, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 4, 5, 5, 5, 2, 3, 0, 5, 5, 4, 1, 5, 4, 3, 1, 5, 4, 3, 4, 4, 3, 3, 4, 3, 3, 0, 3, 2, 0, 2, 3, 0, 3, 0, 0, 3, 3, 0, 5, 3, 2, 3, 3, 0, 3, 0, 3, 0, 3, 4, 5, 4, 5, 3, 0, 4, 3, 0, 3, 4),
+    (0, 3, 0, 3, 0, 3, 0, 3, 0, 3, 3, 4, 3, 2, 3, 2, 3, 0, 4, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 2, 4, 3, 3, 1, 3, 4, 3, 4, 4, 4, 3, 4, 4, 3, 2, 4, 4, 1, 0, 2, 0, 0, 1, 1, 0, 2, 0, 0, 3, 1, 0, 5, 3, 2, 1, 3, 0, 3, 0, 1, 2, 4, 3, 2, 4, 3, 3, 0, 3, 2, 0, 4, 4),
+    (0, 3, 0, 3, 0, 1, 0, 0, 0, 1, 4, 3, 3, 2, 3, 1, 3, 1, 4, 2, 3, 2, 4, 2, 3, 4, 3, 0, 2, 2, 3, 3, 3, 0, 3, 3, 3, 0, 3, 4, 1, 3, 3, 0, 3, 4, 3, 3, 0, 1, 1, 0, 1, 0, 0, 0, 4, 0, 3, 0, 0, 3, 1, 2, 1, 3, 0, 4, 0, 1, 0, 4, 3, 3, 4, 3, 3, 0, 2, 0, 0, 3, 3),
+    (0, 3, 0, 4, 0, 1, 0, 3, 0, 3, 4, 3, 3, 0, 3, 3, 3, 1, 3, 1, 3, 3, 4, 3, 3, 3, 0, 0, 3, 1, 5, 3, 3, 1, 3, 3, 2, 5, 4, 3, 3, 4, 5, 3, 2, 5, 3, 4, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 0, 4, 2, 2, 1, 3, 0, 3, 0, 2, 0, 4, 4, 3, 5, 3, 2, 0, 1, 1, 0, 3, 4),
+    (0, 5, 0, 4, 0, 5, 0, 2, 0, 4, 4, 3, 3, 2, 3, 3, 3, 1, 4, 3, 4, 1, 5, 3, 4, 3, 4, 0, 4, 2, 4, 3, 4, 1, 5, 4, 0, 4, 4, 4, 4, 5, 4, 1, 3, 5, 4, 2, 1, 4, 1, 1, 3, 2, 0, 3, 1, 0, 3, 2, 1, 4, 3, 3, 3, 4, 0, 4, 0, 3, 0, 4, 4, 4, 3, 3, 3, 0, 4, 2, 0, 3, 4),
+    (1, 4, 0, 4, 0, 3, 0, 1, 0, 3, 3, 3, 1, 1, 3, 3, 2, 2, 3, 3, 1, 0, 3, 2, 2, 1, 2, 0, 3, 1, 2, 1, 2, 0, 3, 2, 0, 2, 2, 3, 3, 4, 3, 0, 3, 3, 1, 2, 0, 1, 1, 3, 1, 2, 0, 0, 3, 0, 1, 1, 0, 3, 2, 2, 3, 3, 0, 3, 0, 0, 0, 2, 3, 3, 4, 3, 3, 0, 1, 0, 0, 1, 4),
+    (0, 4, 0, 4, 0, 4, 0, 0, 0, 3, 4, 4, 3, 1, 4, 2, 3, 2, 3, 3, 3, 1, 4, 3, 4, 0, 3, 0, 4, 2, 3, 3, 2, 2, 5, 4, 2, 1, 3, 4, 3, 4, 3, 1, 3, 3, 4, 2, 0, 2, 1, 0, 3, 3, 0, 0, 2, 0, 3, 1, 0, 4, 4, 3, 4, 3, 0, 4, 0, 1, 0, 2, 4, 4, 4, 4, 4, 0, 3, 2, 0, 3, 3),
+    (0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2),
+    (0, 2, 0, 3, 0, 4, 0, 4, 0, 1, 3, 3, 3, 0, 4, 0, 2, 1, 2, 1, 1, 1, 2, 0, 3, 1, 1, 0, 1, 0, 3, 1, 0, 0, 3, 3, 2, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 2, 0, 3, 1, 0, 0, 1, 0, 1, 1, 0, 1, 2, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 3, 4, 3, 1, 0, 1, 0, 3, 0, 2),
+    (0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 1, 0, 2, 0, 3, 1, 0, 1, 3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4, 0, 0, 0, 2, 3, 0, 1, 4, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 3),
+    (0, 2, 0, 5, 0, 5, 0, 1, 0, 2, 4, 3, 3, 2, 5, 1, 3, 2, 3, 3, 3, 0, 4, 1, 2, 0, 3, 0, 4, 0, 2, 2, 1, 1, 5, 3, 0, 0, 1, 4, 2, 3, 2, 0, 3, 3, 3, 2, 0, 2, 4, 1, 1, 2, 0, 1, 1, 0, 3, 1, 0, 1, 3, 1, 2, 3, 0, 2, 0, 0, 0, 1, 3, 5, 4, 4, 4, 0, 3, 0, 0, 1, 3),
+    (0, 4, 0, 5, 0, 4, 0, 4, 0, 4, 5, 4, 3, 3, 4, 3, 3, 3, 4, 3, 4, 4, 5, 3, 4, 5, 4, 2, 4, 2, 3, 4, 3, 1, 4, 4, 1, 3, 5, 4, 4, 5, 5, 4, 4, 5, 5, 5, 2, 3, 3, 1, 4, 3, 1, 3, 3, 0, 3, 3, 1, 4, 3, 4, 4, 4, 0, 3, 0, 4, 0, 3, 3, 4, 4, 5, 0, 0, 4, 3, 0, 4, 5),
+    (0, 4, 0, 4, 0, 3, 0, 3, 0, 3, 4, 4, 4, 3, 3, 2, 4, 3, 4, 3, 4, 3, 5, 3, 4, 3, 2, 1, 4, 2, 4, 4, 3, 1, 3, 4, 2, 4, 5, 5, 3, 4, 5, 4, 1, 5, 4, 3, 0, 3, 2, 2, 3, 2, 1, 3, 1, 0, 3, 3, 3, 5, 3, 3, 3, 5, 4, 4, 2, 3, 3, 4, 3, 3, 3, 2, 1, 0, 3, 2, 1, 4, 3),
+    (0, 4, 0, 5, 0, 4, 0, 3, 0, 3, 5, 5, 3, 2, 4, 3, 4, 0, 5, 4, 4, 1, 4, 4, 4, 3, 3, 3, 4, 3, 5, 5, 2, 3, 3, 4, 1, 2, 5, 5, 3, 5, 5, 2, 3, 5, 5, 4, 0, 3, 2, 0, 3, 3, 1, 1, 5, 1, 4, 1, 0, 4, 3, 2, 3, 5, 0, 4, 0, 3, 0, 5, 4, 3, 4, 3, 0, 0, 4, 1, 0, 4, 4),
+    (1, 3, 0, 4, 0, 2, 0, 2, 0, 2, 5, 5, 3, 3, 3, 3, 3, 0, 4, 2, 3, 4, 4, 4, 3, 4, 0, 0, 3, 4, 5, 4, 3, 3, 3, 3, 2, 5, 5, 4, 5, 5, 5, 4, 3, 5, 5, 5, 1, 3, 1, 0, 1, 0, 0, 3, 2, 0, 4, 2, 0, 5, 2, 3, 2, 4, 1, 3, 0, 3, 0, 4, 5, 4, 5, 4, 3, 0, 4, 2, 0, 5, 4),
+    (0, 3, 0, 4, 0, 5, 0, 3, 0, 3, 4, 4, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 4, 3, 3, 2, 2, 0, 3, 3, 3, 3, 3, 1, 3, 3, 3, 0, 4, 4, 3, 4, 4, 1, 1, 4, 4, 2, 0, 3, 1, 0, 1, 1, 0, 4, 1, 0, 2, 3, 1, 3, 3, 1, 3, 4, 0, 3, 0, 1, 0, 3, 1, 3, 0, 0, 1, 0, 2, 0, 0, 4, 4),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+    (0, 3, 0, 3, 0, 2, 0, 3, 0, 1, 5, 4, 3, 3, 3, 1, 4, 2, 1, 2, 3, 4, 4, 2, 4, 4, 5, 0, 3, 1, 4, 3, 4, 0, 4, 3, 3, 3, 2, 3, 2, 5, 3, 4, 3, 2, 2, 3, 0, 0, 3, 0, 2, 1, 0, 1, 2, 0, 0, 0, 0, 2, 1, 1, 3, 1, 0, 2, 0, 4, 0, 3, 4, 4, 4, 5, 2, 0, 2, 0, 0, 1, 3),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 4, 2, 1, 1, 0, 1, 0, 3, 2, 0, 0, 3, 1, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 4, 0, 4, 2, 1, 0, 0, 0, 0, 0, 1),
+    (0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 2, 0, 2, 1, 0, 0, 1, 2, 1, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2),
+    (0, 4, 0, 4, 0, 4, 0, 3, 0, 4, 4, 3, 4, 2, 4, 3, 2, 0, 4, 4, 4, 3, 5, 3, 5, 3, 3, 2, 4, 2, 4, 3, 4, 3, 1, 4, 0, 2, 3, 4, 4, 4, 3, 3, 3, 4, 4, 4, 3, 4, 1, 3, 4, 3, 2, 1, 2, 1, 3, 3, 3, 4, 4, 3, 3, 5, 0, 4, 0, 3, 0, 4, 3, 3, 3, 2, 1, 0, 3, 0, 0, 3, 3),
+    (0, 4, 0, 3, 0, 3, 0, 3, 0, 3, 5, 5, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 4, 4, 4, 3, 3, 3, 3, 4, 3, 5, 3, 3, 1, 3, 2, 4, 5, 5, 5, 5, 4, 3, 4, 5, 5, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 1, 2, 3, 2, 4, 3, 3, 3, 4, 0, 4, 0, 2, 0, 4, 3, 2, 2, 1, 2, 0, 3, 0, 0, 4, 1),
 )
+# fmt: on

-class JapaneseContextAnalysis(object):
+
+class JapaneseContextAnalysis:
    NUM_OF_CATEGORY = 6
    DONT_KNOW = -1
    ENOUGH_REL_THRESHOLD = 100
    MAX_REL_THRESHOLD = 1000
    MINIMUM_DATA_THRESHOLD = 4

-    def __init__(self):
-        self._total_rel = None
-        self._rel_sample = None
-        self._need_to_skip_char_num = None
-        self._last_char_order = None
-        self._done = None
+    def __init__(self) -> None:
+        self._total_rel = 0
+        self._rel_sample: List[int] = []
+        self._need_to_skip_char_num = 0
+        self._last_char_order = -1
+        self._done = False
        self.reset()

-    def reset(self):
+    def reset(self) -> None:
        self._total_rel = 0  # total sequence received
        # category counters, each integer counts sequence in its category
        self._rel_sample = [0] * self.NUM_OF_CATEGORY
@ -140,7 +144,7 @@ class JapaneseContextAnalysis(object):
        # been made
        self._done = False

-    def feed(self, byte_str, num_bytes):
+    def feed(self, byte_str: Union[bytes, bytearray], num_bytes: int) -> None:
        if self._done:
            return

@ -153,7 +157,7 @@ class JapaneseContextAnalysis(object):
        # this character will simply our logic and improve performance.
        i = self._need_to_skip_char_num
        while i < num_bytes:
-            order, char_len = self.get_order(byte_str[i:i + 2])
+            order, char_len = self.get_order(byte_str[i : i + 2])
            i += char_len
            if i > num_bytes:
                self._need_to_skip_char_num = i - num_bytes
@ -164,32 +168,34 @@ class JapaneseContextAnalysis(object):
                    if self._total_rel > self.MAX_REL_THRESHOLD:
                        self._done = True
                        break
-                    self._rel_sample[jp2CharContext[self._last_char_order][order]] += 1
+                    self._rel_sample[
+                        jp2_char_context[self._last_char_order][order]
+                    ] += 1
                self._last_char_order = order

-    def got_enough_data(self):
+    def got_enough_data(self) -> bool:
        return self._total_rel > self.ENOUGH_REL_THRESHOLD

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        # This is just one way to calculate confidence. It works well for me.
        if self._total_rel > self.MINIMUM_DATA_THRESHOLD:
            return (self._total_rel - self._rel_sample[0]) / self._total_rel
-        else:
-            return self.DONT_KNOW
+        return self.DONT_KNOW

-    def get_order(self, byte_str):
+    def get_order(self, _: Union[bytes, bytearray]) -> Tuple[int, int]:
        return -1, 1

+
 class SJISContextAnalysis(JapaneseContextAnalysis):
-    def __init__(self):
-        super(SJISContextAnalysis, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self._charset_name = "SHIFT_JIS"

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return self._charset_name

-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
        if not byte_str:
            return -1, 1
        # find out current char's byte length
@ -209,8 +215,9 @@ class SJISContextAnalysis(JapaneseContextAnalysis):

        return -1, char_len

+
 class EUCJPContextAnalysis(JapaneseContextAnalysis):
-    def get_order(self, byte_str):
+    def get_order(self, byte_str: Union[bytes, bytearray]) -> Tuple[int, int]:
        if not byte_str:
            return -1, 1
        # find out current char's byte length
@ -229,5 +236,3 @@ class EUCJPContextAnalysis(JapaneseContextAnalysis):
                return second_char - 0xA1, char_len

        return -1, char_len
-
-
--- a/libs/common/chardet/langbulgarianmodel.py
+++ b/libs/common/chardet/langbulgarianmodel.py
--- a/libs/common/chardet/langgreekmodel.py
+++ b/libs/common/chardet/langgreekmodel.py
--- a/libs/common/chardet/langhebrewmodel.py
+++ b/libs/common/chardet/langhebrewmodel.py
@ -1,9 +1,5 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from chardet.sbcharsetprober import SingleByteCharSetModel

-
 # 3: Positive
 # 2: Likely
 # 1: Unlikely
@ -4115,269 +4111,270 @@ HEBREW_LANG_MODEL = {

 # Character Mapping Table(s):
 WINDOWS_1255_HEBREW_CHAR_TO_ORDER = {
-     0: 255,  # '\x00'
-     1: 255,  # '\x01'
-     2: 255,  # '\x02'
-     3: 255,  # '\x03'
-     4: 255,  # '\x04'
-     5: 255,  # '\x05'
-     6: 255,  # '\x06'
-     7: 255,  # '\x07'
-     8: 255,  # '\x08'
-     9: 255,  # '\t'
-     10: 254,  # '\n'
-     11: 255,  # '\x0b'
-     12: 255,  # '\x0c'
-     13: 254,  # '\r'
-     14: 255,  # '\x0e'
-     15: 255,  # '\x0f'
-     16: 255,  # '\x10'
-     17: 255,  # '\x11'
-     18: 255,  # '\x12'
-     19: 255,  # '\x13'
-     20: 255,  # '\x14'
-     21: 255,  # '\x15'
-     22: 255,  # '\x16'
-     23: 255,  # '\x17'
-     24: 255,  # '\x18'
-     25: 255,  # '\x19'
-     26: 255,  # '\x1a'
-     27: 255,  # '\x1b'
-     28: 255,  # '\x1c'
-     29: 255,  # '\x1d'
-     30: 255,  # '\x1e'
-     31: 255,  # '\x1f'
-     32: 253,  # ' '
-     33: 253,  # '!'
-     34: 253,  # '"'
-     35: 253,  # '#'
-     36: 253,  # '$'
-     37: 253,  # '%'
-     38: 253,  # '&'
-     39: 253,  # "'"
-     40: 253,  # '('
-     41: 253,  # ')'
-     42: 253,  # '*'
-     43: 253,  # '+'
-     44: 253,  # ','
-     45: 253,  # '-'
-     46: 253,  # '.'
-     47: 253,  # '/'
-     48: 252,  # '0'
-     49: 252,  # '1'
-     50: 252,  # '2'
-     51: 252,  # '3'
-     52: 252,  # '4'
-     53: 252,  # '5'
-     54: 252,  # '6'
-     55: 252,  # '7'
-     56: 252,  # '8'
-     57: 252,  # '9'
-     58: 253,  # ':'
-     59: 253,  # ';'
-     60: 253,  # '<'
-     61: 253,  # '='
-     62: 253,  # '>'
-     63: 253,  # '?'
-     64: 253,  # '@'
-     65: 69,  # 'A'
-     66: 91,  # 'B'
-     67: 79,  # 'C'
-     68: 80,  # 'D'
-     69: 92,  # 'E'
-     70: 89,  # 'F'
-     71: 97,  # 'G'
-     72: 90,  # 'H'
-     73: 68,  # 'I'
-     74: 111,  # 'J'
-     75: 112,  # 'K'
-     76: 82,  # 'L'
-     77: 73,  # 'M'
-     78: 95,  # 'N'
-     79: 85,  # 'O'
-     80: 78,  # 'P'
-     81: 121,  # 'Q'
-     82: 86,  # 'R'
-     83: 71,  # 'S'
-     84: 67,  # 'T'
-     85: 102,  # 'U'
-     86: 107,  # 'V'
-     87: 84,  # 'W'
-     88: 114,  # 'X'
-     89: 103,  # 'Y'
-     90: 115,  # 'Z'
-     91: 253,  # '['
-     92: 253,  # '\\'
-     93: 253,  # ']'
-     94: 253,  # '^'
-     95: 253,  # '_'
-     96: 253,  # '`'
-     97: 50,  # 'a'
-     98: 74,  # 'b'
-     99: 60,  # 'c'
-     100: 61,  # 'd'
-     101: 42,  # 'e'
-     102: 76,  # 'f'
-     103: 70,  # 'g'
-     104: 64,  # 'h'
-     105: 53,  # 'i'
-     106: 105,  # 'j'
-     107: 93,  # 'k'
-     108: 56,  # 'l'
-     109: 65,  # 'm'
-     110: 54,  # 'n'
-     111: 49,  # 'o'
-     112: 66,  # 'p'
-     113: 110,  # 'q'
-     114: 51,  # 'r'
-     115: 43,  # 's'
-     116: 44,  # 't'
-     117: 63,  # 'u'
-     118: 81,  # 'v'
-     119: 77,  # 'w'
-     120: 98,  # 'x'
-     121: 75,  # 'y'
-     122: 108,  # 'z'
-     123: 253,  # '{'
-     124: 253,  # '|'
-     125: 253,  # '}'
-     126: 253,  # '~'
-     127: 253,  # '\x7f'
-     128: 124,  # '€'
-     129: 202,  # None
-     130: 203,  # '‚'
-     131: 204,  # 'ƒ'
-     132: 205,  # '„'
-     133: 40,  # '…'
-     134: 58,  # '†'
-     135: 206,  # '‡'
-     136: 207,  # 'ˆ'
-     137: 208,  # '‰'
-     138: 209,  # None
-     139: 210,  # '‹'
-     140: 211,  # None
-     141: 212,  # None
-     142: 213,  # None
-     143: 214,  # None
-     144: 215,  # None
-     145: 83,  # '‘'
-     146: 52,  # '’'
-     147: 47,  # '“'
-     148: 46,  # '”'
-     149: 72,  # '•'
-     150: 32,  # '–'
-     151: 94,  # '—'
-     152: 216,  # '˜'
-     153: 113,  # '™'
-     154: 217,  # None
-     155: 109,  # '›'
-     156: 218,  # None
-     157: 219,  # None
-     158: 220,  # None
-     159: 221,  # None
-     160: 34,  # '\xa0'
-     161: 116,  # '¡'
-     162: 222,  # '¢'
-     163: 118,  # '£'
-     164: 100,  # '₪'
-     165: 223,  # '¥'
-     166: 224,  # '¦'
-     167: 117,  # '§'
-     168: 119,  # '¨'
-     169: 104,  # '©'
-     170: 125,  # '×'
-     171: 225,  # '«'
-     172: 226,  # '¬'
-     173: 87,  # '\xad'
-     174: 99,  # '®'
-     175: 227,  # '¯'
-     176: 106,  # '°'
-     177: 122,  # '±'
-     178: 123,  # '²'
-     179: 228,  # '³'
-     180: 55,  # '´'
-     181: 229,  # 'µ'
-     182: 230,  # '¶'
-     183: 101,  # '·'
-     184: 231,  # '¸'
-     185: 232,  # '¹'
-     186: 120,  # '÷'
-     187: 233,  # '»'
-     188: 48,  # '¼'
-     189: 39,  # '½'
-     190: 57,  # '¾'
-     191: 234,  # '¿'
-     192: 30,  # 'ְ'
-     193: 59,  # 'ֱ'
-     194: 41,  # 'ֲ'
-     195: 88,  # 'ֳ'
-     196: 33,  # 'ִ'
-     197: 37,  # 'ֵ'
-     198: 36,  # 'ֶ'
-     199: 31,  # 'ַ'
-     200: 29,  # 'ָ'
-     201: 35,  # 'ֹ'
-     202: 235,  # None
-     203: 62,  # 'ֻ'
-     204: 28,  # 'ּ'
-     205: 236,  # 'ֽ'
-     206: 126,  # '־'
-     207: 237,  # 'ֿ'
-     208: 238,  # '׀'
-     209: 38,  # 'ׁ'
-     210: 45,  # 'ׂ'
-     211: 239,  # '׃'
-     212: 240,  # 'װ'
-     213: 241,  # 'ױ'
-     214: 242,  # 'ײ'
-     215: 243,  # '׳'
-     216: 127,  # '״'
-     217: 244,  # None
-     218: 245,  # None
-     219: 246,  # None
-     220: 247,  # None
-     221: 248,  # None
-     222: 249,  # None
-     223: 250,  # None
-     224: 9,  # 'א'
-     225: 8,  # 'ב'
-     226: 20,  # 'ג'
-     227: 16,  # 'ד'
-     228: 3,  # 'ה'
-     229: 2,  # 'ו'
-     230: 24,  # 'ז'
-     231: 14,  # 'ח'
-     232: 22,  # 'ט'
-     233: 1,  # 'י'
-     234: 25,  # 'ך'
-     235: 15,  # 'כ'
-     236: 4,  # 'ל'
-     237: 11,  # 'ם'
-     238: 6,  # 'מ'
-     239: 23,  # 'ן'
-     240: 12,  # 'נ'
-     241: 19,  # 'ס'
-     242: 13,  # 'ע'
-     243: 26,  # 'ף'
-     244: 18,  # 'פ'
-     245: 27,  # 'ץ'
-     246: 21,  # 'צ'
-     247: 17,  # 'ק'
-     248: 7,  # 'ר'
-     249: 10,  # 'ש'
-     250: 5,  # 'ת'
-     251: 251,  # None
-     252: 252,  # None
-     253: 128,  # '\u200e'
-     254: 96,  # '\u200f'
-     255: 253,  # None
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 69,  # 'A'
+    66: 91,  # 'B'
+    67: 79,  # 'C'
+    68: 80,  # 'D'
+    69: 92,  # 'E'
+    70: 89,  # 'F'
+    71: 97,  # 'G'
+    72: 90,  # 'H'
+    73: 68,  # 'I'
+    74: 111,  # 'J'
+    75: 112,  # 'K'
+    76: 82,  # 'L'
+    77: 73,  # 'M'
+    78: 95,  # 'N'
+    79: 85,  # 'O'
+    80: 78,  # 'P'
+    81: 121,  # 'Q'
+    82: 86,  # 'R'
+    83: 71,  # 'S'
+    84: 67,  # 'T'
+    85: 102,  # 'U'
+    86: 107,  # 'V'
+    87: 84,  # 'W'
+    88: 114,  # 'X'
+    89: 103,  # 'Y'
+    90: 115,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 50,  # 'a'
+    98: 74,  # 'b'
+    99: 60,  # 'c'
+    100: 61,  # 'd'
+    101: 42,  # 'e'
+    102: 76,  # 'f'
+    103: 70,  # 'g'
+    104: 64,  # 'h'
+    105: 53,  # 'i'
+    106: 105,  # 'j'
+    107: 93,  # 'k'
+    108: 56,  # 'l'
+    109: 65,  # 'm'
+    110: 54,  # 'n'
+    111: 49,  # 'o'
+    112: 66,  # 'p'
+    113: 110,  # 'q'
+    114: 51,  # 'r'
+    115: 43,  # 's'
+    116: 44,  # 't'
+    117: 63,  # 'u'
+    118: 81,  # 'v'
+    119: 77,  # 'w'
+    120: 98,  # 'x'
+    121: 75,  # 'y'
+    122: 108,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 124,  # '€'
+    129: 202,  # None
+    130: 203,  # '‚'
+    131: 204,  # 'ƒ'
+    132: 205,  # '„'
+    133: 40,  # '…'
+    134: 58,  # '†'
+    135: 206,  # '‡'
+    136: 207,  # 'ˆ'
+    137: 208,  # '‰'
+    138: 209,  # None
+    139: 210,  # '‹'
+    140: 211,  # None
+    141: 212,  # None
+    142: 213,  # None
+    143: 214,  # None
+    144: 215,  # None
+    145: 83,  # '‘'
+    146: 52,  # '’'
+    147: 47,  # '“'
+    148: 46,  # '”'
+    149: 72,  # '•'
+    150: 32,  # '–'
+    151: 94,  # '—'
+    152: 216,  # '˜'
+    153: 113,  # '™'
+    154: 217,  # None
+    155: 109,  # '›'
+    156: 218,  # None
+    157: 219,  # None
+    158: 220,  # None
+    159: 221,  # None
+    160: 34,  # '\xa0'
+    161: 116,  # '¡'
+    162: 222,  # '¢'
+    163: 118,  # '£'
+    164: 100,  # '₪'
+    165: 223,  # '¥'
+    166: 224,  # '¦'
+    167: 117,  # '§'
+    168: 119,  # '¨'
+    169: 104,  # '©'
+    170: 125,  # '×'
+    171: 225,  # '«'
+    172: 226,  # '¬'
+    173: 87,  # '\xad'
+    174: 99,  # '®'
+    175: 227,  # '¯'
+    176: 106,  # '°'
+    177: 122,  # '±'
+    178: 123,  # '²'
+    179: 228,  # '³'
+    180: 55,  # '´'
+    181: 229,  # 'µ'
+    182: 230,  # '¶'
+    183: 101,  # '·'
+    184: 231,  # '¸'
+    185: 232,  # '¹'
+    186: 120,  # '÷'
+    187: 233,  # '»'
+    188: 48,  # '¼'
+    189: 39,  # '½'
+    190: 57,  # '¾'
+    191: 234,  # '¿'
+    192: 30,  # 'ְ'
+    193: 59,  # 'ֱ'
+    194: 41,  # 'ֲ'
+    195: 88,  # 'ֳ'
+    196: 33,  # 'ִ'
+    197: 37,  # 'ֵ'
+    198: 36,  # 'ֶ'
+    199: 31,  # 'ַ'
+    200: 29,  # 'ָ'
+    201: 35,  # 'ֹ'
+    202: 235,  # None
+    203: 62,  # 'ֻ'
+    204: 28,  # 'ּ'
+    205: 236,  # 'ֽ'
+    206: 126,  # '־'
+    207: 237,  # 'ֿ'
+    208: 238,  # '׀'
+    209: 38,  # 'ׁ'
+    210: 45,  # 'ׂ'
+    211: 239,  # '׃'
+    212: 240,  # 'װ'
+    213: 241,  # 'ױ'
+    214: 242,  # 'ײ'
+    215: 243,  # '׳'
+    216: 127,  # '״'
+    217: 244,  # None
+    218: 245,  # None
+    219: 246,  # None
+    220: 247,  # None
+    221: 248,  # None
+    222: 249,  # None
+    223: 250,  # None
+    224: 9,  # 'א'
+    225: 8,  # 'ב'
+    226: 20,  # 'ג'
+    227: 16,  # 'ד'
+    228: 3,  # 'ה'
+    229: 2,  # 'ו'
+    230: 24,  # 'ז'
+    231: 14,  # 'ח'
+    232: 22,  # 'ט'
+    233: 1,  # 'י'
+    234: 25,  # 'ך'
+    235: 15,  # 'כ'
+    236: 4,  # 'ל'
+    237: 11,  # 'ם'
+    238: 6,  # 'מ'
+    239: 23,  # 'ן'
+    240: 12,  # 'נ'
+    241: 19,  # 'ס'
+    242: 13,  # 'ע'
+    243: 26,  # 'ף'
+    244: 18,  # 'פ'
+    245: 27,  # 'ץ'
+    246: 21,  # 'צ'
+    247: 17,  # 'ק'
+    248: 7,  # 'ר'
+    249: 10,  # 'ש'
+    250: 5,  # 'ת'
+    251: 251,  # None
+    252: 252,  # None
+    253: 128,  # '\u200e'
+    254: 96,  # '\u200f'
+    255: 253,  # None
 }

-WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(charset_name='windows-1255',
-                                                   language='Hebrew',
-                                                   char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER,
-                                                   language_model=HEBREW_LANG_MODEL,
-                                                   typical_positive_ratio=0.984004,
-                                                   keep_ascii_letters=False,
-                                                   alphabet='אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ')
-
+WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(
+    charset_name="windows-1255",
+    language="Hebrew",
+    char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER,
+    language_model=HEBREW_LANG_MODEL,
+    typical_positive_ratio=0.984004,
+    keep_ascii_letters=False,
+    alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
+)
--- a/libs/common/chardet/langhungarianmodel.py
+++ b/libs/common/chardet/langhungarianmodel.py
--- a/libs/common/chardet/langrussianmodel.py
+++ b/libs/common/chardet/langrussianmodel.py
--- a/libs/common/chardet/langthaimodel.py
+++ b/libs/common/chardet/langthaimodel.py
@ -1,9 +1,5 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from chardet.sbcharsetprober import SingleByteCharSetModel

-
 # 3: Positive
 # 2: Likely
 # 1: Unlikely
@ -4115,269 +4111,270 @@ THAI_LANG_MODEL = {

 # Character Mapping Table(s):
 TIS_620_THAI_CHAR_TO_ORDER = {
-     0: 255,  # '\x00'
-     1: 255,  # '\x01'
-     2: 255,  # '\x02'
-     3: 255,  # '\x03'
-     4: 255,  # '\x04'
-     5: 255,  # '\x05'
-     6: 255,  # '\x06'
-     7: 255,  # '\x07'
-     8: 255,  # '\x08'
-     9: 255,  # '\t'
-     10: 254,  # '\n'
-     11: 255,  # '\x0b'
-     12: 255,  # '\x0c'
-     13: 254,  # '\r'
-     14: 255,  # '\x0e'
-     15: 255,  # '\x0f'
-     16: 255,  # '\x10'
-     17: 255,  # '\x11'
-     18: 255,  # '\x12'
-     19: 255,  # '\x13'
-     20: 255,  # '\x14'
-     21: 255,  # '\x15'
-     22: 255,  # '\x16'
-     23: 255,  # '\x17'
-     24: 255,  # '\x18'
-     25: 255,  # '\x19'
-     26: 255,  # '\x1a'
-     27: 255,  # '\x1b'
-     28: 255,  # '\x1c'
-     29: 255,  # '\x1d'
-     30: 255,  # '\x1e'
-     31: 255,  # '\x1f'
-     32: 253,  # ' '
-     33: 253,  # '!'
-     34: 253,  # '"'
-     35: 253,  # '#'
-     36: 253,  # '$'
-     37: 253,  # '%'
-     38: 253,  # '&'
-     39: 253,  # "'"
-     40: 253,  # '('
-     41: 253,  # ')'
-     42: 253,  # '*'
-     43: 253,  # '+'
-     44: 253,  # ','
-     45: 253,  # '-'
-     46: 253,  # '.'
-     47: 253,  # '/'
-     48: 252,  # '0'
-     49: 252,  # '1'
-     50: 252,  # '2'
-     51: 252,  # '3'
-     52: 252,  # '4'
-     53: 252,  # '5'
-     54: 252,  # '6'
-     55: 252,  # '7'
-     56: 252,  # '8'
-     57: 252,  # '9'
-     58: 253,  # ':'
-     59: 253,  # ';'
-     60: 253,  # '<'
-     61: 253,  # '='
-     62: 253,  # '>'
-     63: 253,  # '?'
-     64: 253,  # '@'
-     65: 182,  # 'A'
-     66: 106,  # 'B'
-     67: 107,  # 'C'
-     68: 100,  # 'D'
-     69: 183,  # 'E'
-     70: 184,  # 'F'
-     71: 185,  # 'G'
-     72: 101,  # 'H'
-     73: 94,  # 'I'
-     74: 186,  # 'J'
-     75: 187,  # 'K'
-     76: 108,  # 'L'
-     77: 109,  # 'M'
-     78: 110,  # 'N'
-     79: 111,  # 'O'
-     80: 188,  # 'P'
-     81: 189,  # 'Q'
-     82: 190,  # 'R'
-     83: 89,  # 'S'
-     84: 95,  # 'T'
-     85: 112,  # 'U'
-     86: 113,  # 'V'
-     87: 191,  # 'W'
-     88: 192,  # 'X'
-     89: 193,  # 'Y'
-     90: 194,  # 'Z'
-     91: 253,  # '['
-     92: 253,  # '\\'
-     93: 253,  # ']'
-     94: 253,  # '^'
-     95: 253,  # '_'
-     96: 253,  # '`'
-     97: 64,  # 'a'
-     98: 72,  # 'b'
-     99: 73,  # 'c'
-     100: 114,  # 'd'
-     101: 74,  # 'e'
-     102: 115,  # 'f'
-     103: 116,  # 'g'
-     104: 102,  # 'h'
-     105: 81,  # 'i'
-     106: 201,  # 'j'
-     107: 117,  # 'k'
-     108: 90,  # 'l'
-     109: 103,  # 'm'
-     110: 78,  # 'n'
-     111: 82,  # 'o'
-     112: 96,  # 'p'
-     113: 202,  # 'q'
-     114: 91,  # 'r'
-     115: 79,  # 's'
-     116: 84,  # 't'
-     117: 104,  # 'u'
-     118: 105,  # 'v'
-     119: 97,  # 'w'
-     120: 98,  # 'x'
-     121: 92,  # 'y'
-     122: 203,  # 'z'
-     123: 253,  # '{'
-     124: 253,  # '|'
-     125: 253,  # '}'
-     126: 253,  # '~'
-     127: 253,  # '\x7f'
-     128: 209,  # '\x80'
-     129: 210,  # '\x81'
-     130: 211,  # '\x82'
-     131: 212,  # '\x83'
-     132: 213,  # '\x84'
-     133: 88,  # '\x85'
-     134: 214,  # '\x86'
-     135: 215,  # '\x87'
-     136: 216,  # '\x88'
-     137: 217,  # '\x89'
-     138: 218,  # '\x8a'
-     139: 219,  # '\x8b'
-     140: 220,  # '\x8c'
-     141: 118,  # '\x8d'
-     142: 221,  # '\x8e'
-     143: 222,  # '\x8f'
-     144: 223,  # '\x90'
-     145: 224,  # '\x91'
-     146: 99,  # '\x92'
-     147: 85,  # '\x93'
-     148: 83,  # '\x94'
-     149: 225,  # '\x95'
-     150: 226,  # '\x96'
-     151: 227,  # '\x97'
-     152: 228,  # '\x98'
-     153: 229,  # '\x99'
-     154: 230,  # '\x9a'
-     155: 231,  # '\x9b'
-     156: 232,  # '\x9c'
-     157: 233,  # '\x9d'
-     158: 234,  # '\x9e'
-     159: 235,  # '\x9f'
-     160: 236,  # None
-     161: 5,  # 'ก'
-     162: 30,  # 'ข'
-     163: 237,  # 'ฃ'
-     164: 24,  # 'ค'
-     165: 238,  # 'ฅ'
-     166: 75,  # 'ฆ'
-     167: 8,  # 'ง'
-     168: 26,  # 'จ'
-     169: 52,  # 'ฉ'
-     170: 34,  # 'ช'
-     171: 51,  # 'ซ'
-     172: 119,  # 'ฌ'
-     173: 47,  # 'ญ'
-     174: 58,  # 'ฎ'
-     175: 57,  # 'ฏ'
-     176: 49,  # 'ฐ'
-     177: 53,  # 'ฑ'
-     178: 55,  # 'ฒ'
-     179: 43,  # 'ณ'
-     180: 20,  # 'ด'
-     181: 19,  # 'ต'
-     182: 44,  # 'ถ'
-     183: 14,  # 'ท'
-     184: 48,  # 'ธ'
-     185: 3,  # 'น'
-     186: 17,  # 'บ'
-     187: 25,  # 'ป'
-     188: 39,  # 'ผ'
-     189: 62,  # 'ฝ'
-     190: 31,  # 'พ'
-     191: 54,  # 'ฟ'
-     192: 45,  # 'ภ'
-     193: 9,  # 'ม'
-     194: 16,  # 'ย'
-     195: 2,  # 'ร'
-     196: 61,  # 'ฤ'
-     197: 15,  # 'ล'
-     198: 239,  # 'ฦ'
-     199: 12,  # 'ว'
-     200: 42,  # 'ศ'
-     201: 46,  # 'ษ'
-     202: 18,  # 'ส'
-     203: 21,  # 'ห'
-     204: 76,  # 'ฬ'
-     205: 4,  # 'อ'
-     206: 66,  # 'ฮ'
-     207: 63,  # 'ฯ'
-     208: 22,  # 'ะ'
-     209: 10,  # 'ั'
-     210: 1,  # 'า'
-     211: 36,  # 'ำ'
-     212: 23,  # 'ิ'
-     213: 13,  # 'ี'
-     214: 40,  # 'ึ'
-     215: 27,  # 'ื'
-     216: 32,  # 'ุ'
-     217: 35,  # 'ู'
-     218: 86,  # 'ฺ'
-     219: 240,  # None
-     220: 241,  # None
-     221: 242,  # None
-     222: 243,  # None
-     223: 244,  # '฿'
-     224: 11,  # 'เ'
-     225: 28,  # 'แ'
-     226: 41,  # 'โ'
-     227: 29,  # 'ใ'
-     228: 33,  # 'ไ'
-     229: 245,  # 'ๅ'
-     230: 50,  # 'ๆ'
-     231: 37,  # '็'
-     232: 6,  # '่'
-     233: 7,  # '้'
-     234: 67,  # '๊'
-     235: 77,  # '๋'
-     236: 38,  # '์'
-     237: 93,  # 'ํ'
-     238: 246,  # '๎'
-     239: 247,  # '๏'
-     240: 68,  # '๐'
-     241: 56,  # '๑'
-     242: 59,  # '๒'
-     243: 65,  # '๓'
-     244: 69,  # '๔'
-     245: 60,  # '๕'
-     246: 70,  # '๖'
-     247: 80,  # '๗'
-     248: 71,  # '๘'
-     249: 87,  # '๙'
-     250: 248,  # '๚'
-     251: 249,  # '๛'
-     252: 250,  # None
-     253: 251,  # None
-     254: 252,  # None
-     255: 253,  # None
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 254,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 254,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 253,  # ' '
+    33: 253,  # '!'
+    34: 253,  # '"'
+    35: 253,  # '#'
+    36: 253,  # '$'
+    37: 253,  # '%'
+    38: 253,  # '&'
+    39: 253,  # "'"
+    40: 253,  # '('
+    41: 253,  # ')'
+    42: 253,  # '*'
+    43: 253,  # '+'
+    44: 253,  # ','
+    45: 253,  # '-'
+    46: 253,  # '.'
+    47: 253,  # '/'
+    48: 252,  # '0'
+    49: 252,  # '1'
+    50: 252,  # '2'
+    51: 252,  # '3'
+    52: 252,  # '4'
+    53: 252,  # '5'
+    54: 252,  # '6'
+    55: 252,  # '7'
+    56: 252,  # '8'
+    57: 252,  # '9'
+    58: 253,  # ':'
+    59: 253,  # ';'
+    60: 253,  # '<'
+    61: 253,  # '='
+    62: 253,  # '>'
+    63: 253,  # '?'
+    64: 253,  # '@'
+    65: 182,  # 'A'
+    66: 106,  # 'B'
+    67: 107,  # 'C'
+    68: 100,  # 'D'
+    69: 183,  # 'E'
+    70: 184,  # 'F'
+    71: 185,  # 'G'
+    72: 101,  # 'H'
+    73: 94,  # 'I'
+    74: 186,  # 'J'
+    75: 187,  # 'K'
+    76: 108,  # 'L'
+    77: 109,  # 'M'
+    78: 110,  # 'N'
+    79: 111,  # 'O'
+    80: 188,  # 'P'
+    81: 189,  # 'Q'
+    82: 190,  # 'R'
+    83: 89,  # 'S'
+    84: 95,  # 'T'
+    85: 112,  # 'U'
+    86: 113,  # 'V'
+    87: 191,  # 'W'
+    88: 192,  # 'X'
+    89: 193,  # 'Y'
+    90: 194,  # 'Z'
+    91: 253,  # '['
+    92: 253,  # '\\'
+    93: 253,  # ']'
+    94: 253,  # '^'
+    95: 253,  # '_'
+    96: 253,  # '`'
+    97: 64,  # 'a'
+    98: 72,  # 'b'
+    99: 73,  # 'c'
+    100: 114,  # 'd'
+    101: 74,  # 'e'
+    102: 115,  # 'f'
+    103: 116,  # 'g'
+    104: 102,  # 'h'
+    105: 81,  # 'i'
+    106: 201,  # 'j'
+    107: 117,  # 'k'
+    108: 90,  # 'l'
+    109: 103,  # 'm'
+    110: 78,  # 'n'
+    111: 82,  # 'o'
+    112: 96,  # 'p'
+    113: 202,  # 'q'
+    114: 91,  # 'r'
+    115: 79,  # 's'
+    116: 84,  # 't'
+    117: 104,  # 'u'
+    118: 105,  # 'v'
+    119: 97,  # 'w'
+    120: 98,  # 'x'
+    121: 92,  # 'y'
+    122: 203,  # 'z'
+    123: 253,  # '{'
+    124: 253,  # '|'
+    125: 253,  # '}'
+    126: 253,  # '~'
+    127: 253,  # '\x7f'
+    128: 209,  # '\x80'
+    129: 210,  # '\x81'
+    130: 211,  # '\x82'
+    131: 212,  # '\x83'
+    132: 213,  # '\x84'
+    133: 88,  # '\x85'
+    134: 214,  # '\x86'
+    135: 215,  # '\x87'
+    136: 216,  # '\x88'
+    137: 217,  # '\x89'
+    138: 218,  # '\x8a'
+    139: 219,  # '\x8b'
+    140: 220,  # '\x8c'
+    141: 118,  # '\x8d'
+    142: 221,  # '\x8e'
+    143: 222,  # '\x8f'
+    144: 223,  # '\x90'
+    145: 224,  # '\x91'
+    146: 99,  # '\x92'
+    147: 85,  # '\x93'
+    148: 83,  # '\x94'
+    149: 225,  # '\x95'
+    150: 226,  # '\x96'
+    151: 227,  # '\x97'
+    152: 228,  # '\x98'
+    153: 229,  # '\x99'
+    154: 230,  # '\x9a'
+    155: 231,  # '\x9b'
+    156: 232,  # '\x9c'
+    157: 233,  # '\x9d'
+    158: 234,  # '\x9e'
+    159: 235,  # '\x9f'
+    160: 236,  # None
+    161: 5,  # 'ก'
+    162: 30,  # 'ข'
+    163: 237,  # 'ฃ'
+    164: 24,  # 'ค'
+    165: 238,  # 'ฅ'
+    166: 75,  # 'ฆ'
+    167: 8,  # 'ง'
+    168: 26,  # 'จ'
+    169: 52,  # 'ฉ'
+    170: 34,  # 'ช'
+    171: 51,  # 'ซ'
+    172: 119,  # 'ฌ'
+    173: 47,  # 'ญ'
+    174: 58,  # 'ฎ'
+    175: 57,  # 'ฏ'
+    176: 49,  # 'ฐ'
+    177: 53,  # 'ฑ'
+    178: 55,  # 'ฒ'
+    179: 43,  # 'ณ'
+    180: 20,  # 'ด'
+    181: 19,  # 'ต'
+    182: 44,  # 'ถ'
+    183: 14,  # 'ท'
+    184: 48,  # 'ธ'
+    185: 3,  # 'น'
+    186: 17,  # 'บ'
+    187: 25,  # 'ป'
+    188: 39,  # 'ผ'
+    189: 62,  # 'ฝ'
+    190: 31,  # 'พ'
+    191: 54,  # 'ฟ'
+    192: 45,  # 'ภ'
+    193: 9,  # 'ม'
+    194: 16,  # 'ย'
+    195: 2,  # 'ร'
+    196: 61,  # 'ฤ'
+    197: 15,  # 'ล'
+    198: 239,  # 'ฦ'
+    199: 12,  # 'ว'
+    200: 42,  # 'ศ'
+    201: 46,  # 'ษ'
+    202: 18,  # 'ส'
+    203: 21,  # 'ห'
+    204: 76,  # 'ฬ'
+    205: 4,  # 'อ'
+    206: 66,  # 'ฮ'
+    207: 63,  # 'ฯ'
+    208: 22,  # 'ะ'
+    209: 10,  # 'ั'
+    210: 1,  # 'า'
+    211: 36,  # 'ำ'
+    212: 23,  # 'ิ'
+    213: 13,  # 'ี'
+    214: 40,  # 'ึ'
+    215: 27,  # 'ื'
+    216: 32,  # 'ุ'
+    217: 35,  # 'ู'
+    218: 86,  # 'ฺ'
+    219: 240,  # None
+    220: 241,  # None
+    221: 242,  # None
+    222: 243,  # None
+    223: 244,  # '฿'
+    224: 11,  # 'เ'
+    225: 28,  # 'แ'
+    226: 41,  # 'โ'
+    227: 29,  # 'ใ'
+    228: 33,  # 'ไ'
+    229: 245,  # 'ๅ'
+    230: 50,  # 'ๆ'
+    231: 37,  # '็'
+    232: 6,  # '่'
+    233: 7,  # '้'
+    234: 67,  # '๊'
+    235: 77,  # '๋'
+    236: 38,  # '์'
+    237: 93,  # 'ํ'
+    238: 246,  # '๎'
+    239: 247,  # '๏'
+    240: 68,  # '๐'
+    241: 56,  # '๑'
+    242: 59,  # '๒'
+    243: 65,  # '๓'
+    244: 69,  # '๔'
+    245: 60,  # '๕'
+    246: 70,  # '๖'
+    247: 80,  # '๗'
+    248: 71,  # '๘'
+    249: 87,  # '๙'
+    250: 248,  # '๚'
+    251: 249,  # '๛'
+    252: 250,  # None
+    253: 251,  # None
+    254: 252,  # None
+    255: 253,  # None
 }

-TIS_620_THAI_MODEL = SingleByteCharSetModel(charset_name='TIS-620',
-                                            language='Thai',
-                                            char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER,
-                                            language_model=THAI_LANG_MODEL,
-                                            typical_positive_ratio=0.926386,
-                                            keep_ascii_letters=False,
-                                            alphabet='กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛')
-
+TIS_620_THAI_MODEL = SingleByteCharSetModel(
+    charset_name="TIS-620",
+    language="Thai",
+    char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER,
+    language_model=THAI_LANG_MODEL,
+    typical_positive_ratio=0.926386,
+    keep_ascii_letters=False,
+    alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
+)
--- a/libs/common/chardet/langturkishmodel.py
+++ b/libs/common/chardet/langturkishmodel.py
@ -1,9 +1,5 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from chardet.sbcharsetprober import SingleByteCharSetModel

-
 # 3: Positive
 # 2: Likely
 # 1: Unlikely
@ -4115,269 +4111,270 @@ TURKISH_LANG_MODEL = {

 # Character Mapping Table(s):
 ISO_8859_9_TURKISH_CHAR_TO_ORDER = {
-     0: 255,  # '\x00'
-     1: 255,  # '\x01'
-     2: 255,  # '\x02'
-     3: 255,  # '\x03'
-     4: 255,  # '\x04'
-     5: 255,  # '\x05'
-     6: 255,  # '\x06'
-     7: 255,  # '\x07'
-     8: 255,  # '\x08'
-     9: 255,  # '\t'
-     10: 255,  # '\n'
-     11: 255,  # '\x0b'
-     12: 255,  # '\x0c'
-     13: 255,  # '\r'
-     14: 255,  # '\x0e'
-     15: 255,  # '\x0f'
-     16: 255,  # '\x10'
-     17: 255,  # '\x11'
-     18: 255,  # '\x12'
-     19: 255,  # '\x13'
-     20: 255,  # '\x14'
-     21: 255,  # '\x15'
-     22: 255,  # '\x16'
-     23: 255,  # '\x17'
-     24: 255,  # '\x18'
-     25: 255,  # '\x19'
-     26: 255,  # '\x1a'
-     27: 255,  # '\x1b'
-     28: 255,  # '\x1c'
-     29: 255,  # '\x1d'
-     30: 255,  # '\x1e'
-     31: 255,  # '\x1f'
-     32: 255,  # ' '
-     33: 255,  # '!'
-     34: 255,  # '"'
-     35: 255,  # '#'
-     36: 255,  # '$'
-     37: 255,  # '%'
-     38: 255,  # '&'
-     39: 255,  # "'"
-     40: 255,  # '('
-     41: 255,  # ')'
-     42: 255,  # '*'
-     43: 255,  # '+'
-     44: 255,  # ','
-     45: 255,  # '-'
-     46: 255,  # '.'
-     47: 255,  # '/'
-     48: 255,  # '0'
-     49: 255,  # '1'
-     50: 255,  # '2'
-     51: 255,  # '3'
-     52: 255,  # '4'
-     53: 255,  # '5'
-     54: 255,  # '6'
-     55: 255,  # '7'
-     56: 255,  # '8'
-     57: 255,  # '9'
-     58: 255,  # ':'
-     59: 255,  # ';'
-     60: 255,  # '<'
-     61: 255,  # '='
-     62: 255,  # '>'
-     63: 255,  # '?'
-     64: 255,  # '@'
-     65: 23,  # 'A'
-     66: 37,  # 'B'
-     67: 47,  # 'C'
-     68: 39,  # 'D'
-     69: 29,  # 'E'
-     70: 52,  # 'F'
-     71: 36,  # 'G'
-     72: 45,  # 'H'
-     73: 53,  # 'I'
-     74: 60,  # 'J'
-     75: 16,  # 'K'
-     76: 49,  # 'L'
-     77: 20,  # 'M'
-     78: 46,  # 'N'
-     79: 42,  # 'O'
-     80: 48,  # 'P'
-     81: 69,  # 'Q'
-     82: 44,  # 'R'
-     83: 35,  # 'S'
-     84: 31,  # 'T'
-     85: 51,  # 'U'
-     86: 38,  # 'V'
-     87: 62,  # 'W'
-     88: 65,  # 'X'
-     89: 43,  # 'Y'
-     90: 56,  # 'Z'
-     91: 255,  # '['
-     92: 255,  # '\\'
-     93: 255,  # ']'
-     94: 255,  # '^'
-     95: 255,  # '_'
-     96: 255,  # '`'
-     97: 1,  # 'a'
-     98: 21,  # 'b'
-     99: 28,  # 'c'
-     100: 12,  # 'd'
-     101: 2,  # 'e'
-     102: 18,  # 'f'
-     103: 27,  # 'g'
-     104: 25,  # 'h'
-     105: 3,  # 'i'
-     106: 24,  # 'j'
-     107: 10,  # 'k'
-     108: 5,  # 'l'
-     109: 13,  # 'm'
-     110: 4,  # 'n'
-     111: 15,  # 'o'
-     112: 26,  # 'p'
-     113: 64,  # 'q'
-     114: 7,  # 'r'
-     115: 8,  # 's'
-     116: 9,  # 't'
-     117: 14,  # 'u'
-     118: 32,  # 'v'
-     119: 57,  # 'w'
-     120: 58,  # 'x'
-     121: 11,  # 'y'
-     122: 22,  # 'z'
-     123: 255,  # '{'
-     124: 255,  # '|'
-     125: 255,  # '}'
-     126: 255,  # '~'
-     127: 255,  # '\x7f'
-     128: 180,  # '\x80'
-     129: 179,  # '\x81'
-     130: 178,  # '\x82'
-     131: 177,  # '\x83'
-     132: 176,  # '\x84'
-     133: 175,  # '\x85'
-     134: 174,  # '\x86'
-     135: 173,  # '\x87'
-     136: 172,  # '\x88'
-     137: 171,  # '\x89'
-     138: 170,  # '\x8a'
-     139: 169,  # '\x8b'
-     140: 168,  # '\x8c'
-     141: 167,  # '\x8d'
-     142: 166,  # '\x8e'
-     143: 165,  # '\x8f'
-     144: 164,  # '\x90'
-     145: 163,  # '\x91'
-     146: 162,  # '\x92'
-     147: 161,  # '\x93'
-     148: 160,  # '\x94'
-     149: 159,  # '\x95'
-     150: 101,  # '\x96'
-     151: 158,  # '\x97'
-     152: 157,  # '\x98'
-     153: 156,  # '\x99'
-     154: 155,  # '\x9a'
-     155: 154,  # '\x9b'
-     156: 153,  # '\x9c'
-     157: 152,  # '\x9d'
-     158: 151,  # '\x9e'
-     159: 106,  # '\x9f'
-     160: 150,  # '\xa0'
-     161: 149,  # '¡'
-     162: 148,  # '¢'
-     163: 147,  # '£'
-     164: 146,  # '¤'
-     165: 145,  # '¥'
-     166: 144,  # '¦'
-     167: 100,  # '§'
-     168: 143,  # '¨'
-     169: 142,  # '©'
-     170: 141,  # 'ª'
-     171: 140,  # '«'
-     172: 139,  # '¬'
-     173: 138,  # '\xad'
-     174: 137,  # '®'
-     175: 136,  # '¯'
-     176: 94,  # '°'
-     177: 80,  # '±'
-     178: 93,  # '²'
-     179: 135,  # '³'
-     180: 105,  # '´'
-     181: 134,  # 'µ'
-     182: 133,  # '¶'
-     183: 63,  # '·'
-     184: 132,  # '¸'
-     185: 131,  # '¹'
-     186: 130,  # 'º'
-     187: 129,  # '»'
-     188: 128,  # '¼'
-     189: 127,  # '½'
-     190: 126,  # '¾'
-     191: 125,  # '¿'
-     192: 124,  # 'À'
-     193: 104,  # 'Á'
-     194: 73,  # 'Â'
-     195: 99,  # 'Ã'
-     196: 79,  # 'Ä'
-     197: 85,  # 'Å'
-     198: 123,  # 'Æ'
-     199: 54,  # 'Ç'
-     200: 122,  # 'È'
-     201: 98,  # 'É'
-     202: 92,  # 'Ê'
-     203: 121,  # 'Ë'
-     204: 120,  # 'Ì'
-     205: 91,  # 'Í'
-     206: 103,  # 'Î'
-     207: 119,  # 'Ï'
-     208: 68,  # 'Ğ'
-     209: 118,  # 'Ñ'
-     210: 117,  # 'Ò'
-     211: 97,  # 'Ó'
-     212: 116,  # 'Ô'
-     213: 115,  # 'Õ'
-     214: 50,  # 'Ö'
-     215: 90,  # '×'
-     216: 114,  # 'Ø'
-     217: 113,  # 'Ù'
-     218: 112,  # 'Ú'
-     219: 111,  # 'Û'
-     220: 55,  # 'Ü'
-     221: 41,  # 'İ'
-     222: 40,  # 'Ş'
-     223: 86,  # 'ß'
-     224: 89,  # 'à'
-     225: 70,  # 'á'
-     226: 59,  # 'â'
-     227: 78,  # 'ã'
-     228: 71,  # 'ä'
-     229: 82,  # 'å'
-     230: 88,  # 'æ'
-     231: 33,  # 'ç'
-     232: 77,  # 'è'
-     233: 66,  # 'é'
-     234: 84,  # 'ê'
-     235: 83,  # 'ë'
-     236: 110,  # 'ì'
-     237: 75,  # 'í'
-     238: 61,  # 'î'
-     239: 96,  # 'ï'
-     240: 30,  # 'ğ'
-     241: 67,  # 'ñ'
-     242: 109,  # 'ò'
-     243: 74,  # 'ó'
-     244: 87,  # 'ô'
-     245: 102,  # 'õ'
-     246: 34,  # 'ö'
-     247: 95,  # '÷'
-     248: 81,  # 'ø'
-     249: 108,  # 'ù'
-     250: 76,  # 'ú'
-     251: 72,  # 'û'
-     252: 17,  # 'ü'
-     253: 6,  # 'ı'
-     254: 19,  # 'ş'
-     255: 107,  # 'ÿ'
+    0: 255,  # '\x00'
+    1: 255,  # '\x01'
+    2: 255,  # '\x02'
+    3: 255,  # '\x03'
+    4: 255,  # '\x04'
+    5: 255,  # '\x05'
+    6: 255,  # '\x06'
+    7: 255,  # '\x07'
+    8: 255,  # '\x08'
+    9: 255,  # '\t'
+    10: 255,  # '\n'
+    11: 255,  # '\x0b'
+    12: 255,  # '\x0c'
+    13: 255,  # '\r'
+    14: 255,  # '\x0e'
+    15: 255,  # '\x0f'
+    16: 255,  # '\x10'
+    17: 255,  # '\x11'
+    18: 255,  # '\x12'
+    19: 255,  # '\x13'
+    20: 255,  # '\x14'
+    21: 255,  # '\x15'
+    22: 255,  # '\x16'
+    23: 255,  # '\x17'
+    24: 255,  # '\x18'
+    25: 255,  # '\x19'
+    26: 255,  # '\x1a'
+    27: 255,  # '\x1b'
+    28: 255,  # '\x1c'
+    29: 255,  # '\x1d'
+    30: 255,  # '\x1e'
+    31: 255,  # '\x1f'
+    32: 255,  # ' '
+    33: 255,  # '!'
+    34: 255,  # '"'
+    35: 255,  # '#'
+    36: 255,  # '$'
+    37: 255,  # '%'
+    38: 255,  # '&'
+    39: 255,  # "'"
+    40: 255,  # '('
+    41: 255,  # ')'
+    42: 255,  # '*'
+    43: 255,  # '+'
+    44: 255,  # ','
+    45: 255,  # '-'
+    46: 255,  # '.'
+    47: 255,  # '/'
+    48: 255,  # '0'
+    49: 255,  # '1'
+    50: 255,  # '2'
+    51: 255,  # '3'
+    52: 255,  # '4'
+    53: 255,  # '5'
+    54: 255,  # '6'
+    55: 255,  # '7'
+    56: 255,  # '8'
+    57: 255,  # '9'
+    58: 255,  # ':'
+    59: 255,  # ';'
+    60: 255,  # '<'
+    61: 255,  # '='
+    62: 255,  # '>'
+    63: 255,  # '?'
+    64: 255,  # '@'
+    65: 23,  # 'A'
+    66: 37,  # 'B'
+    67: 47,  # 'C'
+    68: 39,  # 'D'
+    69: 29,  # 'E'
+    70: 52,  # 'F'
+    71: 36,  # 'G'
+    72: 45,  # 'H'
+    73: 53,  # 'I'
+    74: 60,  # 'J'
+    75: 16,  # 'K'
+    76: 49,  # 'L'
+    77: 20,  # 'M'
+    78: 46,  # 'N'
+    79: 42,  # 'O'
+    80: 48,  # 'P'
+    81: 69,  # 'Q'
+    82: 44,  # 'R'
+    83: 35,  # 'S'
+    84: 31,  # 'T'
+    85: 51,  # 'U'
+    86: 38,  # 'V'
+    87: 62,  # 'W'
+    88: 65,  # 'X'
+    89: 43,  # 'Y'
+    90: 56,  # 'Z'
+    91: 255,  # '['
+    92: 255,  # '\\'
+    93: 255,  # ']'
+    94: 255,  # '^'
+    95: 255,  # '_'
+    96: 255,  # '`'
+    97: 1,  # 'a'
+    98: 21,  # 'b'
+    99: 28,  # 'c'
+    100: 12,  # 'd'
+    101: 2,  # 'e'
+    102: 18,  # 'f'
+    103: 27,  # 'g'
+    104: 25,  # 'h'
+    105: 3,  # 'i'
+    106: 24,  # 'j'
+    107: 10,  # 'k'
+    108: 5,  # 'l'
+    109: 13,  # 'm'
+    110: 4,  # 'n'
+    111: 15,  # 'o'
+    112: 26,  # 'p'
+    113: 64,  # 'q'
+    114: 7,  # 'r'
+    115: 8,  # 's'
+    116: 9,  # 't'
+    117: 14,  # 'u'
+    118: 32,  # 'v'
+    119: 57,  # 'w'
+    120: 58,  # 'x'
+    121: 11,  # 'y'
+    122: 22,  # 'z'
+    123: 255,  # '{'
+    124: 255,  # '|'
+    125: 255,  # '}'
+    126: 255,  # '~'
+    127: 255,  # '\x7f'
+    128: 180,  # '\x80'
+    129: 179,  # '\x81'
+    130: 178,  # '\x82'
+    131: 177,  # '\x83'
+    132: 176,  # '\x84'
+    133: 175,  # '\x85'
+    134: 174,  # '\x86'
+    135: 173,  # '\x87'
+    136: 172,  # '\x88'
+    137: 171,  # '\x89'
+    138: 170,  # '\x8a'
+    139: 169,  # '\x8b'
+    140: 168,  # '\x8c'
+    141: 167,  # '\x8d'
+    142: 166,  # '\x8e'
+    143: 165,  # '\x8f'
+    144: 164,  # '\x90'
+    145: 163,  # '\x91'
+    146: 162,  # '\x92'
+    147: 161,  # '\x93'
+    148: 160,  # '\x94'
+    149: 159,  # '\x95'
+    150: 101,  # '\x96'
+    151: 158,  # '\x97'
+    152: 157,  # '\x98'
+    153: 156,  # '\x99'
+    154: 155,  # '\x9a'
+    155: 154,  # '\x9b'
+    156: 153,  # '\x9c'
+    157: 152,  # '\x9d'
+    158: 151,  # '\x9e'
+    159: 106,  # '\x9f'
+    160: 150,  # '\xa0'
+    161: 149,  # '¡'
+    162: 148,  # '¢'
+    163: 147,  # '£'
+    164: 146,  # '¤'
+    165: 145,  # '¥'
+    166: 144,  # '¦'
+    167: 100,  # '§'
+    168: 143,  # '¨'
+    169: 142,  # '©'
+    170: 141,  # 'ª'
+    171: 140,  # '«'
+    172: 139,  # '¬'
+    173: 138,  # '\xad'
+    174: 137,  # '®'
+    175: 136,  # '¯'
+    176: 94,  # '°'
+    177: 80,  # '±'
+    178: 93,  # '²'
+    179: 135,  # '³'
+    180: 105,  # '´'
+    181: 134,  # 'µ'
+    182: 133,  # '¶'
+    183: 63,  # '·'
+    184: 132,  # '¸'
+    185: 131,  # '¹'
+    186: 130,  # 'º'
+    187: 129,  # '»'
+    188: 128,  # '¼'
+    189: 127,  # '½'
+    190: 126,  # '¾'
+    191: 125,  # '¿'
+    192: 124,  # 'À'
+    193: 104,  # 'Á'
+    194: 73,  # 'Â'
+    195: 99,  # 'Ã'
+    196: 79,  # 'Ä'
+    197: 85,  # 'Å'
+    198: 123,  # 'Æ'
+    199: 54,  # 'Ç'
+    200: 122,  # 'È'
+    201: 98,  # 'É'
+    202: 92,  # 'Ê'
+    203: 121,  # 'Ë'
+    204: 120,  # 'Ì'
+    205: 91,  # 'Í'
+    206: 103,  # 'Î'
+    207: 119,  # 'Ï'
+    208: 68,  # 'Ğ'
+    209: 118,  # 'Ñ'
+    210: 117,  # 'Ò'
+    211: 97,  # 'Ó'
+    212: 116,  # 'Ô'
+    213: 115,  # 'Õ'
+    214: 50,  # 'Ö'
+    215: 90,  # '×'
+    216: 114,  # 'Ø'
+    217: 113,  # 'Ù'
+    218: 112,  # 'Ú'
+    219: 111,  # 'Û'
+    220: 55,  # 'Ü'
+    221: 41,  # 'İ'
+    222: 40,  # 'Ş'
+    223: 86,  # 'ß'
+    224: 89,  # 'à'
+    225: 70,  # 'á'
+    226: 59,  # 'â'
+    227: 78,  # 'ã'
+    228: 71,  # 'ä'
+    229: 82,  # 'å'
+    230: 88,  # 'æ'
+    231: 33,  # 'ç'
+    232: 77,  # 'è'
+    233: 66,  # 'é'
+    234: 84,  # 'ê'
+    235: 83,  # 'ë'
+    236: 110,  # 'ì'
+    237: 75,  # 'í'
+    238: 61,  # 'î'
+    239: 96,  # 'ï'
+    240: 30,  # 'ğ'
+    241: 67,  # 'ñ'
+    242: 109,  # 'ò'
+    243: 74,  # 'ó'
+    244: 87,  # 'ô'
+    245: 102,  # 'õ'
+    246: 34,  # 'ö'
+    247: 95,  # '÷'
+    248: 81,  # 'ø'
+    249: 108,  # 'ù'
+    250: 76,  # 'ú'
+    251: 72,  # 'û'
+    252: 17,  # 'ü'
+    253: 6,  # 'ı'
+    254: 19,  # 'ş'
+    255: 107,  # 'ÿ'
 }

-ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-9',
-                                                  language='Turkish',
-                                                  char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER,
-                                                  language_model=TURKISH_LANG_MODEL,
-                                                  typical_positive_ratio=0.97029,
-                                                  keep_ascii_letters=True,
-                                                  alphabet='ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş')
-
+ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(
+    charset_name="ISO-8859-9",
+    language="Turkish",
+    char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER,
+    language_model=TURKISH_LANG_MODEL,
+    typical_positive_ratio=0.97029,
+    keep_ascii_letters=True,
+    alphabet="ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş",
+)
--- a/libs/common/chardet/latin1prober.py
+++ b/libs/common/chardet/latin1prober.py
@ -26,6 +26,8 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import List, Union
+
 from .charsetprober import CharSetProber
 from .enums import ProbingState

@ -41,6 +43,7 @@ ASV = 6  # accent small vowel
 ASO = 7  # accent small other
 CLASS_NUM = 8  # total classes

+# fmt: off
 Latin1_CharToClass = (
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
@ -91,34 +94,34 @@ Latin1ClassModel = (
    0,  3,  1,  3,  1,  1,  1,  3,  # ASV
    0,  3,  1,  3,  1,  1,  3,  3,  # ASO
 )
+# fmt: on


 class Latin1Prober(CharSetProber):
-    def __init__(self):
-        super(Latin1Prober, self).__init__()
-        self._last_char_class = None
-        self._freq_counter = None
+    def __init__(self) -> None:
+        super().__init__()
+        self._last_char_class = OTH
+        self._freq_counter: List[int] = []
        self.reset()

-    def reset(self):
+    def reset(self) -> None:
        self._last_char_class = OTH
        self._freq_counter = [0] * FREQ_CAT_NUM
-        CharSetProber.reset(self)
+        super().reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "ISO-8859-1"

    @property
-    def language(self):
+    def language(self) -> str:
        return ""

-    def feed(self, byte_str):
-        byte_str = self.filter_with_english_letters(byte_str)
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        byte_str = self.remove_xml_tags(byte_str)
        for c in byte_str:
            char_class = Latin1_CharToClass[c]
-            freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM)
-                                    + char_class]
+            freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM) + char_class]
            if freq == 0:
                self._state = ProbingState.NOT_ME
                break
@ -127,19 +130,18 @@ class Latin1Prober(CharSetProber):

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        if self.state == ProbingState.NOT_ME:
            return 0.01

        total = sum(self._freq_counter)
-        if total < 0.01:
-            confidence = 0.0
-        else:
-            confidence = ((self._freq_counter[3] - self._freq_counter[1] * 20.0)
-                          / total)
-        if confidence < 0.0:
-            confidence = 0.0
+        confidence = (
+            0.0
+            if total < 0.01
+            else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
+        )
+        confidence = max(confidence, 0.0)
        # lower the confidence of latin1 so that other more accurate
        # detector can take priority.
-        confidence = confidence * 0.73
+        confidence *= 0.73
        return confidence
--- a/libs/common/chardet/macromanprober.py
+++ b/libs/common/chardet/macromanprober.py
@ -0,0 +1,162 @@
+######################## BEGIN LICENSE BLOCK ########################
+# This code was modified from latin1prober.py by Rob Speer <rob@lumino.so>.
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Rob Speer - adapt to MacRoman encoding
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from typing import List, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+
+FREQ_CAT_NUM = 4
+
+UDF = 0  # undefined
+OTH = 1  # other
+ASC = 2  # ascii capital letter
+ASS = 3  # ascii small letter
+ACV = 4  # accent capital vowel
+ACO = 5  # accent capital other
+ASV = 6  # accent small vowel
+ASO = 7  # accent small other
+ODD = 8  # character that is unlikely to appear
+CLASS_NUM = 9  # total classes
+
+# The change from Latin1 is that we explicitly look for extended characters
+# that are infrequently-occurring symbols, and consider them to always be
+# improbable. This should let MacRoman get out of the way of more likely
+# encodings in most situations.
+
+# fmt: off
+MacRoman_CharToClass = (
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 00 - 07
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 08 - 0F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 10 - 17
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 18 - 1F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 20 - 27
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 28 - 2F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 30 - 37
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # 38 - 3F
+    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 40 - 47
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 48 - 4F
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,  # 50 - 57
+    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,  # 58 - 5F
+    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 60 - 67
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 68 - 6F
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,  # 70 - 77
+    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,  # 78 - 7F
+    ACV, ACV, ACO, ACV, ACO, ACV, ACV, ASV,  # 80 - 87
+    ASV, ASV, ASV, ASV, ASV, ASO, ASV, ASV,  # 88 - 8F
+    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASV,  # 90 - 97
+    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,  # 98 - 9F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, ASO,  # A0 - A7
+    OTH, OTH, ODD, ODD, OTH, OTH, ACV, ACV,  # A8 - AF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,  # B0 - B7
+    OTH, OTH, OTH, OTH, OTH, OTH, ASV, ASV,  # B8 - BF
+    OTH, OTH, ODD, OTH, ODD, OTH, OTH, OTH,  # C0 - C7
+    OTH, OTH, OTH, ACV, ACV, ACV, ACV, ASV,  # C8 - CF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, ODD,  # D0 - D7
+    ASV, ACV, ODD, OTH, OTH, OTH, OTH, OTH,  # D8 - DF
+    OTH, OTH, OTH, OTH, OTH, ACV, ACV, ACV,  # E0 - E7
+    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,  # E8 - EF
+    ODD, ACV, ACV, ACV, ACV, ASV, ODD, ODD,  # F0 - F7
+    ODD, ODD, ODD, ODD, ODD, ODD, ODD, ODD,  # F8 - FF
+)
+
+# 0 : illegal
+# 1 : very unlikely
+# 2 : normal
+# 3 : very likely
+MacRomanClassModel = (
+# UDF OTH ASC ASS ACV ACO ASV ASO ODD
+    0,  0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # OTH
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # ASC
+    0,  3,  3,  3,  1,  1,  3,  3,  1,  # ASS
+    0,  3,  3,  3,  1,  2,  1,  2,  1,  # ACV
+    0,  3,  3,  3,  3,  3,  3,  3,  1,  # ACO
+    0,  3,  1,  3,  1,  1,  1,  3,  1,  # ASV
+    0,  3,  1,  3,  1,  1,  3,  3,  1,  # ASO
+    0,  1,  1,  1,  1,  1,  1,  1,  1,  # ODD
+)
+# fmt: on
+
+
+class MacRomanProber(CharSetProber):
+    def __init__(self) -> None:
+        super().__init__()
+        self._last_char_class = OTH
+        self._freq_counter: List[int] = []
+        self.reset()
+
+    def reset(self) -> None:
+        self._last_char_class = OTH
+        self._freq_counter = [0] * FREQ_CAT_NUM
+
+        # express the prior that MacRoman is a somewhat rare encoding;
+        # this can be done by starting out in a slightly improbable state
+        # that must be overcome
+        self._freq_counter[2] = 10
+
+        super().reset()
+
+    @property
+    def charset_name(self) -> str:
+        return "MacRoman"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        byte_str = self.remove_xml_tags(byte_str)
+        for c in byte_str:
+            char_class = MacRoman_CharToClass[c]
+            freq = MacRomanClassModel[(self._last_char_class * CLASS_NUM) + char_class]
+            if freq == 0:
+                self._state = ProbingState.NOT_ME
+                break
+            self._freq_counter[freq] += 1
+            self._last_char_class = char_class
+
+        return self.state
+
+    def get_confidence(self) -> float:
+        if self.state == ProbingState.NOT_ME:
+            return 0.01
+
+        total = sum(self._freq_counter)
+        confidence = (
+            0.0
+            if total < 0.01
+            else (self._freq_counter[3] - self._freq_counter[1] * 20.0) / total
+        )
+        confidence = max(confidence, 0.0)
+        # lower the confidence of MacRoman so that other more accurate
+        # detector can take priority.
+        confidence *= 0.73
+        return confidence
--- a/libs/common/chardet/mbcharsetprober.py
+++ b/libs/common/chardet/mbcharsetprober.py
@ -27,8 +27,12 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from typing import Optional, Union
+
+from .chardistribution import CharDistributionAnalysis
 from .charsetprober import CharSetProber
-from .enums import ProbingState, MachineState
+from .codingstatemachine import CodingStateMachine
+from .enums import LanguageFilter, MachineState, ProbingState


 class MultiByteCharSetProber(CharSetProber):
@ -36,56 +40,56 @@ class MultiByteCharSetProber(CharSetProber):
    MultiByteCharSetProber
    """

-    def __init__(self, lang_filter=None):
-        super(MultiByteCharSetProber, self).__init__(lang_filter=lang_filter)
-        self.distribution_analyzer = None
-        self.coding_sm = None
-        self._last_char = [0, 0]
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
+        self.distribution_analyzer: Optional[CharDistributionAnalysis] = None
+        self.coding_sm: Optional[CodingStateMachine] = None
+        self._last_char = bytearray(b"\0\0")

-    def reset(self):
-        super(MultiByteCharSetProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        if self.coding_sm:
            self.coding_sm.reset()
        if self.distribution_analyzer:
            self.distribution_analyzer.reset()
-        self._last_char = [0, 0]
+        self._last_char = bytearray(b"\0\0")

-    @property
-    def charset_name(self):
-        raise NotImplementedError
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None

-    @property
-    def language(self):
-        raise NotImplementedError
-
-    def feed(self, byte_str):
-        for i in range(len(byte_str)):
-            coding_state = self.coding_sm.next_state(byte_str[i])
+        for i, byte in enumerate(byte_str):
+            coding_state = self.coding_sm.next_state(byte)
            if coding_state == MachineState.ERROR:
-                self.logger.debug('%s %s prober hit error at byte %s',
-                                  self.charset_name, self.language, i)
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
                self._state = ProbingState.NOT_ME
                break
-            elif coding_state == MachineState.ITS_ME:
+            if coding_state == MachineState.ITS_ME:
                self._state = ProbingState.FOUND_IT
                break
-            elif coding_state == MachineState.START:
+            if coding_state == MachineState.START:
                char_len = self.coding_sm.get_current_charlen()
                if i == 0:
-                    self._last_char[1] = byte_str[0]
+                    self._last_char[1] = byte
                    self.distribution_analyzer.feed(self._last_char, char_len)
                else:
-                    self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
-                                                    char_len)
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)

        self._last_char[0] = byte_str[-1]

        if self.state == ProbingState.DETECTING:
-            if (self.distribution_analyzer.got_enough_data() and
-                    (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
+            if self.distribution_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
                self._state = ProbingState.FOUND_IT

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
        return self.distribution_analyzer.get_confidence()
--- a/libs/common/chardet/mbcsgroupprober.py
+++ b/libs/common/chardet/mbcsgroupprober.py
@ -27,20 +27,22 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .charsetgroupprober import CharSetGroupProber
-from .utf8prober import UTF8Prober
-from .sjisprober import SJISProber
-from .eucjpprober import EUCJPProber
-from .gb2312prober import GB2312Prober
-from .euckrprober import EUCKRProber
-from .cp949prober import CP949Prober
 from .big5prober import Big5Prober
+from .charsetgroupprober import CharSetGroupProber
+from .cp949prober import CP949Prober
+from .enums import LanguageFilter
+from .eucjpprober import EUCJPProber
+from .euckrprober import EUCKRProber
 from .euctwprober import EUCTWProber
+from .gb2312prober import GB2312Prober
+from .johabprober import JOHABProber
+from .sjisprober import SJISProber
+from .utf8prober import UTF8Prober


 class MBCSGroupProber(CharSetGroupProber):
-    def __init__(self, lang_filter=None):
-        super(MBCSGroupProber, self).__init__(lang_filter=lang_filter)
+    def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
+        super().__init__(lang_filter=lang_filter)
        self.probers = [
            UTF8Prober(),
            SJISProber(),
@ -49,6 +51,7 @@ class MBCSGroupProber(CharSetGroupProber):
            EUCKRProber(),
            CP949Prober(),
            Big5Prober(),
-            EUCTWProber()
+            EUCTWProber(),
+            JOHABProber(),
        ]
        self.reset()
--- a/libs/common/chardet/mbcssm.py
+++ b/libs/common/chardet/mbcssm.py
@ -25,43 +25,45 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

+from .codingstatemachinedict import CodingStateMachineDict
 from .enums import MachineState

 # BIG5

+# fmt: off
 BIG5_CLS = (
-    1,1,1,1,1,1,1,1,  # 00 - 07    #allow 0x00 as legal value
-    1,1,1,1,1,1,0,0,  # 08 - 0f
-    1,1,1,1,1,1,1,1,  # 10 - 17
-    1,1,1,0,1,1,1,1,  # 18 - 1f
-    1,1,1,1,1,1,1,1,  # 20 - 27
-    1,1,1,1,1,1,1,1,  # 28 - 2f
-    1,1,1,1,1,1,1,1,  # 30 - 37
-    1,1,1,1,1,1,1,1,  # 38 - 3f
-    2,2,2,2,2,2,2,2,  # 40 - 47
-    2,2,2,2,2,2,2,2,  # 48 - 4f
-    2,2,2,2,2,2,2,2,  # 50 - 57
-    2,2,2,2,2,2,2,2,  # 58 - 5f
-    2,2,2,2,2,2,2,2,  # 60 - 67
-    2,2,2,2,2,2,2,2,  # 68 - 6f
-    2,2,2,2,2,2,2,2,  # 70 - 77
-    2,2,2,2,2,2,2,1,  # 78 - 7f
-    4,4,4,4,4,4,4,4,  # 80 - 87
-    4,4,4,4,4,4,4,4,  # 88 - 8f
-    4,4,4,4,4,4,4,4,  # 90 - 97
-    4,4,4,4,4,4,4,4,  # 98 - 9f
-    4,3,3,3,3,3,3,3,  # a0 - a7
-    3,3,3,3,3,3,3,3,  # a8 - af
-    3,3,3,3,3,3,3,3,  # b0 - b7
-    3,3,3,3,3,3,3,3,  # b8 - bf
-    3,3,3,3,3,3,3,3,  # c0 - c7
-    3,3,3,3,3,3,3,3,  # c8 - cf
-    3,3,3,3,3,3,3,3,  # d0 - d7
-    3,3,3,3,3,3,3,3,  # d8 - df
-    3,3,3,3,3,3,3,3,  # e0 - e7
-    3,3,3,3,3,3,3,3,  # e8 - ef
-    3,3,3,3,3,3,3,3,  # f0 - f7
-    3,3,3,3,3,3,3,0  # f8 - ff
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07    #allow 0x00 as legal value
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 1,  # 78 - 7f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 80 - 87
+    4, 4, 4, 4, 4, 4, 4, 4,  # 88 - 8f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 90 - 97
+    4, 4, 4, 4, 4, 4, 4, 4,  # 98 - 9f
+    4, 3, 3, 3, 3, 3, 3, 3,  # a0 - a7
+    3, 3, 3, 3, 3, 3, 3, 3,  # a8 - af
+    3, 3, 3, 3, 3, 3, 3, 3,  # b0 - b7
+    3, 3, 3, 3, 3, 3, 3, 3,  # b8 - bf
+    3, 3, 3, 3, 3, 3, 3, 3,  # c0 - c7
+    3, 3, 3, 3, 3, 3, 3, 3,  # c8 - cf
+    3, 3, 3, 3, 3, 3, 3, 3,  # d0 - d7
+    3, 3, 3, 3, 3, 3, 3, 3,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 3, 3, 3,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 3, 3, 0  # f8 - ff
 )

 BIG5_ST = (
@ -69,34 +71,37 @@ BIG5_ST = (
    MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,#08-0f
    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START#10-17
 )
+# fmt: on

 BIG5_CHAR_LEN_TABLE = (0, 1, 1, 2, 0)

-BIG5_SM_MODEL = {'class_table': BIG5_CLS,
-                 'class_factor': 5,
-                 'state_table': BIG5_ST,
-                 'char_len_table': BIG5_CHAR_LEN_TABLE,
-                 'name': 'Big5'}
+BIG5_SM_MODEL: CodingStateMachineDict = {
+    "class_table": BIG5_CLS,
+    "class_factor": 5,
+    "state_table": BIG5_ST,
+    "char_len_table": BIG5_CHAR_LEN_TABLE,
+    "name": "Big5",
+}

 # CP949
-
+# fmt: off
 CP949_CLS  = (
-    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0,  # 00 - 0f
-    1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1,  # 10 - 1f
-    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 20 - 2f
-    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 30 - 3f
-    1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,  # 40 - 4f
-    4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 50 - 5f
-    1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5,  # 60 - 6f
-    5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 70 - 7f
-    0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 80 - 8f
-    6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 90 - 9f
-    6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8,  # a0 - af
-    7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,  # b0 - bf
-    7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2,  # c0 - cf
-    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # d0 - df
-    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # e0 - ef
-    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0,  # f0 - ff
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,  # 00 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,  # 10 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 3f
+    1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,  # 40 - 4f
+    4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1,  # 50 - 5f
+    1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,  # 60 - 6f
+    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 1,  # 70 - 7f
+    0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,  # 80 - 8f
+    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,  # 90 - 9f
+    6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8,  # a0 - af
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,  # b0 - bf
+    7, 7, 7, 7, 7, 7, 9, 2, 2, 3, 2, 2, 2, 2, 2, 2,  # c0 - cf
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  # d0 - df
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  # e0 - ef
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0,  # f0 - ff
 )

 CP949_ST = (
@ -109,50 +114,53 @@ CP949_ST = (
    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START, # 5
    MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START, # 6
 )
+# fmt: on

 CP949_CHAR_LEN_TABLE = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)

-CP949_SM_MODEL = {'class_table': CP949_CLS,
-                  'class_factor': 10,
-                  'state_table': CP949_ST,
-                  'char_len_table': CP949_CHAR_LEN_TABLE,
-                  'name': 'CP949'}
+CP949_SM_MODEL: CodingStateMachineDict = {
+    "class_table": CP949_CLS,
+    "class_factor": 10,
+    "state_table": CP949_ST,
+    "char_len_table": CP949_CHAR_LEN_TABLE,
+    "name": "CP949",
+}

 # EUC-JP
-
+# fmt: off
 EUCJP_CLS = (
-    4,4,4,4,4,4,4,4,  # 00 - 07
-    4,4,4,4,4,4,5,5,  # 08 - 0f
-    4,4,4,4,4,4,4,4,  # 10 - 17
-    4,4,4,5,4,4,4,4,  # 18 - 1f
-    4,4,4,4,4,4,4,4,  # 20 - 27
-    4,4,4,4,4,4,4,4,  # 28 - 2f
-    4,4,4,4,4,4,4,4,  # 30 - 37
-    4,4,4,4,4,4,4,4,  # 38 - 3f
-    4,4,4,4,4,4,4,4,  # 40 - 47
-    4,4,4,4,4,4,4,4,  # 48 - 4f
-    4,4,4,4,4,4,4,4,  # 50 - 57
-    4,4,4,4,4,4,4,4,  # 58 - 5f
-    4,4,4,4,4,4,4,4,  # 60 - 67
-    4,4,4,4,4,4,4,4,  # 68 - 6f
-    4,4,4,4,4,4,4,4,  # 70 - 77
-    4,4,4,4,4,4,4,4,  # 78 - 7f
-    5,5,5,5,5,5,5,5,  # 80 - 87
-    5,5,5,5,5,5,1,3,  # 88 - 8f
-    5,5,5,5,5,5,5,5,  # 90 - 97
-    5,5,5,5,5,5,5,5,  # 98 - 9f
-    5,2,2,2,2,2,2,2,  # a0 - a7
-    2,2,2,2,2,2,2,2,  # a8 - af
-    2,2,2,2,2,2,2,2,  # b0 - b7
-    2,2,2,2,2,2,2,2,  # b8 - bf
-    2,2,2,2,2,2,2,2,  # c0 - c7
-    2,2,2,2,2,2,2,2,  # c8 - cf
-    2,2,2,2,2,2,2,2,  # d0 - d7
-    2,2,2,2,2,2,2,2,  # d8 - df
-    0,0,0,0,0,0,0,0,  # e0 - e7
-    0,0,0,0,0,0,0,0,  # e8 - ef
-    0,0,0,0,0,0,0,0,  # f0 - f7
-    0,0,0,0,0,0,0,5  # f8 - ff
+    4, 4, 4, 4, 4, 4, 4, 4,  # 00 - 07
+    4, 4, 4, 4, 4, 4, 5, 5,  # 08 - 0f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 10 - 17
+    4, 4, 4, 5, 4, 4, 4, 4,  # 18 - 1f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 20 - 27
+    4, 4, 4, 4, 4, 4, 4, 4,  # 28 - 2f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 30 - 37
+    4, 4, 4, 4, 4, 4, 4, 4,  # 38 - 3f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 40 - 47
+    4, 4, 4, 4, 4, 4, 4, 4,  # 48 - 4f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 50 - 57
+    4, 4, 4, 4, 4, 4, 4, 4,  # 58 - 5f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 60 - 67
+    4, 4, 4, 4, 4, 4, 4, 4,  # 68 - 6f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 70 - 77
+    4, 4, 4, 4, 4, 4, 4, 4,  # 78 - 7f
+    5, 5, 5, 5, 5, 5, 5, 5,  # 80 - 87
+    5, 5, 5, 5, 5, 5, 1, 3,  # 88 - 8f
+    5, 5, 5, 5, 5, 5, 5, 5,  # 90 - 97
+    5, 5, 5, 5, 5, 5, 5, 5,  # 98 - 9f
+    5, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 0, 5  # f8 - ff
 )

 EUCJP_ST = (
@ -162,100 +170,163 @@ EUCJP_ST = (
     MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     3,MachineState.ERROR,#18-1f
          3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START#20-27
 )
+# fmt: on

 EUCJP_CHAR_LEN_TABLE = (2, 2, 2, 3, 1, 0)

-EUCJP_SM_MODEL = {'class_table': EUCJP_CLS,
-                  'class_factor': 6,
-                  'state_table': EUCJP_ST,
-                  'char_len_table': EUCJP_CHAR_LEN_TABLE,
-                  'name': 'EUC-JP'}
+EUCJP_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCJP_CLS,
+    "class_factor": 6,
+    "state_table": EUCJP_ST,
+    "char_len_table": EUCJP_CHAR_LEN_TABLE,
+    "name": "EUC-JP",
+}

 # EUC-KR
-
+# fmt: off
 EUCKR_CLS  = (
-    1,1,1,1,1,1,1,1,  # 00 - 07
-    1,1,1,1,1,1,0,0,  # 08 - 0f
-    1,1,1,1,1,1,1,1,  # 10 - 17
-    1,1,1,0,1,1,1,1,  # 18 - 1f
-    1,1,1,1,1,1,1,1,  # 20 - 27
-    1,1,1,1,1,1,1,1,  # 28 - 2f
-    1,1,1,1,1,1,1,1,  # 30 - 37
-    1,1,1,1,1,1,1,1,  # 38 - 3f
-    1,1,1,1,1,1,1,1,  # 40 - 47
-    1,1,1,1,1,1,1,1,  # 48 - 4f
-    1,1,1,1,1,1,1,1,  # 50 - 57
-    1,1,1,1,1,1,1,1,  # 58 - 5f
-    1,1,1,1,1,1,1,1,  # 60 - 67
-    1,1,1,1,1,1,1,1,  # 68 - 6f
-    1,1,1,1,1,1,1,1,  # 70 - 77
-    1,1,1,1,1,1,1,1,  # 78 - 7f
-    0,0,0,0,0,0,0,0,  # 80 - 87
-    0,0,0,0,0,0,0,0,  # 88 - 8f
-    0,0,0,0,0,0,0,0,  # 90 - 97
-    0,0,0,0,0,0,0,0,  # 98 - 9f
-    0,2,2,2,2,2,2,2,  # a0 - a7
-    2,2,2,2,2,3,3,3,  # a8 - af
-    2,2,2,2,2,2,2,2,  # b0 - b7
-    2,2,2,2,2,2,2,2,  # b8 - bf
-    2,2,2,2,2,2,2,2,  # c0 - c7
-    2,3,2,2,2,2,2,2,  # c8 - cf
-    2,2,2,2,2,2,2,2,  # d0 - d7
-    2,2,2,2,2,2,2,2,  # d8 - df
-    2,2,2,2,2,2,2,2,  # e0 - e7
-    2,2,2,2,2,2,2,2,  # e8 - ef
-    2,2,2,2,2,2,2,2,  # f0 - f7
-    2,2,2,2,2,2,2,0   # f8 - ff
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 40 - 47
+    1, 1, 1, 1, 1, 1, 1, 1,  # 48 - 4f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 50 - 57
+    1, 1, 1, 1, 1, 1, 1, 1,  # 58 - 5f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 60 - 67
+    1, 1, 1, 1, 1, 1, 1, 1,  # 68 - 6f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 70 - 77
+    1, 1, 1, 1, 1, 1, 1, 1,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 3, 3, 3,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 3, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    2, 2, 2, 2, 2, 2, 2, 2,  # e0 - e7
+    2, 2, 2, 2, 2, 2, 2, 2,  # e8 - ef
+    2, 2, 2, 2, 2, 2, 2, 2,  # f0 - f7
+    2, 2, 2, 2, 2, 2, 2, 0   # f8 - ff
 )

 EUCKR_ST = (
    MachineState.ERROR,MachineState.START,     3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #08-0f
 )
+# fmt: on

 EUCKR_CHAR_LEN_TABLE = (0, 1, 2, 0)

-EUCKR_SM_MODEL = {'class_table': EUCKR_CLS,
-                'class_factor': 4,
-                'state_table': EUCKR_ST,
-                'char_len_table': EUCKR_CHAR_LEN_TABLE,
-                'name': 'EUC-KR'}
+EUCKR_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCKR_CLS,
+    "class_factor": 4,
+    "state_table": EUCKR_ST,
+    "char_len_table": EUCKR_CHAR_LEN_TABLE,
+    "name": "EUC-KR",
+}
+
+# JOHAB
+# fmt: off
+JOHAB_CLS = (
+    4,4,4,4,4,4,4,4,  # 00 - 07
+    4,4,4,4,4,4,0,0,  # 08 - 0f
+    4,4,4,4,4,4,4,4,  # 10 - 17
+    4,4,4,0,4,4,4,4,  # 18 - 1f
+    4,4,4,4,4,4,4,4,  # 20 - 27
+    4,4,4,4,4,4,4,4,  # 28 - 2f
+    4,3,3,3,3,3,3,3,  # 30 - 37
+    3,3,3,3,3,3,3,3,  # 38 - 3f
+    3,1,1,1,1,1,1,1,  # 40 - 47
+    1,1,1,1,1,1,1,1,  # 48 - 4f
+    1,1,1,1,1,1,1,1,  # 50 - 57
+    1,1,1,1,1,1,1,1,  # 58 - 5f
+    1,1,1,1,1,1,1,1,  # 60 - 67
+    1,1,1,1,1,1,1,1,  # 68 - 6f
+    1,1,1,1,1,1,1,1,  # 70 - 77
+    1,1,1,1,1,1,1,2,  # 78 - 7f
+    6,6,6,6,8,8,8,8,  # 80 - 87
+    8,8,8,8,8,8,8,8,  # 88 - 8f
+    8,7,7,7,7,7,7,7,  # 90 - 97
+    7,7,7,7,7,7,7,7,  # 98 - 9f
+    7,7,7,7,7,7,7,7,  # a0 - a7
+    7,7,7,7,7,7,7,7,  # a8 - af
+    7,7,7,7,7,7,7,7,  # b0 - b7
+    7,7,7,7,7,7,7,7,  # b8 - bf
+    7,7,7,7,7,7,7,7,  # c0 - c7
+    7,7,7,7,7,7,7,7,  # c8 - cf
+    7,7,7,7,5,5,5,5,  # d0 - d7
+    5,9,9,9,9,9,9,5,  # d8 - df
+    9,9,9,9,9,9,9,9,  # e0 - e7
+    9,9,9,9,9,9,9,9,  # e8 - ef
+    9,9,9,9,9,9,9,9,  # f0 - f7
+    9,9,5,5,5,5,5,0   # f8 - ff
+)
+
+JOHAB_ST = (
+# cls = 0                   1                   2                   3                   4                   5                   6                   7                   8                   9
+    MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,3                  ,3                  ,4                  ,  # MachineState.START
+    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,  # MachineState.ITS_ME
+    MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,MachineState.ERROR ,  # MachineState.ERROR
+    MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.ERROR ,MachineState.ERROR ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,MachineState.START ,  # 3
+    MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,MachineState.ERROR ,MachineState.START ,  # 4
+)
+# fmt: on
+
+JOHAB_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 0, 0, 2, 2, 2)
+
+JOHAB_SM_MODEL: CodingStateMachineDict = {
+    "class_table": JOHAB_CLS,
+    "class_factor": 10,
+    "state_table": JOHAB_ST,
+    "char_len_table": JOHAB_CHAR_LEN_TABLE,
+    "name": "Johab",
+}

 # EUC-TW
-
+# fmt: off
 EUCTW_CLS = (
-    2,2,2,2,2,2,2,2,  # 00 - 07
-    2,2,2,2,2,2,0,0,  # 08 - 0f
-    2,2,2,2,2,2,2,2,  # 10 - 17
-    2,2,2,0,2,2,2,2,  # 18 - 1f
-    2,2,2,2,2,2,2,2,  # 20 - 27
-    2,2,2,2,2,2,2,2,  # 28 - 2f
-    2,2,2,2,2,2,2,2,  # 30 - 37
-    2,2,2,2,2,2,2,2,  # 38 - 3f
-    2,2,2,2,2,2,2,2,  # 40 - 47
-    2,2,2,2,2,2,2,2,  # 48 - 4f
-    2,2,2,2,2,2,2,2,  # 50 - 57
-    2,2,2,2,2,2,2,2,  # 58 - 5f
-    2,2,2,2,2,2,2,2,  # 60 - 67
-    2,2,2,2,2,2,2,2,  # 68 - 6f
-    2,2,2,2,2,2,2,2,  # 70 - 77
-    2,2,2,2,2,2,2,2,  # 78 - 7f
-    0,0,0,0,0,0,0,0,  # 80 - 87
-    0,0,0,0,0,0,6,0,  # 88 - 8f
-    0,0,0,0,0,0,0,0,  # 90 - 97
-    0,0,0,0,0,0,0,0,  # 98 - 9f
-    0,3,4,4,4,4,4,4,  # a0 - a7
-    5,5,1,1,1,1,1,1,  # a8 - af
-    1,1,1,1,1,1,1,1,  # b0 - b7
-    1,1,1,1,1,1,1,1,  # b8 - bf
-    1,1,3,1,3,3,3,3,  # c0 - c7
-    3,3,3,3,3,3,3,3,  # c8 - cf
-    3,3,3,3,3,3,3,3,  # d0 - d7
-    3,3,3,3,3,3,3,3,  # d8 - df
-    3,3,3,3,3,3,3,3,  # e0 - e7
-    3,3,3,3,3,3,3,3,  # e8 - ef
-    3,3,3,3,3,3,3,3,  # f0 - f7
-    3,3,3,3,3,3,3,0   # f8 - ff
+    2, 2, 2, 2, 2, 2, 2, 2,  # 00 - 07
+    2, 2, 2, 2, 2, 2, 0, 0,  # 08 - 0f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 10 - 17
+    2, 2, 2, 0, 2, 2, 2, 2,  # 18 - 1f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 20 - 27
+    2, 2, 2, 2, 2, 2, 2, 2,  # 28 - 2f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 30 - 37
+    2, 2, 2, 2, 2, 2, 2, 2,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 2,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 6, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 3, 4, 4, 4, 4, 4, 4,  # a0 - a7
+    5, 5, 1, 1, 1, 1, 1, 1,  # a8 - af
+    1, 1, 1, 1, 1, 1, 1, 1,  # b0 - b7
+    1, 1, 1, 1, 1, 1, 1, 1,  # b8 - bf
+    1, 1, 3, 1, 3, 3, 3, 3,  # c0 - c7
+    3, 3, 3, 3, 3, 3, 3, 3,  # c8 - cf
+    3, 3, 3, 3, 3, 3, 3, 3,  # d0 - d7
+    3, 3, 3, 3, 3, 3, 3, 3,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 3, 3, 3,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 3, 3, 0   # f8 - ff
 )

 EUCTW_ST = (
@ -266,50 +337,53 @@ EUCTW_ST = (
         5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,#20-27
    MachineState.START,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
 )
+# fmt: on

 EUCTW_CHAR_LEN_TABLE = (0, 0, 1, 2, 2, 2, 3)

-EUCTW_SM_MODEL = {'class_table': EUCTW_CLS,
-                'class_factor': 7,
-                'state_table': EUCTW_ST,
-                'char_len_table': EUCTW_CHAR_LEN_TABLE,
-                'name': 'x-euc-tw'}
+EUCTW_SM_MODEL: CodingStateMachineDict = {
+    "class_table": EUCTW_CLS,
+    "class_factor": 7,
+    "state_table": EUCTW_ST,
+    "char_len_table": EUCTW_CHAR_LEN_TABLE,
+    "name": "x-euc-tw",
+}

 # GB2312
-
+# fmt: off
 GB2312_CLS = (
-    1,1,1,1,1,1,1,1,  # 00 - 07
-    1,1,1,1,1,1,0,0,  # 08 - 0f
-    1,1,1,1,1,1,1,1,  # 10 - 17
-    1,1,1,0,1,1,1,1,  # 18 - 1f
-    1,1,1,1,1,1,1,1,  # 20 - 27
-    1,1,1,1,1,1,1,1,  # 28 - 2f
-    3,3,3,3,3,3,3,3,  # 30 - 37
-    3,3,1,1,1,1,1,1,  # 38 - 3f
-    2,2,2,2,2,2,2,2,  # 40 - 47
-    2,2,2,2,2,2,2,2,  # 48 - 4f
-    2,2,2,2,2,2,2,2,  # 50 - 57
-    2,2,2,2,2,2,2,2,  # 58 - 5f
-    2,2,2,2,2,2,2,2,  # 60 - 67
-    2,2,2,2,2,2,2,2,  # 68 - 6f
-    2,2,2,2,2,2,2,2,  # 70 - 77
-    2,2,2,2,2,2,2,4,  # 78 - 7f
-    5,6,6,6,6,6,6,6,  # 80 - 87
-    6,6,6,6,6,6,6,6,  # 88 - 8f
-    6,6,6,6,6,6,6,6,  # 90 - 97
-    6,6,6,6,6,6,6,6,  # 98 - 9f
-    6,6,6,6,6,6,6,6,  # a0 - a7
-    6,6,6,6,6,6,6,6,  # a8 - af
-    6,6,6,6,6,6,6,6,  # b0 - b7
-    6,6,6,6,6,6,6,6,  # b8 - bf
-    6,6,6,6,6,6,6,6,  # c0 - c7
-    6,6,6,6,6,6,6,6,  # c8 - cf
-    6,6,6,6,6,6,6,6,  # d0 - d7
-    6,6,6,6,6,6,6,6,  # d8 - df
-    6,6,6,6,6,6,6,6,  # e0 - e7
-    6,6,6,6,6,6,6,6,  # e8 - ef
-    6,6,6,6,6,6,6,6,  # f0 - f7
-    6,6,6,6,6,6,6,0   # f8 - ff
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    3, 3, 3, 3, 3, 3, 3, 3,  # 30 - 37
+    3, 3, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 4,  # 78 - 7f
+    5, 6, 6, 6, 6, 6, 6, 6,  # 80 - 87
+    6, 6, 6, 6, 6, 6, 6, 6,  # 88 - 8f
+    6, 6, 6, 6, 6, 6, 6, 6,  # 90 - 97
+    6, 6, 6, 6, 6, 6, 6, 6,  # 98 - 9f
+    6, 6, 6, 6, 6, 6, 6, 6,  # a0 - a7
+    6, 6, 6, 6, 6, 6, 6, 6,  # a8 - af
+    6, 6, 6, 6, 6, 6, 6, 6,  # b0 - b7
+    6, 6, 6, 6, 6, 6, 6, 6,  # b8 - bf
+    6, 6, 6, 6, 6, 6, 6, 6,  # c0 - c7
+    6, 6, 6, 6, 6, 6, 6, 6,  # c8 - cf
+    6, 6, 6, 6, 6, 6, 6, 6,  # d0 - d7
+    6, 6, 6, 6, 6, 6, 6, 6,  # d8 - df
+    6, 6, 6, 6, 6, 6, 6, 6,  # e0 - e7
+    6, 6, 6, 6, 6, 6, 6, 6,  # e8 - ef
+    6, 6, 6, 6, 6, 6, 6, 6,  # f0 - f7
+    6, 6, 6, 6, 6, 6, 6, 0   # f8 - ff
 )

 GB2312_ST = (
@ -320,6 +394,7 @@ GB2312_ST = (
    MachineState.ERROR,MachineState.ERROR,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ERROR,#20-27
    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.START #28-2f
 )
+# fmt: on

 # To be accurate, the length of class 6 can be either 2 or 4.
 # But it is not necessary to discriminate between the two since
@ -328,100 +403,105 @@ GB2312_ST = (
 # 2 here.
 GB2312_CHAR_LEN_TABLE = (0, 1, 1, 1, 1, 1, 2)

-GB2312_SM_MODEL = {'class_table': GB2312_CLS,
-                   'class_factor': 7,
-                   'state_table': GB2312_ST,
-                   'char_len_table': GB2312_CHAR_LEN_TABLE,
-                   'name': 'GB2312'}
+GB2312_SM_MODEL: CodingStateMachineDict = {
+    "class_table": GB2312_CLS,
+    "class_factor": 7,
+    "state_table": GB2312_ST,
+    "char_len_table": GB2312_CHAR_LEN_TABLE,
+    "name": "GB2312",
+}

 # Shift_JIS
-
+# fmt: off
 SJIS_CLS = (
-    1,1,1,1,1,1,1,1,  # 00 - 07
-    1,1,1,1,1,1,0,0,  # 08 - 0f
-    1,1,1,1,1,1,1,1,  # 10 - 17
-    1,1,1,0,1,1,1,1,  # 18 - 1f
-    1,1,1,1,1,1,1,1,  # 20 - 27
-    1,1,1,1,1,1,1,1,  # 28 - 2f
-    1,1,1,1,1,1,1,1,  # 30 - 37
-    1,1,1,1,1,1,1,1,  # 38 - 3f
-    2,2,2,2,2,2,2,2,  # 40 - 47
-    2,2,2,2,2,2,2,2,  # 48 - 4f
-    2,2,2,2,2,2,2,2,  # 50 - 57
-    2,2,2,2,2,2,2,2,  # 58 - 5f
-    2,2,2,2,2,2,2,2,  # 60 - 67
-    2,2,2,2,2,2,2,2,  # 68 - 6f
-    2,2,2,2,2,2,2,2,  # 70 - 77
-    2,2,2,2,2,2,2,1,  # 78 - 7f
-    3,3,3,3,3,2,2,3,  # 80 - 87
-    3,3,3,3,3,3,3,3,  # 88 - 8f
-    3,3,3,3,3,3,3,3,  # 90 - 97
-    3,3,3,3,3,3,3,3,  # 98 - 9f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 40 - 47
+    2, 2, 2, 2, 2, 2, 2, 2,  # 48 - 4f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 50 - 57
+    2, 2, 2, 2, 2, 2, 2, 2,  # 58 - 5f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 60 - 67
+    2, 2, 2, 2, 2, 2, 2, 2,  # 68 - 6f
+    2, 2, 2, 2, 2, 2, 2, 2,  # 70 - 77
+    2, 2, 2, 2, 2, 2, 2, 1,  # 78 - 7f
+    3, 3, 3, 3, 3, 2, 2, 3,  # 80 - 87
+    3, 3, 3, 3, 3, 3, 3, 3,  # 88 - 8f
+    3, 3, 3, 3, 3, 3, 3, 3,  # 90 - 97
+    3, 3, 3, 3, 3, 3, 3, 3,  # 98 - 9f
    #0xa0 is illegal in sjis encoding, but some pages does
    #contain such byte. We need to be more error forgiven.
-    2,2,2,2,2,2,2,2,  # a0 - a7
-    2,2,2,2,2,2,2,2,  # a8 - af
-    2,2,2,2,2,2,2,2,  # b0 - b7
-    2,2,2,2,2,2,2,2,  # b8 - bf
-    2,2,2,2,2,2,2,2,  # c0 - c7
-    2,2,2,2,2,2,2,2,  # c8 - cf
-    2,2,2,2,2,2,2,2,  # d0 - d7
-    2,2,2,2,2,2,2,2,  # d8 - df
-    3,3,3,3,3,3,3,3,  # e0 - e7
-    3,3,3,3,3,4,4,4,  # e8 - ef
-    3,3,3,3,3,3,3,3,  # f0 - f7
-    3,3,3,3,3,0,0,0)  # f8 - ff
-
+    2, 2, 2, 2, 2, 2, 2, 2,  # a0 - a7
+    2, 2, 2, 2, 2, 2, 2, 2,  # a8 - af
+    2, 2, 2, 2, 2, 2, 2, 2,  # b0 - b7
+    2, 2, 2, 2, 2, 2, 2, 2,  # b8 - bf
+    2, 2, 2, 2, 2, 2, 2, 2,  # c0 - c7
+    2, 2, 2, 2, 2, 2, 2, 2,  # c8 - cf
+    2, 2, 2, 2, 2, 2, 2, 2,  # d0 - d7
+    2, 2, 2, 2, 2, 2, 2, 2,  # d8 - df
+    3, 3, 3, 3, 3, 3, 3, 3,  # e0 - e7
+    3, 3, 3, 3, 3, 4, 4, 4,  # e8 - ef
+    3, 3, 3, 3, 3, 3, 3, 3,  # f0 - f7
+    3, 3, 3, 3, 3, 0, 0, 0,  # f8 - ff
+)

 SJIS_ST = (
    MachineState.ERROR,MachineState.START,MachineState.START,     3,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,#00-07
    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ITS_ME,#08-0f
    MachineState.ITS_ME,MachineState.ITS_ME,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START #10-17
 )
+# fmt: on

 SJIS_CHAR_LEN_TABLE = (0, 1, 1, 2, 0, 0)

-SJIS_SM_MODEL = {'class_table': SJIS_CLS,
-               'class_factor': 6,
-               'state_table': SJIS_ST,
-               'char_len_table': SJIS_CHAR_LEN_TABLE,
-               'name': 'Shift_JIS'}
+SJIS_SM_MODEL: CodingStateMachineDict = {
+    "class_table": SJIS_CLS,
+    "class_factor": 6,
+    "state_table": SJIS_ST,
+    "char_len_table": SJIS_CHAR_LEN_TABLE,
+    "name": "Shift_JIS",
+}

 # UCS2-BE
-
+# fmt: off
 UCS2BE_CLS = (
-    0,0,0,0,0,0,0,0,  # 00 - 07
-    0,0,1,0,0,2,0,0,  # 08 - 0f
-    0,0,0,0,0,0,0,0,  # 10 - 17
-    0,0,0,3,0,0,0,0,  # 18 - 1f
-    0,0,0,0,0,0,0,0,  # 20 - 27
-    0,3,3,3,3,3,0,0,  # 28 - 2f
-    0,0,0,0,0,0,0,0,  # 30 - 37
-    0,0,0,0,0,0,0,0,  # 38 - 3f
-    0,0,0,0,0,0,0,0,  # 40 - 47
-    0,0,0,0,0,0,0,0,  # 48 - 4f
-    0,0,0,0,0,0,0,0,  # 50 - 57
-    0,0,0,0,0,0,0,0,  # 58 - 5f
-    0,0,0,0,0,0,0,0,  # 60 - 67
-    0,0,0,0,0,0,0,0,  # 68 - 6f
-    0,0,0,0,0,0,0,0,  # 70 - 77
-    0,0,0,0,0,0,0,0,  # 78 - 7f
-    0,0,0,0,0,0,0,0,  # 80 - 87
-    0,0,0,0,0,0,0,0,  # 88 - 8f
-    0,0,0,0,0,0,0,0,  # 90 - 97
-    0,0,0,0,0,0,0,0,  # 98 - 9f
-    0,0,0,0,0,0,0,0,  # a0 - a7
-    0,0,0,0,0,0,0,0,  # a8 - af
-    0,0,0,0,0,0,0,0,  # b0 - b7
-    0,0,0,0,0,0,0,0,  # b8 - bf
-    0,0,0,0,0,0,0,0,  # c0 - c7
-    0,0,0,0,0,0,0,0,  # c8 - cf
-    0,0,0,0,0,0,0,0,  # d0 - d7
-    0,0,0,0,0,0,0,0,  # d8 - df
-    0,0,0,0,0,0,0,0,  # e0 - e7
-    0,0,0,0,0,0,0,0,  # e8 - ef
-    0,0,0,0,0,0,0,0,  # f0 - f7
-    0,0,0,0,0,0,4,5   # f8 - ff
+    0, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 1, 0, 0, 2, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 3, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 3, 3, 3, 3, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 0, 0, 0, 0, 0, 0, 0,  # a0 - a7
+    0, 0, 0, 0, 0, 0, 0, 0,  # a8 - af
+    0, 0, 0, 0, 0, 0, 0, 0,  # b0 - b7
+    0, 0, 0, 0, 0, 0, 0, 0,  # b8 - bf
+    0, 0, 0, 0, 0, 0, 0, 0,  # c0 - c7
+    0, 0, 0, 0, 0, 0, 0, 0,  # c8 - cf
+    0, 0, 0, 0, 0, 0, 0, 0,  # d0 - d7
+    0, 0, 0, 0, 0, 0, 0, 0,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 4, 5   # f8 - ff
 )

 UCS2BE_ST  = (
@ -433,50 +513,53 @@ UCS2BE_ST  = (
          5,     8,     6,     6,MachineState.ERROR,     6,     6,     6,#28-2f
          6,     6,     6,     6,MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START #30-37
 )
+# fmt: on

 UCS2BE_CHAR_LEN_TABLE = (2, 2, 2, 0, 2, 2)

-UCS2BE_SM_MODEL = {'class_table': UCS2BE_CLS,
-                   'class_factor': 6,
-                   'state_table': UCS2BE_ST,
-                   'char_len_table': UCS2BE_CHAR_LEN_TABLE,
-                   'name': 'UTF-16BE'}
+UCS2BE_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UCS2BE_CLS,
+    "class_factor": 6,
+    "state_table": UCS2BE_ST,
+    "char_len_table": UCS2BE_CHAR_LEN_TABLE,
+    "name": "UTF-16BE",
+}

 # UCS2-LE
-
+# fmt: off
 UCS2LE_CLS = (
-    0,0,0,0,0,0,0,0,  # 00 - 07
-    0,0,1,0,0,2,0,0,  # 08 - 0f
-    0,0,0,0,0,0,0,0,  # 10 - 17
-    0,0,0,3,0,0,0,0,  # 18 - 1f
-    0,0,0,0,0,0,0,0,  # 20 - 27
-    0,3,3,3,3,3,0,0,  # 28 - 2f
-    0,0,0,0,0,0,0,0,  # 30 - 37
-    0,0,0,0,0,0,0,0,  # 38 - 3f
-    0,0,0,0,0,0,0,0,  # 40 - 47
-    0,0,0,0,0,0,0,0,  # 48 - 4f
-    0,0,0,0,0,0,0,0,  # 50 - 57
-    0,0,0,0,0,0,0,0,  # 58 - 5f
-    0,0,0,0,0,0,0,0,  # 60 - 67
-    0,0,0,0,0,0,0,0,  # 68 - 6f
-    0,0,0,0,0,0,0,0,  # 70 - 77
-    0,0,0,0,0,0,0,0,  # 78 - 7f
-    0,0,0,0,0,0,0,0,  # 80 - 87
-    0,0,0,0,0,0,0,0,  # 88 - 8f
-    0,0,0,0,0,0,0,0,  # 90 - 97
-    0,0,0,0,0,0,0,0,  # 98 - 9f
-    0,0,0,0,0,0,0,0,  # a0 - a7
-    0,0,0,0,0,0,0,0,  # a8 - af
-    0,0,0,0,0,0,0,0,  # b0 - b7
-    0,0,0,0,0,0,0,0,  # b8 - bf
-    0,0,0,0,0,0,0,0,  # c0 - c7
-    0,0,0,0,0,0,0,0,  # c8 - cf
-    0,0,0,0,0,0,0,0,  # d0 - d7
-    0,0,0,0,0,0,0,0,  # d8 - df
-    0,0,0,0,0,0,0,0,  # e0 - e7
-    0,0,0,0,0,0,0,0,  # e8 - ef
-    0,0,0,0,0,0,0,0,  # f0 - f7
-    0,0,0,0,0,0,4,5   # f8 - ff
+    0, 0, 0, 0, 0, 0, 0, 0,  # 00 - 07
+    0, 0, 1, 0, 0, 2, 0, 0,  # 08 - 0f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 10 - 17
+    0, 0, 0, 3, 0, 0, 0, 0,  # 18 - 1f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 20 - 27
+    0, 3, 3, 3, 3, 3, 0, 0,  # 28 - 2f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 30 - 37
+    0, 0, 0, 0, 0, 0, 0, 0,  # 38 - 3f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 40 - 47
+    0, 0, 0, 0, 0, 0, 0, 0,  # 48 - 4f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 50 - 57
+    0, 0, 0, 0, 0, 0, 0, 0,  # 58 - 5f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 60 - 67
+    0, 0, 0, 0, 0, 0, 0, 0,  # 68 - 6f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 70 - 77
+    0, 0, 0, 0, 0, 0, 0, 0,  # 78 - 7f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 80 - 87
+    0, 0, 0, 0, 0, 0, 0, 0,  # 88 - 8f
+    0, 0, 0, 0, 0, 0, 0, 0,  # 90 - 97
+    0, 0, 0, 0, 0, 0, 0, 0,  # 98 - 9f
+    0, 0, 0, 0, 0, 0, 0, 0,  # a0 - a7
+    0, 0, 0, 0, 0, 0, 0, 0,  # a8 - af
+    0, 0, 0, 0, 0, 0, 0, 0,  # b0 - b7
+    0, 0, 0, 0, 0, 0, 0, 0,  # b8 - bf
+    0, 0, 0, 0, 0, 0, 0, 0,  # c0 - c7
+    0, 0, 0, 0, 0, 0, 0, 0,  # c8 - cf
+    0, 0, 0, 0, 0, 0, 0, 0,  # d0 - d7
+    0, 0, 0, 0, 0, 0, 0, 0,  # d8 - df
+    0, 0, 0, 0, 0, 0, 0, 0,  # e0 - e7
+    0, 0, 0, 0, 0, 0, 0, 0,  # e8 - ef
+    0, 0, 0, 0, 0, 0, 0, 0,  # f0 - f7
+    0, 0, 0, 0, 0, 0, 4, 5   # f8 - ff
 )

 UCS2LE_ST = (
@ -488,50 +571,53 @@ UCS2LE_ST = (
          5,     5,     5,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,     5,     5,#28-2f
          5,     5,     5,MachineState.ERROR,     5,MachineState.ERROR,MachineState.START,MachineState.START #30-37
 )
+# fmt: on

 UCS2LE_CHAR_LEN_TABLE = (2, 2, 2, 2, 2, 2)

-UCS2LE_SM_MODEL = {'class_table': UCS2LE_CLS,
-                 'class_factor': 6,
-                 'state_table': UCS2LE_ST,
-                 'char_len_table': UCS2LE_CHAR_LEN_TABLE,
-                 'name': 'UTF-16LE'}
+UCS2LE_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UCS2LE_CLS,
+    "class_factor": 6,
+    "state_table": UCS2LE_ST,
+    "char_len_table": UCS2LE_CHAR_LEN_TABLE,
+    "name": "UTF-16LE",
+}

 # UTF-8
-
+# fmt: off
 UTF8_CLS = (
-    1,1,1,1,1,1,1,1,  # 00 - 07  #allow 0x00 as a legal value
-    1,1,1,1,1,1,0,0,  # 08 - 0f
-    1,1,1,1,1,1,1,1,  # 10 - 17
-    1,1,1,0,1,1,1,1,  # 18 - 1f
-    1,1,1,1,1,1,1,1,  # 20 - 27
-    1,1,1,1,1,1,1,1,  # 28 - 2f
-    1,1,1,1,1,1,1,1,  # 30 - 37
-    1,1,1,1,1,1,1,1,  # 38 - 3f
-    1,1,1,1,1,1,1,1,  # 40 - 47
-    1,1,1,1,1,1,1,1,  # 48 - 4f
-    1,1,1,1,1,1,1,1,  # 50 - 57
-    1,1,1,1,1,1,1,1,  # 58 - 5f
-    1,1,1,1,1,1,1,1,  # 60 - 67
-    1,1,1,1,1,1,1,1,  # 68 - 6f
-    1,1,1,1,1,1,1,1,  # 70 - 77
-    1,1,1,1,1,1,1,1,  # 78 - 7f
-    2,2,2,2,3,3,3,3,  # 80 - 87
-    4,4,4,4,4,4,4,4,  # 88 - 8f
-    4,4,4,4,4,4,4,4,  # 90 - 97
-    4,4,4,4,4,4,4,4,  # 98 - 9f
-    5,5,5,5,5,5,5,5,  # a0 - a7
-    5,5,5,5,5,5,5,5,  # a8 - af
-    5,5,5,5,5,5,5,5,  # b0 - b7
-    5,5,5,5,5,5,5,5,  # b8 - bf
-    0,0,6,6,6,6,6,6,  # c0 - c7
-    6,6,6,6,6,6,6,6,  # c8 - cf
-    6,6,6,6,6,6,6,6,  # d0 - d7
-    6,6,6,6,6,6,6,6,  # d8 - df
-    7,8,8,8,8,8,8,8,  # e0 - e7
-    8,8,8,8,8,9,8,8,  # e8 - ef
-    10,11,11,11,11,11,11,11,  # f0 - f7
-    12,13,13,13,14,15,0,0    # f8 - ff
+    1, 1, 1, 1, 1, 1, 1, 1,  # 00 - 07  #allow 0x00 as a legal value
+    1, 1, 1, 1, 1, 1, 0, 0,  # 08 - 0f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 10 - 17
+    1, 1, 1, 0, 1, 1, 1, 1,  # 18 - 1f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 20 - 27
+    1, 1, 1, 1, 1, 1, 1, 1,  # 28 - 2f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 30 - 37
+    1, 1, 1, 1, 1, 1, 1, 1,  # 38 - 3f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 40 - 47
+    1, 1, 1, 1, 1, 1, 1, 1,  # 48 - 4f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 50 - 57
+    1, 1, 1, 1, 1, 1, 1, 1,  # 58 - 5f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 60 - 67
+    1, 1, 1, 1, 1, 1, 1, 1,  # 68 - 6f
+    1, 1, 1, 1, 1, 1, 1, 1,  # 70 - 77
+    1, 1, 1, 1, 1, 1, 1, 1,  # 78 - 7f
+    2, 2, 2, 2, 3, 3, 3, 3,  # 80 - 87
+    4, 4, 4, 4, 4, 4, 4, 4,  # 88 - 8f
+    4, 4, 4, 4, 4, 4, 4, 4,  # 90 - 97
+    4, 4, 4, 4, 4, 4, 4, 4,  # 98 - 9f
+    5, 5, 5, 5, 5, 5, 5, 5,  # a0 - a7
+    5, 5, 5, 5, 5, 5, 5, 5,  # a8 - af
+    5, 5, 5, 5, 5, 5, 5, 5,  # b0 - b7
+    5, 5, 5, 5, 5, 5, 5, 5,  # b8 - bf
+    0, 0, 6, 6, 6, 6, 6, 6,  # c0 - c7
+    6, 6, 6, 6, 6, 6, 6, 6,  # c8 - cf
+    6, 6, 6, 6, 6, 6, 6, 6,  # d0 - d7
+    6, 6, 6, 6, 6, 6, 6, 6,  # d8 - df
+    7, 8, 8, 8, 8, 8, 8, 8,  # e0 - e7
+    8, 8, 8, 8, 8, 9, 8, 8,  # e8 - ef
+    10, 11, 11, 11, 11, 11, 11, 11,  # f0 - f7
+    12, 13, 13, 13, 14, 15, 0, 0    # f8 - ff
 )

 UTF8_ST = (
@ -562,11 +648,14 @@ UTF8_ST = (
    MachineState.ERROR,MachineState.ERROR,MachineState.START,MachineState.START,MachineState.START,MachineState.START,MachineState.ERROR,MachineState.ERROR,#c0-c7
    MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR,MachineState.ERROR #c8-cf
 )
+# fmt: on

 UTF8_CHAR_LEN_TABLE = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6)

-UTF8_SM_MODEL = {'class_table': UTF8_CLS,
-                 'class_factor': 16,
-                 'state_table': UTF8_ST,
-                 'char_len_table': UTF8_CHAR_LEN_TABLE,
-                 'name': 'UTF-8'}
+UTF8_SM_MODEL: CodingStateMachineDict = {
+    "class_table": UTF8_CLS,
+    "class_factor": 16,
+    "state_table": UTF8_ST,
+    "char_len_table": UTF8_CHAR_LEN_TABLE,
+    "name": "UTF-8",
+}
--- a/libs/common/chardet/metadata/languages.py
+++ b/libs/common/chardet/metadata/languages.py
@ -1,19 +1,17 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 """
 Metadata about languages used by our model training code for our
 SingleByteCharSetProbers.  Could be used for other things in the future.

 This code is based on the language metadata from the uchardet project.
 """
-from __future__ import absolute_import, print_function

 from string import ascii_letters
+from typing import List, Optional
+
+# TODO: Add Ukrainian (KOI8-U)


-# TODO: Add Ukranian (KOI8-U)
-
-class Language(object):
+class Language:
    """Metadata about a language useful for training models

    :ivar name: The human name for the language, in English.
@ -33,9 +31,17 @@ class Language(object):
                            Wikipedia for training data.
    :type wiki_start_pages: list of str
    """
-    def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
-                 alphabet=None, wiki_start_pages=None):
-        super(Language, self).__init__()
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        iso_code: Optional[str] = None,
+        use_ascii: bool = True,
+        charsets: Optional[List[str]] = None,
+        alphabet: Optional[str] = None,
+        wiki_start_pages: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__()
        self.name = name
        self.iso_code = iso_code
        self.use_ascii = use_ascii
@ -46,265 +52,301 @@ class Language(object):
            else:
                alphabet = ascii_letters
        elif not alphabet:
-            raise ValueError('Must supply alphabet if use_ascii is False')
-        self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None
+            raise ValueError("Must supply alphabet if use_ascii is False")
+        self.alphabet = "".join(sorted(set(alphabet))) if alphabet else None
        self.wiki_start_pages = wiki_start_pages

-    def __repr__(self):
-        return '{}({})'.format(self.__class__.__name__,
-                               ', '.join('{}={!r}'.format(k, v)
-                                         for k, v in self.__dict__.items()
-                                         if not k.startswith('_')))
+    def __repr__(self) -> str:
+        param_str = ", ".join(
+            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
+        )
+        return f"{self.__class__.__name__}({param_str})"


-LANGUAGES = {'Arabic': Language(name='Arabic',
-                                iso_code='ar',
-                                use_ascii=False,
-                                # We only support encodings that use isolated
-                                # forms, because the current recommendation is
-                                # that the rendering system handles presentation
-                                # forms. This means we purposefully skip IBM864.
-                                charsets=['ISO-8859-6', 'WINDOWS-1256',
-                                          'CP720', 'CP864'],
-                                alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ',
-                                wiki_start_pages=[u'الصفحة_الرئيسية']),
-             'Belarusian': Language(name='Belarusian',
-                                    iso_code='be',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                              'IBM866', 'MacCyrillic'],
-                                    alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ'
-                                              u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'),
-                                    wiki_start_pages=[u'Галоўная_старонка']),
-             'Bulgarian': Language(name='Bulgarian',
-                                   iso_code='bg',
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                             'IBM855'],
-                                   alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ'
-                                             u'абвгдежзийклмнопрстуфхцчшщъьюя'),
-                                   wiki_start_pages=[u'Начална_страница']),
-             'Czech': Language(name='Czech',
-                               iso_code='cz',
-                               use_ascii=True,
-                               charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                               alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ',
-                               wiki_start_pages=[u'Hlavní_strana']),
-             'Danish': Language(name='Danish',
-                                iso_code='da',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'ISO-8859-15',
-                                          'WINDOWS-1252'],
-                                alphabet=u'æøåÆØÅ',
-                                wiki_start_pages=[u'Forside']),
-             'German': Language(name='German',
-                                iso_code='de',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                                alphabet=u'äöüßÄÖÜ',
-                                wiki_start_pages=[u'Wikipedia:Hauptseite']),
-             'Greek': Language(name='Greek',
-                               iso_code='el',
-                               use_ascii=False,
-                               charsets=['ISO-8859-7', 'WINDOWS-1253'],
-                               alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ'
-                                         u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'),
-                               wiki_start_pages=[u'Πύλη:Κύρια']),
-             'English': Language(name='English',
-                                 iso_code='en',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                                 wiki_start_pages=[u'Main_Page']),
-             'Esperanto': Language(name='Esperanto',
-                                   iso_code='eo',
-                                   # Q, W, X, and Y not used at all
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-3'],
-                                   alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz'
-                                             u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'),
-                                   wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']),
-             'Spanish': Language(name='Spanish',
-                                 iso_code='es',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ',
-                                 wiki_start_pages=[u'Wikipedia:Portada']),
-             'Estonian': Language(name='Estonian',
-                                  iso_code='et',
-                                  use_ascii=False,
-                                  charsets=['ISO-8859-4', 'ISO-8859-13',
-                                            'WINDOWS-1257'],
-                                  # C, F, Š, Q, W, X, Y, Z, Ž are only for
-                                  # loanwords
-                                  alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ'
-                                            u'abdeghijklmnoprstuvõäöü'),
-                                  wiki_start_pages=[u'Esileht']),
-             'Finnish': Language(name='Finnish',
-                                 iso_code='fi',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÅÄÖŠŽåäöšž',
-                                 wiki_start_pages=[u'Wikipedia:Etusivu']),
-             'French': Language(name='French',
-                                iso_code='fr',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'ISO-8859-15',
-                                          'WINDOWS-1252'],
-                                alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ',
-                                wiki_start_pages=[u'Wikipédia:Accueil_principal',
-                                                  u'Bœuf (animal)']),
-             'Hebrew': Language(name='Hebrew',
-                                iso_code='he',
-                                use_ascii=False,
-                                charsets=['ISO-8859-8', 'WINDOWS-1255'],
-                                alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ',
-                                wiki_start_pages=[u'עמוד_ראשי']),
-             'Croatian': Language(name='Croatian',
-                                  iso_code='hr',
-                                  # Q, W, X, Y are only used for foreign words.
-                                  use_ascii=False,
-                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                  alphabet=(u'abcčćdđefghijklmnoprsštuvzž'
-                                            u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'),
-                                  wiki_start_pages=[u'Glavna_stranica']),
-             'Hungarian': Language(name='Hungarian',
-                                   iso_code='hu',
-                                   # Q, W, X, Y are only used for foreign words.
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                   alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű'
-                                             u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'),
-                                   wiki_start_pages=[u'Kezdőlap']),
-             'Italian': Language(name='Italian',
-                                 iso_code='it',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÀÈÉÌÒÓÙàèéìòóù',
-                                 wiki_start_pages=[u'Pagina_principale']),
-             'Lithuanian': Language(name='Lithuanian',
-                                    iso_code='lt',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-13', 'WINDOWS-1257',
-                                              'ISO-8859-4'],
-                                    # Q, W, and X not used at all
-                                    alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ'
-                                              u'aąbcčdeęėfghiįyjklmnoprsštuųūvzž'),
-                                    wiki_start_pages=[u'Pagrindinis_puslapis']),
-             'Latvian': Language(name='Latvian',
-                                 iso_code='lv',
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-13', 'WINDOWS-1257',
-                                           'ISO-8859-4'],
-                                 # Q, W, X, Y are only for loanwords
-                                 alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ'
-                                           u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'),
-                                 wiki_start_pages=[u'Sākumlapa']),
-             'Macedonian': Language(name='Macedonian',
-                                    iso_code='mk',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                              'MacCyrillic', 'IBM855'],
-                                    alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ'
-                                              u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'),
-                                    wiki_start_pages=[u'Главна_страница']),
-             'Dutch': Language(name='Dutch',
-                               iso_code='nl',
-                               use_ascii=True,
-                               charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                               wiki_start_pages=[u'Hoofdpagina']),
-             'Polish': Language(name='Polish',
-                                iso_code='pl',
-                                # Q and X are only used for foreign words.
-                                use_ascii=False,
-                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ'
-                                          u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'),
-                                wiki_start_pages=[u'Wikipedia:Strona_główna']),
-             'Portuguese': Language(name='Portuguese',
-                                 iso_code='pt',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú',
-                                 wiki_start_pages=[u'Wikipédia:Página_principal']),
-             'Romanian': Language(name='Romanian',
-                                  iso_code='ro',
-                                  use_ascii=True,
-                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                  alphabet=u'ăâîșțĂÂÎȘȚ',
-                                  wiki_start_pages=[u'Pagina_principală']),
-             'Russian': Language(name='Russian',
-                                 iso_code='ru',
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                           'KOI8-R', 'MacCyrillic', 'IBM866',
-                                           'IBM855'],
-                                 alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
-                                           u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
-                                 wiki_start_pages=[u'Заглавная_страница']),
-             'Slovak': Language(name='Slovak',
-                                iso_code='sk',
-                                use_ascii=True,
-                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ',
-                                wiki_start_pages=[u'Hlavná_stránka']),
-             'Slovene': Language(name='Slovene',
-                                 iso_code='sl',
-                                 # Q, W, X, Y are only used for foreign words.
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                 alphabet=(u'abcčdefghijklmnoprsštuvzž'
-                                           u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'),
-                                 wiki_start_pages=[u'Glavna_stran']),
-             # Serbian can be written in both Latin and Cyrillic, but there's no
-             # simple way to get the Latin alphabet pages from Wikipedia through
-             # the API, so for now we just support Cyrillic.
-             'Serbian': Language(name='Serbian',
-                                 iso_code='sr',
-                                 alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ'
-                                           u'абвгдђежзијклљмнњопрстћуфхцчџш'),
-                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                           'MacCyrillic', 'IBM855'],
-                                 wiki_start_pages=[u'Главна_страна']),
-             'Thai': Language(name='Thai',
-                              iso_code='th',
-                              use_ascii=False,
-                              charsets=['ISO-8859-11', 'TIS-620', 'CP874'],
-                              alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛',
-                              wiki_start_pages=[u'หน้าหลัก']),
-             'Turkish': Language(name='Turkish',
-                                 iso_code='tr',
-                                 # Q, W, and X are not used by Turkish
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-3', 'ISO-8859-9',
-                                           'WINDOWS-1254'],
-                                 alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû'
-                                           u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'),
-                                 wiki_start_pages=[u'Ana_Sayfa']),
-             'Vietnamese': Language(name='Vietnamese',
-                                    iso_code='vi',
-                                    use_ascii=False,
-                                    # Windows-1258 is the only common 8-bit
-                                    # Vietnamese encoding supported by Python.
-                                    # From Wikipedia:
-                                    # For systems that lack support for Unicode,
-                                    # dozens of 8-bit Vietnamese code pages are
-                                    # available.[1] The most common are VISCII
-                                    # (TCVN 5712:1993), VPS, and Windows-1258.[3]
-                                    # Where ASCII is required, such as when
-                                    # ensuring readability in plain text e-mail,
-                                    # Vietnamese letters are often encoded
-                                    # according to Vietnamese Quoted-Readable
-                                    # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
-                                    # though usage of either variable-width
-                                    # scheme has declined dramatically following
-                                    # the adoption of Unicode on the World Wide
-                                    # Web.
-                                    charsets=['WINDOWS-1258'],
-                                    alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy'
-                                              u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'),
-                                    wiki_start_pages=[u'Chữ_Quốc_ngữ']),
-            }
+LANGUAGES = {
+    "Arabic": Language(
+        name="Arabic",
+        iso_code="ar",
+        use_ascii=False,
+        # We only support encodings that use isolated
+        # forms, because the current recommendation is
+        # that the rendering system handles presentation
+        # forms. This means we purposefully skip IBM864.
+        charsets=["ISO-8859-6", "WINDOWS-1256", "CP720", "CP864"],
+        alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ",
+        wiki_start_pages=["الصفحة_الرئيسية"],
+    ),
+    "Belarusian": Language(
+        name="Belarusian",
+        iso_code="be",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "IBM866", "MacCyrillic"],
+        alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ",
+        wiki_start_pages=["Галоўная_старонка"],
+    ),
+    "Bulgarian": Language(
+        name="Bulgarian",
+        iso_code="bg",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "IBM855"],
+        alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
+        wiki_start_pages=["Начална_страница"],
+    ),
+    "Czech": Language(
+        name="Czech",
+        iso_code="cz",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ",
+        wiki_start_pages=["Hlavní_strana"],
+    ),
+    "Danish": Language(
+        name="Danish",
+        iso_code="da",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="æøåÆØÅ",
+        wiki_start_pages=["Forside"],
+    ),
+    "German": Language(
+        name="German",
+        iso_code="de",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="äöüßẞÄÖÜ",
+        wiki_start_pages=["Wikipedia:Hauptseite"],
+    ),
+    "Greek": Language(
+        name="Greek",
+        iso_code="el",
+        use_ascii=False,
+        charsets=["ISO-8859-7", "WINDOWS-1253"],
+        alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ",
+        wiki_start_pages=["Πύλη:Κύρια"],
+    ),
+    "English": Language(
+        name="English",
+        iso_code="en",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
+        wiki_start_pages=["Main_Page"],
+    ),
+    "Esperanto": Language(
+        name="Esperanto",
+        iso_code="eo",
+        # Q, W, X, and Y not used at all
+        use_ascii=False,
+        charsets=["ISO-8859-3"],
+        alphabet="abcĉdefgĝhĥijĵklmnoprsŝtuŭvzABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ",
+        wiki_start_pages=["Vikipedio:Ĉefpaĝo"],
+    ),
+    "Spanish": Language(
+        name="Spanish",
+        iso_code="es",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ñáéíóúüÑÁÉÍÓÚÜ",
+        wiki_start_pages=["Wikipedia:Portada"],
+    ),
+    "Estonian": Language(
+        name="Estonian",
+        iso_code="et",
+        use_ascii=False,
+        charsets=["ISO-8859-4", "ISO-8859-13", "WINDOWS-1257"],
+        # C, F, Š, Q, W, X, Y, Z, Ž are only for
+        # loanwords
+        alphabet="ABDEGHIJKLMNOPRSTUVÕÄÖÜabdeghijklmnoprstuvõäöü",
+        wiki_start_pages=["Esileht"],
+    ),
+    "Finnish": Language(
+        name="Finnish",
+        iso_code="fi",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÅÄÖŠŽåäöšž",
+        wiki_start_pages=["Wikipedia:Etusivu"],
+    ),
+    "French": Language(
+        name="French",
+        iso_code="fr",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ",
+        wiki_start_pages=["Wikipédia:Accueil_principal", "Bœuf (animal)"],
+    ),
+    "Hebrew": Language(
+        name="Hebrew",
+        iso_code="he",
+        use_ascii=False,
+        charsets=["ISO-8859-8", "WINDOWS-1255"],
+        alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
+        wiki_start_pages=["עמוד_ראשי"],
+    ),
+    "Croatian": Language(
+        name="Croatian",
+        iso_code="hr",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcčćdđefghijklmnoprsštuvzžABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ",
+        wiki_start_pages=["Glavna_stranica"],
+    ),
+    "Hungarian": Language(
+        name="Hungarian",
+        iso_code="hu",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcdefghijklmnoprstuvzáéíóöőúüűABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ",
+        wiki_start_pages=["Kezdőlap"],
+    ),
+    "Italian": Language(
+        name="Italian",
+        iso_code="it",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÀÈÉÌÒÓÙàèéìòóù",
+        wiki_start_pages=["Pagina_principale"],
+    ),
+    "Lithuanian": Language(
+        name="Lithuanian",
+        iso_code="lt",
+        use_ascii=False,
+        charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
+        # Q, W, and X not used at all
+        alphabet="AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽaąbcčdeęėfghiįyjklmnoprsštuųūvzž",
+        wiki_start_pages=["Pagrindinis_puslapis"],
+    ),
+    "Latvian": Language(
+        name="Latvian",
+        iso_code="lv",
+        use_ascii=False,
+        charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"],
+        # Q, W, X, Y are only for loanwords
+        alphabet="AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž",
+        wiki_start_pages=["Sākumlapa"],
+    ),
+    "Macedonian": Language(
+        name="Macedonian",
+        iso_code="mk",
+        use_ascii=False,
+        charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
+        alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш",
+        wiki_start_pages=["Главна_страница"],
+    ),
+    "Dutch": Language(
+        name="Dutch",
+        iso_code="nl",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"],
+        wiki_start_pages=["Hoofdpagina"],
+    ),
+    "Polish": Language(
+        name="Polish",
+        iso_code="pl",
+        # Q and X are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻaąbcćdeęfghijklłmnńoóprsśtuwyzźż",
+        wiki_start_pages=["Wikipedia:Strona_główna"],
+    ),
+    "Portuguese": Language(
+        name="Portuguese",
+        iso_code="pt",
+        use_ascii=True,
+        charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"],
+        alphabet="ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú",
+        wiki_start_pages=["Wikipédia:Página_principal"],
+    ),
+    "Romanian": Language(
+        name="Romanian",
+        iso_code="ro",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="ăâîșțĂÂÎȘȚ",
+        wiki_start_pages=["Pagina_principală"],
+    ),
+    "Russian": Language(
+        name="Russian",
+        iso_code="ru",
+        use_ascii=False,
+        charsets=[
+            "ISO-8859-5",
+            "WINDOWS-1251",
+            "KOI8-R",
+            "MacCyrillic",
+            "IBM866",
+            "IBM855",
+        ],
+        alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
+        wiki_start_pages=["Заглавная_страница"],
+    ),
+    "Slovak": Language(
+        name="Slovak",
+        iso_code="sk",
+        use_ascii=True,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ",
+        wiki_start_pages=["Hlavná_stránka"],
+    ),
+    "Slovene": Language(
+        name="Slovene",
+        iso_code="sl",
+        # Q, W, X, Y are only used for foreign words.
+        use_ascii=False,
+        charsets=["ISO-8859-2", "WINDOWS-1250"],
+        alphabet="abcčdefghijklmnoprsštuvzžABCČDEFGHIJKLMNOPRSŠTUVZŽ",
+        wiki_start_pages=["Glavna_stran"],
+    ),
+    # Serbian can be written in both Latin and Cyrillic, but there's no
+    # simple way to get the Latin alphabet pages from Wikipedia through
+    # the API, so for now we just support Cyrillic.
+    "Serbian": Language(
+        name="Serbian",
+        iso_code="sr",
+        alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш",
+        charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"],
+        wiki_start_pages=["Главна_страна"],
+    ),
+    "Thai": Language(
+        name="Thai",
+        iso_code="th",
+        use_ascii=False,
+        charsets=["ISO-8859-11", "TIS-620", "CP874"],
+        alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
+        wiki_start_pages=["หน้าหลัก"],
+    ),
+    "Turkish": Language(
+        name="Turkish",
+        iso_code="tr",
+        # Q, W, and X are not used by Turkish
+        use_ascii=False,
+        charsets=["ISO-8859-3", "ISO-8859-9", "WINDOWS-1254"],
+        alphabet="abcçdefgğhıijklmnoöprsştuüvyzâîûABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ",
+        wiki_start_pages=["Ana_Sayfa"],
+    ),
+    "Vietnamese": Language(
+        name="Vietnamese",
+        iso_code="vi",
+        use_ascii=False,
+        # Windows-1258 is the only common 8-bit
+        # Vietnamese encoding supported by Python.
+        # From Wikipedia:
+        # For systems that lack support for Unicode,
+        # dozens of 8-bit Vietnamese code pages are
+        # available.[1] The most common are VISCII
+        # (TCVN 5712:1993), VPS, and Windows-1258.[3]
+        # Where ASCII is required, such as when
+        # ensuring readability in plain text e-mail,
+        # Vietnamese letters are often encoded
+        # according to Vietnamese Quoted-Readable
+        # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
+        # though usage of either variable-width
+        # scheme has declined dramatically following
+        # the adoption of Unicode on the World Wide
+        # Web.
+        charsets=["WINDOWS-1258"],
+        alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY",
+        wiki_start_pages=["Chữ_Quốc_ngữ"],
+    ),
+}
--- a/libs/common/chardet/py.typed
+++ b/libs/common/chardet/py.typed
--- a/libs/common/chardet/resultdict.py
+++ b/libs/common/chardet/resultdict.py
@ -0,0 +1,16 @@
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    # TypedDict was introduced in Python 3.8.
+    #
+    # TODO: Remove the else block and TYPE_CHECKING check when dropping support
+    # for Python 3.7.
+    from typing import TypedDict
+
+    class ResultDict(TypedDict):
+        encoding: Optional[str]
+        confidence: float
+        language: Optional[str]
+
+else:
+    ResultDict = dict
--- a/libs/common/chardet/sbcharsetprober.py
+++ b/libs/common/chardet/sbcharsetprober.py
@ -26,70 +26,77 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from collections import namedtuple
+from typing import Dict, List, NamedTuple, Optional, Union

 from .charsetprober import CharSetProber
 from .enums import CharacterCategory, ProbingState, SequenceLikelihood


-SingleByteCharSetModel = namedtuple('SingleByteCharSetModel',
-                                    ['charset_name',
-                                     'language',
-                                     'char_to_order_map',
-                                     'language_model',
-                                     'typical_positive_ratio',
-                                     'keep_ascii_letters',
-                                     'alphabet'])
+class SingleByteCharSetModel(NamedTuple):
+    charset_name: str
+    language: str
+    char_to_order_map: Dict[int, int]
+    language_model: Dict[int, Dict[int, int]]
+    typical_positive_ratio: float
+    keep_ascii_letters: bool
+    alphabet: str


 class SingleByteCharSetProber(CharSetProber):
    SAMPLE_SIZE = 64
-    SB_ENOUGH_REL_THRESHOLD = 1024  #  0.25 * SAMPLE_SIZE^2
+    SB_ENOUGH_REL_THRESHOLD = 1024  # 0.25 * SAMPLE_SIZE^2
    POSITIVE_SHORTCUT_THRESHOLD = 0.95
    NEGATIVE_SHORTCUT_THRESHOLD = 0.05

-    def __init__(self, model, reversed=False, name_prober=None):
-        super(SingleByteCharSetProber, self).__init__()
+    def __init__(
+        self,
+        model: SingleByteCharSetModel,
+        is_reversed: bool = False,
+        name_prober: Optional[CharSetProber] = None,
+    ) -> None:
+        super().__init__()
        self._model = model
        # TRUE if we need to reverse every pair in the model lookup
-        self._reversed = reversed
+        self._reversed = is_reversed
        # Optional auxiliary prober for name decision
        self._name_prober = name_prober
-        self._last_order = None
-        self._seq_counters = None
-        self._total_seqs = None
-        self._total_char = None
-        self._freq_char = None
+        self._last_order = 255
+        self._seq_counters: List[int] = []
+        self._total_seqs = 0
+        self._total_char = 0
+        self._control_char = 0
+        self._freq_char = 0
        self.reset()

-    def reset(self):
-        super(SingleByteCharSetProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        # char order of last character
        self._last_order = 255
        self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
        self._total_seqs = 0
        self._total_char = 0
+        self._control_char = 0
        # characters that fall in our sampling range
        self._freq_char = 0

    @property
-    def charset_name(self):
+    def charset_name(self) -> Optional[str]:
        if self._name_prober:
            return self._name_prober.charset_name
-        else:
-            return self._model.charset_name
+        return self._model.charset_name

    @property
-    def language(self):
+    def language(self) -> Optional[str]:
        if self._name_prober:
            return self._name_prober.language
-        else:
-            return self._model.language
+        return self._model.language

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        # TODO: Make filter_international_words keep things in self.alphabet
        if not self._model.keep_ascii_letters:
            byte_str = self.filter_international_words(byte_str)
+        else:
+            byte_str = self.remove_xml_tags(byte_str)
        if not byte_str:
            return self.state
        char_to_order_map = self._model.char_to_order_map
@ -103,9 +110,6 @@ class SingleByteCharSetProber(CharSetProber):
            #      _total_char purposes.
            if order < CharacterCategory.CONTROL:
                self._total_char += 1
-            # TODO: Follow uchardet's lead and discount confidence for frequent
-            #       control characters.
-            #       See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
            if order < self.SAMPLE_SIZE:
                self._freq_char += 1
                if self._last_order < self.SAMPLE_SIZE:
@ -122,23 +126,36 @@ class SingleByteCharSetProber(CharSetProber):
            if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
                confidence = self.get_confidence()
                if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
-                    self.logger.debug('%s confidence = %s, we have a winner',
-                                      charset_name, confidence)
+                    self.logger.debug(
+                        "%s confidence = %s, we have a winner", charset_name, confidence
+                    )
                    self._state = ProbingState.FOUND_IT
                elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
-                    self.logger.debug('%s confidence = %s, below negative '
-                                      'shortcut threshhold %s', charset_name,
-                                      confidence,
-                                      self.NEGATIVE_SHORTCUT_THRESHOLD)
+                    self.logger.debug(
+                        "%s confidence = %s, below negative shortcut threshold %s",
+                        charset_name,
+                        confidence,
+                        self.NEGATIVE_SHORTCUT_THRESHOLD,
+                    )
                    self._state = ProbingState.NOT_ME

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        r = 0.01
        if self._total_seqs > 0:
-            r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
-                 self._total_seqs / self._model.typical_positive_ratio)
+            r = (
+                (
+                    self._seq_counters[SequenceLikelihood.POSITIVE]
+                    + 0.25 * self._seq_counters[SequenceLikelihood.LIKELY]
+                )
+                / self._total_seqs
+                / self._model.typical_positive_ratio
+            )
+            # The more control characters (proportionnaly to the size
+            # of the text), the less confident we become in the current
+            # charset.
+            r = r * (self._total_char - self._control_char) / self._total_char
            r = r * self._freq_char / self._total_char
            if r >= 1.0:
                r = 0.99
--- a/libs/common/chardet/sbcsgroupprober.py
+++ b/libs/common/chardet/sbcsgroupprober.py
@ -28,33 +28,38 @@

 from .charsetgroupprober import CharSetGroupProber
 from .hebrewprober import HebrewProber
-from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
-                                 WINDOWS_1251_BULGARIAN_MODEL)
+from .langbulgarianmodel import ISO_8859_5_BULGARIAN_MODEL, WINDOWS_1251_BULGARIAN_MODEL
 from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
 from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
+
 # from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
 #                                  WINDOWS_1250_HUNGARIAN_MODEL)
-from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
-                               ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
-                               MACCYRILLIC_RUSSIAN_MODEL,
-                               WINDOWS_1251_RUSSIAN_MODEL)
+from .langrussianmodel import (
+    IBM855_RUSSIAN_MODEL,
+    IBM866_RUSSIAN_MODEL,
+    ISO_8859_5_RUSSIAN_MODEL,
+    KOI8_R_RUSSIAN_MODEL,
+    MACCYRILLIC_RUSSIAN_MODEL,
+    WINDOWS_1251_RUSSIAN_MODEL,
+)
 from .langthaimodel import TIS_620_THAI_MODEL
 from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
 from .sbcharsetprober import SingleByteCharSetProber


 class SBCSGroupProber(CharSetGroupProber):
-    def __init__(self):
-        super(SBCSGroupProber, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        hebrew_prober = HebrewProber()
-        logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
-                                                        False, hebrew_prober)
+        logical_hebrew_prober = SingleByteCharSetProber(
+            WINDOWS_1255_HEBREW_MODEL, is_reversed=False, name_prober=hebrew_prober
+        )
        # TODO: See if using ISO-8859-8 Hebrew model works better here, since
        #       it's actually the visual one
-        visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
-                                                       True, hebrew_prober)
-        hebrew_prober.set_model_probers(logical_hebrew_prober,
-                                        visual_hebrew_prober)
+        visual_hebrew_prober = SingleByteCharSetProber(
+            WINDOWS_1255_HEBREW_MODEL, is_reversed=True, name_prober=hebrew_prober
+        )
+        hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
        # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
        #       and several tests failed that did not before. Some thought
        #       should be put into the ordering, and we should consider making
--- a/libs/common/chardet/sjisprober.py
+++ b/libs/common/chardet/sjisprober.py
@ -25,68 +25,81 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
+from typing import Union
+
 from .chardistribution import SJISDistributionAnalysis
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
 from .jpcntx import SJISContextAnalysis
+from .mbcharsetprober import MultiByteCharSetProber
 from .mbcssm import SJIS_SM_MODEL
-from .enums import ProbingState, MachineState


 class SJISProber(MultiByteCharSetProber):
-    def __init__(self):
-        super(SJISProber, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
        self.distribution_analyzer = SJISDistributionAnalysis()
        self.context_analyzer = SJISContextAnalysis()
        self.reset()

-    def reset(self):
-        super(SJISProber, self).reset()
+    def reset(self) -> None:
+        super().reset()
        self.context_analyzer.reset()

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return self.context_analyzer.charset_name

    @property
-    def language(self):
+    def language(self) -> str:
        return "Japanese"

-    def feed(self, byte_str):
-        for i in range(len(byte_str)):
-            coding_state = self.coding_sm.next_state(byte_str[i])
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        assert self.coding_sm is not None
+        assert self.distribution_analyzer is not None
+
+        for i, byte in enumerate(byte_str):
+            coding_state = self.coding_sm.next_state(byte)
            if coding_state == MachineState.ERROR:
-                self.logger.debug('%s %s prober hit error at byte %s',
-                                  self.charset_name, self.language, i)
+                self.logger.debug(
+                    "%s %s prober hit error at byte %s",
+                    self.charset_name,
+                    self.language,
+                    i,
+                )
                self._state = ProbingState.NOT_ME
                break
-            elif coding_state == MachineState.ITS_ME:
+            if coding_state == MachineState.ITS_ME:
                self._state = ProbingState.FOUND_IT
                break
-            elif coding_state == MachineState.START:
+            if coding_state == MachineState.START:
                char_len = self.coding_sm.get_current_charlen()
                if i == 0:
-                    self._last_char[1] = byte_str[0]
-                    self.context_analyzer.feed(self._last_char[2 - char_len:],
-                                               char_len)
+                    self._last_char[1] = byte
+                    self.context_analyzer.feed(
+                        self._last_char[2 - char_len :], char_len
+                    )
                    self.distribution_analyzer.feed(self._last_char, char_len)
                else:
-                    self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3
-                                                        - char_len], char_len)
-                    self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
-                                                    char_len)
+                    self.context_analyzer.feed(
+                        byte_str[i + 1 - char_len : i + 3 - char_len], char_len
+                    )
+                    self.distribution_analyzer.feed(byte_str[i - 1 : i + 1], char_len)

        self._last_char[0] = byte_str[-1]

        if self.state == ProbingState.DETECTING:
-            if (self.context_analyzer.got_enough_data() and
-               (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
+            if self.context_analyzer.got_enough_data() and (
+                self.get_confidence() > self.SHORTCUT_THRESHOLD
+            ):
                self._state = ProbingState.FOUND_IT

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
+        assert self.distribution_analyzer is not None
+
        context_conf = self.context_analyzer.get_confidence()
        distrib_conf = self.distribution_analyzer.get_confidence()
        return max(context_conf, distrib_conf)
--- a/libs/common/chardet/universaldetector.py
+++ b/libs/common/chardet/universaldetector.py
@ -39,16 +39,21 @@ class a user of ``chardet`` should use.
 import codecs
 import logging
 import re
+from typing import List, Optional, Union

 from .charsetgroupprober import CharSetGroupProber
+from .charsetprober import CharSetProber
 from .enums import InputState, LanguageFilter, ProbingState
 from .escprober import EscCharSetProber
 from .latin1prober import Latin1Prober
+from .macromanprober import MacRomanProber
 from .mbcsgroupprober import MBCSGroupProber
+from .resultdict import ResultDict
 from .sbcsgroupprober import SBCSGroupProber
+from .utf1632prober import UTF1632Prober


-class UniversalDetector(object):
+class UniversalDetector:
    """
    The ``UniversalDetector`` class underlies the ``chardet.detect`` function
    and coordinates all of the different charset probers.
@ -66,49 +71,87 @@ class UniversalDetector(object):
    """

    MINIMUM_THRESHOLD = 0.20
-    HIGH_BYTE_DETECTOR = re.compile(b'[\x80-\xFF]')
-    ESC_DETECTOR = re.compile(b'(\033|~{)')
-    WIN_BYTE_DETECTOR = re.compile(b'[\x80-\x9F]')
-    ISO_WIN_MAP = {'iso-8859-1': 'Windows-1252',
-                   'iso-8859-2': 'Windows-1250',
-                   'iso-8859-5': 'Windows-1251',
-                   'iso-8859-6': 'Windows-1256',
-                   'iso-8859-7': 'Windows-1253',
-                   'iso-8859-8': 'Windows-1255',
-                   'iso-8859-9': 'Windows-1254',
-                   'iso-8859-13': 'Windows-1257'}
+    HIGH_BYTE_DETECTOR = re.compile(b"[\x80-\xFF]")
+    ESC_DETECTOR = re.compile(b"(\033|~{)")
+    WIN_BYTE_DETECTOR = re.compile(b"[\x80-\x9F]")
+    ISO_WIN_MAP = {
+        "iso-8859-1": "Windows-1252",
+        "iso-8859-2": "Windows-1250",
+        "iso-8859-5": "Windows-1251",
+        "iso-8859-6": "Windows-1256",
+        "iso-8859-7": "Windows-1253",
+        "iso-8859-8": "Windows-1255",
+        "iso-8859-9": "Windows-1254",
+        "iso-8859-13": "Windows-1257",
+    }
+    # Based on https://encoding.spec.whatwg.org/#names-and-labels
+    # but altered to match Python names for encodings and remove mappings
+    # that break tests.
+    LEGACY_MAP = {
+        "ascii": "Windows-1252",
+        "iso-8859-1": "Windows-1252",
+        "tis-620": "ISO-8859-11",
+        "iso-8859-9": "Windows-1254",
+        "gb2312": "GB18030",
+        "euc-kr": "CP949",
+        "utf-16le": "UTF-16",
+    }

-    def __init__(self, lang_filter=LanguageFilter.ALL):
-        self._esc_charset_prober = None
-        self._charset_probers = []
-        self.result = None
-        self.done = None
-        self._got_data = None
-        self._input_state = None
-        self._last_char = None
+    def __init__(
+        self,
+        lang_filter: LanguageFilter = LanguageFilter.ALL,
+        should_rename_legacy: bool = False,
+    ) -> None:
+        self._esc_charset_prober: Optional[EscCharSetProber] = None
+        self._utf1632_prober: Optional[UTF1632Prober] = None
+        self._charset_probers: List[CharSetProber] = []
+        self.result: ResultDict = {
+            "encoding": None,
+            "confidence": 0.0,
+            "language": None,
+        }
+        self.done = False
+        self._got_data = False
+        self._input_state = InputState.PURE_ASCII
+        self._last_char = b""
        self.lang_filter = lang_filter
        self.logger = logging.getLogger(__name__)
-        self._has_win_bytes = None
+        self._has_win_bytes = False
+        self.should_rename_legacy = should_rename_legacy
        self.reset()

-    def reset(self):
+    @property
+    def input_state(self) -> int:
+        return self._input_state
+
+    @property
+    def has_win_bytes(self) -> bool:
+        return self._has_win_bytes
+
+    @property
+    def charset_probers(self) -> List[CharSetProber]:
+        return self._charset_probers
+
+    def reset(self) -> None:
        """
        Reset the UniversalDetector and all of its probers back to their
        initial states.  This is called by ``__init__``, so you only need to
        call this directly in between analyses of different documents.
        """
-        self.result = {'encoding': None, 'confidence': 0.0, 'language': None}
+        self.result = {"encoding": None, "confidence": 0.0, "language": None}
        self.done = False
        self._got_data = False
        self._has_win_bytes = False
        self._input_state = InputState.PURE_ASCII
-        self._last_char = b''
+        self._last_char = b""
        if self._esc_charset_prober:
            self._esc_charset_prober.reset()
+        if self._utf1632_prober:
+            self._utf1632_prober.reset()
        for prober in self._charset_probers:
            prober.reset()

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> None:
        """
        Takes a chunk of a document and feeds it through all of the relevant
        charset probers.
@ -125,7 +168,7 @@ class UniversalDetector(object):
        if self.done:
            return

-        if not len(byte_str):
+        if not byte_str:
            return

        if not isinstance(byte_str, bytearray):
@ -136,35 +179,38 @@ class UniversalDetector(object):
            # If the data starts with BOM, we know it is UTF
            if byte_str.startswith(codecs.BOM_UTF8):
                # EF BB BF  UTF-8 with BOM
-                self.result = {'encoding': "UTF-8-SIG",
-                               'confidence': 1.0,
-                               'language': ''}
-            elif byte_str.startswith((codecs.BOM_UTF32_LE,
-                                      codecs.BOM_UTF32_BE)):
+                self.result = {
+                    "encoding": "UTF-8-SIG",
+                    "confidence": 1.0,
+                    "language": "",
+                }
+            elif byte_str.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
                # FF FE 00 00  UTF-32, little-endian BOM
                # 00 00 FE FF  UTF-32, big-endian BOM
-                self.result = {'encoding': "UTF-32",
-                               'confidence': 1.0,
-                               'language': ''}
-            elif byte_str.startswith(b'\xFE\xFF\x00\x00'):
+                self.result = {"encoding": "UTF-32", "confidence": 1.0, "language": ""}
+            elif byte_str.startswith(b"\xFE\xFF\x00\x00"):
                # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
-                self.result = {'encoding': "X-ISO-10646-UCS-4-3412",
-                               'confidence': 1.0,
-                               'language': ''}
-            elif byte_str.startswith(b'\x00\x00\xFF\xFE'):
+                self.result = {
+                    # TODO: This encoding is not supported by Python. Should remove?
+                    "encoding": "X-ISO-10646-UCS-4-3412",
+                    "confidence": 1.0,
+                    "language": "",
+                }
+            elif byte_str.startswith(b"\x00\x00\xFF\xFE"):
                # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
-                self.result = {'encoding': "X-ISO-10646-UCS-4-2143",
-                               'confidence': 1.0,
-                               'language': ''}
+                self.result = {
+                    # TODO: This encoding is not supported by Python. Should remove?
+                    "encoding": "X-ISO-10646-UCS-4-2143",
+                    "confidence": 1.0,
+                    "language": "",
+                }
            elif byte_str.startswith((codecs.BOM_LE, codecs.BOM_BE)):
                # FF FE  UTF-16, little endian BOM
                # FE FF  UTF-16, big endian BOM
-                self.result = {'encoding': "UTF-16",
-                               'confidence': 1.0,
-                               'language': ''}
+                self.result = {"encoding": "UTF-16", "confidence": 1.0, "language": ""}

            self._got_data = True
-            if self.result['encoding'] is not None:
+            if self.result["encoding"] is not None:
                self.done = True
                return

@ -173,12 +219,29 @@ class UniversalDetector(object):
        if self._input_state == InputState.PURE_ASCII:
            if self.HIGH_BYTE_DETECTOR.search(byte_str):
                self._input_state = InputState.HIGH_BYTE
-            elif self._input_state == InputState.PURE_ASCII and \
-                    self.ESC_DETECTOR.search(self._last_char + byte_str):
+            elif (
+                self._input_state == InputState.PURE_ASCII
+                and self.ESC_DETECTOR.search(self._last_char + byte_str)
+            ):
                self._input_state = InputState.ESC_ASCII

        self._last_char = byte_str[-1:]

+        # next we will look to see if it is appears to be either a UTF-16 or
+        # UTF-32 encoding
+        if not self._utf1632_prober:
+            self._utf1632_prober = UTF1632Prober()
+
+        if self._utf1632_prober.state == ProbingState.DETECTING:
+            if self._utf1632_prober.feed(byte_str) == ProbingState.FOUND_IT:
+                self.result = {
+                    "encoding": self._utf1632_prober.charset_name,
+                    "confidence": self._utf1632_prober.get_confidence(),
+                    "language": "",
+                }
+                self.done = True
+                return
+
        # If we've seen escape sequences, use the EscCharSetProber, which
        # uses a simple state machine to check for known escape sequences in
        # HZ and ISO-2022 encodings, since those are the only encodings that
@ -187,12 +250,11 @@ class UniversalDetector(object):
            if not self._esc_charset_prober:
                self._esc_charset_prober = EscCharSetProber(self.lang_filter)
            if self._esc_charset_prober.feed(byte_str) == ProbingState.FOUND_IT:
-                self.result = {'encoding':
-                               self._esc_charset_prober.charset_name,
-                               'confidence':
-                               self._esc_charset_prober.get_confidence(),
-                               'language':
-                               self._esc_charset_prober.language}
+                self.result = {
+                    "encoding": self._esc_charset_prober.charset_name,
+                    "confidence": self._esc_charset_prober.get_confidence(),
+                    "language": self._esc_charset_prober.language,
+                }
                self.done = True
        # If we've seen high bytes (i.e., those with values greater than 127),
        # we need to do more complicated checks using all our multi-byte and
@ -207,17 +269,20 @@ class UniversalDetector(object):
                if self.lang_filter & LanguageFilter.NON_CJK:
                    self._charset_probers.append(SBCSGroupProber())
                self._charset_probers.append(Latin1Prober())
+                self._charset_probers.append(MacRomanProber())
            for prober in self._charset_probers:
                if prober.feed(byte_str) == ProbingState.FOUND_IT:
-                    self.result = {'encoding': prober.charset_name,
-                                   'confidence': prober.get_confidence(),
-                                   'language': prober.language}
+                    self.result = {
+                        "encoding": prober.charset_name,
+                        "confidence": prober.get_confidence(),
+                        "language": prober.language,
+                    }
                    self.done = True
                    break
            if self.WIN_BYTE_DETECTOR.search(byte_str):
                self._has_win_bytes = True

-    def close(self):
+    def close(self) -> ResultDict:
        """
        Stop analyzing the current document and come up with a final
        prediction.
@ -231,13 +296,11 @@ class UniversalDetector(object):
        self.done = True

        if not self._got_data:
-            self.logger.debug('no data received!')
+            self.logger.debug("no data received!")

        # Default to ASCII if it is all we've seen so far
        elif self._input_state == InputState.PURE_ASCII:
-            self.result = {'encoding': 'ascii',
-                           'confidence': 1.0,
-                           'language': ''}
+            self.result = {"encoding": "ascii", "confidence": 1.0, "language": ""}

        # If we have seen non-ASCII, return the best that met MINIMUM_THRESHOLD
        elif self._input_state == InputState.HIGH_BYTE:
@ -253,34 +316,47 @@ class UniversalDetector(object):
                    max_prober = prober
            if max_prober and (max_prober_confidence > self.MINIMUM_THRESHOLD):
                charset_name = max_prober.charset_name
-                lower_charset_name = max_prober.charset_name.lower()
+                assert charset_name is not None
+                lower_charset_name = charset_name.lower()
                confidence = max_prober.get_confidence()
                # Use Windows encoding name instead of ISO-8859 if we saw any
                # extra Windows-specific bytes
-                if lower_charset_name.startswith('iso-8859'):
+                if lower_charset_name.startswith("iso-8859"):
                    if self._has_win_bytes:
-                        charset_name = self.ISO_WIN_MAP.get(lower_charset_name,
-                                                            charset_name)
-                self.result = {'encoding': charset_name,
-                               'confidence': confidence,
-                               'language': max_prober.language}
+                        charset_name = self.ISO_WIN_MAP.get(
+                            lower_charset_name, charset_name
+                        )
+                # Rename legacy encodings with superset encodings if asked
+                if self.should_rename_legacy:
+                    charset_name = self.LEGACY_MAP.get(
+                        (charset_name or "").lower(), charset_name
+                    )
+                self.result = {
+                    "encoding": charset_name,
+                    "confidence": confidence,
+                    "language": max_prober.language,
+                }

        # Log all prober confidences if none met MINIMUM_THRESHOLD
        if self.logger.getEffectiveLevel() <= logging.DEBUG:
-            if self.result['encoding'] is None:
-                self.logger.debug('no probers hit minimum threshold')
+            if self.result["encoding"] is None:
+                self.logger.debug("no probers hit minimum threshold")
                for group_prober in self._charset_probers:
                    if not group_prober:
                        continue
                    if isinstance(group_prober, CharSetGroupProber):
                        for prober in group_prober.probers:
-                            self.logger.debug('%s %s confidence = %s',
-                                              prober.charset_name,
-                                              prober.language,
-                                              prober.get_confidence())
+                            self.logger.debug(
+                                "%s %s confidence = %s",
+                                prober.charset_name,
+                                prober.language,
+                                prober.get_confidence(),
+                            )
                    else:
-                        self.logger.debug('%s %s confidence = %s',
-                                          group_prober.charset_name,
-                                          group_prober.language,
-                                          group_prober.get_confidence())
+                        self.logger.debug(
+                            "%s %s confidence = %s",
+                            group_prober.charset_name,
+                            group_prober.language,
+                            group_prober.get_confidence(),
+                        )
        return self.result
--- a/libs/common/chardet/utf1632prober.py
+++ b/libs/common/chardet/utf1632prober.py
@ -0,0 +1,225 @@
+######################## BEGIN LICENSE BLOCK ########################
+#
+# Contributor(s):
+#   Jason Zavaglia
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+from typing import List, Union
+
+from .charsetprober import CharSetProber
+from .enums import ProbingState
+
+
+class UTF1632Prober(CharSetProber):
+    """
+    This class simply looks for occurrences of zero bytes, and infers
+    whether the file is UTF16 or UTF32 (low-endian or big-endian)
+    For instance, files looking like ( \0 \0 \0 [nonzero] )+
+    have a good probability to be UTF32BE.  Files looking like ( \0 [nonzero] )+
+    may be guessed to be UTF16BE, and inversely for little-endian varieties.
+    """
+
+    # how many logical characters to scan before feeling confident of prediction
+    MIN_CHARS_FOR_DETECTION = 20
+    # a fixed constant ratio of expected zeros or non-zeros in modulo-position.
+    EXPECTED_RATIO = 0.94
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.position = 0
+        self.zeros_at_mod = [0] * 4
+        self.nonzeros_at_mod = [0] * 4
+        self._state = ProbingState.DETECTING
+        self.quad = [0, 0, 0, 0]
+        self.invalid_utf16be = False
+        self.invalid_utf16le = False
+        self.invalid_utf32be = False
+        self.invalid_utf32le = False
+        self.first_half_surrogate_pair_detected_16be = False
+        self.first_half_surrogate_pair_detected_16le = False
+        self.reset()
+
+    def reset(self) -> None:
+        super().reset()
+        self.position = 0
+        self.zeros_at_mod = [0] * 4
+        self.nonzeros_at_mod = [0] * 4
+        self._state = ProbingState.DETECTING
+        self.invalid_utf16be = False
+        self.invalid_utf16le = False
+        self.invalid_utf32be = False
+        self.invalid_utf32le = False
+        self.first_half_surrogate_pair_detected_16be = False
+        self.first_half_surrogate_pair_detected_16le = False
+        self.quad = [0, 0, 0, 0]
+
+    @property
+    def charset_name(self) -> str:
+        if self.is_likely_utf32be():
+            return "utf-32be"
+        if self.is_likely_utf32le():
+            return "utf-32le"
+        if self.is_likely_utf16be():
+            return "utf-16be"
+        if self.is_likely_utf16le():
+            return "utf-16le"
+        # default to something valid
+        return "utf-16"
+
+    @property
+    def language(self) -> str:
+        return ""
+
+    def approx_32bit_chars(self) -> float:
+        return max(1.0, self.position / 4.0)
+
+    def approx_16bit_chars(self) -> float:
+        return max(1.0, self.position / 2.0)
+
+    def is_likely_utf32be(self) -> bool:
+        approx_chars = self.approx_32bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
+            and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
+            and not self.invalid_utf32be
+        )
+
+    def is_likely_utf32le(self) -> bool:
+        approx_chars = self.approx_32bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
+            and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
+            and not self.invalid_utf32le
+        )
+
+    def is_likely_utf16be(self) -> bool:
+        approx_chars = self.approx_16bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            (self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars
+            > self.EXPECTED_RATIO
+            and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars
+            > self.EXPECTED_RATIO
+            and not self.invalid_utf16be
+        )
+
+    def is_likely_utf16le(self) -> bool:
+        approx_chars = self.approx_16bit_chars()
+        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
+            (self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars
+            > self.EXPECTED_RATIO
+            and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars
+            > self.EXPECTED_RATIO
+            and not self.invalid_utf16le
+        )
+
+    def validate_utf32_characters(self, quad: List[int]) -> None:
+        """
+        Validate if the quad of bytes is valid UTF-32.
+
+        UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
+        excluding 0x0000D800 - 0x0000DFFF
+
+        https://en.wikipedia.org/wiki/UTF-32
+        """
+        if (
+            quad[0] != 0
+            or quad[1] > 0x10
+            or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF)
+        ):
+            self.invalid_utf32be = True
+        if (
+            quad[3] != 0
+            or quad[2] > 0x10
+            or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF)
+        ):
+            self.invalid_utf32le = True
+
+    def validate_utf16_characters(self, pair: List[int]) -> None:
+        """
+        Validate if the pair of bytes is  valid UTF-16.
+
+        UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
+        with an exception for surrogate pairs, which must be in the range
+        0xD800-0xDBFF followed by 0xDC00-0xDFFF
+
+        https://en.wikipedia.org/wiki/UTF-16
+        """
+        if not self.first_half_surrogate_pair_detected_16be:
+            if 0xD8 <= pair[0] <= 0xDB:
+                self.first_half_surrogate_pair_detected_16be = True
+            elif 0xDC <= pair[0] <= 0xDF:
+                self.invalid_utf16be = True
+        else:
+            if 0xDC <= pair[0] <= 0xDF:
+                self.first_half_surrogate_pair_detected_16be = False
+            else:
+                self.invalid_utf16be = True
+
+        if not self.first_half_surrogate_pair_detected_16le:
+            if 0xD8 <= pair[1] <= 0xDB:
+                self.first_half_surrogate_pair_detected_16le = True
+            elif 0xDC <= pair[1] <= 0xDF:
+                self.invalid_utf16le = True
+        else:
+            if 0xDC <= pair[1] <= 0xDF:
+                self.first_half_surrogate_pair_detected_16le = False
+            else:
+                self.invalid_utf16le = True
+
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
+        for c in byte_str:
+            mod4 = self.position % 4
+            self.quad[mod4] = c
+            if mod4 == 3:
+                self.validate_utf32_characters(self.quad)
+                self.validate_utf16_characters(self.quad[0:2])
+                self.validate_utf16_characters(self.quad[2:4])
+            if c == 0:
+                self.zeros_at_mod[mod4] += 1
+            else:
+                self.nonzeros_at_mod[mod4] += 1
+            self.position += 1
+        return self.state
+
+    @property
+    def state(self) -> ProbingState:
+        if self._state in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
+            # terminal, decided states
+            return self._state
+        if self.get_confidence() > 0.80:
+            self._state = ProbingState.FOUND_IT
+        elif self.position > 4 * 1024:
+            # if we get to 4kb into the file, and we can't conclude it's UTF,
+            # let's give up
+            self._state = ProbingState.NOT_ME
+        return self._state
+
+    def get_confidence(self) -> float:
+        return (
+            0.85
+            if (
+                self.is_likely_utf16le()
+                or self.is_likely_utf16be()
+                or self.is_likely_utf32le()
+                or self.is_likely_utf32be()
+            )
+            else 0.00
+        )
--- a/libs/common/chardet/utf8prober.py
+++ b/libs/common/chardet/utf8prober.py
@ -25,45 +25,46 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################

-from .charsetprober import CharSetProber
-from .enums import ProbingState, MachineState
-from .codingstatemachine import CodingStateMachine
-from .mbcssm import UTF8_SM_MODEL
+from typing import Union

+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .enums import MachineState, ProbingState
+from .mbcssm import UTF8_SM_MODEL


 class UTF8Prober(CharSetProber):
    ONE_CHAR_PROB = 0.5

-    def __init__(self):
-        super(UTF8Prober, self).__init__()
+    def __init__(self) -> None:
+        super().__init__()
        self.coding_sm = CodingStateMachine(UTF8_SM_MODEL)
-        self._num_mb_chars = None
+        self._num_mb_chars = 0
        self.reset()

-    def reset(self):
-        super(UTF8Prober, self).reset()
+    def reset(self) -> None:
+        super().reset()
        self.coding_sm.reset()
        self._num_mb_chars = 0

    @property
-    def charset_name(self):
+    def charset_name(self) -> str:
        return "utf-8"

    @property
-    def language(self):
+    def language(self) -> str:
        return ""

-    def feed(self, byte_str):
+    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        for c in byte_str:
            coding_state = self.coding_sm.next_state(c)
            if coding_state == MachineState.ERROR:
                self._state = ProbingState.NOT_ME
                break
-            elif coding_state == MachineState.ITS_ME:
+            if coding_state == MachineState.ITS_ME:
                self._state = ProbingState.FOUND_IT
                break
-            elif coding_state == MachineState.START:
+            if coding_state == MachineState.START:
                if self.coding_sm.get_current_charlen() >= 2:
                    self._num_mb_chars += 1

@ -73,10 +74,9 @@ class UTF8Prober(CharSetProber):

        return self.state

-    def get_confidence(self):
+    def get_confidence(self) -> float:
        unlike = 0.99
        if self._num_mb_chars < 6:
-            unlike *= self.ONE_CHAR_PROB ** self._num_mb_chars
+            unlike *= self.ONE_CHAR_PROB**self._num_mb_chars
            return 1.0 - unlike
-        else:
-            return unlike
+        return unlike
--- a/libs/common/chardet/version.py
+++ b/libs/common/chardet/version.py
@ -1,9 +1,9 @@
 """
 This module exists only to simplify retrieving the version number of chardet
-from within setup.py and from chardet subpackages.
+from within setuptools and from chardet subpackages.

 :author: Dan Blanchard (dan.blanchard@gmail.com)
 """

-__version__ = "4.0.0"
-VERSION = __version__.split('.')
+__version__ = "5.1.0"
+VERSION = __version__.split(".")
--- a/libs/common/charset_normalizer/init.py
+++ b/libs/common/charset_normalizer/init.py
@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+"""
+Charset-Normalizer
+~~~~~~~~~~~~~~
+The Real First Universal Charset Detector.
+A library that helps you read text from an unknown charset encoding.
+Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
+All IANA character set names for which the Python core library provides codecs are supported.
+
+Basic usage:
+   >>> from charset_normalizer import from_bytes
+   >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
+   >>> best_guess = results.best()
+   >>> str(best_guess)
+   'Bсеки човек има право на образование. Oбразованието!'
+
+Others methods and usages are available - see the full documentation
+at <https://github.com/Ousret/charset_normalizer>.
+:copyright: (c) 2021 by Ahmed TAHRI
+:license: MIT, see LICENSE for more details.
+"""
+import logging
+
+from .api import from_bytes, from_fp, from_path, normalize
+from .legacy import (
+    CharsetDetector,
+    CharsetDoctor,
+    CharsetNormalizerMatch,
+    CharsetNormalizerMatches,
+    detect,
+)
+from .models import CharsetMatch, CharsetMatches
+from .utils import set_logging_handler
+from .version import VERSION, __version__
+
+__all__ = (
+    "from_fp",
+    "from_path",
+    "from_bytes",
+    "normalize",
+    "detect",
+    "CharsetMatch",
+    "CharsetMatches",
+    "CharsetNormalizerMatch",
+    "CharsetNormalizerMatches",
+    "CharsetDetector",
+    "CharsetDoctor",
+    "__version__",
+    "VERSION",
+    "set_logging_handler",
+)
+
+# Attach a NullHandler to the top level logger by default
+# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
+
+logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())
--- a/libs/common/charset_normalizer/api.py
+++ b/libs/common/charset_normalizer/api.py
@ -0,0 +1,584 @@
+import logging
+import warnings
+from os import PathLike
+from os.path import basename, splitext
+from typing import Any, BinaryIO, List, Optional, Set
+
+from .cd import (
+    coherence_ratio,
+    encoding_languages,
+    mb_encoding_languages,
+    merge_coherence_ratios,
+)
+from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE
+from .md import mess_ratio
+from .models import CharsetMatch, CharsetMatches
+from .utils import (
+    any_specified_encoding,
+    cut_sequence_chunks,
+    iana_name,
+    identify_sig_or_bom,
+    is_cp_similar,
+    is_multi_byte_encoding,
+    should_strip_sig_or_bom,
+)
+
+# Will most likely be controversial
+# logging.addLevelName(TRACE, "TRACE")
+logger = logging.getLogger("charset_normalizer")
+explain_handler = logging.StreamHandler()
+explain_handler.setFormatter(
+    logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
+)
+
+
+def from_bytes(
+    sequences: bytes,
+    steps: int = 5,
+    chunk_size: int = 512,
+    threshold: float = 0.2,
+    cp_isolation: Optional[List[str]] = None,
+    cp_exclusion: Optional[List[str]] = None,
+    preemptive_behaviour: bool = True,
+    explain: bool = False,
+) -> CharsetMatches:
+    """
+    Given a raw bytes sequence, return the best possibles charset usable to render str objects.
+    If there is no results, it is a strong indicator that the source is binary/not text.
+    By default, the process will extract 5 blocs of 512o each to assess the mess and coherence of a given sequence.
+    And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
+
+    The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
+    but never take it for granted. Can improve the performance.
+
+    You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
+    purpose.
+
+    This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
+    By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
+    toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
+    Custom logging format and handler can be set manually.
+    """
+
+    if not isinstance(sequences, (bytearray, bytes)):
+        raise TypeError(
+            "Expected object of type bytes or bytearray, got: {0}".format(
+                type(sequences)
+            )
+        )
+
+    if explain:
+        previous_logger_level: int = logger.level
+        logger.addHandler(explain_handler)
+        logger.setLevel(TRACE)
+
+    length: int = len(sequences)
+
+    if length == 0:
+        logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
+        if explain:
+            logger.removeHandler(explain_handler)
+            logger.setLevel(previous_logger_level or logging.WARNING)
+        return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
+
+    if cp_isolation is not None:
+        logger.log(
+            TRACE,
+            "cp_isolation is set. use this flag for debugging purpose. "
+            "limited list of encoding allowed : %s.",
+            ", ".join(cp_isolation),
+        )
+        cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
+    else:
+        cp_isolation = []
+
+    if cp_exclusion is not None:
+        logger.log(
+            TRACE,
+            "cp_exclusion is set. use this flag for debugging purpose. "
+            "limited list of encoding excluded : %s.",
+            ", ".join(cp_exclusion),
+        )
+        cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
+    else:
+        cp_exclusion = []
+
+    if length <= (chunk_size * steps):
+        logger.log(
+            TRACE,
+            "override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
+            steps,
+            chunk_size,
+            length,
+        )
+        steps = 1
+        chunk_size = length
+
+    if steps > 1 and length / steps < chunk_size:
+        chunk_size = int(length / steps)
+
+    is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
+    is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
+
+    if is_too_small_sequence:
+        logger.log(
+            TRACE,
+            "Trying to detect encoding from a tiny portion of ({}) byte(s).".format(
+                length
+            ),
+        )
+    elif is_too_large_sequence:
+        logger.log(
+            TRACE,
+            "Using lazy str decoding because the payload is quite large, ({}) byte(s).".format(
+                length
+            ),
+        )
+
+    prioritized_encodings: List[str] = []
+
+    specified_encoding: Optional[str] = (
+        any_specified_encoding(sequences) if preemptive_behaviour else None
+    )
+
+    if specified_encoding is not None:
+        prioritized_encodings.append(specified_encoding)
+        logger.log(
+            TRACE,
+            "Detected declarative mark in sequence. Priority +1 given for %s.",
+            specified_encoding,
+        )
+
+    tested: Set[str] = set()
+    tested_but_hard_failure: List[str] = []
+    tested_but_soft_failure: List[str] = []
+
+    fallback_ascii: Optional[CharsetMatch] = None
+    fallback_u8: Optional[CharsetMatch] = None
+    fallback_specified: Optional[CharsetMatch] = None
+
+    results: CharsetMatches = CharsetMatches()
+
+    sig_encoding, sig_payload = identify_sig_or_bom(sequences)
+
+    if sig_encoding is not None:
+        prioritized_encodings.append(sig_encoding)
+        logger.log(
+            TRACE,
+            "Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
+            len(sig_payload),
+            sig_encoding,
+        )
+
+    prioritized_encodings.append("ascii")
+
+    if "utf_8" not in prioritized_encodings:
+        prioritized_encodings.append("utf_8")
+
+    for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
+
+        if cp_isolation and encoding_iana not in cp_isolation:
+            continue
+
+        if cp_exclusion and encoding_iana in cp_exclusion:
+            continue
+
+        if encoding_iana in tested:
+            continue
+
+        tested.add(encoding_iana)
+
+        decoded_payload: Optional[str] = None
+        bom_or_sig_available: bool = sig_encoding == encoding_iana
+        strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
+            encoding_iana
+        )
+
+        if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
+            logger.log(
+                TRACE,
+                "Encoding %s wont be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
+                encoding_iana,
+            )
+            continue
+
+        try:
+            is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
+        except (ModuleNotFoundError, ImportError):
+            logger.log(
+                TRACE,
+                "Encoding %s does not provide an IncrementalDecoder",
+                encoding_iana,
+            )
+            continue
+
+        try:
+            if is_too_large_sequence and is_multi_byte_decoder is False:
+                str(
+                    sequences[: int(50e4)]
+                    if strip_sig_or_bom is False
+                    else sequences[len(sig_payload) : int(50e4)],
+                    encoding=encoding_iana,
+                )
+            else:
+                decoded_payload = str(
+                    sequences
+                    if strip_sig_or_bom is False
+                    else sequences[len(sig_payload) :],
+                    encoding=encoding_iana,
+                )
+        except (UnicodeDecodeError, LookupError) as e:
+            if not isinstance(e, LookupError):
+                logger.log(
+                    TRACE,
+                    "Code page %s does not fit given bytes sequence at ALL. %s",
+                    encoding_iana,
+                    str(e),
+                )
+            tested_but_hard_failure.append(encoding_iana)
+            continue
+
+        similar_soft_failure_test: bool = False
+
+        for encoding_soft_failed in tested_but_soft_failure:
+            if is_cp_similar(encoding_iana, encoding_soft_failed):
+                similar_soft_failure_test = True
+                break
+
+        if similar_soft_failure_test:
+            logger.log(
+                TRACE,
+                "%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
+                encoding_iana,
+                encoding_soft_failed,
+            )
+            continue
+
+        r_ = range(
+            0 if not bom_or_sig_available else len(sig_payload),
+            length,
+            int(length / steps),
+        )
+
+        multi_byte_bonus: bool = (
+            is_multi_byte_decoder
+            and decoded_payload is not None
+            and len(decoded_payload) < length
+        )
+
+        if multi_byte_bonus:
+            logger.log(
+                TRACE,
+                "Code page %s is a multi byte encoding table and it appear that at least one character "
+                "was encoded using n-bytes.",
+                encoding_iana,
+            )
+
+        max_chunk_gave_up: int = int(len(r_) / 4)
+
+        max_chunk_gave_up = max(max_chunk_gave_up, 2)
+        early_stop_count: int = 0
+        lazy_str_hard_failure = False
+
+        md_chunks: List[str] = []
+        md_ratios = []
+
+        try:
+            for chunk in cut_sequence_chunks(
+                sequences,
+                encoding_iana,
+                r_,
+                chunk_size,
+                bom_or_sig_available,
+                strip_sig_or_bom,
+                sig_payload,
+                is_multi_byte_decoder,
+                decoded_payload,
+            ):
+                md_chunks.append(chunk)
+
+                md_ratios.append(mess_ratio(chunk, threshold))
+
+                if md_ratios[-1] >= threshold:
+                    early_stop_count += 1
+
+                if (early_stop_count >= max_chunk_gave_up) or (
+                    bom_or_sig_available and strip_sig_or_bom is False
+                ):
+                    break
+        except UnicodeDecodeError as e:  # Lazy str loading may have missed something there
+            logger.log(
+                TRACE,
+                "LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
+                encoding_iana,
+                str(e),
+            )
+            early_stop_count = max_chunk_gave_up
+            lazy_str_hard_failure = True
+
+        # We might want to check the sequence again with the whole content
+        # Only if initial MD tests passes
+        if (
+            not lazy_str_hard_failure
+            and is_too_large_sequence
+            and not is_multi_byte_decoder
+        ):
+            try:
+                sequences[int(50e3) :].decode(encoding_iana, errors="strict")
+            except UnicodeDecodeError as e:
+                logger.log(
+                    TRACE,
+                    "LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
+                    encoding_iana,
+                    str(e),
+                )
+                tested_but_hard_failure.append(encoding_iana)
+                continue
+
+        mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
+        if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
+            tested_but_soft_failure.append(encoding_iana)
+            logger.log(
+                TRACE,
+                "%s was excluded because of initial chaos probing. Gave up %i time(s). "
+                "Computed mean chaos is %f %%.",
+                encoding_iana,
+                early_stop_count,
+                round(mean_mess_ratio * 100, ndigits=3),
+            )
+            # Preparing those fallbacks in case we got nothing.
+            if (
+                encoding_iana in ["ascii", "utf_8", specified_encoding]
+                and not lazy_str_hard_failure
+            ):
+                fallback_entry = CharsetMatch(
+                    sequences, encoding_iana, threshold, False, [], decoded_payload
+                )
+                if encoding_iana == specified_encoding:
+                    fallback_specified = fallback_entry
+                elif encoding_iana == "ascii":
+                    fallback_ascii = fallback_entry
+                else:
+                    fallback_u8 = fallback_entry
+            continue
+
+        logger.log(
+            TRACE,
+            "%s passed initial chaos probing. Mean measured chaos is %f %%",
+            encoding_iana,
+            round(mean_mess_ratio * 100, ndigits=3),
+        )
+
+        if not is_multi_byte_decoder:
+            target_languages: List[str] = encoding_languages(encoding_iana)
+        else:
+            target_languages = mb_encoding_languages(encoding_iana)
+
+        if target_languages:
+            logger.log(
+                TRACE,
+                "{} should target any language(s) of {}".format(
+                    encoding_iana, str(target_languages)
+                ),
+            )
+
+        cd_ratios = []
+
+        # We shall skip the CD when its about ASCII
+        # Most of the time its not relevant to run "language-detection" on it.
+        if encoding_iana != "ascii":
+            for chunk in md_chunks:
+                chunk_languages = coherence_ratio(
+                    chunk, 0.1, ",".join(target_languages) if target_languages else None
+                )
+
+                cd_ratios.append(chunk_languages)
+
+        cd_ratios_merged = merge_coherence_ratios(cd_ratios)
+
+        if cd_ratios_merged:
+            logger.log(
+                TRACE,
+                "We detected language {} using {}".format(
+                    cd_ratios_merged, encoding_iana
+                ),
+            )
+
+        results.append(
+            CharsetMatch(
+                sequences,
+                encoding_iana,
+                mean_mess_ratio,
+                bom_or_sig_available,
+                cd_ratios_merged,
+                decoded_payload,
+            )
+        )
+
+        if (
+            encoding_iana in [specified_encoding, "ascii", "utf_8"]
+            and mean_mess_ratio < 0.1
+        ):
+            logger.debug(
+                "Encoding detection: %s is most likely the one.", encoding_iana
+            )
+            if explain:
+                logger.removeHandler(explain_handler)
+                logger.setLevel(previous_logger_level)
+            return CharsetMatches([results[encoding_iana]])
+
+        if encoding_iana == sig_encoding:
+            logger.debug(
+                "Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
+                "the beginning of the sequence.",
+                encoding_iana,
+            )
+            if explain:
+                logger.removeHandler(explain_handler)
+                logger.setLevel(previous_logger_level)
+            return CharsetMatches([results[encoding_iana]])
+
+    if len(results) == 0:
+        if fallback_u8 or fallback_ascii or fallback_specified:
+            logger.log(
+                TRACE,
+                "Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
+            )
+
+        if fallback_specified:
+            logger.debug(
+                "Encoding detection: %s will be used as a fallback match",
+                fallback_specified.encoding,
+            )
+            results.append(fallback_specified)
+        elif (
+            (fallback_u8 and fallback_ascii is None)
+            or (
+                fallback_u8
+                and fallback_ascii
+                and fallback_u8.fingerprint != fallback_ascii.fingerprint
+            )
+            or (fallback_u8 is not None)
+        ):
+            logger.debug("Encoding detection: utf_8 will be used as a fallback match")
+            results.append(fallback_u8)
+        elif fallback_ascii:
+            logger.debug("Encoding detection: ascii will be used as a fallback match")
+            results.append(fallback_ascii)
+
+    if results:
+        logger.debug(
+            "Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
+            results.best().encoding,  # type: ignore
+            len(results) - 1,
+        )
+    else:
+        logger.debug("Encoding detection: Unable to determine any suitable charset.")
+
+    if explain:
+        logger.removeHandler(explain_handler)
+        logger.setLevel(previous_logger_level)
+
+    return results
+
+
+def from_fp(
+    fp: BinaryIO,
+    steps: int = 5,
+    chunk_size: int = 512,
+    threshold: float = 0.20,
+    cp_isolation: Optional[List[str]] = None,
+    cp_exclusion: Optional[List[str]] = None,
+    preemptive_behaviour: bool = True,
+    explain: bool = False,
+) -> CharsetMatches:
+    """
+    Same thing than the function from_bytes but using a file pointer that is already ready.
+    Will not close the file pointer.
+    """
+    return from_bytes(
+        fp.read(),
+        steps,
+        chunk_size,
+        threshold,
+        cp_isolation,
+        cp_exclusion,
+        preemptive_behaviour,
+        explain,
+    )
+
+
+def from_path(
+    path: "PathLike[Any]",
+    steps: int = 5,
+    chunk_size: int = 512,
+    threshold: float = 0.20,
+    cp_isolation: Optional[List[str]] = None,
+    cp_exclusion: Optional[List[str]] = None,
+    preemptive_behaviour: bool = True,
+    explain: bool = False,
+) -> CharsetMatches:
+    """
+    Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
+    Can raise IOError.
+    """
+    with open(path, "rb") as fp:
+        return from_fp(
+            fp,
+            steps,
+            chunk_size,
+            threshold,
+            cp_isolation,
+            cp_exclusion,
+            preemptive_behaviour,
+            explain,
+        )
+
+
+def normalize(
+    path: "PathLike[Any]",
+    steps: int = 5,
+    chunk_size: int = 512,
+    threshold: float = 0.20,
+    cp_isolation: Optional[List[str]] = None,
+    cp_exclusion: Optional[List[str]] = None,
+    preemptive_behaviour: bool = True,
+) -> CharsetMatch:
+    """
+    Take a (text-based) file path and try to create another file next to it, this time using UTF-8.
+    """
+    warnings.warn(
+        "normalize is deprecated and will be removed in 3.0",
+        DeprecationWarning,
+    )
+
+    results = from_path(
+        path,
+        steps,
+        chunk_size,
+        threshold,
+        cp_isolation,
+        cp_exclusion,
+        preemptive_behaviour,
+    )
+
+    filename = basename(path)
+    target_extensions = list(splitext(filename))
+
+    if len(results) == 0:
+        raise IOError(
+            'Unable to normalize "{}", no encoding charset seems to fit.'.format(
+                filename
+            )
+        )
+
+    result = results.best()
+
+    target_extensions[0] += "-" + result.encoding  # type: ignore
+
+    with open(
+        "{}".format(str(path).replace(filename, "".join(target_extensions))), "wb"
+    ) as fp:
+        fp.write(result.output())  # type: ignore
+
+    return result  # type: ignore
--- a/libs/common/charset_normalizer/assets/init.py
+++ b/libs/common/charset_normalizer/assets/init.py
--- a/libs/common/charset_normalizer/cd.py
+++ b/libs/common/charset_normalizer/cd.py
@ -0,0 +1,339 @@
+import importlib
+from codecs import IncrementalDecoder
+from collections import Counter
+from functools import lru_cache
+from typing import Counter as TypeCounter, Dict, List, Optional, Tuple
+
+from .assets import FREQUENCIES
+from .constant import KO_NAMES, LANGUAGE_SUPPORTED_COUNT, TOO_SMALL_SEQUENCE, ZH_NAMES
+from .md import is_suspiciously_successive_range
+from .models import CoherenceMatches
+from .utils import (
+    is_accentuated,
+    is_latin,
+    is_multi_byte_encoding,
+    is_unicode_range_secondary,
+    unicode_range,
+)
+
+
+def encoding_unicode_range(iana_name: str) -> List[str]:
+    """
+    Return associated unicode ranges in a single byte code page.
+    """
+    if is_multi_byte_encoding(iana_name):
+        raise IOError("Function not supported on multi-byte code page")
+
+    decoder = importlib.import_module(
+        "encodings.{}".format(iana_name)
+    ).IncrementalDecoder
+
+    p: IncrementalDecoder = decoder(errors="ignore")
+    seen_ranges: Dict[str, int] = {}
+    character_count: int = 0
+
+    for i in range(0x40, 0xFF):
+        chunk: str = p.decode(bytes([i]))
+
+        if chunk:
+            character_range: Optional[str] = unicode_range(chunk)
+
+            if character_range is None:
+                continue
+
+            if is_unicode_range_secondary(character_range) is False:
+                if character_range not in seen_ranges:
+                    seen_ranges[character_range] = 0
+                seen_ranges[character_range] += 1
+                character_count += 1
+
+    return sorted(
+        [
+            character_range
+            for character_range in seen_ranges
+            if seen_ranges[character_range] / character_count >= 0.15
+        ]
+    )
+
+
+def unicode_range_languages(primary_range: str) -> List[str]:
+    """
+    Return inferred languages used with a unicode range.
+    """
+    languages: List[str] = []
+
+    for language, characters in FREQUENCIES.items():
+        for character in characters:
+            if unicode_range(character) == primary_range:
+                languages.append(language)
+                break
+
+    return languages
+
+
+@lru_cache()
+def encoding_languages(iana_name: str) -> List[str]:
+    """
+    Single-byte encoding language association. Some code page are heavily linked to particular language(s).
+    This function does the correspondence.
+    """
+    unicode_ranges: List[str] = encoding_unicode_range(iana_name)
+    primary_range: Optional[str] = None
+
+    for specified_range in unicode_ranges:
+        if "Latin" not in specified_range:
+            primary_range = specified_range
+            break
+
+    if primary_range is None:
+        return ["Latin Based"]
+
+    return unicode_range_languages(primary_range)
+
+
+@lru_cache()
+def mb_encoding_languages(iana_name: str) -> List[str]:
+    """
+    Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
+    This function does the correspondence.
+    """
+    if (
+        iana_name.startswith("shift_")
+        or iana_name.startswith("iso2022_jp")
+        or iana_name.startswith("euc_j")
+        or iana_name == "cp932"
+    ):
+        return ["Japanese"]
+    if iana_name.startswith("gb") or iana_name in ZH_NAMES:
+        return ["Chinese", "Classical Chinese"]
+    if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
+        return ["Korean"]
+
+    return []
+
+
+@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
+def get_target_features(language: str) -> Tuple[bool, bool]:
+    """
+    Determine main aspects from a supported language if it contains accents and if is pure Latin.
+    """
+    target_have_accents: bool = False
+    target_pure_latin: bool = True
+
+    for character in FREQUENCIES[language]:
+        if not target_have_accents and is_accentuated(character):
+            target_have_accents = True
+        if target_pure_latin and is_latin(character) is False:
+            target_pure_latin = False
+
+    return target_have_accents, target_pure_latin
+
+
+def alphabet_languages(
+    characters: List[str], ignore_non_latin: bool = False
+) -> List[str]:
+    """
+    Return associated languages associated to given characters.
+    """
+    languages: List[Tuple[str, float]] = []
+
+    source_have_accents = any(is_accentuated(character) for character in characters)
+
+    for language, language_characters in FREQUENCIES.items():
+
+        target_have_accents, target_pure_latin = get_target_features(language)
+
+        if ignore_non_latin and target_pure_latin is False:
+            continue
+
+        if target_have_accents is False and source_have_accents:
+            continue
+
+        character_count: int = len(language_characters)
+
+        character_match_count: int = len(
+            [c for c in language_characters if c in characters]
+        )
+
+        ratio: float = character_match_count / character_count
+
+        if ratio >= 0.2:
+            languages.append((language, ratio))
+
+    languages = sorted(languages, key=lambda x: x[1], reverse=True)
+
+    return [compatible_language[0] for compatible_language in languages]
+
+
+def characters_popularity_compare(
+    language: str, ordered_characters: List[str]
+) -> float:
+    """
+    Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
+    The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
+    Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
+    """
+    if language not in FREQUENCIES:
+        raise ValueError("{} not available".format(language))
+
+    character_approved_count: int = 0
+    FREQUENCIES_language_set = set(FREQUENCIES[language])
+
+    for character in ordered_characters:
+        if character not in FREQUENCIES_language_set:
+            continue
+
+        characters_before_source: List[str] = FREQUENCIES[language][
+            0 : FREQUENCIES[language].index(character)
+        ]
+        characters_after_source: List[str] = FREQUENCIES[language][
+            FREQUENCIES[language].index(character) :
+        ]
+        characters_before: List[str] = ordered_characters[
+            0 : ordered_characters.index(character)
+        ]
+        characters_after: List[str] = ordered_characters[
+            ordered_characters.index(character) :
+        ]
+
+        before_match_count: int = len(
+            set(characters_before) & set(characters_before_source)
+        )
+
+        after_match_count: int = len(
+            set(characters_after) & set(characters_after_source)
+        )
+
+        if len(characters_before_source) == 0 and before_match_count <= 4:
+            character_approved_count += 1
+            continue
+
+        if len(characters_after_source) == 0 and after_match_count <= 4:
+            character_approved_count += 1
+            continue
+
+        if (
+            before_match_count / len(characters_before_source) >= 0.4
+            or after_match_count / len(characters_after_source) >= 0.4
+        ):
+            character_approved_count += 1
+            continue
+
+    return character_approved_count / len(ordered_characters)
+
+
+def alpha_unicode_split(decoded_sequence: str) -> List[str]:
+    """
+    Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
+    Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
+    One containing the latin letters and the other hebrew.
+    """
+    layers: Dict[str, str] = {}
+
+    for character in decoded_sequence:
+        if character.isalpha() is False:
+            continue
+
+        character_range: Optional[str] = unicode_range(character)
+
+        if character_range is None:
+            continue
+
+        layer_target_range: Optional[str] = None
+
+        for discovered_range in layers:
+            if (
+                is_suspiciously_successive_range(discovered_range, character_range)
+                is False
+            ):
+                layer_target_range = discovered_range
+                break
+
+        if layer_target_range is None:
+            layer_target_range = character_range
+
+        if layer_target_range not in layers:
+            layers[layer_target_range] = character.lower()
+            continue
+
+        layers[layer_target_range] += character.lower()
+
+    return list(layers.values())
+
+
+def merge_coherence_ratios(results: List[CoherenceMatches]) -> CoherenceMatches:
+    """
+    This function merge results previously given by the function coherence_ratio.
+    The return type is the same as coherence_ratio.
+    """
+    per_language_ratios: Dict[str, List[float]] = {}
+    for result in results:
+        for sub_result in result:
+            language, ratio = sub_result
+            if language not in per_language_ratios:
+                per_language_ratios[language] = [ratio]
+                continue
+            per_language_ratios[language].append(ratio)
+
+    merge = [
+        (
+            language,
+            round(
+                sum(per_language_ratios[language]) / len(per_language_ratios[language]),
+                4,
+            ),
+        )
+        for language in per_language_ratios
+    ]
+
+    return sorted(merge, key=lambda x: x[1], reverse=True)
+
+
+@lru_cache(maxsize=2048)
+def coherence_ratio(
+    decoded_sequence: str, threshold: float = 0.1, lg_inclusion: Optional[str] = None
+) -> CoherenceMatches:
+    """
+    Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
+    A layer = Character extraction by alphabets/ranges.
+    """
+
+    results: List[Tuple[str, float]] = []
+    ignore_non_latin: bool = False
+
+    sufficient_match_count: int = 0
+
+    lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
+    if "Latin Based" in lg_inclusion_list:
+        ignore_non_latin = True
+        lg_inclusion_list.remove("Latin Based")
+
+    for layer in alpha_unicode_split(decoded_sequence):
+        sequence_frequencies: TypeCounter[str] = Counter(layer)
+        most_common = sequence_frequencies.most_common()
+
+        character_count: int = sum(o for c, o in most_common)
+
+        if character_count <= TOO_SMALL_SEQUENCE:
+            continue
+
+        popular_character_ordered: List[str] = [c for c, o in most_common]
+
+        for language in lg_inclusion_list or alphabet_languages(
+            popular_character_ordered, ignore_non_latin
+        ):
+            ratio: float = characters_popularity_compare(
+                language, popular_character_ordered
+            )
+
+            if ratio < threshold:
+                continue
+            elif ratio >= 0.8:
+                sufficient_match_count += 1
+
+            results.append((language, round(ratio, 4)))
+
+            if sufficient_match_count >= 3:
+                break
+
+    return sorted(results, key=lambda x: x[1], reverse=True)
--- a/libs/common/charset_normalizer/cli/init.py
+++ b/libs/common/charset_normalizer/cli/init.py
--- a/libs/common/charset_normalizer/cli/normalizer.py
+++ b/libs/common/charset_normalizer/cli/normalizer.py
@ -0,0 +1,295 @@
+import argparse
+import sys
+from json import dumps
+from os.path import abspath
+from platform import python_version
+from typing import List, Optional
+
+try:
+    from unicodedata2 import unidata_version
+except ImportError:
+    from unicodedata import unidata_version
+
+from charset_normalizer import from_fp
+from charset_normalizer.models import CliDetectionResult
+from charset_normalizer.version import __version__
+
+
+def query_yes_no(question: str, default: str = "yes") -> bool:
+    """Ask a yes/no question via input() and return their answer.
+
+    "question" is a string that is presented to the user.
+    "default" is the presumed answer if the user just hits <Enter>.
+        It must be "yes" (the default), "no" or None (meaning
+        an answer is required of the user).
+
+    The "answer" return value is True for "yes" or False for "no".
+
+    Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input
+    """
+    valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
+    if default is None:
+        prompt = " [y/n] "
+    elif default == "yes":
+        prompt = " [Y/n] "
+    elif default == "no":
+        prompt = " [y/N] "
+    else:
+        raise ValueError("invalid default answer: '%s'" % default)
+
+    while True:
+        sys.stdout.write(question + prompt)
+        choice = input().lower()
+        if default is not None and choice == "":
+            return valid[default]
+        elif choice in valid:
+            return valid[choice]
+        else:
+            sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n")
+
+
+def cli_detect(argv: Optional[List[str]] = None) -> int:
+    """
+    CLI assistant using ARGV and ArgumentParser
+    :param argv:
+    :return: 0 if everything is fine, anything else equal trouble
+    """
+    parser = argparse.ArgumentParser(
+        description="The Real First Universal Charset Detector. "
+        "Discover originating encoding used on text file. "
+        "Normalize text to unicode."
+    )
+
+    parser.add_argument(
+        "files", type=argparse.FileType("rb"), nargs="+", help="File(s) to be analysed"
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        dest="verbose",
+        help="Display complementary information about file if any. "
+        "Stdout will contain logs about the detection process.",
+    )
+    parser.add_argument(
+        "-a",
+        "--with-alternative",
+        action="store_true",
+        default=False,
+        dest="alternatives",
+        help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--normalize",
+        action="store_true",
+        default=False,
+        dest="normalize",
+        help="Permit to normalize input file. If not set, program does not write anything.",
+    )
+    parser.add_argument(
+        "-m",
+        "--minimal",
+        action="store_true",
+        default=False,
+        dest="minimal",
+        help="Only output the charset detected to STDOUT. Disabling JSON output.",
+    )
+    parser.add_argument(
+        "-r",
+        "--replace",
+        action="store_true",
+        default=False,
+        dest="replace",
+        help="Replace file when trying to normalize it instead of creating a new one.",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        dest="force",
+        help="Replace file without asking if you are sure, use this flag with caution.",
+    )
+    parser.add_argument(
+        "-t",
+        "--threshold",
+        action="store",
+        default=0.2,
+        type=float,
+        dest="threshold",
+        help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="Charset-Normalizer {} - Python {} - Unicode {}".format(
+            __version__, python_version(), unidata_version
+        ),
+        help="Show version information and exit.",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.replace is True and args.normalize is False:
+        print("Use --replace in addition of --normalize only.", file=sys.stderr)
+        return 1
+
+    if args.force is True and args.replace is False:
+        print("Use --force in addition of --replace only.", file=sys.stderr)
+        return 1
+
+    if args.threshold < 0.0 or args.threshold > 1.0:
+        print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
+        return 1
+
+    x_ = []
+
+    for my_file in args.files:
+
+        matches = from_fp(my_file, threshold=args.threshold, explain=args.verbose)
+
+        best_guess = matches.best()
+
+        if best_guess is None:
+            print(
+                'Unable to identify originating encoding for "{}". {}'.format(
+                    my_file.name,
+                    "Maybe try increasing maximum amount of chaos."
+                    if args.threshold < 1.0
+                    else "",
+                ),
+                file=sys.stderr,
+            )
+            x_.append(
+                CliDetectionResult(
+                    abspath(my_file.name),
+                    None,
+                    [],
+                    [],
+                    "Unknown",
+                    [],
+                    False,
+                    1.0,
+                    0.0,
+                    None,
+                    True,
+                )
+            )
+        else:
+            x_.append(
+                CliDetectionResult(
+                    abspath(my_file.name),
+                    best_guess.encoding,
+                    best_guess.encoding_aliases,
+                    [
+                        cp
+                        for cp in best_guess.could_be_from_charset
+                        if cp != best_guess.encoding
+                    ],
+                    best_guess.language,
+                    best_guess.alphabets,
+                    best_guess.bom,
+                    best_guess.percent_chaos,
+                    best_guess.percent_coherence,
+                    None,
+                    True,
+                )
+            )
+
+            if len(matches) > 1 and args.alternatives:
+                for el in matches:
+                    if el != best_guess:
+                        x_.append(
+                            CliDetectionResult(
+                                abspath(my_file.name),
+                                el.encoding,
+                                el.encoding_aliases,
+                                [
+                                    cp
+                                    for cp in el.could_be_from_charset
+                                    if cp != el.encoding
+                                ],
+                                el.language,
+                                el.alphabets,
+                                el.bom,
+                                el.percent_chaos,
+                                el.percent_coherence,
+                                None,
+                                False,
+                            )
+                        )
+
+            if args.normalize is True:
+
+                if best_guess.encoding.startswith("utf") is True:
+                    print(
+                        '"{}" file does not need to be normalized, as it already came from unicode.'.format(
+                            my_file.name
+                        ),
+                        file=sys.stderr,
+                    )
+                    if my_file.closed is False:
+                        my_file.close()
+                    continue
+
+                o_: List[str] = my_file.name.split(".")
+
+                if args.replace is False:
+                    o_.insert(-1, best_guess.encoding)
+                    if my_file.closed is False:
+                        my_file.close()
+                elif (
+                    args.force is False
+                    and query_yes_no(
+                        'Are you sure to normalize "{}" by replacing it ?'.format(
+                            my_file.name
+                        ),
+                        "no",
+                    )
+                    is False
+                ):
+                    if my_file.closed is False:
+                        my_file.close()
+                    continue
+
+                try:
+                    x_[0].unicode_path = abspath("./{}".format(".".join(o_)))
+
+                    with open(x_[0].unicode_path, "w", encoding="utf-8") as fp:
+                        fp.write(str(best_guess))
+                except IOError as e:
+                    print(str(e), file=sys.stderr)
+                    if my_file.closed is False:
+                        my_file.close()
+                    return 2
+
+        if my_file.closed is False:
+            my_file.close()
+
+    if args.minimal is False:
+        print(
+            dumps(
+                [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
+                ensure_ascii=True,
+                indent=4,
+            )
+        )
+    else:
+        for my_file in args.files:
+            print(
+                ", ".join(
+                    [
+                        el.encoding or "undefined"
+                        for el in x_
+                        if el.path == abspath(my_file.name)
+                    ]
+                )
+            )
+
+    return 0
+
+
+if __name__ == "__main__":
+    cli_detect()
--- a/libs/common/charset_normalizer/constant.py
+++ b/libs/common/charset_normalizer/constant.py
@ -0,0 +1,497 @@
+from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
+from encodings.aliases import aliases
+from re import IGNORECASE, compile as re_compile
+from typing import Dict, List, Set, Union
+
+from .assets import FREQUENCIES
+
+# Contain for each eligible encoding a list of/item bytes SIG/BOM
+ENCODING_MARKS: Dict[str, Union[bytes, List[bytes]]] = {
+    "utf_8": BOM_UTF8,
+    "utf_7": [
+        b"\x2b\x2f\x76\x38",
+        b"\x2b\x2f\x76\x39",
+        b"\x2b\x2f\x76\x2b",
+        b"\x2b\x2f\x76\x2f",
+        b"\x2b\x2f\x76\x38\x2d",
+    ],
+    "gb18030": b"\x84\x31\x95\x33",
+    "utf_32": [BOM_UTF32_BE, BOM_UTF32_LE],
+    "utf_16": [BOM_UTF16_BE, BOM_UTF16_LE],
+}
+
+TOO_SMALL_SEQUENCE: int = 32
+TOO_BIG_SEQUENCE: int = int(10e6)
+
+UTF8_MAXIMAL_ALLOCATION: int = 1112064
+
+UNICODE_RANGES_COMBINED: Dict[str, range] = {
+    "Control character": range(31 + 1),
+    "Basic Latin": range(32, 127 + 1),
+    "Latin-1 Supplement": range(128, 255 + 1),
+    "Latin Extended-A": range(256, 383 + 1),
+    "Latin Extended-B": range(384, 591 + 1),
+    "IPA Extensions": range(592, 687 + 1),
+    "Spacing Modifier Letters": range(688, 767 + 1),
+    "Combining Diacritical Marks": range(768, 879 + 1),
+    "Greek and Coptic": range(880, 1023 + 1),
+    "Cyrillic": range(1024, 1279 + 1),
+    "Cyrillic Supplement": range(1280, 1327 + 1),
+    "Armenian": range(1328, 1423 + 1),
+    "Hebrew": range(1424, 1535 + 1),
+    "Arabic": range(1536, 1791 + 1),
+    "Syriac": range(1792, 1871 + 1),
+    "Arabic Supplement": range(1872, 1919 + 1),
+    "Thaana": range(1920, 1983 + 1),
+    "NKo": range(1984, 2047 + 1),
+    "Samaritan": range(2048, 2111 + 1),
+    "Mandaic": range(2112, 2143 + 1),
+    "Syriac Supplement": range(2144, 2159 + 1),
+    "Arabic Extended-A": range(2208, 2303 + 1),
+    "Devanagari": range(2304, 2431 + 1),
+    "Bengali": range(2432, 2559 + 1),
+    "Gurmukhi": range(2560, 2687 + 1),
+    "Gujarati": range(2688, 2815 + 1),
+    "Oriya": range(2816, 2943 + 1),
+    "Tamil": range(2944, 3071 + 1),
+    "Telugu": range(3072, 3199 + 1),
+    "Kannada": range(3200, 3327 + 1),
+    "Malayalam": range(3328, 3455 + 1),
+    "Sinhala": range(3456, 3583 + 1),
+    "Thai": range(3584, 3711 + 1),
+    "Lao": range(3712, 3839 + 1),
+    "Tibetan": range(3840, 4095 + 1),
+    "Myanmar": range(4096, 4255 + 1),
+    "Georgian": range(4256, 4351 + 1),
+    "Hangul Jamo": range(4352, 4607 + 1),
+    "Ethiopic": range(4608, 4991 + 1),
+    "Ethiopic Supplement": range(4992, 5023 + 1),
+    "Cherokee": range(5024, 5119 + 1),
+    "Unified Canadian Aboriginal Syllabics": range(5120, 5759 + 1),
+    "Ogham": range(5760, 5791 + 1),
+    "Runic": range(5792, 5887 + 1),
+    "Tagalog": range(5888, 5919 + 1),
+    "Hanunoo": range(5920, 5951 + 1),
+    "Buhid": range(5952, 5983 + 1),
+    "Tagbanwa": range(5984, 6015 + 1),
+    "Khmer": range(6016, 6143 + 1),
+    "Mongolian": range(6144, 6319 + 1),
+    "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6399 + 1),
+    "Limbu": range(6400, 6479 + 1),
+    "Tai Le": range(6480, 6527 + 1),
+    "New Tai Lue": range(6528, 6623 + 1),
+    "Khmer Symbols": range(6624, 6655 + 1),
+    "Buginese": range(6656, 6687 + 1),
+    "Tai Tham": range(6688, 6831 + 1),
+    "Combining Diacritical Marks Extended": range(6832, 6911 + 1),
+    "Balinese": range(6912, 7039 + 1),
+    "Sundanese": range(7040, 7103 + 1),
+    "Batak": range(7104, 7167 + 1),
+    "Lepcha": range(7168, 7247 + 1),
+    "Ol Chiki": range(7248, 7295 + 1),
+    "Cyrillic Extended C": range(7296, 7311 + 1),
+    "Sundanese Supplement": range(7360, 7375 + 1),
+    "Vedic Extensions": range(7376, 7423 + 1),
+    "Phonetic Extensions": range(7424, 7551 + 1),
+    "Phonetic Extensions Supplement": range(7552, 7615 + 1),
+    "Combining Diacritical Marks Supplement": range(7616, 7679 + 1),
+    "Latin Extended Additional": range(7680, 7935 + 1),
+    "Greek Extended": range(7936, 8191 + 1),
+    "General Punctuation": range(8192, 8303 + 1),
+    "Superscripts and Subscripts": range(8304, 8351 + 1),
+    "Currency Symbols": range(8352, 8399 + 1),
+    "Combining Diacritical Marks for Symbols": range(8400, 8447 + 1),
+    "Letterlike Symbols": range(8448, 8527 + 1),
+    "Number Forms": range(8528, 8591 + 1),
+    "Arrows": range(8592, 8703 + 1),
+    "Mathematical Operators": range(8704, 8959 + 1),
+    "Miscellaneous Technical": range(8960, 9215 + 1),
+    "Control Pictures": range(9216, 9279 + 1),
+    "Optical Character Recognition": range(9280, 9311 + 1),
+    "Enclosed Alphanumerics": range(9312, 9471 + 1),
+    "Box Drawing": range(9472, 9599 + 1),
+    "Block Elements": range(9600, 9631 + 1),
+    "Geometric Shapes": range(9632, 9727 + 1),
+    "Miscellaneous Symbols": range(9728, 9983 + 1),
+    "Dingbats": range(9984, 10175 + 1),
+    "Miscellaneous Mathematical Symbols-A": range(10176, 10223 + 1),
+    "Supplemental Arrows-A": range(10224, 10239 + 1),
+    "Braille Patterns": range(10240, 10495 + 1),
+    "Supplemental Arrows-B": range(10496, 10623 + 1),
+    "Miscellaneous Mathematical Symbols-B": range(10624, 10751 + 1),
+    "Supplemental Mathematical Operators": range(10752, 11007 + 1),
+    "Miscellaneous Symbols and Arrows": range(11008, 11263 + 1),
+    "Glagolitic": range(11264, 11359 + 1),
+    "Latin Extended-C": range(11360, 11391 + 1),
+    "Coptic": range(11392, 11519 + 1),
+    "Georgian Supplement": range(11520, 11567 + 1),
+    "Tifinagh": range(11568, 11647 + 1),
+    "Ethiopic Extended": range(11648, 11743 + 1),
+    "Cyrillic Extended-A": range(11744, 11775 + 1),
+    "Supplemental Punctuation": range(11776, 11903 + 1),
+    "CJK Radicals Supplement": range(11904, 12031 + 1),
+    "Kangxi Radicals": range(12032, 12255 + 1),
+    "Ideographic Description Characters": range(12272, 12287 + 1),
+    "CJK Symbols and Punctuation": range(12288, 12351 + 1),
+    "Hiragana": range(12352, 12447 + 1),
+    "Katakana": range(12448, 12543 + 1),
+    "Bopomofo": range(12544, 12591 + 1),
+    "Hangul Compatibility Jamo": range(12592, 12687 + 1),
+    "Kanbun": range(12688, 12703 + 1),
+    "Bopomofo Extended": range(12704, 12735 + 1),
+    "CJK Strokes": range(12736, 12783 + 1),
+    "Katakana Phonetic Extensions": range(12784, 12799 + 1),
+    "Enclosed CJK Letters and Months": range(12800, 13055 + 1),
+    "CJK Compatibility": range(13056, 13311 + 1),
+    "CJK Unified Ideographs Extension A": range(13312, 19903 + 1),
+    "Yijing Hexagram Symbols": range(19904, 19967 + 1),
+    "CJK Unified Ideographs": range(19968, 40959 + 1),
+    "Yi Syllables": range(40960, 42127 + 1),
+    "Yi Radicals": range(42128, 42191 + 1),
+    "Lisu": range(42192, 42239 + 1),
+    "Vai": range(42240, 42559 + 1),
+    "Cyrillic Extended-B": range(42560, 42655 + 1),
+    "Bamum": range(42656, 42751 + 1),
+    "Modifier Tone Letters": range(42752, 42783 + 1),
+    "Latin Extended-D": range(42784, 43007 + 1),
+    "Syloti Nagri": range(43008, 43055 + 1),
+    "Common Indic Number Forms": range(43056, 43071 + 1),
+    "Phags-pa": range(43072, 43135 + 1),
+    "Saurashtra": range(43136, 43231 + 1),
+    "Devanagari Extended": range(43232, 43263 + 1),
+    "Kayah Li": range(43264, 43311 + 1),
+    "Rejang": range(43312, 43359 + 1),
+    "Hangul Jamo Extended-A": range(43360, 43391 + 1),
+    "Javanese": range(43392, 43487 + 1),
+    "Myanmar Extended-B": range(43488, 43519 + 1),
+    "Cham": range(43520, 43615 + 1),
+    "Myanmar Extended-A": range(43616, 43647 + 1),
+    "Tai Viet": range(43648, 43743 + 1),
+    "Meetei Mayek Extensions": range(43744, 43775 + 1),
+    "Ethiopic Extended-A": range(43776, 43823 + 1),
+    "Latin Extended-E": range(43824, 43887 + 1),
+    "Cherokee Supplement": range(43888, 43967 + 1),
+    "Meetei Mayek": range(43968, 44031 + 1),
+    "Hangul Syllables": range(44032, 55215 + 1),
+    "Hangul Jamo Extended-B": range(55216, 55295 + 1),
+    "High Surrogates": range(55296, 56191 + 1),
+    "High Private Use Surrogates": range(56192, 56319 + 1),
+    "Low Surrogates": range(56320, 57343 + 1),
+    "Private Use Area": range(57344, 63743 + 1),
+    "CJK Compatibility Ideographs": range(63744, 64255 + 1),
+    "Alphabetic Presentation Forms": range(64256, 64335 + 1),
+    "Arabic Presentation Forms-A": range(64336, 65023 + 1),
+    "Variation Selectors": range(65024, 65039 + 1),
+    "Vertical Forms": range(65040, 65055 + 1),
+    "Combining Half Marks": range(65056, 65071 + 1),
+    "CJK Compatibility Forms": range(65072, 65103 + 1),
+    "Small Form Variants": range(65104, 65135 + 1),
+    "Arabic Presentation Forms-B": range(65136, 65279 + 1),
+    "Halfwidth and Fullwidth Forms": range(65280, 65519 + 1),
+    "Specials": range(65520, 65535 + 1),
+    "Linear B Syllabary": range(65536, 65663 + 1),
+    "Linear B Ideograms": range(65664, 65791 + 1),
+    "Aegean Numbers": range(65792, 65855 + 1),
+    "Ancient Greek Numbers": range(65856, 65935 + 1),
+    "Ancient Symbols": range(65936, 65999 + 1),
+    "Phaistos Disc": range(66000, 66047 + 1),
+    "Lycian": range(66176, 66207 + 1),
+    "Carian": range(66208, 66271 + 1),
+    "Coptic Epact Numbers": range(66272, 66303 + 1),
+    "Old Italic": range(66304, 66351 + 1),
+    "Gothic": range(66352, 66383 + 1),
+    "Old Permic": range(66384, 66431 + 1),
+    "Ugaritic": range(66432, 66463 + 1),
+    "Old Persian": range(66464, 66527 + 1),
+    "Deseret": range(66560, 66639 + 1),
+    "Shavian": range(66640, 66687 + 1),
+    "Osmanya": range(66688, 66735 + 1),
+    "Osage": range(66736, 66815 + 1),
+    "Elbasan": range(66816, 66863 + 1),
+    "Caucasian Albanian": range(66864, 66927 + 1),
+    "Linear A": range(67072, 67455 + 1),
+    "Cypriot Syllabary": range(67584, 67647 + 1),
+    "Imperial Aramaic": range(67648, 67679 + 1),
+    "Palmyrene": range(67680, 67711 + 1),
+    "Nabataean": range(67712, 67759 + 1),
+    "Hatran": range(67808, 67839 + 1),
+    "Phoenician": range(67840, 67871 + 1),
+    "Lydian": range(67872, 67903 + 1),
+    "Meroitic Hieroglyphs": range(67968, 67999 + 1),
+    "Meroitic Cursive": range(68000, 68095 + 1),
+    "Kharoshthi": range(68096, 68191 + 1),
+    "Old South Arabian": range(68192, 68223 + 1),
+    "Old North Arabian": range(68224, 68255 + 1),
+    "Manichaean": range(68288, 68351 + 1),
+    "Avestan": range(68352, 68415 + 1),
+    "Inscriptional Parthian": range(68416, 68447 + 1),
+    "Inscriptional Pahlavi": range(68448, 68479 + 1),
+    "Psalter Pahlavi": range(68480, 68527 + 1),
+    "Old Turkic": range(68608, 68687 + 1),
+    "Old Hungarian": range(68736, 68863 + 1),
+    "Rumi Numeral Symbols": range(69216, 69247 + 1),
+    "Brahmi": range(69632, 69759 + 1),
+    "Kaithi": range(69760, 69839 + 1),
+    "Sora Sompeng": range(69840, 69887 + 1),
+    "Chakma": range(69888, 69967 + 1),
+    "Mahajani": range(69968, 70015 + 1),
+    "Sharada": range(70016, 70111 + 1),
+    "Sinhala Archaic Numbers": range(70112, 70143 + 1),
+    "Khojki": range(70144, 70223 + 1),
+    "Multani": range(70272, 70319 + 1),
+    "Khudawadi": range(70320, 70399 + 1),
+    "Grantha": range(70400, 70527 + 1),
+    "Newa": range(70656, 70783 + 1),
+    "Tirhuta": range(70784, 70879 + 1),
+    "Siddham": range(71040, 71167 + 1),
+    "Modi": range(71168, 71263 + 1),
+    "Mongolian Supplement": range(71264, 71295 + 1),
+    "Takri": range(71296, 71375 + 1),
+    "Ahom": range(71424, 71487 + 1),
+    "Warang Citi": range(71840, 71935 + 1),
+    "Zanabazar Square": range(72192, 72271 + 1),
+    "Soyombo": range(72272, 72367 + 1),
+    "Pau Cin Hau": range(72384, 72447 + 1),
+    "Bhaiksuki": range(72704, 72815 + 1),
+    "Marchen": range(72816, 72895 + 1),
+    "Masaram Gondi": range(72960, 73055 + 1),
+    "Cuneiform": range(73728, 74751 + 1),
+    "Cuneiform Numbers and Punctuation": range(74752, 74879 + 1),
+    "Early Dynastic Cuneiform": range(74880, 75087 + 1),
+    "Egyptian Hieroglyphs": range(77824, 78895 + 1),
+    "Anatolian Hieroglyphs": range(82944, 83583 + 1),
+    "Bamum Supplement": range(92160, 92735 + 1),
+    "Mro": range(92736, 92783 + 1),
+    "Bassa Vah": range(92880, 92927 + 1),
+    "Pahawh Hmong": range(92928, 93071 + 1),
+    "Miao": range(93952, 94111 + 1),
+    "Ideographic Symbols and Punctuation": range(94176, 94207 + 1),
+    "Tangut": range(94208, 100351 + 1),
+    "Tangut Components": range(100352, 101119 + 1),
+    "Kana Supplement": range(110592, 110847 + 1),
+    "Kana Extended-A": range(110848, 110895 + 1),
+    "Nushu": range(110960, 111359 + 1),
+    "Duployan": range(113664, 113823 + 1),
+    "Shorthand Format Controls": range(113824, 113839 + 1),
+    "Byzantine Musical Symbols": range(118784, 119039 + 1),
+    "Musical Symbols": range(119040, 119295 + 1),
+    "Ancient Greek Musical Notation": range(119296, 119375 + 1),
+    "Tai Xuan Jing Symbols": range(119552, 119647 + 1),
+    "Counting Rod Numerals": range(119648, 119679 + 1),
+    "Mathematical Alphanumeric Symbols": range(119808, 120831 + 1),
+    "Sutton SignWriting": range(120832, 121519 + 1),
+    "Glagolitic Supplement": range(122880, 122927 + 1),
+    "Mende Kikakui": range(124928, 125151 + 1),
+    "Adlam": range(125184, 125279 + 1),
+    "Arabic Mathematical Alphabetic Symbols": range(126464, 126719 + 1),
+    "Mahjong Tiles": range(126976, 127023 + 1),
+    "Domino Tiles": range(127024, 127135 + 1),
+    "Playing Cards": range(127136, 127231 + 1),
+    "Enclosed Alphanumeric Supplement": range(127232, 127487 + 1),
+    "Enclosed Ideographic Supplement": range(127488, 127743 + 1),
+    "Miscellaneous Symbols and Pictographs": range(127744, 128511 + 1),
+    "Emoticons range(Emoji)": range(128512, 128591 + 1),
+    "Ornamental Dingbats": range(128592, 128639 + 1),
+    "Transport and Map Symbols": range(128640, 128767 + 1),
+    "Alchemical Symbols": range(128768, 128895 + 1),
+    "Geometric Shapes Extended": range(128896, 129023 + 1),
+    "Supplemental Arrows-C": range(129024, 129279 + 1),
+    "Supplemental Symbols and Pictographs": range(129280, 129535 + 1),
+    "CJK Unified Ideographs Extension B": range(131072, 173791 + 1),
+    "CJK Unified Ideographs Extension C": range(173824, 177983 + 1),
+    "CJK Unified Ideographs Extension D": range(177984, 178207 + 1),
+    "CJK Unified Ideographs Extension E": range(178208, 183983 + 1),
+    "CJK Unified Ideographs Extension F": range(183984, 191471 + 1),
+    "CJK Compatibility Ideographs Supplement": range(194560, 195103 + 1),
+    "Tags": range(917504, 917631 + 1),
+    "Variation Selectors Supplement": range(917760, 917999 + 1),
+}
+
+
+UNICODE_SECONDARY_RANGE_KEYWORD: List[str] = [
+    "Supplement",
+    "Extended",
+    "Extensions",
+    "Modifier",
+    "Marks",
+    "Punctuation",
+    "Symbols",
+    "Forms",
+    "Operators",
+    "Miscellaneous",
+    "Drawing",
+    "Block",
+    "Shapes",
+    "Supplemental",
+    "Tags",
+]
+
+RE_POSSIBLE_ENCODING_INDICATION = re_compile(
+    r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)",
+    IGNORECASE,
+)
+
+IANA_SUPPORTED: List[str] = sorted(
+    filter(
+        lambda x: x.endswith("_codec") is False
+        and x not in {"rot_13", "tactis", "mbcs"},
+        list(set(aliases.values())),
+    )
+)
+
+IANA_SUPPORTED_COUNT: int = len(IANA_SUPPORTED)
+
+# pre-computed code page that are similar using the function cp_similarity.
+IANA_SUPPORTED_SIMILAR: Dict[str, List[str]] = {
+    "cp037": ["cp1026", "cp1140", "cp273", "cp500"],
+    "cp1026": ["cp037", "cp1140", "cp273", "cp500"],
+    "cp1125": ["cp866"],
+    "cp1140": ["cp037", "cp1026", "cp273", "cp500"],
+    "cp1250": ["iso8859_2"],
+    "cp1251": ["kz1048", "ptcp154"],
+    "cp1252": ["iso8859_15", "iso8859_9", "latin_1"],
+    "cp1253": ["iso8859_7"],
+    "cp1254": ["iso8859_15", "iso8859_9", "latin_1"],
+    "cp1257": ["iso8859_13"],
+    "cp273": ["cp037", "cp1026", "cp1140", "cp500"],
+    "cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"],
+    "cp500": ["cp037", "cp1026", "cp1140", "cp273"],
+    "cp850": ["cp437", "cp857", "cp858", "cp865"],
+    "cp857": ["cp850", "cp858", "cp865"],
+    "cp858": ["cp437", "cp850", "cp857", "cp865"],
+    "cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"],
+    "cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"],
+    "cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"],
+    "cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"],
+    "cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"],
+    "cp866": ["cp1125"],
+    "iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"],
+    "iso8859_11": ["tis_620"],
+    "iso8859_13": ["cp1257"],
+    "iso8859_14": [
+        "iso8859_10",
+        "iso8859_15",
+        "iso8859_16",
+        "iso8859_3",
+        "iso8859_9",
+        "latin_1",
+    ],
+    "iso8859_15": [
+        "cp1252",
+        "cp1254",
+        "iso8859_10",
+        "iso8859_14",
+        "iso8859_16",
+        "iso8859_3",
+        "iso8859_9",
+        "latin_1",
+    ],
+    "iso8859_16": [
+        "iso8859_14",
+        "iso8859_15",
+        "iso8859_2",
+        "iso8859_3",
+        "iso8859_9",
+        "latin_1",
+    ],
+    "iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"],
+    "iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"],
+    "iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"],
+    "iso8859_7": ["cp1253"],
+    "iso8859_9": [
+        "cp1252",
+        "cp1254",
+        "cp1258",
+        "iso8859_10",
+        "iso8859_14",
+        "iso8859_15",
+        "iso8859_16",
+        "iso8859_3",
+        "iso8859_4",
+        "latin_1",
+    ],
+    "kz1048": ["cp1251", "ptcp154"],
+    "latin_1": [
+        "cp1252",
+        "cp1254",
+        "cp1258",
+        "iso8859_10",
+        "iso8859_14",
+        "iso8859_15",
+        "iso8859_16",
+        "iso8859_3",
+        "iso8859_4",
+        "iso8859_9",
+    ],
+    "mac_iceland": ["mac_roman", "mac_turkish"],
+    "mac_roman": ["mac_iceland", "mac_turkish"],
+    "mac_turkish": ["mac_iceland", "mac_roman"],
+    "ptcp154": ["cp1251", "kz1048"],
+    "tis_620": ["iso8859_11"],
+}
+
+
+CHARDET_CORRESPONDENCE: Dict[str, str] = {
+    "iso2022_kr": "ISO-2022-KR",
+    "iso2022_jp": "ISO-2022-JP",
+    "euc_kr": "EUC-KR",
+    "tis_620": "TIS-620",
+    "utf_32": "UTF-32",
+    "euc_jp": "EUC-JP",
+    "koi8_r": "KOI8-R",
+    "iso8859_1": "ISO-8859-1",
+    "iso8859_2": "ISO-8859-2",
+    "iso8859_5": "ISO-8859-5",
+    "iso8859_6": "ISO-8859-6",
+    "iso8859_7": "ISO-8859-7",
+    "iso8859_8": "ISO-8859-8",
+    "utf_16": "UTF-16",
+    "cp855": "IBM855",
+    "mac_cyrillic": "MacCyrillic",
+    "gb2312": "GB2312",
+    "gb18030": "GB18030",
+    "cp932": "CP932",
+    "cp866": "IBM866",
+    "utf_8": "utf-8",
+    "utf_8_sig": "UTF-8-SIG",
+    "shift_jis": "SHIFT_JIS",
+    "big5": "Big5",
+    "cp1250": "windows-1250",
+    "cp1251": "windows-1251",
+    "cp1252": "Windows-1252",
+    "cp1253": "windows-1253",
+    "cp1255": "windows-1255",
+    "cp1256": "windows-1256",
+    "cp1254": "Windows-1254",
+    "cp949": "CP949",
+}
+
+
+COMMON_SAFE_ASCII_CHARACTERS: Set[str] = {
+    "<",
+    ">",
+    "=",
+    ":",
+    "/",
+    "&",
+    ";",
+    "{",
+    "}",
+    "[",
+    "]",
+    ",",
+    "|",
+    '"',
+    "-",
+}
+
+
+KO_NAMES: Set[str] = {"johab", "cp949", "euc_kr"}
+ZH_NAMES: Set[str] = {"big5", "cp950", "big5hkscs", "hz"}
+
+NOT_PRINTABLE_PATTERN = re_compile(r"[0-9\W\n\r\t]+")
+
+LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES)
+
+# Logging LEVEL bellow DEBUG
+TRACE: int = 5
--- a/libs/common/charset_normalizer/legacy.py
+++ b/libs/common/charset_normalizer/legacy.py
@ -0,0 +1,95 @@
+import warnings
+from typing import Dict, Optional, Union
+
+from .api import from_bytes, from_fp, from_path, normalize
+from .constant import CHARDET_CORRESPONDENCE
+from .models import CharsetMatch, CharsetMatches
+
+
+def detect(byte_str: bytes) -> Dict[str, Optional[Union[str, float]]]:
+    """
+    chardet legacy method
+    Detect the encoding of the given byte string. It should be mostly backward-compatible.
+    Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it)
+    This function is deprecated and should be used to migrate your project easily, consult the documentation for
+    further information. Not planned for removal.
+
+    :param byte_str:     The byte sequence to examine.
+    """
+    if not isinstance(byte_str, (bytearray, bytes)):
+        raise TypeError(  # pragma: nocover
+            "Expected object of type bytes or bytearray, got: "
+            "{0}".format(type(byte_str))
+        )
+
+    if isinstance(byte_str, bytearray):
+        byte_str = bytes(byte_str)
+
+    r = from_bytes(byte_str).best()
+
+    encoding = r.encoding if r is not None else None
+    language = r.language if r is not None and r.language != "Unknown" else ""
+    confidence = 1.0 - r.chaos if r is not None else None
+
+    # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process
+    # but chardet does return 'utf-8-sig' and it is a valid codec name.
+    if r is not None and encoding == "utf_8" and r.bom:
+        encoding += "_sig"
+
+    return {
+        "encoding": encoding
+        if encoding not in CHARDET_CORRESPONDENCE
+        else CHARDET_CORRESPONDENCE[encoding],
+        "language": language,
+        "confidence": confidence,
+    }
+
+
+class CharsetNormalizerMatch(CharsetMatch):
+    pass
+
+
+class CharsetNormalizerMatches(CharsetMatches):
+    @staticmethod
+    def from_fp(*args, **kwargs):  # type: ignore
+        warnings.warn(  # pragma: nocover
+            "staticmethod from_fp, from_bytes, from_path and normalize are deprecated "
+            "and scheduled to be removed in 3.0",
+            DeprecationWarning,
+        )
+        return from_fp(*args, **kwargs)  # pragma: nocover
+
+    @staticmethod
+    def from_bytes(*args, **kwargs):  # type: ignore
+        warnings.warn(  # pragma: nocover
+            "staticmethod from_fp, from_bytes, from_path and normalize are deprecated "
+            "and scheduled to be removed in 3.0",
+            DeprecationWarning,
+        )
+        return from_bytes(*args, **kwargs)  # pragma: nocover
+
+    @staticmethod
+    def from_path(*args, **kwargs):  # type: ignore
+        warnings.warn(  # pragma: nocover
+            "staticmethod from_fp, from_bytes, from_path and normalize are deprecated "
+            "and scheduled to be removed in 3.0",
+            DeprecationWarning,
+        )
+        return from_path(*args, **kwargs)  # pragma: nocover
+
+    @staticmethod
+    def normalize(*args, **kwargs):  # type: ignore
+        warnings.warn(  # pragma: nocover
+            "staticmethod from_fp, from_bytes, from_path and normalize are deprecated "
+            "and scheduled to be removed in 3.0",
+            DeprecationWarning,
+        )
+        return normalize(*args, **kwargs)  # pragma: nocover
+
+
+class CharsetDetector(CharsetNormalizerMatches):
+    pass
+
+
+class CharsetDoctor(CharsetNormalizerMatches):
+    pass
--- a/libs/common/charset_normalizer/md.py
+++ b/libs/common/charset_normalizer/md.py
@ -0,0 +1,553 @@
+from functools import lru_cache
+from typing import List, Optional
+
+from .constant import COMMON_SAFE_ASCII_CHARACTERS, UNICODE_SECONDARY_RANGE_KEYWORD
+from .utils import (
+    is_accentuated,
+    is_ascii,
+    is_case_variable,
+    is_cjk,
+    is_emoticon,
+    is_hangul,
+    is_hiragana,
+    is_katakana,
+    is_latin,
+    is_punctuation,
+    is_separator,
+    is_symbol,
+    is_thai,
+    is_unprintable,
+    remove_accent,
+    unicode_range,
+)
+
+
+class MessDetectorPlugin:
+    """
+    Base abstract class used for mess detection plugins.
+    All detectors MUST extend and implement given methods.
+    """
+
+    def eligible(self, character: str) -> bool:
+        """
+        Determine if given character should be fed in.
+        """
+        raise NotImplementedError  # pragma: nocover
+
+    def feed(self, character: str) -> None:
+        """
+        The main routine to be executed upon character.
+        Insert the logic in witch the text would be considered chaotic.
+        """
+        raise NotImplementedError  # pragma: nocover
+
+    def reset(self) -> None:  # pragma: no cover
+        """
+        Permit to reset the plugin to the initial state.
+        """
+        raise NotImplementedError
+
+    @property
+    def ratio(self) -> float:
+        """
+        Compute the chaos ratio based on what your feed() has seen.
+        Must NOT be lower than 0.; No restriction gt 0.
+        """
+        raise NotImplementedError  # pragma: nocover
+
+
+class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._punctuation_count: int = 0
+        self._symbol_count: int = 0
+        self._character_count: int = 0
+
+        self._last_printable_char: Optional[str] = None
+        self._frenzy_symbol_in_word: bool = False
+
+    def eligible(self, character: str) -> bool:
+        return character.isprintable()
+
+    def feed(self, character: str) -> None:
+        self._character_count += 1
+
+        if (
+            character != self._last_printable_char
+            and character not in COMMON_SAFE_ASCII_CHARACTERS
+        ):
+            if is_punctuation(character):
+                self._punctuation_count += 1
+            elif (
+                character.isdigit() is False
+                and is_symbol(character)
+                and is_emoticon(character) is False
+            ):
+                self._symbol_count += 2
+
+        self._last_printable_char = character
+
+    def reset(self) -> None:  # pragma: no cover
+        self._punctuation_count = 0
+        self._character_count = 0
+        self._symbol_count = 0
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+
+        ratio_of_punctuation: float = (
+            self._punctuation_count + self._symbol_count
+        ) / self._character_count
+
+        return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0
+
+
+class TooManyAccentuatedPlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._character_count: int = 0
+        self._accentuated_count: int = 0
+
+    def eligible(self, character: str) -> bool:
+        return character.isalpha()
+
+    def feed(self, character: str) -> None:
+        self._character_count += 1
+
+        if is_accentuated(character):
+            self._accentuated_count += 1
+
+    def reset(self) -> None:  # pragma: no cover
+        self._character_count = 0
+        self._accentuated_count = 0
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+        ratio_of_accentuation: float = self._accentuated_count / self._character_count
+        return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
+
+
+class UnprintablePlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._unprintable_count: int = 0
+        self._character_count: int = 0
+
+    def eligible(self, character: str) -> bool:
+        return True
+
+    def feed(self, character: str) -> None:
+        if is_unprintable(character):
+            self._unprintable_count += 1
+        self._character_count += 1
+
+    def reset(self) -> None:  # pragma: no cover
+        self._unprintable_count = 0
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+
+        return (self._unprintable_count * 8) / self._character_count
+
+
+class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._successive_count: int = 0
+        self._character_count: int = 0
+
+        self._last_latin_character: Optional[str] = None
+
+    def eligible(self, character: str) -> bool:
+        return character.isalpha() and is_latin(character)
+
+    def feed(self, character: str) -> None:
+        self._character_count += 1
+        if (
+            self._last_latin_character is not None
+            and is_accentuated(character)
+            and is_accentuated(self._last_latin_character)
+        ):
+            if character.isupper() and self._last_latin_character.isupper():
+                self._successive_count += 1
+            # Worse if its the same char duplicated with different accent.
+            if remove_accent(character) == remove_accent(self._last_latin_character):
+                self._successive_count += 1
+        self._last_latin_character = character
+
+    def reset(self) -> None:  # pragma: no cover
+        self._successive_count = 0
+        self._character_count = 0
+        self._last_latin_character = None
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+
+        return (self._successive_count * 2) / self._character_count
+
+
+class SuspiciousRange(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._suspicious_successive_range_count: int = 0
+        self._character_count: int = 0
+        self._last_printable_seen: Optional[str] = None
+
+    def eligible(self, character: str) -> bool:
+        return character.isprintable()
+
+    def feed(self, character: str) -> None:
+        self._character_count += 1
+
+        if (
+            character.isspace()
+            or is_punctuation(character)
+            or character in COMMON_SAFE_ASCII_CHARACTERS
+        ):
+            self._last_printable_seen = None
+            return
+
+        if self._last_printable_seen is None:
+            self._last_printable_seen = character
+            return
+
+        unicode_range_a: Optional[str] = unicode_range(self._last_printable_seen)
+        unicode_range_b: Optional[str] = unicode_range(character)
+
+        if is_suspiciously_successive_range(unicode_range_a, unicode_range_b):
+            self._suspicious_successive_range_count += 1
+
+        self._last_printable_seen = character
+
+    def reset(self) -> None:  # pragma: no cover
+        self._character_count = 0
+        self._suspicious_successive_range_count = 0
+        self._last_printable_seen = None
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+
+        ratio_of_suspicious_range_usage: float = (
+            self._suspicious_successive_range_count * 2
+        ) / self._character_count
+
+        if ratio_of_suspicious_range_usage < 0.1:
+            return 0.0
+
+        return ratio_of_suspicious_range_usage
+
+
+class SuperWeirdWordPlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._word_count: int = 0
+        self._bad_word_count: int = 0
+        self._foreign_long_count: int = 0
+
+        self._is_current_word_bad: bool = False
+        self._foreign_long_watch: bool = False
+
+        self._character_count: int = 0
+        self._bad_character_count: int = 0
+
+        self._buffer: str = ""
+        self._buffer_accent_count: int = 0
+
+    def eligible(self, character: str) -> bool:
+        return True
+
+    def feed(self, character: str) -> None:
+        if character.isalpha():
+            self._buffer += character
+            if is_accentuated(character):
+                self._buffer_accent_count += 1
+            if (
+                self._foreign_long_watch is False
+                and (is_latin(character) is False or is_accentuated(character))
+                and is_cjk(character) is False
+                and is_hangul(character) is False
+                and is_katakana(character) is False
+                and is_hiragana(character) is False
+                and is_thai(character) is False
+            ):
+                self._foreign_long_watch = True
+            return
+        if not self._buffer:
+            return
+        if (
+            character.isspace() or is_punctuation(character) or is_separator(character)
+        ) and self._buffer:
+            self._word_count += 1
+            buffer_length: int = len(self._buffer)
+
+            self._character_count += buffer_length
+
+            if buffer_length >= 4:
+                if self._buffer_accent_count / buffer_length > 0.34:
+                    self._is_current_word_bad = True
+                # Word/Buffer ending with a upper case accentuated letter are so rare,
+                # that we will consider them all as suspicious. Same weight as foreign_long suspicious.
+                if is_accentuated(self._buffer[-1]) and self._buffer[-1].isupper():
+                    self._foreign_long_count += 1
+                    self._is_current_word_bad = True
+            if buffer_length >= 24 and self._foreign_long_watch:
+                self._foreign_long_count += 1
+                self._is_current_word_bad = True
+
+            if self._is_current_word_bad:
+                self._bad_word_count += 1
+                self._bad_character_count += len(self._buffer)
+                self._is_current_word_bad = False
+
+            self._foreign_long_watch = False
+            self._buffer = ""
+            self._buffer_accent_count = 0
+        elif (
+            character not in {"<", ">", "-", "=", "~", "|", "_"}
+            and character.isdigit() is False
+            and is_symbol(character)
+        ):
+            self._is_current_word_bad = True
+            self._buffer += character
+
+    def reset(self) -> None:  # pragma: no cover
+        self._buffer = ""
+        self._is_current_word_bad = False
+        self._foreign_long_watch = False
+        self._bad_word_count = 0
+        self._word_count = 0
+        self._character_count = 0
+        self._bad_character_count = 0
+        self._foreign_long_count = 0
+
+    @property
+    def ratio(self) -> float:
+        if self._word_count <= 10 and self._foreign_long_count == 0:
+            return 0.0
+
+        return self._bad_character_count / self._character_count
+
+
+class CjkInvalidStopPlugin(MessDetectorPlugin):
+    """
+    GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
+    can be easily detected. Searching for the overuse of '丅' and '丄'.
+    """
+
+    def __init__(self) -> None:
+        self._wrong_stop_count: int = 0
+        self._cjk_character_count: int = 0
+
+    def eligible(self, character: str) -> bool:
+        return True
+
+    def feed(self, character: str) -> None:
+        if character in {"丅", "丄"}:
+            self._wrong_stop_count += 1
+            return
+        if is_cjk(character):
+            self._cjk_character_count += 1
+
+    def reset(self) -> None:  # pragma: no cover
+        self._wrong_stop_count = 0
+        self._cjk_character_count = 0
+
+    @property
+    def ratio(self) -> float:
+        if self._cjk_character_count < 16:
+            return 0.0
+        return self._wrong_stop_count / self._cjk_character_count
+
+
+class ArchaicUpperLowerPlugin(MessDetectorPlugin):
+    def __init__(self) -> None:
+        self._buf: bool = False
+
+        self._character_count_since_last_sep: int = 0
+
+        self._successive_upper_lower_count: int = 0
+        self._successive_upper_lower_count_final: int = 0
+
+        self._character_count: int = 0
+
+        self._last_alpha_seen: Optional[str] = None
+        self._current_ascii_only: bool = True
+
+    def eligible(self, character: str) -> bool:
+        return True
+
+    def feed(self, character: str) -> None:
+        is_concerned = character.isalpha() and is_case_variable(character)
+        chunk_sep = is_concerned is False
+
+        if chunk_sep and self._character_count_since_last_sep > 0:
+            if (
+                self._character_count_since_last_sep <= 64
+                and character.isdigit() is False
+                and self._current_ascii_only is False
+            ):
+                self._successive_upper_lower_count_final += (
+                    self._successive_upper_lower_count
+                )
+
+            self._successive_upper_lower_count = 0
+            self._character_count_since_last_sep = 0
+            self._last_alpha_seen = None
+            self._buf = False
+            self._character_count += 1
+            self._current_ascii_only = True
+
+            return
+
+        if self._current_ascii_only is True and is_ascii(character) is False:
+            self._current_ascii_only = False
+
+        if self._last_alpha_seen is not None:
+            if (character.isupper() and self._last_alpha_seen.islower()) or (
+                character.islower() and self._last_alpha_seen.isupper()
+            ):
+                if self._buf is True:
+                    self._successive_upper_lower_count += 2
+                    self._buf = False
+                else:
+                    self._buf = True
+            else:
+                self._buf = False
+
+        self._character_count += 1
+        self._character_count_since_last_sep += 1
+        self._last_alpha_seen = character
+
+    def reset(self) -> None:  # pragma: no cover
+        self._character_count = 0
+        self._character_count_since_last_sep = 0
+        self._successive_upper_lower_count = 0
+        self._successive_upper_lower_count_final = 0
+        self._last_alpha_seen = None
+        self._buf = False
+        self._current_ascii_only = True
+
+    @property
+    def ratio(self) -> float:
+        if self._character_count == 0:
+            return 0.0
+
+        return self._successive_upper_lower_count_final / self._character_count
+
+
+@lru_cache(maxsize=1024)
+def is_suspiciously_successive_range(
+    unicode_range_a: Optional[str], unicode_range_b: Optional[str]
+) -> bool:
+    """
+    Determine if two Unicode range seen next to each other can be considered as suspicious.
+    """
+    if unicode_range_a is None or unicode_range_b is None:
+        return True
+
+    if unicode_range_a == unicode_range_b:
+        return False
+
+    if "Latin" in unicode_range_a and "Latin" in unicode_range_b:
+        return False
+
+    if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b:
+        return False
+
+    # Latin characters can be accompanied with a combining diacritical mark
+    # eg. Vietnamese.
+    if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and (
+        "Combining" in unicode_range_a or "Combining" in unicode_range_b
+    ):
+        return False
+
+    keywords_range_a, keywords_range_b = unicode_range_a.split(
+        " "
+    ), unicode_range_b.split(" ")
+
+    for el in keywords_range_a:
+        if el in UNICODE_SECONDARY_RANGE_KEYWORD:
+            continue
+        if el in keywords_range_b:
+            return False
+
+    # Japanese Exception
+    range_a_jp_chars, range_b_jp_chars = (
+        unicode_range_a
+        in (
+            "Hiragana",
+            "Katakana",
+        ),
+        unicode_range_b in ("Hiragana", "Katakana"),
+    )
+    if (range_a_jp_chars or range_b_jp_chars) and (
+        "CJK" in unicode_range_a or "CJK" in unicode_range_b
+    ):
+        return False
+    if range_a_jp_chars and range_b_jp_chars:
+        return False
+
+    if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b:
+        if "CJK" in unicode_range_a or "CJK" in unicode_range_b:
+            return False
+        if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
+            return False
+
+    # Chinese/Japanese use dedicated range for punctuation and/or separators.
+    if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or (
+        unicode_range_a in ["Katakana", "Hiragana"]
+        and unicode_range_b in ["Katakana", "Hiragana"]
+    ):
+        if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b:
+            return False
+        if "Forms" in unicode_range_a or "Forms" in unicode_range_b:
+            return False
+
+    return True
+
+
+@lru_cache(maxsize=2048)
+def mess_ratio(
+    decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False
+) -> float:
+    """
+    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
+    """
+
+    detectors: List[MessDetectorPlugin] = [
+        md_class() for md_class in MessDetectorPlugin.__subclasses__()
+    ]
+
+    length: int = len(decoded_sequence) + 1
+
+    mean_mess_ratio: float = 0.0
+
+    if length < 512:
+        intermediary_mean_mess_ratio_calc: int = 32
+    elif length <= 1024:
+        intermediary_mean_mess_ratio_calc = 64
+    else:
+        intermediary_mean_mess_ratio_calc = 128
+
+    for character, index in zip(decoded_sequence + "\n", range(length)):
+        for detector in detectors:
+            if detector.eligible(character):
+                detector.feed(character)
+
+        if (
+            index > 0 and index % intermediary_mean_mess_ratio_calc == 0
+        ) or index == length - 1:
+            mean_mess_ratio = sum(dt.ratio for dt in detectors)
+
+            if mean_mess_ratio >= maximum_threshold:
+                break
+
+    if debug:
+        for dt in detectors:  # pragma: nocover
+            print(dt.__class__, dt.ratio)
+
+    return round(mean_mess_ratio, 3)
--- a/libs/common/charset_normalizer/models.py
+++ b/libs/common/charset_normalizer/models.py
@ -0,0 +1,401 @@
+import warnings
+from collections import Counter
+from encodings.aliases import aliases
+from hashlib import sha256
+from json import dumps
+from re import sub
+from typing import (
+    Any,
+    Counter as TypeCounter,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from .constant import NOT_PRINTABLE_PATTERN, TOO_BIG_SEQUENCE
+from .md import mess_ratio
+from .utils import iana_name, is_multi_byte_encoding, unicode_range
+
+
+class CharsetMatch:
+    def __init__(
+        self,
+        payload: bytes,
+        guessed_encoding: str,
+        mean_mess_ratio: float,
+        has_sig_or_bom: bool,
+        languages: "CoherenceMatches",
+        decoded_payload: Optional[str] = None,
+    ):
+        self._payload: bytes = payload
+
+        self._encoding: str = guessed_encoding
+        self._mean_mess_ratio: float = mean_mess_ratio
+        self._languages: CoherenceMatches = languages
+        self._has_sig_or_bom: bool = has_sig_or_bom
+        self._unicode_ranges: Optional[List[str]] = None
+
+        self._leaves: List[CharsetMatch] = []
+        self._mean_coherence_ratio: float = 0.0
+
+        self._output_payload: Optional[bytes] = None
+        self._output_encoding: Optional[str] = None
+
+        self._string: Optional[str] = decoded_payload
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, CharsetMatch):
+            raise TypeError(
+                "__eq__ cannot be invoked on {} and {}.".format(
+                    str(other.__class__), str(self.__class__)
+                )
+            )
+        return self.encoding == other.encoding and self.fingerprint == other.fingerprint
+
+    def __lt__(self, other: object) -> bool:
+        """
+        Implemented to make sorted available upon CharsetMatches items.
+        """
+        if not isinstance(other, CharsetMatch):
+            raise ValueError
+
+        chaos_difference: float = abs(self.chaos - other.chaos)
+        coherence_difference: float = abs(self.coherence - other.coherence)
+
+        # Bellow 1% difference --> Use Coherence
+        if chaos_difference < 0.01 and coherence_difference > 0.02:
+            # When having a tough decision, use the result that decoded as many multi-byte as possible.
+            if chaos_difference == 0.0 and self.coherence == other.coherence:
+                return self.multi_byte_usage > other.multi_byte_usage
+            return self.coherence > other.coherence
+
+        return self.chaos < other.chaos
+
+    @property
+    def multi_byte_usage(self) -> float:
+        return 1.0 - len(str(self)) / len(self.raw)
+
+    @property
+    def chaos_secondary_pass(self) -> float:
+        """
+        Check once again chaos in decoded text, except this time, with full content.
+        Use with caution, this can be very slow.
+        Notice: Will be removed in 3.0
+        """
+        warnings.warn(
+            "chaos_secondary_pass is deprecated and will be removed in 3.0",
+            DeprecationWarning,
+        )
+        return mess_ratio(str(self), 1.0)
+
+    @property
+    def coherence_non_latin(self) -> float:
+        """
+        Coherence ratio on the first non-latin language detected if ANY.
+        Notice: Will be removed in 3.0
+        """
+        warnings.warn(
+            "coherence_non_latin is deprecated and will be removed in 3.0",
+            DeprecationWarning,
+        )
+        return 0.0
+
+    @property
+    def w_counter(self) -> TypeCounter[str]:
+        """
+        Word counter instance on decoded text.
+        Notice: Will be removed in 3.0
+        """
+        warnings.warn(
+            "w_counter is deprecated and will be removed in 3.0", DeprecationWarning
+        )
+
+        string_printable_only = sub(NOT_PRINTABLE_PATTERN, " ", str(self).lower())
+
+        return Counter(string_printable_only.split())
+
+    def __str__(self) -> str:
+        # Lazy Str Loading
+        if self._string is None:
+            self._string = str(self._payload, self._encoding, "strict")
+        return self._string
+
+    def __repr__(self) -> str:
+        return "<CharsetMatch '{}' bytes({})>".format(self.encoding, self.fingerprint)
+
+    def add_submatch(self, other: "CharsetMatch") -> None:
+        if not isinstance(other, CharsetMatch) or other == self:
+            raise ValueError(
+                "Unable to add instance <{}> as a submatch of a CharsetMatch".format(
+                    other.__class__
+                )
+            )
+
+        other._string = None  # Unload RAM usage; dirty trick.
+        self._leaves.append(other)
+
+    @property
+    def encoding(self) -> str:
+        return self._encoding
+
+    @property
+    def encoding_aliases(self) -> List[str]:
+        """
+        Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
+        """
+        also_known_as: List[str] = []
+        for u, p in aliases.items():
+            if self.encoding == u:
+                also_known_as.append(p)
+            elif self.encoding == p:
+                also_known_as.append(u)
+        return also_known_as
+
+    @property
+    def bom(self) -> bool:
+        return self._has_sig_or_bom
+
+    @property
+    def byte_order_mark(self) -> bool:
+        return self._has_sig_or_bom
+
+    @property
+    def languages(self) -> List[str]:
+        """
+        Return the complete list of possible languages found in decoded sequence.
+        Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
+        """
+        return [e[0] for e in self._languages]
+
+    @property
+    def language(self) -> str:
+        """
+        Most probable language found in decoded sequence. If none were detected or inferred, the property will return
+        "Unknown".
+        """
+        if not self._languages:
+            # Trying to infer the language based on the given encoding
+            # Its either English or we should not pronounce ourselves in certain cases.
+            if "ascii" in self.could_be_from_charset:
+                return "English"
+
+            # doing it there to avoid circular import
+            from charset_normalizer.cd import encoding_languages, mb_encoding_languages
+
+            languages = (
+                mb_encoding_languages(self.encoding)
+                if is_multi_byte_encoding(self.encoding)
+                else encoding_languages(self.encoding)
+            )
+
+            if len(languages) == 0 or "Latin Based" in languages:
+                return "Unknown"
+
+            return languages[0]
+
+        return self._languages[0][0]
+
+    @property
+    def chaos(self) -> float:
+        return self._mean_mess_ratio
+
+    @property
+    def coherence(self) -> float:
+        if not self._languages:
+            return 0.0
+        return self._languages[0][1]
+
+    @property
+    def percent_chaos(self) -> float:
+        return round(self.chaos * 100, ndigits=3)
+
+    @property
+    def percent_coherence(self) -> float:
+        return round(self.coherence * 100, ndigits=3)
+
+    @property
+    def raw(self) -> bytes:
+        """
+        Original untouched bytes.
+        """
+        return self._payload
+
+    @property
+    def submatch(self) -> List["CharsetMatch"]:
+        return self._leaves
+
+    @property
+    def has_submatch(self) -> bool:
+        return len(self._leaves) > 0
+
+    @property
+    def alphabets(self) -> List[str]:
+        if self._unicode_ranges is not None:
+            return self._unicode_ranges
+        # list detected ranges
+        detected_ranges: List[Optional[str]] = [
+            unicode_range(char) for char in str(self)
+        ]
+        # filter and sort
+        self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
+        return self._unicode_ranges
+
+    @property
+    def could_be_from_charset(self) -> List[str]:
+        """
+        The complete list of encoding that output the exact SAME str result and therefore could be the originating
+        encoding.
+        This list does include the encoding available in property 'encoding'.
+        """
+        return [self._encoding] + [m.encoding for m in self._leaves]
+
+    def first(self) -> "CharsetMatch":
+        """
+        Kept for BC reasons. Will be removed in 3.0.
+        """
+        return self
+
+    def best(self) -> "CharsetMatch":
+        """
+        Kept for BC reasons. Will be removed in 3.0.
+        """
+        return self
+
+    def output(self, encoding: str = "utf_8") -> bytes:
+        """
+        Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
+        Any errors will be simply ignored by the encoder NOT replaced.
+        """
+        if self._output_encoding is None or self._output_encoding != encoding:
+            self._output_encoding = encoding
+            self._output_payload = str(self).encode(encoding, "replace")
+
+        return self._output_payload  # type: ignore
+
+    @property
+    def fingerprint(self) -> str:
+        """
+        Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
+        """
+        return sha256(self.output()).hexdigest()
+
+
+class CharsetMatches:
+    """
+    Container with every CharsetMatch items ordered by default from most probable to the less one.
+    Act like a list(iterable) but does not implements all related methods.
+    """
+
+    def __init__(self, results: Optional[List[CharsetMatch]] = None):
+        self._results: List[CharsetMatch] = sorted(results) if results else []
+
+    def __iter__(self) -> Iterator[CharsetMatch]:
+        yield from self._results
+
+    def __getitem__(self, item: Union[int, str]) -> CharsetMatch:
+        """
+        Retrieve a single item either by its position or encoding name (alias may be used here).
+        Raise KeyError upon invalid index or encoding not present in results.
+        """
+        if isinstance(item, int):
+            return self._results[item]
+        if isinstance(item, str):
+            item = iana_name(item, False)
+            for result in self._results:
+                if item in result.could_be_from_charset:
+                    return result
+        raise KeyError
+
+    def __len__(self) -> int:
+        return len(self._results)
+
+    def __bool__(self) -> bool:
+        return len(self._results) > 0
+
+    def append(self, item: CharsetMatch) -> None:
+        """
+        Insert a single match. Will be inserted accordingly to preserve sort.
+        Can be inserted as a submatch.
+        """
+        if not isinstance(item, CharsetMatch):
+            raise ValueError(
+                "Cannot append instance '{}' to CharsetMatches".format(
+                    str(item.__class__)
+                )
+            )
+        # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
+        if len(item.raw) <= TOO_BIG_SEQUENCE:
+            for match in self._results:
+                if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
+                    match.add_submatch(item)
+                    return
+        self._results.append(item)
+        self._results = sorted(self._results)
+
+    def best(self) -> Optional["CharsetMatch"]:
+        """
+        Simply return the first match. Strict equivalent to matches[0].
+        """
+        if not self._results:
+            return None
+        return self._results[0]
+
+    def first(self) -> Optional["CharsetMatch"]:
+        """
+        Redundant method, call the method best(). Kept for BC reasons.
+        """
+        return self.best()
+
+
+CoherenceMatch = Tuple[str, float]
+CoherenceMatches = List[CoherenceMatch]
+
+
+class CliDetectionResult:
+    def __init__(
+        self,
+        path: str,
+        encoding: Optional[str],
+        encoding_aliases: List[str],
+        alternative_encodings: List[str],
+        language: str,
+        alphabets: List[str],
+        has_sig_or_bom: bool,
+        chaos: float,
+        coherence: float,
+        unicode_path: Optional[str],
+        is_preferred: bool,
+    ):
+        self.path: str = path
+        self.unicode_path: Optional[str] = unicode_path
+        self.encoding: Optional[str] = encoding
+        self.encoding_aliases: List[str] = encoding_aliases
+        self.alternative_encodings: List[str] = alternative_encodings
+        self.language: str = language
+        self.alphabets: List[str] = alphabets
+        self.has_sig_or_bom: bool = has_sig_or_bom
+        self.chaos: float = chaos
+        self.coherence: float = coherence
+        self.is_preferred: bool = is_preferred
+
+    @property
+    def __dict__(self) -> Dict[str, Any]:  # type: ignore
+        return {
+            "path": self.path,
+            "encoding": self.encoding,
+            "encoding_aliases": self.encoding_aliases,
+            "alternative_encodings": self.alternative_encodings,
+            "language": self.language,
+            "alphabets": self.alphabets,
+            "has_sig_or_bom": self.has_sig_or_bom,
+            "chaos": self.chaos,
+            "coherence": self.coherence,
+            "unicode_path": self.unicode_path,
+            "is_preferred": self.is_preferred,
+        }
+
+    def to_json(self) -> str:
+        return dumps(self.__dict__, ensure_ascii=True, indent=4)
--- a/libs/common/charset_normalizer/py.typed
+++ b/libs/common/charset_normalizer/py.typed
--- a/libs/common/charset_normalizer/utils.py
+++ b/libs/common/charset_normalizer/utils.py
@ -0,0 +1,424 @@
+try:
+    # WARNING: unicodedata2 support is going to be removed in 3.0
+    # Python is quickly catching up.
+    import unicodedata2 as unicodedata
+except ImportError:
+    import unicodedata  # type: ignore[no-redef]
+
+import importlib
+import logging
+from codecs import IncrementalDecoder
+from encodings.aliases import aliases
+from functools import lru_cache
+from re import findall
+from typing import Generator, List, Optional, Set, Tuple, Union
+
+from _multibytecodec import MultibyteIncrementalDecoder
+
+from .constant import (
+    ENCODING_MARKS,
+    IANA_SUPPORTED_SIMILAR,
+    RE_POSSIBLE_ENCODING_INDICATION,
+    UNICODE_RANGES_COMBINED,
+    UNICODE_SECONDARY_RANGE_KEYWORD,
+    UTF8_MAXIMAL_ALLOCATION,
+)
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_accentuated(character: str) -> bool:
+    try:
+        description: str = unicodedata.name(character)
+    except ValueError:
+        return False
+    return (
+        "WITH GRAVE" in description
+        or "WITH ACUTE" in description
+        or "WITH CEDILLA" in description
+        or "WITH DIAERESIS" in description
+        or "WITH CIRCUMFLEX" in description
+        or "WITH TILDE" in description
+    )
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def remove_accent(character: str) -> str:
+    decomposed: str = unicodedata.decomposition(character)
+    if not decomposed:
+        return character
+
+    codes: List[str] = decomposed.split(" ")
+
+    return chr(int(codes[0], 16))
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def unicode_range(character: str) -> Optional[str]:
+    """
+    Retrieve the Unicode range official name from a single character.
+    """
+    character_ord: int = ord(character)
+
+    for range_name, ord_range in UNICODE_RANGES_COMBINED.items():
+        if character_ord in ord_range:
+            return range_name
+
+    return None
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_latin(character: str) -> bool:
+    try:
+        description: str = unicodedata.name(character)
+    except ValueError:
+        return False
+    return "LATIN" in description
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_ascii(character: str) -> bool:
+    try:
+        character.encode("ascii")
+    except UnicodeEncodeError:
+        return False
+    return True
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_punctuation(character: str) -> bool:
+    character_category: str = unicodedata.category(character)
+
+    if "P" in character_category:
+        return True
+
+    character_range: Optional[str] = unicode_range(character)
+
+    if character_range is None:
+        return False
+
+    return "Punctuation" in character_range
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_symbol(character: str) -> bool:
+    character_category: str = unicodedata.category(character)
+
+    if "S" in character_category or "N" in character_category:
+        return True
+
+    character_range: Optional[str] = unicode_range(character)
+
+    if character_range is None:
+        return False
+
+    return "Forms" in character_range
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_emoticon(character: str) -> bool:
+    character_range: Optional[str] = unicode_range(character)
+
+    if character_range is None:
+        return False
+
+    return "Emoticons" in character_range
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_separator(character: str) -> bool:
+    if character.isspace() or character in {"｜", "+", ",", ";", "<", ">"}:
+        return True
+
+    character_category: str = unicodedata.category(character)
+
+    return "Z" in character_category
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_case_variable(character: str) -> bool:
+    return character.islower() != character.isupper()
+
+
+def is_private_use_only(character: str) -> bool:
+    character_category: str = unicodedata.category(character)
+
+    return character_category == "Co"
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_cjk(character: str) -> bool:
+    try:
+        character_name = unicodedata.name(character)
+    except ValueError:
+        return False
+
+    return "CJK" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_hiragana(character: str) -> bool:
+    try:
+        character_name = unicodedata.name(character)
+    except ValueError:
+        return False
+
+    return "HIRAGANA" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_katakana(character: str) -> bool:
+    try:
+        character_name = unicodedata.name(character)
+    except ValueError:
+        return False
+
+    return "KATAKANA" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_hangul(character: str) -> bool:
+    try:
+        character_name = unicodedata.name(character)
+    except ValueError:
+        return False
+
+    return "HANGUL" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_thai(character: str) -> bool:
+    try:
+        character_name = unicodedata.name(character)
+    except ValueError:
+        return False
+
+    return "THAI" in character_name
+
+
+@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
+def is_unicode_range_secondary(range_name: str) -> bool:
+    return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_unprintable(character: str) -> bool:
+    return (
+        character.isspace() is False  # includes \n \t \r \v
+        and character.isprintable() is False
+        and character != "\x1A"  # Why? Its the ASCII substitute character.
+        and character != "\ufeff"  # bug discovered in Python,
+        # Zero Width No-Break Space located in 	Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
+    )
+
+
+def any_specified_encoding(sequence: bytes, search_zone: int = 4096) -> Optional[str]:
+    """
+    Extract using ASCII-only decoder any specified encoding in the first n-bytes.
+    """
+    if not isinstance(sequence, bytes):
+        raise TypeError
+
+    seq_len: int = len(sequence)
+
+    results: List[str] = findall(
+        RE_POSSIBLE_ENCODING_INDICATION,
+        sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"),
+    )
+
+    if len(results) == 0:
+        return None
+
+    for specified_encoding in results:
+        specified_encoding = specified_encoding.lower().replace("-", "_")
+
+        encoding_alias: str
+        encoding_iana: str
+
+        for encoding_alias, encoding_iana in aliases.items():
+            if encoding_alias == specified_encoding:
+                return encoding_iana
+            if encoding_iana == specified_encoding:
+                return encoding_iana
+
+    return None
+
+
+@lru_cache(maxsize=128)
+def is_multi_byte_encoding(name: str) -> bool:
+    """
+    Verify is a specific encoding is a multi byte one based on it IANA name
+    """
+    return name in {
+        "utf_8",
+        "utf_8_sig",
+        "utf_16",
+        "utf_16_be",
+        "utf_16_le",
+        "utf_32",
+        "utf_32_le",
+        "utf_32_be",
+        "utf_7",
+    } or issubclass(
+        importlib.import_module("encodings.{}".format(name)).IncrementalDecoder,
+        MultibyteIncrementalDecoder,
+    )
+
+
+def identify_sig_or_bom(sequence: bytes) -> Tuple[Optional[str], bytes]:
+    """
+    Identify and extract SIG/BOM in given sequence.
+    """
+
+    for iana_encoding in ENCODING_MARKS:
+        marks: Union[bytes, List[bytes]] = ENCODING_MARKS[iana_encoding]
+
+        if isinstance(marks, bytes):
+            marks = [marks]
+
+        for mark in marks:
+            if sequence.startswith(mark):
+                return iana_encoding, mark
+
+    return None, b""
+
+
+def should_strip_sig_or_bom(iana_encoding: str) -> bool:
+    return iana_encoding not in {"utf_16", "utf_32"}
+
+
+def iana_name(cp_name: str, strict: bool = True) -> str:
+    cp_name = cp_name.lower().replace("-", "_")
+
+    encoding_alias: str
+    encoding_iana: str
+
+    for encoding_alias, encoding_iana in aliases.items():
+        if cp_name in [encoding_alias, encoding_iana]:
+            return encoding_iana
+
+    if strict:
+        raise ValueError("Unable to retrieve IANA for '{}'".format(cp_name))
+
+    return cp_name
+
+
+def range_scan(decoded_sequence: str) -> List[str]:
+    ranges: Set[str] = set()
+
+    for character in decoded_sequence:
+        character_range: Optional[str] = unicode_range(character)
+
+        if character_range is None:
+            continue
+
+        ranges.add(character_range)
+
+    return list(ranges)
+
+
+def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
+
+    if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
+        return 0.0
+
+    decoder_a = importlib.import_module(
+        "encodings.{}".format(iana_name_a)
+    ).IncrementalDecoder
+    decoder_b = importlib.import_module(
+        "encodings.{}".format(iana_name_b)
+    ).IncrementalDecoder
+
+    id_a: IncrementalDecoder = decoder_a(errors="ignore")
+    id_b: IncrementalDecoder = decoder_b(errors="ignore")
+
+    character_match_count: int = 0
+
+    for i in range(255):
+        to_be_decoded: bytes = bytes([i])
+        if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
+            character_match_count += 1
+
+    return character_match_count / 254
+
+
+def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
+    """
+    Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
+    the function cp_similarity.
+    """
+    return (
+        iana_name_a in IANA_SUPPORTED_SIMILAR
+        and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
+    )
+
+
+def set_logging_handler(
+    name: str = "charset_normalizer",
+    level: int = logging.INFO,
+    format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
+) -> None:
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(format_string))
+    logger.addHandler(handler)
+
+
+def cut_sequence_chunks(
+    sequences: bytes,
+    encoding_iana: str,
+    offsets: range,
+    chunk_size: int,
+    bom_or_sig_available: bool,
+    strip_sig_or_bom: bool,
+    sig_payload: bytes,
+    is_multi_byte_decoder: bool,
+    decoded_payload: Optional[str] = None,
+) -> Generator[str, None, None]:
+
+    if decoded_payload and is_multi_byte_decoder is False:
+        for i in offsets:
+            chunk = decoded_payload[i : i + chunk_size]
+            if not chunk:
+                break
+            yield chunk
+    else:
+        for i in offsets:
+            chunk_end = i + chunk_size
+            if chunk_end > len(sequences) + 8:
+                continue
+
+            cut_sequence = sequences[i : i + chunk_size]
+
+            if bom_or_sig_available and strip_sig_or_bom is False:
+                cut_sequence = sig_payload + cut_sequence
+
+            chunk = cut_sequence.decode(
+                encoding_iana,
+                errors="ignore" if is_multi_byte_decoder else "strict",
+            )
+
+            # multi-byte bad cutting detector and adjustment
+            # not the cleanest way to perform that fix but clever enough for now.
+            if is_multi_byte_decoder and i > 0 and sequences[i] >= 0x80:
+
+                chunk_partial_size_chk: int = min(chunk_size, 16)
+
+                if (
+                    decoded_payload
+                    and chunk[:chunk_partial_size_chk] not in decoded_payload
+                ):
+                    for j in range(i, i - 4, -1):
+                        cut_sequence = sequences[j:chunk_end]
+
+                        if bom_or_sig_available and strip_sig_or_bom is False:
+                            cut_sequence = sig_payload + cut_sequence
+
+                        chunk = cut_sequence.decode(encoding_iana, errors="ignore")
+
+                        if chunk[:chunk_partial_size_chk] in decoded_payload:
+                            break
+
+            yield chunk
--- a/libs/common/charset_normalizer/version.py
+++ b/libs/common/charset_normalizer/version.py
@ -0,0 +1,6 @@
+"""
+Expose version
+"""
+
+__version__ = "2.1.1"
+VERSION = __version__.split(".")
--- a/libs/common/confuse/init.py
+++ b/libs/common/confuse/init.py
@ -1,9 +1,7 @@
 """Painless YAML configuration.
 """

-from __future__ import division, absolute_import, print_function
-
-__version__ = '1.7.0'
+__version__ = '2.0.0'

 from .exceptions import * # NOQA
 from .util import * # NOQA
--- a/libs/common/confuse/core.py
+++ b/libs/common/confuse/core.py
@ -15,8 +15,6 @@

 """Worry-free YAML configuration files.
 """
-from __future__ import division, absolute_import, print_function
-
 import errno
 import os
 import yaml
@ -165,23 +163,10 @@ class ConfigView(object):
    def __str__(self):
        """Get the value for this view as a bytestring.
        """
-        if util.PY3:
-            return self.__unicode__()
-        else:
-            return bytes(self.get())
-
-    def __unicode__(self):
-        """Get the value for this view as a Unicode string.
-        """
-        return util.STRING(self.get())
-
-    def __nonzero__(self):
-        """Gets the value for this view as a boolean. (Python 2 only.)
-        """
-        return self.__bool__()
+        return str(self.get())

    def __bool__(self):
-        """Gets the value for this view as a boolean. (Python 3 only.)
+        """Gets the value for this view as a bool.
        """
        return bool(self.get())

@ -441,7 +426,7 @@ class Subview(ConfigView):
            self.name += u'#{0}'.format(self.key)
        elif isinstance(self.key, bytes):
            self.name += self.key.decode('utf-8')
-        elif isinstance(self.key, util.STRING):
+        elif isinstance(self.key, str):
            self.name += self.key
        else:
            self.name += repr(self.key)
--- a/libs/common/confuse/exceptions.py
+++ b/libs/common/confuse/exceptions.py
@ -1,5 +1,3 @@
-from __future__ import division, absolute_import, print_function
-
 import yaml

 __all__ = [
--- a/libs/common/confuse/sources.py
+++ b/libs/common/confuse/sources.py
@ -1,6 +1,4 @@
-from __future__ import division, absolute_import, print_function
-
-from .util import BASESTRING, build_dict
+from .util import build_dict
 from . import yaml_util
 import os

@ -27,8 +25,7 @@ class ConfigSource(dict):
        behavior.
        """
        super(ConfigSource, self).__init__(value)
-        if (filename is not None
-                and not isinstance(filename, BASESTRING)):
+        if filename is not None and not isinstance(filename, str):
            raise TypeError(u'filename must be a string or None')
        self.filename = filename
        self.default = default
--- a/libs/common/confuse/templates.py
+++ b/libs/common/confuse/templates.py
@ -1,29 +1,12 @@
-from __future__ import division, absolute_import, print_function
-
 import os
 import re
-import sys
+import enum
+import pathlib
+from collections import abc

 from . import util
 from . import exceptions

-try:
-    import enum
-    SUPPORTS_ENUM = True
-except ImportError:
-    SUPPORTS_ENUM = False
-
-try:
-    import pathlib
-    SUPPORTS_PATHLIB = True
-except ImportError:
-    SUPPORTS_PATHLIB = False
-
-if sys.version_info >= (3, 3):
-    from collections import abc
-else:
-    import collections as abc
-

 REQUIRED = object()
 """A sentinel indicating that there is no default value and an exception
@ -130,7 +113,7 @@ class Number(Template):
    def convert(self, value, view):
        """Check that the value is an int or a float.
        """
-        if isinstance(value, util.NUMERIC_TYPES):
+        if isinstance(value, (int, float)):
            return value
        else:
            self.fail(
@ -243,7 +226,7 @@ class String(Template):
    def convert(self, value, view):
        """Check that the value is a string and matches the pattern.
        """
-        if not isinstance(value, util.BASESTRING):
+        if not isinstance(value, str):
            self.fail(u'must be a string', view, True)

        if self.pattern and not self.regex.match(value):
@ -278,7 +261,7 @@ class Choice(Template):
        """Ensure that the value is among the choices (and remap if the
        choices are a mapping).
        """
-        if (SUPPORTS_ENUM and isinstance(self.choices, type)
+        if (isinstance(self.choices, type)
                and issubclass(self.choices, enum.Enum)):
            try:
                return self.choices(value)
@ -385,7 +368,7 @@ class StrSeq(Template):
        self.split = split

    def _convert_value(self, x, view):
-        if isinstance(x, util.STRING):
+        if isinstance(x, str):
            return x
        elif isinstance(x, bytes):
            return x.decode('utf-8', 'ignore')
@ -396,7 +379,7 @@ class StrSeq(Template):
        if isinstance(value, bytes):
            value = value.decode('utf-8', 'ignore')

-        if isinstance(value, util.STRING):
+        if isinstance(value, str):
            if self.split:
                value = value.split()
            else:
@ -566,13 +549,13 @@ class Filename(Template):
        except exceptions.NotFoundError:
            return self.get_default_value(view.name)

-        if not isinstance(path, util.BASESTRING):
+        if not isinstance(path, str):
            self.fail(
                u'must be a filename, not {0}'.format(type(path).__name__),
                view,
                True
            )
-        path = os.path.expanduser(util.STRING(path))
+        path = os.path.expanduser(str(path))

        if not os.path.isabs(path):
            if self.cwd is not None:
@ -602,9 +585,6 @@ class Path(Filename):

    Filenames are parsed equivalent to the `Filename` template and then
    converted to `pathlib.Path` objects.
-
-    For Python 2 it returns the original path as returned by the `Filename`
-    template.
    """
    def value(self, view, template=None):
        value = super(Path, self).value(view, template)
@ -709,15 +689,14 @@ def as_template(value):
        return Integer()
    elif isinstance(value, int):
        return Integer(value)
-    elif isinstance(value, type) and issubclass(value, util.BASESTRING):
+    elif isinstance(value, type) and issubclass(value, str):
        return String()
-    elif isinstance(value, util.BASESTRING):
+    elif isinstance(value, str):
        return String(value)
    elif isinstance(value, set):
        # convert to list to avoid hash related problems
        return Choice(list(value))
-    elif (SUPPORTS_ENUM and isinstance(value, type)
-            and issubclass(value, enum.Enum)):
+    elif isinstance(value, type) and issubclass(value, enum.Enum):
        return Choice(value)
    elif isinstance(value, list):
        return OneOf(value)
@ -725,7 +704,7 @@ def as_template(value):
        return Number()
    elif isinstance(value, float):
        return Number(value)
-    elif SUPPORTS_PATHLIB and isinstance(value, pathlib.PurePath):
+    elif isinstance(value, pathlib.PurePath):
        return Path(value)
    elif value is None:
        return Template(None)
--- a/libs/common/confuse/util.py
+++ b/libs/common/confuse/util.py
@ -1,5 +1,3 @@
-from __future__ import division, absolute_import, print_function
-
 import os
 import sys
 import argparse
@ -8,12 +6,6 @@ import platform
 import pkgutil


-PY3 = sys.version_info[0] == 3
-STRING = str if PY3 else unicode  # noqa: F821
-BASESTRING = str if PY3 else basestring  # noqa: F821
-NUMERIC_TYPES = (int, float) if PY3 else (int, float, long)  # noqa: F821
-
-
 UNIX_DIR_FALLBACK = '~/.config'
 WINDOWS_DIR_VAR = 'APPDATA'
 WINDOWS_DIR_FALLBACK = '~\\AppData\\Roaming'
@ -75,7 +67,7 @@ def build_dict(obj, sep='', keep_none=False):
        return obj

    # Get keys iterator
-    keys = obj.keys() if PY3 else obj.iterkeys()
+    keys = obj.keys()
    if sep:
        # Splitting keys by `sep` needs sorted keys to prevent parents
        # from clobbering children
--- a/libs/common/confuse/yaml_util.py
+++ b/libs/common/confuse/yaml_util.py
@ -1,9 +1,6 @@
-from __future__ import division, absolute_import, print_function
-
 from collections import OrderedDict
 import yaml
 from .exceptions import ConfigReadError
-from .util import BASESTRING

 # YAML loading.

@ -119,7 +116,7 @@ def parse_as_scalar(value, loader=Loader):
      - The empty string '' will return None
    """
    # We only deal with strings
-    if not isinstance(value, BASESTRING):
+    if not isinstance(value, str):
        return value
    try:
        loader = loader('')
--- a/libs/common/decorator.py
+++ b/libs/common/decorator.py
@ -1,6 +1,6 @@
 # #########################     LICENSE     ############################ #

-# Copyright (c) 2005-2018, Michele Simionato
+# Copyright (c) 2005-2021, Michele Simionato
 # All rights reserved.

 # Redistribution and use in source and binary forms, with or without
@ -28,55 +28,26 @@
 # DAMAGE.

 """
-Decorator module, see http://pypi.python.org/pypi/decorator
+Decorator module, see
+https://github.com/micheles/decorator/blob/master/docs/documentation.md
 for the documentation.
 """
-from __future__ import print_function
-
 import re
 import sys
 import inspect
 import operator
 import itertools
-import collections
-
-__version__ = '4.4.2'
-
-if sys.version_info >= (3,):
-    from inspect import getfullargspec
-
-    def get_init(cls):
-        return cls.__init__
-else:
-    FullArgSpec = collections.namedtuple(
-        'FullArgSpec', 'args varargs varkw defaults '
-        'kwonlyargs kwonlydefaults annotations')
-
-    def getfullargspec(f):
-        "A quick and dirty replacement for getfullargspec for Python 2.X"
-        return FullArgSpec._make(inspect.getargspec(f) + ([], None, {}))
-
-    def get_init(cls):
-        return cls.__init__.__func__
-
-try:
-    iscoroutinefunction = inspect.iscoroutinefunction
-except AttributeError:
-    # let's assume there are no coroutine functions in old Python
-    def iscoroutinefunction(f):
-        return False
-try:
-    from inspect import isgeneratorfunction
-except ImportError:
-    # assume no generator function in old Python versions
-    def isgeneratorfunction(caller):
-        return False
+from contextlib import _GeneratorContextManager
+from inspect import getfullargspec, iscoroutinefunction, isgeneratorfunction

+__version__ = '5.1.1'

 DEF = re.compile(r'\s*def\s*([_\w][_\w\d]*)\s*\(')
+POS = inspect.Parameter.POSITIONAL_OR_KEYWORD
+EMPTY = inspect.Parameter.empty


-# basic functionality
+# this is not used anymore in the core, but kept for backward compatibility
 class FunctionMaker(object):
    """
    An object with the ability to create functions with a given signature.
@ -100,7 +71,7 @@ class FunctionMaker(object):
                self.name = '_lambda_'
            self.doc = func.__doc__
            self.module = func.__module__
-            if inspect.isfunction(func):
+            if inspect.isroutine(func):
                argspec = getfullargspec(func)
                self.annotations = getattr(func, '__annotations__', {})
                for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs',
@ -143,7 +114,9 @@ class FunctionMaker(object):
            raise TypeError('You are decorating a non function: %s' % func)

    def update(self, func, **kw):
-        "Update the signature of func with the data in self"
+        """
+        Update the signature of func with the data in self
+        """
        func.__name__ = self.name
        func.__doc__ = getattr(self, 'doc', None)
        func.__dict__ = getattr(self, 'dict', {})
@ -160,7 +133,9 @@ class FunctionMaker(object):
        func.__dict__.update(kw)

    def make(self, src_templ, evaldict=None, addsource=False, **attrs):
-        "Make a new function from a given template and update the signature"
+        """
+        Make a new function from a given template and update the signature
+        """
        src = src_templ % vars(self)  # expand name and signature
        evaldict = evaldict or {}
        mo = DEF.search(src)
@ -221,106 +196,128 @@ class FunctionMaker(object):
        return self.make(body, evaldict, addsource, **attrs)


-def decorate(func, caller, extras=()):
+def fix(args, kwargs, sig):
    """
-    decorate(func, caller) decorates a function using a caller.
-    If the caller is a generator function, the resulting function
-    will be a generator function.
+    Fix args and kwargs to be consistent with the signature
    """
-    evaldict = dict(_call_=caller, _func_=func)
-    es = ''
-    for i, extra in enumerate(extras):
-        ex = '_e%d_' % i
-        evaldict[ex] = extra
-        es += ex + ', '
+    ba = sig.bind(*args, **kwargs)
+    ba.apply_defaults()  # needed for test_dan_schult
+    return ba.args, ba.kwargs

-    if '3.5' <= sys.version < '3.6':
-        # with Python 3.5 isgeneratorfunction returns True for all coroutines
-        # however we know that it is NOT possible to have a generator
-        # coroutine in python 3.5: PEP525 was not there yet
-        generatorcaller = isgeneratorfunction(
-            caller) and not iscoroutinefunction(caller)
+
+def decorate(func, caller, extras=(), kwsyntax=False):
+    """
+    Decorates a function/generator/coroutine using a caller.
+    If kwsyntax is True calling the decorated functions with keyword
+    syntax will pass the named arguments inside the ``kw`` dictionary,
+    even if such argument are positional, similarly to what functools.wraps
+    does. By default kwsyntax is False and the the arguments are untouched.
+    """
+    sig = inspect.signature(func)
+    if iscoroutinefunction(caller):
+        async def fun(*args, **kw):
+            if not kwsyntax:
+                args, kw = fix(args, kw, sig)
+            return await caller(func, *(extras + args), **kw)
+    elif isgeneratorfunction(caller):
+        def fun(*args, **kw):
+            if not kwsyntax:
+                args, kw = fix(args, kw, sig)
+            for res in caller(func, *(extras + args), **kw):
+                yield res
    else:
-        generatorcaller = isgeneratorfunction(caller)
-    if generatorcaller:
-        fun = FunctionMaker.create(
-            func, "for res in _call_(_func_, %s%%(shortsignature)s):\n"
-                  "    yield res" % es, evaldict, __wrapped__=func)
-    else:
-        fun = FunctionMaker.create(
-            func, "return _call_(_func_, %s%%(shortsignature)s)" % es,
-            evaldict, __wrapped__=func)
-    if hasattr(func, '__qualname__'):
-        fun.__qualname__ = func.__qualname__
+        def fun(*args, **kw):
+            if not kwsyntax:
+                args, kw = fix(args, kw, sig)
+            return caller(func, *(extras + args), **kw)
+    fun.__name__ = func.__name__
+    fun.__doc__ = func.__doc__
+    fun.__wrapped__ = func
+    fun.__signature__ = sig
+    fun.__qualname__ = func.__qualname__
+    # builtin functions like defaultdict.__setitem__ lack many attributes
+    try:
+        fun.__defaults__ = func.__defaults__
+    except AttributeError:
+        pass
+    try:
+        fun.__kwdefaults__ = func.__kwdefaults__
+    except AttributeError:
+        pass
+    try:
+        fun.__annotations__ = func.__annotations__
+    except AttributeError:
+        pass
+    try:
+        fun.__module__ = func.__module__
+    except AttributeError:
+        pass
+    try:
+        fun.__dict__.update(func.__dict__)
+    except AttributeError:
+        pass
    return fun


-def decorator(caller, _func=None):
-    """decorator(caller) converts a caller function into a decorator"""
+def decoratorx(caller):
+    """
+    A version of "decorator" implemented via "exec" and not via the
+    Signature object. Use this if you are want to preserve the `.__code__`
+    object properties (https://github.com/micheles/decorator/issues/129).
+    """
+    def dec(func):
+        return FunctionMaker.create(
+            func,
+            "return _call_(_func_, %(shortsignature)s)",
+            dict(_call_=caller, _func_=func),
+            __wrapped__=func, __qualname__=func.__qualname__)
+    return dec
+
+
+def decorator(caller, _func=None, kwsyntax=False):
+    """
+    decorator(caller) converts a caller function into a decorator
+    """
    if _func is not None:  # return a decorated function
        # this is obsolete behavior; you should use decorate instead
-        return decorate(_func, caller)
+        return decorate(_func, caller, (), kwsyntax)
    # else return a decorator function
-    defaultargs, defaults = '', ()
-    if inspect.isclass(caller):
-        name = caller.__name__.lower()
-        doc = 'decorator(%s) converts functions/generators into ' \
-            'factories of %s objects' % (caller.__name__, caller.__name__)
-    elif inspect.isfunction(caller):
-        if caller.__name__ == '<lambda>':
-            name = '_lambda_'
+    sig = inspect.signature(caller)
+    dec_params = [p for p in sig.parameters.values() if p.kind is POS]
+
+    def dec(func=None, *args, **kw):
+        na = len(args) + 1
+        extras = args + tuple(kw.get(p.name, p.default)
+                              for p in dec_params[na:]
+                              if p.default is not EMPTY)
+        if func is None:
+            return lambda func: decorate(func, caller, extras, kwsyntax)
        else:
-            name = caller.__name__
-        doc = caller.__doc__
-        nargs = caller.__code__.co_argcount
-        ndefs = len(caller.__defaults__ or ())
-        defaultargs = ', '.join(caller.__code__.co_varnames[nargs-ndefs:nargs])
-        if defaultargs:
-            defaultargs += ','
-        defaults = caller.__defaults__
-    else:  # assume caller is an object with a __call__ method
-        name = caller.__class__.__name__.lower()
-        doc = caller.__call__.__doc__
-    evaldict = dict(_call=caller, _decorate_=decorate)
-    dec = FunctionMaker.create(
-        '%s(func, %s)' % (name, defaultargs),
-        'if func is None: return lambda func:  _decorate_(func, _call, (%s))\n'
-        'return _decorate_(func, _call, (%s))' % (defaultargs, defaultargs),
-        evaldict, doc=doc, module=caller.__module__, __wrapped__=caller)
-    if defaults:
-        dec.__defaults__ = (None,) + defaults
+            return decorate(func, caller, extras, kwsyntax)
+    dec.__signature__ = sig.replace(parameters=dec_params)
+    dec.__name__ = caller.__name__
+    dec.__doc__ = caller.__doc__
+    dec.__wrapped__ = caller
+    dec.__qualname__ = caller.__qualname__
+    dec.__kwdefaults__ = getattr(caller, '__kwdefaults__', None)
+    dec.__dict__.update(caller.__dict__)
    return dec


 # ####################### contextmanager ####################### #

-try:  # Python >= 3.2
-    from contextlib import _GeneratorContextManager
-except ImportError:  # Python >= 2.5
-    from contextlib import GeneratorContextManager as _GeneratorContextManager
-

 class ContextManager(_GeneratorContextManager):
+    def __init__(self, g, *a, **k):
+        _GeneratorContextManager.__init__(self, g, a, k)
+
    def __call__(self, func):
-        """Context manager decorator"""
-        return FunctionMaker.create(
-            func, "with _self_: return _func_(%(shortsignature)s)",
-            dict(_self_=self, _func_=func), __wrapped__=func)
+        def caller(f, *a, **k):
+            with self.__class__(self.func, *self.args, **self.kwds):
+                return f(*a, **k)
+        return decorate(func, caller)


-init = getfullargspec(_GeneratorContextManager.__init__)
-n_args = len(init.args)
-if n_args == 2 and not init.varargs:  # (self, genobj) Python 2.7
-    def __init__(self, g, *a, **k):
-        return _GeneratorContextManager.__init__(self, g(*a, **k))
-    ContextManager.__init__ = __init__
-elif n_args == 2 and init.varargs:  # (self, gen, *a, **k) Python 3.4
-    pass
-elif n_args == 4:  # (self, gen, args, kwds) Python 3.5
-    def __init__(self, g, *a, **k):
-        return _GeneratorContextManager.__init__(self, g, a, k)
-    ContextManager.__init__ = __init__
-
 _contextmanager = decorator(ContextManager)


--- a/libs/common/distutils-precedence.pth
+++ b/libs/common/distutils-precedence.pth
@ -0,0 +1 @@
+import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim(); 
--- a/libs/common/dogpile/init.py
+++ b/libs/common/dogpile/init.py
@ -1,4 +1,4 @@
-__version__ = '0.9.2'
+__version__ = "1.1.8"

 from .lock import Lock  # noqa
 from .lock import NeedRegenerationException  # noqa
--- a/libs/common/dogpile/cache/api.py
+++ b/libs/common/dogpile/cache/api.py
@ -1,13 +1,19 @@
-import operator
-
-from ..util.compat import py3k
+import abc
+import pickle
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Mapping
+from typing import NamedTuple
+from typing import Optional
+from typing import Sequence
+from typing import Union


-class NoValue(object):
+class NoValue:
    """Describe a missing cache value.

-    The :attr:`.NO_VALUE` module global
-    should be used.
+    The :data:`.NO_VALUE` constant should be used.

    """

@ -22,51 +28,123 @@ class NoValue(object):
        """
        return "<dogpile.cache.api.NoValue object>"

-    if py3k:
-
-        def __bool__(self):  # pragma NO COVERAGE
-            return False
-
-    else:
-
-        def __nonzero__(self):  # pragma NO COVERAGE
-            return False
+    def __bool__(self):  # pragma NO COVERAGE
+        return False


 NO_VALUE = NoValue()
 """Value returned from ``get()`` that describes
 a  key not present."""

+MetaDataType = Mapping[str, Any]

-class CachedValue(tuple):
+
+KeyType = str
+"""A cache key."""
+
+ValuePayload = Any
+"""An object to be placed in the cache against a key."""
+
+
+KeyManglerType = Callable[[KeyType], KeyType]
+Serializer = Callable[[ValuePayload], bytes]
+Deserializer = Callable[[bytes], ValuePayload]
+
+
+class CacheMutex(abc.ABC):
+    """Describes a mutexing object with acquire and release methods.
+
+    This is an abstract base class; any object that has acquire/release
+    methods may be used.
+
+    .. versionadded:: 1.1
+
+
+    .. seealso::
+
+        :meth:`.CacheBackend.get_mutex` - the backend method that optionally
+        returns this locking object.
+
+    """
+
+    @abc.abstractmethod
+    def acquire(self, wait: bool = True) -> bool:
+        """Acquire the mutex.
+
+        :param wait: if True, block until available, else return True/False
+         immediately.
+
+        :return: True if the lock succeeded.
+
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def release(self) -> None:
+        """Release the mutex."""
+
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def locked(self) -> bool:
+        """Check if the mutex was acquired.
+
+        :return: true if the lock is acquired.
+
+        .. versionadded:: 1.1.2
+
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        return hasattr(C, "acquire") and hasattr(C, "release")
+
+
+class CachedValue(NamedTuple):
    """Represent a value stored in the cache.

    :class:`.CachedValue` is a two-tuple of
    ``(payload, metadata)``, where ``metadata``
    is dogpile.cache's tracking information (
-    currently the creation time).  The metadata
-    and tuple structure is pickleable, if
-    the backend requires serialization.
+    currently the creation time).

    """

-    payload = property(operator.itemgetter(0))
-    """Named accessor for the payload."""
+    payload: ValuePayload

-    metadata = property(operator.itemgetter(1))
-    """Named accessor for the dogpile.cache metadata dictionary."""
-
-    def __new__(cls, payload, metadata):
-        return tuple.__new__(cls, (payload, metadata))
-
-    def __reduce__(self):
-        return CachedValue, (self.payload, self.metadata)
+    metadata: MetaDataType


-class CacheBackend(object):
-    """Base class for backend implementations."""
+CacheReturnType = Union[CachedValue, NoValue]
+"""The non-serialized form of what may be returned from a backend
+get method.

-    key_mangler = None
+"""
+
+SerializedReturnType = Union[bytes, NoValue]
+"""the serialized form of what may be returned from a backend get method."""
+
+BackendFormatted = Union[CacheReturnType, SerializedReturnType]
+"""Describes the type returned from the :meth:`.CacheBackend.get` method."""
+
+BackendSetType = Union[CachedValue, bytes]
+"""Describes the value argument passed to the :meth:`.CacheBackend.set`
+method."""
+
+BackendArguments = Mapping[str, Any]
+
+
+class CacheBackend:
+    """Base class for backend implementations.
+
+    Backends which set and get Python object values should subclass this
+    backend.   For backends in which the value that's stored is ultimately
+    a stream of bytes, the :class:`.BytesBackend` should be used.
+
+    """
+
+    key_mangler: Optional[Callable[[KeyType], KeyType]] = None
    """Key mangling function.

    May be None, or otherwise declared
@ -74,7 +152,23 @@ class CacheBackend(object):

    """

-    def __init__(self, arguments):  # pragma NO COVERAGE
+    serializer: Union[None, Serializer] = None
+    """Serializer function that will be used by default if not overridden
+    by the region.
+
+    .. versionadded:: 1.1
+
+    """
+
+    deserializer: Union[None, Deserializer] = None
+    """deserializer function that will be used by default if not overridden
+    by the region.
+
+    .. versionadded:: 1.1
+
+    """
+
+    def __init__(self, arguments: BackendArguments):  # pragma NO COVERAGE
        """Construct a new :class:`.CacheBackend`.

        Subclasses should override this to
@ -97,10 +191,10 @@ class CacheBackend(object):
            )
        )

-    def has_lock_timeout(self):
+    def has_lock_timeout(self) -> bool:
        return False

-    def get_mutex(self, key):
+    def get_mutex(self, key: KeyType) -> Optional[CacheMutex]:
        """Return an optional mutexing object for the given key.

        This object need only provide an ``acquire()``
@ -133,48 +227,141 @@ class CacheBackend(object):
        """
        return None

-    def get(self, key):  # pragma NO COVERAGE
-        """Retrieve a value from the cache.
+    def get(self, key: KeyType) -> BackendFormatted:  # pragma NO COVERAGE
+        """Retrieve an optionally serialized value from the cache.

-        The returned value should be an instance of
-        :class:`.CachedValue`, or ``NO_VALUE`` if
-        not present.
+        :param key: String key that was passed to the :meth:`.CacheRegion.get`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :return: the Python object that corresponds to
+         what was established via the :meth:`.CacheBackend.set` method,
+         or the :data:`.NO_VALUE` constant if not present.
+
+        If a serializer is in use, this method will only be called if the
+        :meth:`.CacheBackend.get_serialized` method is not overridden.

        """
        raise NotImplementedError()

-    def get_multi(self, keys):  # pragma NO COVERAGE
-        """Retrieve multiple values from the cache.
+    def get_multi(
+        self, keys: Sequence[KeyType]
+    ) -> Sequence[BackendFormatted]:  # pragma NO COVERAGE
+        """Retrieve multiple optionally serialized values from the cache.

-        The returned value should be a list, corresponding
-        to the list of keys given.
+        :param keys: sequence of string keys that was passed to the
+         :meth:`.CacheRegion.get_multi` method, which will also be processed
+         by the "key mangling" function if one was present.
+
+        :return a list of values as would be returned
+         individually via the :meth:`.CacheBackend.get` method, corresponding
+         to the list of keys given.
+
+        If a serializer is in use, this method will only be called if the
+        :meth:`.CacheBackend.get_serialized_multi` method is not overridden.

        .. versionadded:: 0.5.0

        """
        raise NotImplementedError()

-    def set(self, key, value):  # pragma NO COVERAGE
-        """Set a value in the cache.
+    def get_serialized(self, key: KeyType) -> SerializedReturnType:
+        """Retrieve a serialized value from the cache.

-        The key will be whatever was passed
-        to the registry, processed by the
-        "key mangling" function, if any.
-        The value will always be an instance
-        of :class:`.CachedValue`.
+        :param key: String key that was passed to the :meth:`.CacheRegion.get`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :return: a bytes object, or :data:`.NO_VALUE`
+         constant if not present.
+
+        The default implementation of this method for :class:`.CacheBackend`
+        returns the value of the :meth:`.CacheBackend.get` method.
+
+        .. versionadded:: 1.1
+
+        .. seealso::
+
+            :class:`.BytesBackend`
+
+        """
+        return cast(SerializedReturnType, self.get(key))
+
+    def get_serialized_multi(
+        self, keys: Sequence[KeyType]
+    ) -> Sequence[SerializedReturnType]:  # pragma NO COVERAGE
+        """Retrieve multiple serialized values from the cache.
+
+        :param keys: sequence of string keys that was passed to the
+         :meth:`.CacheRegion.get_multi` method, which will also be processed
+         by the "key mangling" function if one was present.
+
+        :return: list of bytes objects
+
+        The default implementation of this method for :class:`.CacheBackend`
+        returns the value of the :meth:`.CacheBackend.get_multi` method.
+
+        .. versionadded:: 1.1
+
+        .. seealso::
+
+            :class:`.BytesBackend`
+
+        """
+        return cast(Sequence[SerializedReturnType], self.get_multi(keys))
+
+    def set(
+        self, key: KeyType, value: BackendSetType
+    ) -> None:  # pragma NO COVERAGE
+        """Set an optionally serialized value in the cache.
+
+        :param key: String key that was passed to the :meth:`.CacheRegion.set`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :param value: The optionally serialized :class:`.CachedValue` object.
+         May be an instance of :class:`.CachedValue` or a bytes object
+         depending on if a serializer is in use with the region and if the
+         :meth:`.CacheBackend.set_serialized` method is not overridden.
+
+        .. seealso::
+
+            :meth:`.CacheBackend.set_serialized`

        """
        raise NotImplementedError()

-    def set_multi(self, mapping):  # pragma NO COVERAGE
+    def set_serialized(
+        self, key: KeyType, value: bytes
+    ) -> None:  # pragma NO COVERAGE
+        """Set a serialized value in the cache.
+
+        :param key: String key that was passed to the :meth:`.CacheRegion.set`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :param value: a bytes object to be stored.
+
+        The default implementation of this method for :class:`.CacheBackend`
+        calls upon the :meth:`.CacheBackend.set` method.
+
+        .. versionadded:: 1.1
+
+        .. seealso::
+
+            :class:`.BytesBackend`
+
+        """
+        self.set(key, value)
+
+    def set_multi(
+        self, mapping: Mapping[KeyType, BackendSetType]
+    ) -> None:  # pragma NO COVERAGE
        """Set multiple values in the cache.

-        ``mapping`` is a dict in which
-        the key will be whatever was passed
-        to the registry, processed by the
-        "key mangling" function, if any.
-        The value will always be an instance
-        of :class:`.CachedValue`.
+        :param mapping: a dict in which the key will be whatever was passed to
+         the :meth:`.CacheRegion.set_multi` method, processed by the "key
+         mangling" function, if any.

        When implementing a new :class:`.CacheBackend` or cutomizing via
        :class:`.ProxyBackend`, be aware that when this method is invoked by
@ -184,17 +371,52 @@ class CacheBackend(object):
        -- that will have the undesirable effect of modifying the returned
        values as well.

+        If a serializer is in use, this method will only be called if the
+        :meth:`.CacheBackend.set_serialized_multi` method is not overridden.
+
+
        .. versionadded:: 0.5.0

        """
        raise NotImplementedError()

-    def delete(self, key):  # pragma NO COVERAGE
+    def set_serialized_multi(
+        self, mapping: Mapping[KeyType, bytes]
+    ) -> None:  # pragma NO COVERAGE
+        """Set multiple serialized values in the cache.
+
+        :param mapping: a dict in which the key will be whatever was passed to
+         the :meth:`.CacheRegion.set_multi` method, processed by the "key
+         mangling" function, if any.
+
+        When implementing a new :class:`.CacheBackend` or cutomizing via
+        :class:`.ProxyBackend`, be aware that when this method is invoked by
+        :meth:`.Region.get_or_create_multi`, the ``mapping`` values are the
+        same ones returned to the upstream caller. If the subclass alters the
+        values in any way, it must not do so 'in-place' on the ``mapping`` dict
+        -- that will have the undesirable effect of modifying the returned
+        values as well.
+
+        .. versionadded:: 1.1
+
+        The default implementation of this method for :class:`.CacheBackend`
+        calls upon the :meth:`.CacheBackend.set_multi` method.
+
+        .. seealso::
+
+            :class:`.BytesBackend`
+
+
+        """
+        self.set_multi(mapping)
+
+    def delete(self, key: KeyType) -> None:  # pragma NO COVERAGE
        """Delete a value from the cache.

-        The key will be whatever was passed
-        to the registry, processed by the
-        "key mangling" function, if any.
+        :param key: String key that was passed to the
+         :meth:`.CacheRegion.delete`
+         method, which will also be processed by the "key mangling" function
+         if one was present.

        The behavior here should be idempotent,
        that is, can be called any number of times
@ -203,12 +425,14 @@ class CacheBackend(object):
        """
        raise NotImplementedError()

-    def delete_multi(self, keys):  # pragma NO COVERAGE
+    def delete_multi(
+        self, keys: Sequence[KeyType]
+    ) -> None:  # pragma NO COVERAGE
        """Delete multiple values from the cache.

-        The key will be whatever was passed
-        to the registry, processed by the
-        "key mangling" function, if any.
+        :param keys: sequence of string keys that was passed to the
+         :meth:`.CacheRegion.delete_multi` method, which will also be processed
+         by the "key mangling" function if one was present.

        The behavior here should be idempotent,
        that is, can be called any number of times
@ -219,3 +443,95 @@ class CacheBackend(object):

        """
        raise NotImplementedError()
+
+
+class DefaultSerialization:
+    serializer: Union[None, Serializer] = staticmethod(  # type: ignore
+        pickle.dumps
+    )
+    deserializer: Union[None, Deserializer] = staticmethod(  # type: ignore
+        pickle.loads
+    )
+
+
+class BytesBackend(DefaultSerialization, CacheBackend):
+    """A cache backend that receives and returns series of bytes.
+
+    This backend only supports the "serialized" form of values; subclasses
+    should implement :meth:`.BytesBackend.get_serialized`,
+    :meth:`.BytesBackend.get_serialized_multi`,
+    :meth:`.BytesBackend.set_serialized`,
+    :meth:`.BytesBackend.set_serialized_multi`.
+
+    .. versionadded:: 1.1
+
+    """
+
+    def get_serialized(self, key: KeyType) -> SerializedReturnType:
+        """Retrieve a serialized value from the cache.
+
+        :param key: String key that was passed to the :meth:`.CacheRegion.get`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :return: a bytes object, or :data:`.NO_VALUE`
+         constant if not present.
+
+        .. versionadded:: 1.1
+
+        """
+        raise NotImplementedError()
+
+    def get_serialized_multi(
+        self, keys: Sequence[KeyType]
+    ) -> Sequence[SerializedReturnType]:  # pragma NO COVERAGE
+        """Retrieve multiple serialized values from the cache.
+
+        :param keys: sequence of string keys that was passed to the
+         :meth:`.CacheRegion.get_multi` method, which will also be processed
+         by the "key mangling" function if one was present.
+
+        :return: list of bytes objects
+
+        .. versionadded:: 1.1
+
+        """
+        raise NotImplementedError()
+
+    def set_serialized(
+        self, key: KeyType, value: bytes
+    ) -> None:  # pragma NO COVERAGE
+        """Set a serialized value in the cache.
+
+        :param key: String key that was passed to the :meth:`.CacheRegion.set`
+         method, which will also be processed by the "key mangling" function
+         if one was present.
+
+        :param value: a bytes object to be stored.
+
+        .. versionadded:: 1.1
+
+        """
+        raise NotImplementedError()
+
+    def set_serialized_multi(
+        self, mapping: Mapping[KeyType, bytes]
+    ) -> None:  # pragma NO COVERAGE
+        """Set multiple serialized values in the cache.
+
+        :param mapping: a dict in which the key will be whatever was passed to
+         the :meth:`.CacheRegion.set_multi` method, processed by the "key
+         mangling" function, if any.
+
+        When implementing a new :class:`.CacheBackend` or cutomizing via
+        :class:`.ProxyBackend`, be aware that when this method is invoked by
+        :meth:`.Region.get_or_create_multi`, the ``mapping`` values are the
+        same ones returned to the upstream caller. If the subclass alters the
+        values in any way, it must not do so 'in-place' on the ``mapping`` dict
+        -- that will have the undesirable effect of modifying the returned
+        values as well.
+
+        .. versionadded:: 1.1
+
+        """
+        raise NotImplementedError()
--- a/libs/common/dogpile/cache/backends/init.py
+++ b/libs/common/dogpile/cache/backends/init.py
@ -24,6 +24,11 @@ register_backend(
    "dogpile.cache.backends.memcached",
    "MemcachedBackend",
 )
+register_backend(
+    "dogpile.cache.pymemcache",
+    "dogpile.cache.backends.memcached",
+    "PyMemcacheBackend",
+)
 register_backend(
    "dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend"
 )
@ -35,3 +40,8 @@ register_backend(
 register_backend(
    "dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend"
 )
+register_backend(
+    "dogpile.cache.redis_sentinel",
+    "dogpile.cache.backends.redis",
+    "RedisSentinelBackend",
+)
--- a/libs/common/dogpile/cache/backends/file.py
+++ b/libs/common/dogpile/cache/backends/file.py
@ -9,17 +9,18 @@ Provides backends that deal with local filesystem access.
 from __future__ import with_statement

 from contextlib import contextmanager
+import dbm
 import os
+import threading

-from ..api import CacheBackend
+from ..api import BytesBackend
 from ..api import NO_VALUE
 from ... import util
-from ...util import compat

 __all__ = ["DBMBackend", "FileLock", "AbstractFileLock"]


-class DBMBackend(CacheBackend):
+class DBMBackend(BytesBackend):
    """A file-backend using a dbm file to store keys.

    Basic usage::
@ -156,12 +157,6 @@ class DBMBackend(CacheBackend):
            util.KeyReentrantMutex.factory,
        )

-        # TODO: make this configurable
-        if compat.py3k:
-            import dbm
-        else:
-            import anydbm as dbm
-        self.dbmmodule = dbm
        self._init_dbm_file()

    def _init_lock(self, argument, suffix, basedir, basefile, wrapper=None):
@ -185,7 +180,7 @@ class DBMBackend(CacheBackend):
                    exists = True
                    break
        if not exists:
-            fh = self.dbmmodule.open(self.filename, "c")
+            fh = dbm.open(self.filename, "c")
            fh.close()

    def get_mutex(self, key):
@ -215,57 +210,50 @@ class DBMBackend(CacheBackend):
    @contextmanager
    def _dbm_file(self, write):
        with self._use_rw_lock(write):
-            dbm = self.dbmmodule.open(self.filename, "w" if write else "r")
-            yield dbm
-            dbm.close()
+            with dbm.open(self.filename, "w" if write else "r") as dbm_obj:
+                yield dbm_obj

-    def get(self, key):
-        with self._dbm_file(False) as dbm:
-            if hasattr(dbm, "get"):
-                value = dbm.get(key, NO_VALUE)
+    def get_serialized(self, key):
+        with self._dbm_file(False) as dbm_obj:
+            if hasattr(dbm_obj, "get"):
+                value = dbm_obj.get(key, NO_VALUE)
            else:
                # gdbm objects lack a .get method
                try:
-                    value = dbm[key]
+                    value = dbm_obj[key]
                except KeyError:
                    value = NO_VALUE
-            if value is not NO_VALUE:
-                value = compat.pickle.loads(value)
            return value

-    def get_multi(self, keys):
-        return [self.get(key) for key in keys]
+    def get_serialized_multi(self, keys):
+        return [self.get_serialized(key) for key in keys]

-    def set(self, key, value):
-        with self._dbm_file(True) as dbm:
-            dbm[key] = compat.pickle.dumps(
-                value, compat.pickle.HIGHEST_PROTOCOL
-            )
+    def set_serialized(self, key, value):
+        with self._dbm_file(True) as dbm_obj:
+            dbm_obj[key] = value

-    def set_multi(self, mapping):
-        with self._dbm_file(True) as dbm:
+    def set_serialized_multi(self, mapping):
+        with self._dbm_file(True) as dbm_obj:
            for key, value in mapping.items():
-                dbm[key] = compat.pickle.dumps(
-                    value, compat.pickle.HIGHEST_PROTOCOL
-                )
+                dbm_obj[key] = value

    def delete(self, key):
-        with self._dbm_file(True) as dbm:
+        with self._dbm_file(True) as dbm_obj:
            try:
-                del dbm[key]
+                del dbm_obj[key]
            except KeyError:
                pass

    def delete_multi(self, keys):
-        with self._dbm_file(True) as dbm:
+        with self._dbm_file(True) as dbm_obj:
            for key in keys:
                try:
-                    del dbm[key]
+                    del dbm_obj[key]
                except KeyError:
                    pass


-class AbstractFileLock(object):
+class AbstractFileLock:
    """Coordinate read/write access to a file.

    typically is a file-based lock but doesn't necessarily have to be.
@ -397,7 +385,7 @@ class FileLock(AbstractFileLock):
    """

    def __init__(self, filename):
-        self._filedescriptor = compat.threading.local()
+        self._filedescriptor = threading.local()
        self.filename = filename

    @util.memoized_property
--- a/libs/common/dogpile/cache/backends/memcached.py
+++ b/libs/common/dogpile/cache/backends/memcached.py
@ -7,29 +7,42 @@ Provides backends for talking to `memcached <http://memcached.org>`_.
 """

 import random
+import threading
 import time
+import typing
+from typing import Any
+from typing import Mapping
+import warnings

 from ..api import CacheBackend
 from ..api import NO_VALUE
 from ... import util
-from ...util import compat
+
+
+if typing.TYPE_CHECKING:
+    import bmemcached
+    import memcache
+    import pylibmc
+    import pymemcache
+else:
+    # delayed import
+    bmemcached = None  # noqa F811
+    memcache = None  # noqa F811
+    pylibmc = None  # noqa F811
+    pymemcache = None  # noqa F811

 __all__ = (
    "GenericMemcachedBackend",
    "MemcachedBackend",
    "PylibmcBackend",
+    "PyMemcacheBackend",
    "BMemcachedBackend",
    "MemcachedLock",
 )


 class MemcachedLock(object):
-    """Simple distributed lock using memcached.
-
-    This is an adaptation of the lock featured at
-    http://amix.dk/blog/post/19386
-
-    """
+    """Simple distributed lock using memcached."""

    def __init__(self, client_fn, key, timeout=0):
        self.client_fn = client_fn
@ -45,11 +58,15 @@ class MemcachedLock(object):
            elif not wait:
                return False
            else:
-                sleep_time = (((i + 1) * random.random()) + 2 ** i) / 2.5
+                sleep_time = (((i + 1) * random.random()) + 2**i) / 2.5
                time.sleep(sleep_time)
            if i < 15:
                i += 1

+    def locked(self):
+        client = self.client_fn()
+        return client.get(self.key) is not None
+
    def release(self):
        client = self.client_fn()
        client.delete(self.key)
@ -107,10 +124,17 @@ class GenericMemcachedBackend(CacheBackend):

    """

-    set_arguments = {}
+    set_arguments: Mapping[str, Any] = {}
    """Additional arguments which will be passed
    to the :meth:`set` method."""

+    # No need to override serializer, as all the memcached libraries
+    # handles that themselves. Still, we support customizing the
+    # serializer/deserializer to use better default pickle protocol
+    # or completely different serialization mechanism
+    serializer = None
+    deserializer = None
+
    def __init__(self, arguments):
        self._imports()
        # using a plain threading.local here.   threading.local
@ -138,7 +162,7 @@ class GenericMemcachedBackend(CacheBackend):
    def _clients(self):
        backend = self

-        class ClientPool(compat.threading.local):
+        class ClientPool(threading.local):
            def __init__(self):
                self.memcached = backend._create_client()

@ -173,12 +197,17 @@ class GenericMemcachedBackend(CacheBackend):

    def get_multi(self, keys):
        values = self.client.get_multi(keys)
-        return [NO_VALUE if key not in values else values[key] for key in keys]
+
+        return [
+            NO_VALUE if val is None else val
+            for val in [values.get(key, NO_VALUE) for key in keys]
+        ]

    def set(self, key, value):
        self.client.set(key, value, **self.set_arguments)

    def set_multi(self, mapping):
+        mapping = {key: value for key, value in mapping.items()}
        self.client.set_multi(mapping, **self.set_arguments)

    def delete(self, key):
@ -188,10 +217,9 @@ class GenericMemcachedBackend(CacheBackend):
        self.client.delete_multi(keys)


-class MemcacheArgs(object):
+class MemcacheArgs(GenericMemcachedBackend):
    """Mixin which provides support for the 'time' argument to set(),
    'min_compress_len' to other methods.
-
    """

    def __init__(self, arguments):
@ -207,9 +235,6 @@ class MemcacheArgs(object):
        super(MemcacheArgs, self).__init__(arguments)


-pylibmc = None
-
-
 class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
    """A backend for the
    `pylibmc <http://sendapatch.se/projects/pylibmc/index.html>`_
@ -259,9 +284,6 @@ class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
        )


-memcache = None
-
-
 class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
    """A backend using the standard
    `Python-memcached <http://www.tummy.com/Community/software/\
@ -280,17 +302,39 @@ class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
            }
        )

+    :param dead_retry: Number of seconds memcached server is considered dead
+     before it is tried again. Will be passed to ``memcache.Client``
+     as the ``dead_retry`` parameter.
+
+     .. versionchanged:: 1.1.8  Moved the ``dead_retry`` argument which was
+        erroneously added to "set_parameters" to
+        be part of the Memcached connection arguments.
+
+    :param socket_timeout: Timeout in seconds for every call to a server.
+      Will be passed to ``memcache.Client`` as the ``socket_timeout``
+      parameter.
+
+      .. versionchanged:: 1.1.8  Moved the ``socket_timeout`` argument which
+         was erroneously added to "set_parameters"
+         to be part of the Memcached connection arguments.
+
    """

+    def __init__(self, arguments):
+        self.dead_retry = arguments.get("dead_retry", 30)
+        self.socket_timeout = arguments.get("socket_timeout", 3)
+        super(MemcachedBackend, self).__init__(arguments)
+
    def _imports(self):
        global memcache
        import memcache  # noqa

    def _create_client(self):
-        return memcache.Client(self.url)
-
-
-bmemcached = None
+        return memcache.Client(
+            self.url,
+            dead_retry=self.dead_retry,
+            socket_timeout=self.socket_timeout,
+        )


 class BMemcachedBackend(GenericMemcachedBackend):
@ -299,9 +343,11 @@ class BMemcachedBackend(GenericMemcachedBackend):
    python-binary-memcached>`_
    memcached client.

-    This is a pure Python memcached client which
-    includes the ability to authenticate with a memcached
-    server using SASL.
+    This is a pure Python memcached client which includes
+    security features like SASL and SSL/TLS.
+
+    SASL is a standard for adding authentication mechanisms
+    to protocols in a way that is protocol independent.

    A typical configuration using username/password::

@ -317,6 +363,25 @@ class BMemcachedBackend(GenericMemcachedBackend):
            }
        )

+    A typical configuration using tls_context::
+
+        import ssl
+        from dogpile.cache import make_region
+
+        ctx = ssl.create_default_context(cafile="/path/to/my-ca.pem")
+
+        region = make_region().configure(
+            'dogpile.cache.bmemcached',
+            expiration_time = 3600,
+            arguments = {
+                'url':["127.0.0.1"],
+                'tls_context':ctx,
+            }
+        )
+
+    For advanced ways to configure TLS creating a more complex
+    tls_context visit https://docs.python.org/3/library/ssl.html
+
    Arguments which can be passed to the ``arguments``
    dictionary include:

@ -324,12 +389,17 @@ class BMemcachedBackend(GenericMemcachedBackend):
     SASL authentication.
    :param password: optional password, will be used for
     SASL authentication.
+    :param tls_context: optional TLS context, will be used for
+     TLS connections.
+
+     .. versionadded:: 1.0.2

    """

    def __init__(self, arguments):
        self.username = arguments.get("username", None)
        self.password = arguments.get("password", None)
+        self.tls_context = arguments.get("tls_context", None)
        super(BMemcachedBackend, self).__init__(arguments)

    def _imports(self):
@ -355,10 +425,215 @@ class BMemcachedBackend(GenericMemcachedBackend):

    def _create_client(self):
        return self.Client(
-            self.url, username=self.username, password=self.password
+            self.url,
+            username=self.username,
+            password=self.password,
+            tls_context=self.tls_context,
        )

    def delete_multi(self, keys):
        """python-binary-memcached api does not implements delete_multi"""
        for key in keys:
            self.delete(key)
+
+
+class PyMemcacheBackend(GenericMemcachedBackend):
+    """A backend for the
+    `pymemcache <https://github.com/pinterest/pymemcache>`_
+    memcached client.
+
+    A comprehensive, fast, pure Python memcached client
+
+    .. versionadded:: 1.1.2
+
+    pymemcache supports the following features:
+
+    * Complete implementation of the memcached text protocol.
+    * Configurable timeouts for socket connect and send/recv calls.
+    * Access to the "noreply" flag, which can significantly increase
+      the speed of writes.
+    * Flexible, simple approach to serialization and deserialization.
+    * The (optional) ability to treat network and memcached errors as
+      cache misses.
+
+    dogpile.cache uses the ``HashClient`` from pymemcache in order to reduce
+    API differences when compared to other memcached client drivers.
+    This allows the user to provide a single server or a list of memcached
+    servers.
+
+    Arguments which can be passed to the ``arguments``
+    dictionary include:
+
+    :param tls_context: optional TLS context, will be used for
+     TLS connections.
+
+     A typical configuration using tls_context::
+
+        import ssl
+        from dogpile.cache import make_region
+
+        ctx = ssl.create_default_context(cafile="/path/to/my-ca.pem")
+
+        region = make_region().configure(
+            'dogpile.cache.pymemcache',
+            expiration_time = 3600,
+            arguments = {
+                'url':["127.0.0.1"],
+                'tls_context':ctx,
+            }
+        )
+
+     .. seealso::
+
+        `<https://docs.python.org/3/library/ssl.html>`_ - additional TLS
+        documentation.
+
+    :param serde: optional "serde". Defaults to
+     ``pymemcache.serde.pickle_serde``.
+
+    :param default_noreply:  defaults to False.  When set to True this flag
+     enables the pymemcache "noreply" feature.  See the pymemcache
+     documentation for further details.
+
+    :param socket_keepalive: optional socket keepalive, will be used for
+     TCP keepalive configuration.  Use of this parameter requires pymemcache
+     3.5.0 or greater.  This parameter
+     accepts a
+     `pymemcache.client.base.KeepAliveOpts
+     <https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html#pymemcache.client.base.KeepaliveOpts>`_
+     object.
+
+     A typical configuration using ``socket_keepalive``::
+
+        from pymemcache import KeepaliveOpts
+        from dogpile.cache import make_region
+
+        # Using the default keepalive configuration
+        socket_keepalive = KeepaliveOpts()
+
+        region = make_region().configure(
+            'dogpile.cache.pymemcache',
+            expiration_time = 3600,
+            arguments = {
+                'url':["127.0.0.1"],
+                'socket_keepalive': socket_keepalive
+            }
+        )
+
+     .. versionadded:: 1.1.4 - added support for ``socket_keepalive``.
+
+    :param enable_retry_client: optional flag to enable retry client
+     mechanisms to handle failure.  Defaults to False.  When set to ``True``,
+     the :paramref:`.PyMemcacheBackend.retry_attempts` parameter must also
+     be set, along with optional parameters
+     :paramref:`.PyMemcacheBackend.retry_delay`.
+     :paramref:`.PyMemcacheBackend.retry_for`,
+     :paramref:`.PyMemcacheBackend.do_not_retry_for`.
+
+     .. seealso::
+
+        `<https://pymemcache.readthedocs.io/en/latest/getting_started.html#using-the-built-in-retrying-mechanism>`_ -
+        in the pymemcache documentation
+
+     .. versionadded:: 1.1.4
+
+    :param retry_attempts: how many times to attempt an action with
+     pymemcache's retrying wrapper before failing. Must be 1 or above.
+     Defaults to None.
+
+     .. versionadded:: 1.1.4
+
+    :param retry_delay: optional int|float, how many seconds to sleep between
+     each attempt. Used by the retry wrapper. Defaults to None.
+
+     .. versionadded:: 1.1.4
+
+    :param retry_for: optional None|tuple|set|list, what exceptions to
+     allow retries for. Will allow retries for all exceptions if None.
+     Example: ``(MemcacheClientError, MemcacheUnexpectedCloseError)``
+     Accepts any class that is a subclass of Exception.  Defaults to None.
+
+     .. versionadded:: 1.1.4
+
+    :param do_not_retry_for: optional None|tuple|set|list, what
+     exceptions should be retried. Will not block retries for any Exception if
+     None. Example: ``(IOError, MemcacheIllegalInputError)``
+     Accepts any class that is a subclass of Exception. Defaults to None.
+
+     .. versionadded:: 1.1.4
+
+    :param hashclient_retry_attempts: Amount of times a client should be tried
+     before it is marked dead and removed from the pool in the HashClient's
+     internal mechanisms.
+
+     .. versionadded:: 1.1.5
+
+    :param hashclient_retry_timeout: Time in seconds that should pass between
+     retry attempts in the HashClient's internal mechanisms.
+
+     .. versionadded:: 1.1.5
+
+    :param dead_timeout: Time in seconds before attempting to add a node
+     back in the pool in the HashClient's internal mechanisms.
+
+     .. versionadded:: 1.1.5
+
+    """  # noqa E501
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+
+        self.serde = arguments.get("serde", pymemcache.serde.pickle_serde)
+        self.default_noreply = arguments.get("default_noreply", False)
+        self.tls_context = arguments.get("tls_context", None)
+        self.socket_keepalive = arguments.get("socket_keepalive", None)
+        self.enable_retry_client = arguments.get("enable_retry_client", False)
+        self.retry_attempts = arguments.get("retry_attempts", None)
+        self.retry_delay = arguments.get("retry_delay", None)
+        self.retry_for = arguments.get("retry_for", None)
+        self.do_not_retry_for = arguments.get("do_not_retry_for", None)
+        self.hashclient_retry_attempts = arguments.get(
+            "hashclient_retry_attempts", 2
+        )
+        self.hashclient_retry_timeout = arguments.get(
+            "hashclient_retry_timeout", 1
+        )
+        self.dead_timeout = arguments.get("hashclient_dead_timeout", 60)
+        if (
+            self.retry_delay is not None
+            or self.retry_attempts is not None
+            or self.retry_for is not None
+            or self.do_not_retry_for is not None
+        ) and not self.enable_retry_client:
+            warnings.warn(
+                "enable_retry_client is not set; retry options "
+                "will be ignored"
+            )
+
+    def _imports(self):
+        global pymemcache
+        import pymemcache
+
+    def _create_client(self):
+        _kwargs = {
+            "serde": self.serde,
+            "default_noreply": self.default_noreply,
+            "tls_context": self.tls_context,
+            "retry_attempts": self.hashclient_retry_attempts,
+            "retry_timeout": self.hashclient_retry_timeout,
+            "dead_timeout": self.dead_timeout,
+        }
+        if self.socket_keepalive is not None:
+            _kwargs.update({"socket_keepalive": self.socket_keepalive})
+
+        client = pymemcache.client.hash.HashClient(self.url, **_kwargs)
+        if self.enable_retry_client:
+            return pymemcache.client.retrying.RetryingClient(
+                client,
+                attempts=self.retry_attempts,
+                retry_delay=self.retry_delay,
+                retry_for=self.retry_for,
+                do_not_retry_for=self.do_not_retry_for,
+            )
+
+        return client
--- a/libs/common/dogpile/cache/backends/memory.py
+++ b/libs/common/dogpile/cache/backends/memory.py
@ -10,9 +10,10 @@ places the value as given into the dictionary.

 """

+
 from ..api import CacheBackend
+from ..api import DefaultSerialization
 from ..api import NO_VALUE
-from ...util.compat import pickle


 class MemoryBackend(CacheBackend):
@ -49,36 +50,20 @@ class MemoryBackend(CacheBackend):

    """

-    pickle_values = False
-
    def __init__(self, arguments):
        self._cache = arguments.pop("cache_dict", {})

    def get(self, key):
-        value = self._cache.get(key, NO_VALUE)
-        if value is not NO_VALUE and self.pickle_values:
-            value = pickle.loads(value)
-        return value
+        return self._cache.get(key, NO_VALUE)

    def get_multi(self, keys):
-        ret = [self._cache.get(key, NO_VALUE) for key in keys]
-        if self.pickle_values:
-            ret = [
-                pickle.loads(value) if value is not NO_VALUE else value
-                for value in ret
-            ]
-        return ret
+        return [self._cache.get(key, NO_VALUE) for key in keys]

    def set(self, key, value):
-        if self.pickle_values:
-            value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
        self._cache[key] = value

    def set_multi(self, mapping):
-        pickle_values = self.pickle_values
        for key, value in mapping.items():
-            if pickle_values:
-                value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
            self._cache[key] = value

    def delete(self, key):
@ -89,7 +74,7 @@ class MemoryBackend(CacheBackend):
            self._cache.pop(key, None)


-class MemoryPickleBackend(MemoryBackend):
+class MemoryPickleBackend(DefaultSerialization, MemoryBackend):
    """A backend that uses a plain dictionary, but serializes objects on
    :meth:`.MemoryBackend.set` and deserializes :meth:`.MemoryBackend.get`.

@ -120,5 +105,3 @@ class MemoryPickleBackend(MemoryBackend):
    .. versionadded:: 0.5.3

    """
-
-    pickle_values = True
--- a/libs/common/dogpile/cache/backends/null.py
+++ b/libs/common/dogpile/cache/backends/null.py
@ -24,6 +24,9 @@ class NullLock(object):
    def release(self):
        pass

+    def locked(self):
+        return False
+

 class NullBackend(CacheBackend):
    """A "null" backend that effectively disables all cache operations.
--- a/libs/common/dogpile/cache/backends/redis.py
+++ b/libs/common/dogpile/cache/backends/redis.py
@ -8,20 +8,23 @@ Provides backends for talking to `Redis <http://redis.io>`_.

 from __future__ import absolute_import

+import typing
 import warnings

-from ..api import CacheBackend
+from ..api import BytesBackend
 from ..api import NO_VALUE
-from ...util.compat import pickle
-from ...util.compat import u

-redis = None
+if typing.TYPE_CHECKING:
+    import redis
+else:
+    # delayed import
+    redis = None  # noqa F811

-__all__ = ("RedisBackend",)
+__all__ = ("RedisBackend", "RedisSentinelBackend")


-class RedisBackend(CacheBackend):
-    """A `Redis <http://redis.io/>`_ backend, using the
+class RedisBackend(BytesBackend):
+    r"""A `Redis <http://redis.io/>`_ backend, using the
    `redis-py <http://pypi.python.org/pypi/redis/>`_ backend.

    Example configuration::
@ -46,14 +49,10 @@ class RedisBackend(CacheBackend):
    :param url: string. If provided, will override separate host/port/db
     params.  The format is that accepted by ``StrictRedis.from_url()``.

-     .. versionadded:: 0.4.1
-
    :param host: string, default is ``localhost``.

    :param password: string, default is no password.

-     .. versionadded:: 0.4.1
-
    :param port: integer, default is ``6379``.

    :param db: integer, default is ``0``.
@ -71,33 +70,31 @@ class RedisBackend(CacheBackend):
     Redis should expire it.  This argument is only valid when
     ``distributed_lock`` is ``True``.

-     .. versionadded:: 0.5.0
-
    :param socket_timeout: float, seconds for socket timeout.
     Default is None (no timeout).

-     .. versionadded:: 0.5.4
-
    :param lock_sleep: integer, number of seconds to sleep when failed to
     acquire a lock.  This argument is only valid when
     ``distributed_lock`` is ``True``.

-     .. versionadded:: 0.5.0
-
    :param connection_pool: ``redis.ConnectionPool`` object.  If provided,
     this object supersedes other connection arguments passed to the
     ``redis.StrictRedis`` instance, including url and/or host as well as
     socket_timeout, and will be passed to ``redis.StrictRedis`` as the
     source of connectivity.

-     .. versionadded:: 0.5.4
-
    :param thread_local_lock: bool, whether a thread-local Redis lock object
     should be used. This is the default, but is not compatible with
     asynchronous runners, as they run in a different thread than the one
     used to create the lock.

-     .. versionadded:: 0.9.1
+    :param connection_kwargs: dict, additional keyword arguments are passed
+     along to the
+     ``StrictRedis.from_url()`` method or ``StrictRedis()`` constructor
+     directly, including parameters like ``ssl``, ``ssl_certfile``,
+     ``charset``, etc.
+
+     .. versionadded:: 1.1.6  Added ``connection_kwargs`` parameter.

    """

@ -109,12 +106,12 @@ class RedisBackend(CacheBackend):
        self.password = arguments.pop("password", None)
        self.port = arguments.pop("port", 6379)
        self.db = arguments.pop("db", 0)
-        self.distributed_lock = arguments.get("distributed_lock", False)
+        self.distributed_lock = arguments.pop("distributed_lock", False)
        self.socket_timeout = arguments.pop("socket_timeout", None)
-
-        self.lock_timeout = arguments.get("lock_timeout", None)
-        self.lock_sleep = arguments.get("lock_sleep", 0.1)
-        self.thread_local_lock = arguments.get("thread_local_lock", True)
+        self.lock_timeout = arguments.pop("lock_timeout", None)
+        self.lock_sleep = arguments.pop("lock_sleep", 0.1)
+        self.thread_local_lock = arguments.pop("thread_local_lock", True)
+        self.connection_kwargs = arguments.pop("connection_kwargs", {})

        if self.distributed_lock and self.thread_local_lock:
            warnings.warn(
@ -123,8 +120,8 @@ class RedisBackend(CacheBackend):
            )

        self.redis_expiration_time = arguments.pop("redis_expiration_time", 0)
-        self.connection_pool = arguments.get("connection_pool", None)
-        self.client = self._create_client()
+        self.connection_pool = arguments.pop("connection_pool", None)
+        self._create_client()

    def _imports(self):
        # defer imports until backend is used
@ -136,73 +133,207 @@ class RedisBackend(CacheBackend):
            # the connection pool already has all other connection
            # options present within, so here we disregard socket_timeout
            # and others.
-            return redis.StrictRedis(connection_pool=self.connection_pool)
-
-        args = {}
-        if self.socket_timeout:
-            args["socket_timeout"] = self.socket_timeout
-
-        if self.url is not None:
-            args.update(url=self.url)
-            return redis.StrictRedis.from_url(**args)
-        else:
-            args.update(
-                host=self.host,
-                password=self.password,
-                port=self.port,
-                db=self.db,
+            self.writer_client = redis.StrictRedis(
+                connection_pool=self.connection_pool
            )
-            return redis.StrictRedis(**args)
+            self.reader_client = self.writer_client
+        else:
+            args = {}
+            args.update(self.connection_kwargs)
+            if self.socket_timeout:
+                args["socket_timeout"] = self.socket_timeout
+
+            if self.url is not None:
+                args.update(url=self.url)
+                self.writer_client = redis.StrictRedis.from_url(**args)
+                self.reader_client = self.writer_client
+            else:
+                args.update(
+                    host=self.host,
+                    password=self.password,
+                    port=self.port,
+                    db=self.db,
+                )
+                self.writer_client = redis.StrictRedis(**args)
+                self.reader_client = self.writer_client

    def get_mutex(self, key):
        if self.distributed_lock:
-            return self.client.lock(
-                u("_lock{0}").format(key),
-                timeout=self.lock_timeout,
-                sleep=self.lock_sleep,
-                thread_local=self.thread_local_lock,
+            return _RedisLockWrapper(
+                self.writer_client.lock(
+                    "_lock{0}".format(key),
+                    timeout=self.lock_timeout,
+                    sleep=self.lock_sleep,
+                    thread_local=self.thread_local_lock,
+                )
            )
        else:
            return None

-    def get(self, key):
-        value = self.client.get(key)
+    def get_serialized(self, key):
+        value = self.reader_client.get(key)
        if value is None:
            return NO_VALUE
-        return pickle.loads(value)
+        return value

-    def get_multi(self, keys):
+    def get_serialized_multi(self, keys):
        if not keys:
            return []
-        values = self.client.mget(keys)
-        return [pickle.loads(v) if v is not None else NO_VALUE for v in values]
+        values = self.reader_client.mget(keys)
+        return [v if v is not None else NO_VALUE for v in values]

-    def set(self, key, value):
+    def set_serialized(self, key, value):
        if self.redis_expiration_time:
-            self.client.setex(
-                key,
-                self.redis_expiration_time,
-                pickle.dumps(value, pickle.HIGHEST_PROTOCOL),
-            )
+            self.writer_client.setex(key, self.redis_expiration_time, value)
        else:
-            self.client.set(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
-
-    def set_multi(self, mapping):
-        mapping = dict(
-            (k, pickle.dumps(v, pickle.HIGHEST_PROTOCOL))
-            for k, v in mapping.items()
-        )
+            self.writer_client.set(key, value)

+    def set_serialized_multi(self, mapping):
        if not self.redis_expiration_time:
-            self.client.mset(mapping)
+            self.writer_client.mset(mapping)
        else:
-            pipe = self.client.pipeline()
+            pipe = self.writer_client.pipeline()
            for key, value in mapping.items():
                pipe.setex(key, self.redis_expiration_time, value)
            pipe.execute()

    def delete(self, key):
-        self.client.delete(key)
+        self.writer_client.delete(key)

    def delete_multi(self, keys):
-        self.client.delete(*keys)
+        self.writer_client.delete(*keys)
+
+
+class _RedisLockWrapper:
+    __slots__ = ("mutex", "__weakref__")
+
+    def __init__(self, mutex: typing.Any):
+        self.mutex = mutex
+
+    def acquire(self, wait: bool = True) -> typing.Any:
+        return self.mutex.acquire(blocking=wait)
+
+    def release(self) -> typing.Any:
+        return self.mutex.release()
+
+    def locked(self) -> bool:
+        return self.mutex.locked()  # type: ignore
+
+
+class RedisSentinelBackend(RedisBackend):
+    """A `Redis <http://redis.io/>`_ backend, using the
+    `redis-py <http://pypi.python.org/pypi/redis/>`_ backend.
+    It will use the Sentinel of a Redis cluster.
+
+    .. versionadded:: 1.0.0
+
+    Example configuration::
+
+        from dogpile.cache import make_region
+
+        region = make_region().configure(
+            'dogpile.cache.redis_sentinel',
+            arguments = {
+                'sentinels': [
+                    ['redis_sentinel_1', 26379],
+                    ['redis_sentinel_2', 26379]
+                ],
+                'db': 0,
+                'redis_expiration_time': 60*60*2,   # 2 hours
+                'distributed_lock': True,
+                'thread_local_lock': False
+            }
+        )
+
+
+    Arguments accepted in the arguments dictionary:
+
+    :param db: integer, default is ``0``.
+
+    :param redis_expiration_time: integer, number of seconds after setting
+     a value that Redis should expire it.  This should be larger than dogpile's
+     cache expiration.  By default no expiration is set.
+
+    :param distributed_lock: boolean, when True, will use a
+     redis-lock as the dogpile lock. Use this when multiple processes will be
+     talking to the same redis instance. When False, dogpile will
+     coordinate on a regular threading mutex, Default is True.
+
+    :param lock_timeout: integer, number of seconds after acquiring a lock that
+     Redis should expire it.  This argument is only valid when
+     ``distributed_lock`` is ``True``.
+
+    :param socket_timeout: float, seconds for socket timeout.
+     Default is None (no timeout).
+
+    :param sentinels: is a list of sentinel nodes. Each node is represented by
+     a pair (hostname, port).
+     Default is None (not in sentinel mode).
+
+    :param service_name: str, the service name.
+     Default is 'mymaster'.
+
+    :param sentinel_kwargs: is a dictionary of connection arguments used when
+     connecting to sentinel instances. Any argument that can be passed to
+     a normal Redis connection can be specified here.
+     Default is {}.
+
+    :param connection_kwargs: dict, additional keyword arguments are passed
+     along to the
+     ``StrictRedis.from_url()`` method or ``StrictRedis()`` constructor
+     directly, including parameters like ``ssl``, ``ssl_certfile``,
+     ``charset``, etc.
+
+    :param lock_sleep: integer, number of seconds to sleep when failed to
+     acquire a lock.  This argument is only valid when
+     ``distributed_lock`` is ``True``.
+
+    :param thread_local_lock: bool, whether a thread-local Redis lock object
+     should be used. This is the default, but is not compatible with
+     asynchronous runners, as they run in a different thread than the one
+     used to create the lock.
+
+    """
+
+    def __init__(self, arguments):
+        arguments = arguments.copy()
+
+        self.sentinels = arguments.pop("sentinels", None)
+        self.service_name = arguments.pop("service_name", "mymaster")
+        self.sentinel_kwargs = arguments.pop("sentinel_kwargs", {})
+
+        super().__init__(
+            arguments={
+                "distributed_lock": True,
+                "thread_local_lock": False,
+                **arguments,
+            }
+        )
+
+    def _imports(self):
+        # defer imports until backend is used
+        global redis
+        import redis.sentinel  # noqa
+
+    def _create_client(self):
+        sentinel_kwargs = {}
+        sentinel_kwargs.update(self.sentinel_kwargs)
+        sentinel_kwargs.setdefault("password", self.password)
+
+        connection_kwargs = {}
+        connection_kwargs.update(self.connection_kwargs)
+        connection_kwargs.setdefault("password", self.password)
+
+        if self.db is not None:
+            connection_kwargs.setdefault("db", self.db)
+            sentinel_kwargs.setdefault("db", self.db)
+        if self.socket_timeout is not None:
+            connection_kwargs.setdefault("socket_timeout", self.socket_timeout)
+
+        sentinel = redis.sentinel.Sentinel(
+            self.sentinels,
+            sentinel_kwargs=sentinel_kwargs,
+            **connection_kwargs,
+        )
+        self.writer_client = sentinel.master_for(self.service_name)
+        self.reader_client = sentinel.slave_for(self.service_name)
--- a/libs/common/dogpile/cache/proxy.py
+++ b/libs/common/dogpile/cache/proxy.py
@ -10,7 +10,16 @@ base backend.

 """

+from typing import Mapping
+from typing import Optional
+from typing import Sequence
+
+from .api import BackendFormatted
+from .api import BackendSetType
 from .api import CacheBackend
+from .api import CacheMutex
+from .api import KeyType
+from .api import SerializedReturnType


 class ProxyBackend(CacheBackend):
@ -55,11 +64,11 @@ class ProxyBackend(CacheBackend):

    """

-    def __init__(self, *args, **kwargs):
-        self.proxied = None
+    def __init__(self, *arg, **kw):
+        pass

-    def wrap(self, backend):
-        """ Take a backend as an argument and setup the self.proxied property.
+    def wrap(self, backend: CacheBackend) -> "ProxyBackend":
+        """Take a backend as an argument and setup the self.proxied property.
        Return an object that be used as a backend by a :class:`.CacheRegion`
        object.
        """
@ -73,23 +82,37 @@ class ProxyBackend(CacheBackend):
    # Delegate any functions that are not already overridden to
    # the proxies backend
    #
-    def get(self, key):
+    def get(self, key: KeyType) -> BackendFormatted:
        return self.proxied.get(key)

-    def set(self, key, value):
+    def set(self, key: KeyType, value: BackendSetType) -> None:
        self.proxied.set(key, value)

-    def delete(self, key):
+    def delete(self, key: KeyType) -> None:
        self.proxied.delete(key)

-    def get_multi(self, keys):
+    def get_multi(self, keys: Sequence[KeyType]) -> Sequence[BackendFormatted]:
        return self.proxied.get_multi(keys)

-    def set_multi(self, mapping):
+    def set_multi(self, mapping: Mapping[KeyType, BackendSetType]) -> None:
        self.proxied.set_multi(mapping)

-    def delete_multi(self, keys):
+    def delete_multi(self, keys: Sequence[KeyType]) -> None:
        self.proxied.delete_multi(keys)

-    def get_mutex(self, key):
+    def get_mutex(self, key: KeyType) -> Optional[CacheMutex]:
        return self.proxied.get_mutex(key)
+
+    def get_serialized(self, key: KeyType) -> SerializedReturnType:
+        return self.proxied.get_serialized(key)
+
+    def get_serialized_multi(
+        self, keys: Sequence[KeyType]
+    ) -> Sequence[SerializedReturnType]:
+        return self.proxied.get_serialized_multi(keys)
+
+    def set_serialized(self, key: KeyType, value: bytes) -> None:
+        self.proxied.set_serialized(key, value)
+
+    def set_serialized_multi(self, mapping: Mapping[KeyType, bytes]) -> None:
+        self.proxied.set_serialized_multi(mapping)
--- a/Show more
+++ b/Show more
				`@ -0,0 +1 @@`
				`import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim();`