Update idna-3.3

lib/idna/__init__.py

@@ -1,2 +1,44 @@
 from .package_data import __version__
-from .core import *
+from .core import (
+    IDNABidiError,
+    IDNAError,
+    InvalidCodepoint,
+    InvalidCodepointContext,
+    alabel,
+    check_bidi,
+    check_hyphen_ok,
+    check_initial_combiner,
+    check_label,
+    check_nfc,
+    decode,
+    encode,
+    ulabel,
+    uts46_remap,
+    valid_contextj,
+    valid_contexto,
+    valid_label_length,
+    valid_string_length,
+)
+from .intranges import intranges_contain
+
+__all__ = [
+    "IDNABidiError",
+    "IDNAError",
+    "InvalidCodepoint",
+    "InvalidCodepointContext",
+    "alabel",
+    "check_bidi",
+    "check_hyphen_ok",
+    "check_initial_combiner",
+    "check_label",
+    "check_nfc",
+    "decode",
+    "encode",
+    "intranges_contain",
+    "ulabel",
+    "uts46_remap",
+    "valid_contextj",
+    "valid_contexto",
+    "valid_label_length",
+    "valid_string_length",
+]

lib/idna/codec.py

@@ -1,41 +1,40 @@
 from .core import encode, decode, alabel, ulabel, IDNAError
 import codecs
 import re
+from typing import Tuple, Optional
 
-_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]')
+_unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')
 
 class Codec(codecs.Codec):
 
-    def encode(self, data, errors='strict'):
+    def encode(self, data: str, errors: str = 'strict') -> Tuple[bytes, int]:
-
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return "", 0
+            return b"", 0
 
         return encode(data), len(data)
 
-    def decode(self, data, errors='strict'):
+    def decode(self, data: bytes, errors: str = 'strict') -> Tuple[str, int]:
-
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return u"", 0
+            return '', 0
 
         return decode(data), len(data)
 
 class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
-    def _buffer_encode(self, data, errors, final):
+    def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[str, int]:  # type: ignore
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return ("", 0)
+            return "", 0
 
         labels = _unicode_dots_re.split(data)
-        trailing_dot = u''
+        trailing_dot = ''
         if labels:
             if not labels[-1]:
                 trailing_dot = '.'
@@ -55,37 +54,29 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
             size += len(label)
 
         # Join with U+002E
-        result = ".".join(result) + trailing_dot
+        result_str = '.'.join(result) + trailing_dot  # type: ignore
         size += len(trailing_dot)
-        return (result, size)
+        return result_str, size
 
 class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
-    def _buffer_decode(self, data, errors, final):
+    def _buffer_decode(self, data: str, errors: str, final: bool) -> Tuple[str, int]:  # type: ignore
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return (u"", 0)
+            return ('', 0)
 
-        # IDNA allows decoding to operate on Unicode strings, too.
+        labels = _unicode_dots_re.split(data)
-        if isinstance(data, unicode):
+        trailing_dot = ''
-            labels = _unicode_dots_re.split(data)
-        else:
-            # Must be ASCII string
-            data = str(data)
-            unicode(data, "ascii")
-            labels = data.split(".")
-
-        trailing_dot = u''
         if labels:
             if not labels[-1]:
-                trailing_dot = u'.'
+                trailing_dot = '.'
                 del labels[-1]
             elif not final:
                 # Keep potentially unfinished label until the next call
                 del labels[-1]
                 if labels:
-                    trailing_dot = u'.'
+                    trailing_dot = '.'
 
         result = []
         size = 0
@@ -95,22 +86,25 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
                 size += 1
             size += len(label)
 
-        result = u".".join(result) + trailing_dot
+        result_str = '.'.join(result) + trailing_dot
         size += len(trailing_dot)
-        return (result, size)
+        return (result_str, size)
 
 
 class StreamWriter(Codec, codecs.StreamWriter):
     pass
 
+
 class StreamReader(Codec, codecs.StreamReader):
     pass
 
-def getregentry():
+
+def getregentry() -> codecs.CodecInfo:
+    # Compatibility as a search_function for codecs.register()
     return codecs.CodecInfo(
         name='idna',
-        encode=Codec().encode,
+        encode=Codec().encode,  # type: ignore
-        decode=Codec().decode,
+        decode=Codec().decode,  # type: ignore
         incrementalencoder=IncrementalEncoder,
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,

lib/idna/compat.py

@@ -1,12 +1,13 @@
 from .core import *
 from .codec import *
+from typing import Any, Union
 
-def ToASCII(label):
+def ToASCII(label: str) -> bytes:
     return encode(label)
 
-def ToUnicode(label):
+def ToUnicode(label: Union[bytes, bytearray]) -> str:
     return decode(label)
 
-def nameprep(s):
+def nameprep(s: Any) -> None:
-    raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol")
+    raise NotImplementedError('IDNA 2008 does not utilise nameprep protocol')
 

lib/idna/core.py

@@ -2,16 +2,12 @@ from . import idnadata
 import bisect
 import unicodedata
 import re
-import sys
+from typing import Union, Optional
 from .intranges import intranges_contain
 
 _virama_combining_class = 9
 _alabel_prefix = b'xn--'
-_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]')
+_unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')
-
-if sys.version_info[0] == 3:
-    unicode = str
-    unichr = chr
 
 class IDNAError(UnicodeError):
     """ Base exception for all IDNA-encoding related problems """
@@ -33,45 +29,45 @@ class InvalidCodepointContext(IDNAError):
     pass
 
 
-def _combining_class(cp):
+def _combining_class(cp: int) -> int:
-    return unicodedata.combining(unichr(cp))
+    v = unicodedata.combining(chr(cp))
+    if v == 0:
+        if not unicodedata.name(chr(cp)):
+            raise ValueError('Unknown character in unicodedata')
+    return v
 
-def _is_script(cp, script):
+def _is_script(cp: str, script: str) -> bool:
     return intranges_contain(ord(cp), idnadata.scripts[script])
 
-def _punycode(s):
+def _punycode(s: str) -> bytes:
     return s.encode('punycode')
 
-def _unot(s):
+def _unot(s: int) -> str:
-    return 'U+{0:04X}'.format(s)
+    return 'U+{:04X}'.format(s)
 
 
-def valid_label_length(label):
+def valid_label_length(label: Union[bytes, str]) -> bool:
-
     if len(label) > 63:
         return False
     return True
 
 
-def valid_string_length(label, trailing_dot):
+def valid_string_length(label: Union[bytes, str], trailing_dot: bool) -> bool:
-
     if len(label) > (254 if trailing_dot else 253):
         return False
     return True
 
 
-def check_bidi(label, check_ltr=False):
+def check_bidi(label: str, check_ltr: bool = False) -> bool:
-
     # Bidi rules should only be applied if string contains RTL characters
     bidi_label = False
     for (idx, cp) in enumerate(label, 1):
         direction = unicodedata.bidirectional(cp)
         if direction == '':
             # String likely comes from a newer version of Unicode
-            raise IDNABidiError('Unknown directionality in label {0} at position {1}'.format(repr(label), idx))
+            raise IDNABidiError('Unknown directionality in label {} at position {}'.format(repr(label), idx))
         if direction in ['R', 'AL', 'AN']:
             bidi_label = True
-            break
     if not bidi_label and not check_ltr:
         return True
 
@@ -82,17 +78,17 @@ def check_bidi(label, check_ltr=False):
     elif direction == 'L':
         rtl = False
     else:
-        raise IDNABidiError('First codepoint in label {0} must be directionality L, R or AL'.format(repr(label)))
+        raise IDNABidiError('First codepoint in label {} must be directionality L, R or AL'.format(repr(label)))
 
     valid_ending = False
-    number_type = False
+    number_type = None  # type: Optional[str]
     for (idx, cp) in enumerate(label, 1):
         direction = unicodedata.bidirectional(cp)
 
         if rtl:
             # Bidi rule 2
             if not direction in ['R', 'AL', 'AN', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:
-                raise IDNABidiError('Invalid direction for codepoint at position {0} in a right-to-left label'.format(idx))
+                raise IDNABidiError('Invalid direction for codepoint at position {} in a right-to-left label'.format(idx))
             # Bidi rule 3
             if direction in ['R', 'AL', 'EN', 'AN']:
                 valid_ending = True
@@ -108,7 +104,7 @@ def check_bidi(label, check_ltr=False):
         else:
             # Bidi rule 5
             if not direction in ['L', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:
-                raise IDNABidiError('Invalid direction for codepoint at position {0} in a left-to-right label'.format(idx))
+                raise IDNABidiError('Invalid direction for codepoint at position {} in a left-to-right label'.format(idx))
             # Bidi rule 6
             if direction in ['L', 'EN']:
                 valid_ending = True
@@ -121,15 +117,13 @@ def check_bidi(label, check_ltr=False):
     return True
 
 
-def check_initial_combiner(label):
+def check_initial_combiner(label: str) -> bool:
-
     if unicodedata.category(label[0])[0] == 'M':
         raise IDNAError('Label begins with an illegal combining character')
     return True
 
 
-def check_hyphen_ok(label):
+def check_hyphen_ok(label: str) -> bool:
-
     if label[2:4] == '--':
         raise IDNAError('Label has disallowed hyphens in 3rd and 4th position')
     if label[0] == '-' or label[-1] == '-':
@@ -137,14 +131,12 @@ def check_hyphen_ok(label):
     return True
 
 
-def check_nfc(label):
+def check_nfc(label: str) -> None:
-
     if unicodedata.normalize('NFC', label) != label:
         raise IDNAError('Label must be in Normalization Form C')
 
 
-def valid_contextj(label, pos):
+def valid_contextj(label: str, pos: int) -> bool:
-
     cp_value = ord(label[pos])
 
     if cp_value == 0x200c:
@@ -187,8 +179,7 @@ def valid_contextj(label, pos):
         return False
 
 
-def valid_contexto(label, pos, exception=False):
+def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
-
     cp_value = ord(label[pos])
 
     if cp_value == 0x00b7:
@@ -209,7 +200,7 @@ def valid_contexto(label, pos, exception=False):
 
     elif cp_value == 0x30fb:
         for cp in label:
-            if cp == u'\u30fb':
+            if cp == '\u30fb':
                 continue
             if _is_script(cp, 'Hiragana') or _is_script(cp, 'Katakana') or _is_script(cp, 'Han'):
                 return True
@@ -227,9 +218,10 @@ def valid_contexto(label, pos, exception=False):
                 return False
         return True
 
+    return False
 
-def check_label(label):
 
+def check_label(label: Union[str, bytes, bytearray]) -> None:
     if isinstance(label, (bytes, bytearray)):
         label = label.decode('utf-8')
     if len(label) == 0:
@@ -244,98 +236,110 @@ def check_label(label):
         if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']):
             continue
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']):
-            if not valid_contextj(label, pos):
+            try:
-                raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label)))
+                if not valid_contextj(label, pos):
+                    raise InvalidCodepointContext('Joiner {} not allowed at position {} in {}'.format(
+                        _unot(cp_value), pos+1, repr(label)))
+            except ValueError:
+                raise IDNAError('Unknown codepoint adjacent to joiner {} at position {} in {}'.format(
+                    _unot(cp_value), pos+1, repr(label)))
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']):
             if not valid_contexto(label, pos):
-                raise InvalidCodepointContext('Codepoint {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label)))
+                raise InvalidCodepointContext('Codepoint {} not allowed at position {} in {}'.format(_unot(cp_value), pos+1, repr(label)))
         else:
-            raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
+            raise InvalidCodepoint('Codepoint {} at position {} of {} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
 
     check_bidi(label)
 
 
-def alabel(label):
+def alabel(label: str) -> bytes:
-
     try:
-        label = label.encode('ascii')
+        label_bytes = label.encode('ascii')
-        try:
+        ulabel(label_bytes)
-            ulabel(label)
+        if not valid_label_length(label_bytes):
-        except IDNAError:
-            raise IDNAError('The label {0} is not a valid A-label'.format(label))
-        if not valid_label_length(label):
             raise IDNAError('Label too long')
-        return label
+        return label_bytes
     except UnicodeEncodeError:
         pass
 
     if not label:
         raise IDNAError('No Input')
 
-    label = unicode(label)
+    label = str(label)
     check_label(label)
-    label = _punycode(label)
+    label_bytes = _punycode(label)
-    label = _alabel_prefix + label
+    label_bytes = _alabel_prefix + label_bytes
 
-    if not valid_label_length(label):
+    if not valid_label_length(label_bytes):
         raise IDNAError('Label too long')
 
-    return label
+    return label_bytes
 
 
-def ulabel(label):
+def ulabel(label: Union[str, bytes, bytearray]) -> str:
-
     if not isinstance(label, (bytes, bytearray)):
         try:
-            label = label.encode('ascii')
+            label_bytes = label.encode('ascii')
         except UnicodeEncodeError:
             check_label(label)
             return label
-
-    label = label.lower()
-    if label.startswith(_alabel_prefix):
-        label = label[len(_alabel_prefix):]
     else:
-        check_label(label)
+        label_bytes = label
-        return label.decode('ascii')
 
-    label = label.decode('punycode')
+    label_bytes = label_bytes.lower()
+    if label_bytes.startswith(_alabel_prefix):
+        label_bytes = label_bytes[len(_alabel_prefix):]
+        if not label_bytes:
+            raise IDNAError('Malformed A-label, no Punycode eligible content found')
+        if label_bytes.decode('ascii')[-1] == '-':
+            raise IDNAError('A-label must not end with a hyphen')
+    else:
+        check_label(label_bytes)
+        return label_bytes.decode('ascii')
+
+    try:
+        label = label_bytes.decode('punycode')
+    except UnicodeError:
+        raise IDNAError('Invalid A-label')
     check_label(label)
     return label
 
 
-def uts46_remap(domain, std3_rules=True, transitional=False):
+def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:
     """Re-map the characters in the string according to UTS46 processing."""
     from .uts46data import uts46data
-    output = u""
+    output = ''
-    try:
+
-        for pos, char in enumerate(domain):
+    for pos, char in enumerate(domain):
-            code_point = ord(char)
+        code_point = ord(char)
+        try:
             uts46row = uts46data[code_point if code_point < 256 else
-                bisect.bisect_left(uts46data, (code_point, "Z")) - 1]
+                bisect.bisect_left(uts46data, (code_point, 'Z')) - 1]
             status = uts46row[1]
-            replacement = uts46row[2] if len(uts46row) == 3 else None
+            replacement = None  # type: Optional[str]
-            if (status == "V" or
+            if len(uts46row) == 3:
-                    (status == "D" and not transitional) or
+                replacement = uts46row[2]  # type: ignore
-                    (status == "3" and std3_rules and replacement is None)):
+            if (status == 'V' or
+                    (status == 'D' and not transitional) or
+                    (status == '3' and not std3_rules and replacement is None)):
                 output += char
-            elif replacement is not None and (status == "M" or
+            elif replacement is not None and (status == 'M' or
-                    (status == "3" and std3_rules) or
+                    (status == '3' and not std3_rules) or
-                    (status == "D" and transitional)):
+                    (status == 'D' and transitional)):
                 output += replacement
-            elif status != "I":
+            elif status != 'I':
                 raise IndexError()
-        return unicodedata.normalize("NFC", output)
+        except IndexError:
-    except IndexError:
+            raise InvalidCodepoint(
-        raise InvalidCodepoint(
+                'Codepoint {} not allowed at position {} in {}'.format(
-            "Codepoint {0} not allowed at position {1} in {2}".format(
+                _unot(code_point), pos + 1, repr(domain)))
-            _unot(code_point), pos + 1, repr(domain)))
+
+    return unicodedata.normalize('NFC', output)
 
 
-def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
+def encode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False, transitional: bool = False) -> bytes:
-
     if isinstance(s, (bytes, bytearray)):
-        s = s.decode("ascii")
+        s = s.decode('ascii')
     if uts46:
         s = uts46_remap(s, std3_rules, transitional)
     trailing_dot = False
@@ -344,15 +348,17 @@ def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
         labels = s.split('.')
     else:
         labels = _unicode_dots_re.split(s)
-    while labels and not labels[0]:
+    if not labels or labels == ['']:
-        del labels[0]
-    if not labels:
         raise IDNAError('Empty domain')
     if labels[-1] == '':
         del labels[-1]
         trailing_dot = True
     for label in labels:
-        result.append(alabel(label))
+        s = alabel(label)
+        if s:
+            result.append(s)
+        else:
+            raise IDNAError('Empty label')
     if trailing_dot:
         result.append(b'')
     s = b'.'.join(result)
@@ -361,10 +367,12 @@ def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
     return s
 
 
-def decode(s, strict=False, uts46=False, std3_rules=False):
+def decode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False) -> str:
-
+    try:
-    if isinstance(s, (bytes, bytearray)):
+        if isinstance(s, (bytes, bytearray)):
-        s = s.decode("ascii")
+            s = s.decode('ascii')
+    except UnicodeDecodeError:
+        raise IDNAError('Invalid ASCII in A-label')
     if uts46:
         s = uts46_remap(s, std3_rules, False)
     trailing_dot = False
@@ -372,16 +380,18 @@ def decode(s, strict=False, uts46=False, std3_rules=False):
     if not strict:
         labels = _unicode_dots_re.split(s)
     else:
-        labels = s.split(u'.')
+        labels = s.split('.')
-    while labels and not labels[0]:
+    if not labels or labels == ['']:
-        del labels[0]
-    if not labels:
         raise IDNAError('Empty domain')
     if not labels[-1]:
         del labels[-1]
         trailing_dot = True
     for label in labels:
-        result.append(ulabel(label))
+        s = ulabel(label)
+        if s:
+            result.append(s)
+        else:
+            raise IDNAError('Empty label')
     if trailing_dot:
-        result.append(u'')
+        result.append('')
-    return u'.'.join(result)
+    return '.'.join(result)

lib/idna/intranges.py

@@ -6,8 +6,9 @@ in the original list?" in time O(log(# runs)).
 """
 
 import bisect
+from typing import List, Tuple
 
-def intranges_from_list(list_):
+def intranges_from_list(list_: List[int]) -> Tuple[int, ...]:
     """Represent a list of integers as a sequence of ranges:
     ((start_0, end_0), (start_1, end_1), ...), such that the original
     integers are exactly those x such that start_i <= x < end_i for some i.
@@ -28,14 +29,14 @@ def intranges_from_list(list_):
 
     return tuple(ranges)
 
-def _encode_range(start, end):
+def _encode_range(start: int, end: int) -> int:
     return (start << 32) | end
 
-def _decode_range(r):
+def _decode_range(r: int) -> Tuple[int, int]:
     return (r >> 32), (r & ((1 << 32) - 1))
 
 
-def intranges_contain(int_, ranges):
+def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool:
     """Determine if `int_` falls into one of the ranges in `ranges`."""
     tuple_ = _encode_range(int_, 0)
     pos = bisect.bisect_left(ranges, tuple_)

lib/idna/package_data.py

@@ -1,2 +1,2 @@
-__version__ = '2.6'
+__version__ = '3.3'