diff --git a/lib/urllib3/__init__.py b/lib/urllib3/__init__.py index fe86b59d..c6fa3821 100644 --- a/lib/urllib3/__init__.py +++ b/lib/urllib3/__init__.py @@ -19,6 +19,23 @@ from .util.retry import Retry from .util.timeout import Timeout from .util.url import get_host +# === NOTE TO REPACKAGERS AND VENDORS === +# Please delete this block, this logic is only +# for urllib3 being distributed via PyPI. +# See: https://github.com/urllib3/urllib3/issues/2680 +try: + import urllib3_secure_extra # type: ignore # noqa: F401 +except ImportError: + pass +else: + warnings.warn( + "'urllib3[secure]' extra is deprecated and will be removed " + "in a future release of urllib3 2.x. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2680", + category=DeprecationWarning, + stacklevel=2, + ) + __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" __version__ = __version__ diff --git a/lib/urllib3/_version.py b/lib/urllib3/_version.py index d905b697..6fbc84b3 100644 --- a/lib/urllib3/_version.py +++ b/lib/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.9" +__version__ = "1.26.12" diff --git a/lib/urllib3/connection.py b/lib/urllib3/connection.py index 7bf395bd..10fb36c4 100644 --- a/lib/urllib3/connection.py +++ b/lib/urllib3/connection.py @@ -68,7 +68,7 @@ port_by_scheme = {"http": 80, "https": 443} # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2020, 7, 1) +RECENT_DATE = datetime.date(2022, 1, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") diff --git a/lib/urllib3/connectionpool.py b/lib/urllib3/connectionpool.py index 15bffcb2..96339e90 100644 --- a/lib/urllib3/connectionpool.py +++ b/lib/urllib3/connectionpool.py @@ -767,6 +767,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): isinstance(e, BaseSSLError) and self.proxy and _is_ssl_error_message_from_http_proxy(e) + and conn.proxy + and conn.proxy.scheme == "https" ): e = ProxyError( "Your proxy appears to only use HTTP and not HTTPS, " diff --git a/lib/urllib3/contrib/pyopenssl.py b/lib/urllib3/contrib/pyopenssl.py index def83afd..50a07d59 100644 --- a/lib/urllib3/contrib/pyopenssl.py +++ b/lib/urllib3/contrib/pyopenssl.py @@ -73,11 +73,20 @@ except ImportError: # Platform-specific: Python 3 import logging import ssl import sys +import warnings from .. import util from ..packages import six from ..util.ssl_ import PROTOCOL_TLS_CLIENT +warnings.warn( + "'urllib3.contrib.pyopenssl' module is deprecated and will be removed " + "in a future release of urllib3 2.x. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2680", + category=DeprecationWarning, + stacklevel=2, +) + __all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works. @@ -406,7 +415,6 @@ if _fileobject: # Platform-specific: Python 2 self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) - else: # Platform-specific: Python 3 makefile = backport_makefile diff --git a/lib/urllib3/contrib/securetransport.py b/lib/urllib3/contrib/securetransport.py index 554c015f..6c46a3b9 100644 --- a/lib/urllib3/contrib/securetransport.py +++ b/lib/urllib3/contrib/securetransport.py @@ -770,7 +770,6 @@ if _fileobject: # Platform-specific: Python 2 self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) - else: # Platform-specific: Python 3 def makefile(self, mode="r", buffering=None, *args, **kwargs): diff --git a/lib/urllib3/packages/six.py b/lib/urllib3/packages/six.py index ba50acb0..f099a3dc 100644 --- a/lib/urllib3/packages/six.py +++ b/lib/urllib3/packages/six.py @@ -772,7 +772,6 @@ if PY3: value = None tb = None - else: def exec_(_code_, _globs_=None, _locs_=None): diff --git a/lib/urllib3/response.py b/lib/urllib3/response.py index fdb50ddb..01f08eee 100644 --- a/lib/urllib3/response.py +++ b/lib/urllib3/response.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import io import logging +import sys import zlib from contextlib import contextmanager from socket import error as SocketError @@ -15,6 +16,7 @@ try: except ImportError: brotli = None +from . import util from ._collections import HTTPHeaderDict from .connection import BaseSSLError, HTTPException from .exceptions import ( @@ -481,6 +483,54 @@ class HTTPResponse(io.IOBase): if self._original_response and self._original_response.isclosed(): self.release_conn() + def _fp_read(self, amt): + """ + Read a response with the thought that reading the number of bytes + larger than can fit in a 32-bit int at a time via SSL in some + known cases leads to an overflow error that has to be prevented + if `amt` or `self.length_remaining` indicate that a problem may + happen. + + The known cases: + * 3.8 <= CPython < 3.9.7 because of a bug + https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900. + * urllib3 injected with pyOpenSSL-backed SSL-support. + * CPython < 3.10 only when `amt` does not fit 32-bit int. + """ + assert self._fp + c_int_max = 2 ** 31 - 1 + if ( + ( + (amt and amt > c_int_max) + or (self.length_remaining and self.length_remaining > c_int_max) + ) + and not util.IS_SECURETRANSPORT + and (util.IS_PYOPENSSL or sys.version_info < (3, 10)) + ): + buffer = io.BytesIO() + # Besides `max_chunk_amt` being a maximum chunk size, it + # affects memory overhead of reading a response by this + # method in CPython. + # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum + # chunk size that does not lead to an overflow error, but + # 256 MiB is a compromise. + max_chunk_amt = 2 ** 28 + while amt is None or amt != 0: + if amt is not None: + chunk_amt = min(amt, max_chunk_amt) + amt -= chunk_amt + else: + chunk_amt = max_chunk_amt + data = self._fp.read(chunk_amt) + if not data: + break + buffer.write(data) + del data # to reduce peak memory usage by `max_chunk_amt`. + return buffer.getvalue() + else: + # StringIO doesn't like amt=None + return self._fp.read(amt) if amt is not None else self._fp.read() + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`http.client.HTTPResponse.read`, but with two additional @@ -513,13 +563,11 @@ class HTTPResponse(io.IOBase): fp_closed = getattr(self._fp, "closed", False) with self._error_catcher(): + data = self._fp_read(amt) if not fp_closed else b"" if amt is None: - # cStringIO doesn't like amt=None - data = self._fp.read() if not fp_closed else b"" flush_decoder = True else: cache_content = False - data = self._fp.read(amt) if not fp_closed else b"" if ( amt != 0 and not data ): # Platform-specific: Buggy versions of Python. diff --git a/lib/urllib3/util/url.py b/lib/urllib3/util/url.py index 81a03da9..b667c160 100644 --- a/lib/urllib3/util/url.py +++ b/lib/urllib3/util/url.py @@ -279,6 +279,9 @@ def _normalize_host(host, scheme): if scheme in NORMALIZABLE_SCHEMES: is_ipv6 = IPV6_ADDRZ_RE.match(host) if is_ipv6: + # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as + # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID + # separator as necessary to return a valid RFC 4007 scoped IP. match = ZONE_ID_RE.search(host) if match: start, end = match.span(1) @@ -331,7 +334,7 @@ def parse_url(url): """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. - This parser is RFC 3986 compliant. + This parser is RFC 3986 and RFC 6874 compliant. The parser logic and helper functions are based heavily on work done in the ``rfc3986`` module. diff --git a/lib/urllib3/util/wait.py b/lib/urllib3/util/wait.py index c280646c..21b4590b 100644 --- a/lib/urllib3/util/wait.py +++ b/lib/urllib3/util/wait.py @@ -42,7 +42,6 @@ if sys.version_info >= (3, 5): def _retry_on_intr(fn, timeout): return fn(timeout) - else: # Old and broken Pythons. def _retry_on_intr(fn, timeout):