diff --git a/lib/cheroot/_compat.py b/lib/cheroot/_compat.py index 20c993de..dbe5c6d2 100644 --- a/lib/cheroot/_compat.py +++ b/lib/cheroot/_compat.py @@ -24,6 +24,7 @@ SYS_PLATFORM = platform.system() IS_WINDOWS = SYS_PLATFORM == 'Windows' IS_LINUX = SYS_PLATFORM == 'Linux' IS_MACOS = SYS_PLATFORM == 'Darwin' +IS_SOLARIS = SYS_PLATFORM == 'SunOS' PLATFORM_ARCH = platform.machine() IS_PPC = PLATFORM_ARCH.startswith('ppc') diff --git a/lib/cheroot/_compat.pyi b/lib/cheroot/_compat.pyi index 023bad8c..67d93cf6 100644 --- a/lib/cheroot/_compat.pyi +++ b/lib/cheroot/_compat.pyi @@ -10,6 +10,7 @@ SYS_PLATFORM: str IS_WINDOWS: bool IS_LINUX: bool IS_MACOS: bool +IS_SOLARIS: bool PLATFORM_ARCH: str IS_PPC: bool diff --git a/lib/cheroot/connections.py b/lib/cheroot/connections.py index 9b6366e5..9346bc6a 100644 --- a/lib/cheroot/connections.py +++ b/lib/cheroot/connections.py @@ -274,8 +274,7 @@ class ConnectionManager: # One of the reason on why a socket could cause an error # is that the socket is already closed, ignore the # socket error if we try to close it at this point. - # This is equivalent to OSError in Py3 - with suppress(socket.error): + with suppress(OSError): conn.close() def _from_server_socket(self, server_socket): # noqa: C901 # FIXME @@ -308,7 +307,7 @@ class ConnectionManager: wfile = mf(s, 'wb', io.DEFAULT_BUFFER_SIZE) try: wfile.write(''.join(buf).encode('ISO-8859-1')) - except socket.error as ex: + except OSError as ex: if ex.args[0] not in errors.socket_errors_to_ignore: raise return @@ -343,7 +342,7 @@ class ConnectionManager: # notice keyboard interrupts on Win32, which don't interrupt # accept() by default return - except socket.error as ex: + except OSError as ex: if self.server.stats['Enabled']: self.server.stats['Socket Errors'] += 1 if ex.args[0] in errors.socket_error_eintr: diff --git a/lib/cheroot/errors.py b/lib/cheroot/errors.py index 046263ad..f6b588c2 100644 --- a/lib/cheroot/errors.py +++ b/lib/cheroot/errors.py @@ -77,9 +77,4 @@ Refs: * https://docs.microsoft.com/windows/win32/api/winsock/nf-winsock-shutdown """ -try: # py3 - acceptable_sock_shutdown_exceptions = ( - BrokenPipeError, ConnectionResetError, - ) -except NameError: # py2 - acceptable_sock_shutdown_exceptions = () +acceptable_sock_shutdown_exceptions = (BrokenPipeError, ConnectionResetError) diff --git a/lib/cheroot/server.py b/lib/cheroot/server.py index 6b8e37a9..bceeb2c9 100644 --- a/lib/cheroot/server.py +++ b/lib/cheroot/server.py @@ -1572,6 +1572,9 @@ class HTTPServer: ``PEERCREDS``-provided IDs. """ + reuse_port = False + """If True, set SO_REUSEPORT on the socket.""" + keep_alive_conn_limit = 10 """Maximum number of waiting keep-alive connections that will be kept open. @@ -1581,6 +1584,7 @@ class HTTPServer: self, bind_addr, gateway, minthreads=10, maxthreads=-1, server_name=None, peercreds_enabled=False, peercreds_resolve_enabled=False, + reuse_port=False, ): """Initialize HTTPServer instance. @@ -1591,6 +1595,8 @@ class HTTPServer: maxthreads (int): maximum number of threads for HTTP thread pool server_name (str): web server name to be advertised via Server HTTP header + reuse_port (bool): if True SO_REUSEPORT option would be set to + socket """ self.bind_addr = bind_addr self.gateway = gateway @@ -1606,6 +1612,7 @@ class HTTPServer: self.peercreds_resolve_enabled = ( peercreds_resolve_enabled and peercreds_enabled ) + self.reuse_port = reuse_port self.clear_stats() def clear_stats(self): @@ -1880,6 +1887,7 @@ class HTTPServer: self.bind_addr, family, type, proto, self.nodelay, self.ssl_adapter, + self.reuse_port, ) sock = self.socket = self.bind_socket(sock, self.bind_addr) self.bind_addr = self.resolve_real_bind_addr(sock) @@ -1911,9 +1919,6 @@ class HTTPServer: 'remove() argument 1 must be encoded ' 'string without null bytes, not unicode' not in err_msg - and 'embedded NUL character' not in err_msg # py34 - and 'argument must be a ' - 'string without NUL characters' not in err_msg # pypy2 ): raise except ValueError as val_err: @@ -1931,6 +1936,7 @@ class HTTPServer: bind_addr=bind_addr, family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, nodelay=self.nodelay, ssl_adapter=self.ssl_adapter, + reuse_port=self.reuse_port, ) try: @@ -1971,7 +1977,36 @@ class HTTPServer: return sock @staticmethod - def prepare_socket(bind_addr, family, type, proto, nodelay, ssl_adapter): + def _make_socket_reusable(socket_, bind_addr): + host, port = bind_addr[:2] + IS_EPHEMERAL_PORT = port == 0 + + if socket_.family not in (socket.AF_INET, socket.AF_INET6): + raise ValueError('Cannot reuse a non-IP socket') + + if IS_EPHEMERAL_PORT: + raise ValueError('Cannot reuse an ephemeral port (0)') + + # Most BSD kernels implement SO_REUSEPORT the way that only the + # latest listener can read from socket. Some of BSD kernels also + # have SO_REUSEPORT_LB that works similarly to SO_REUSEPORT + # in Linux. + if hasattr(socket, 'SO_REUSEPORT_LB'): + socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT_LB, 1) + elif hasattr(socket, 'SO_REUSEPORT'): + socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + elif IS_WINDOWS: + socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + else: + raise NotImplementedError( + 'Current platform does not support port reuse', + ) + + @classmethod + def prepare_socket( + cls, bind_addr, family, type, proto, nodelay, ssl_adapter, + reuse_port=False, + ): """Create and prepare the socket object.""" sock = socket.socket(family, type, proto) connections.prevent_socket_inheritance(sock) @@ -1979,6 +2014,9 @@ class HTTPServer: host, port = bind_addr[:2] IS_EPHEMERAL_PORT = port == 0 + if reuse_port: + cls._make_socket_reusable(socket_=sock, bind_addr=bind_addr) + if not (IS_WINDOWS or IS_EPHEMERAL_PORT): """Enable SO_REUSEADDR for the current socket. diff --git a/lib/cheroot/server.pyi b/lib/cheroot/server.pyi index 864adff4..ecbe2f27 100644 --- a/lib/cheroot/server.pyi +++ b/lib/cheroot/server.pyi @@ -130,9 +130,10 @@ class HTTPServer: ssl_adapter: Any peercreds_enabled: bool peercreds_resolve_enabled: bool + reuse_port: bool keep_alive_conn_limit: int requests: Any - def __init__(self, bind_addr, gateway, minthreads: int = ..., maxthreads: int = ..., server_name: Any | None = ..., peercreds_enabled: bool = ..., peercreds_resolve_enabled: bool = ...) -> None: ... + def __init__(self, bind_addr, gateway, minthreads: int = ..., maxthreads: int = ..., server_name: Any | None = ..., peercreds_enabled: bool = ..., peercreds_resolve_enabled: bool = ..., reuse_port: bool = ...) -> None: ... stats: Any def clear_stats(self): ... def runtime(self): ... @@ -152,7 +153,9 @@ class HTTPServer: def bind(self, family, type, proto: int = ...): ... def bind_unix_socket(self, bind_addr): ... @staticmethod - def prepare_socket(bind_addr, family, type, proto, nodelay, ssl_adapter): ... + def _make_socket_reusable(socket_, bind_addr) -> None: ... + @classmethod + def prepare_socket(cls, bind_addr, family, type, proto, nodelay, ssl_adapter, reuse_port: bool = ...): ... @staticmethod def bind_socket(socket_, bind_addr): ... @staticmethod diff --git a/lib/cheroot/ssl/__init__.pyi b/lib/cheroot/ssl/__init__.pyi index a9807660..4801fbdd 100644 --- a/lib/cheroot/ssl/__init__.pyi +++ b/lib/cheroot/ssl/__init__.pyi @@ -1,7 +1,7 @@ -from abc import abstractmethod +from abc import abstractmethod, ABCMeta from typing import Any -class Adapter(): +class Adapter(metaclass=ABCMeta): certificate: Any private_key: Any certificate_chain: Any diff --git a/lib/cheroot/test/_pytest_plugin.py b/lib/cheroot/test/_pytest_plugin.py index 8ff3b02c..61f2efe1 100644 --- a/lib/cheroot/test/_pytest_plugin.py +++ b/lib/cheroot/test/_pytest_plugin.py @@ -4,11 +4,7 @@ Contains hooks, which are tightly bound to the Cheroot framework itself, useless for end-users' app testing. """ -from __future__ import absolute_import, division, print_function -__metaclass__ = type - import pytest -import six pytest_version = tuple(map(int, pytest.__version__.split('.'))) @@ -45,16 +41,3 @@ def pytest_load_initial_conftests(early_config, parser, args): 'type=SocketKind.SOCK_STREAM, proto=.:' 'pytest.PytestUnraisableExceptionWarning:_pytest.unraisableexception', )) - - if six.PY2: - return - - # NOTE: `ResourceWarning` does not exist under Python 2 and so using - # NOTE: it in warning filters results in an `_OptionError` exception - # NOTE: being raised. - early_config._inicache['filterwarnings'].extend(( - # FIXME: Try to figure out what causes this and ensure that the socket - # FIXME: gets closed. - 'ignore:unclosed = resource_limit for fn in native_process_conn.filenos) +@pytest.mark.skipif( + not hasattr(socket, 'SO_REUSEPORT'), + reason='socket.SO_REUSEPORT is not supported on this platform', +) +@pytest.mark.parametrize( + 'ip_addr', + ( + ANY_INTERFACE_IPV4, + ANY_INTERFACE_IPV6, + ), +) +def test_reuse_port(http_server, ip_addr, mocker): + """Check that port initialized externally can be reused.""" + family = socket.getaddrinfo(ip_addr, EPHEMERAL_PORT)[0][0] + s = socket.socket(family) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + s.bind((ip_addr, EPHEMERAL_PORT)) + server = HTTPServer( + bind_addr=s.getsockname()[:2], gateway=Gateway, reuse_port=True, + ) + spy = mocker.spy(server, 'prepare') + server.prepare() + server.stop() + s.close() + assert spy.spy_exception is None + + ISSUE511 = IS_MACOS @@ -439,3 +469,90 @@ def many_open_sockets(request, resource_limit): # Close our open resources for test_socket in test_sockets: test_socket.close() + + +@pytest.mark.parametrize( + ('minthreads', 'maxthreads', 'inited_maxthreads'), + ( + ( + # NOTE: The docstring only mentions -1 to mean "no max", but other + # NOTE: negative numbers should also work. + 1, + -2, + float('inf'), + ), + (1, -1, float('inf')), + (1, 1, 1), + (1, 2, 2), + (1, float('inf'), float('inf')), + (2, -2, float('inf')), + (2, -1, float('inf')), + (2, 2, 2), + (2, float('inf'), float('inf')), + ), +) +def test_threadpool_threadrange_set(minthreads, maxthreads, inited_maxthreads): + """Test setting the number of threads in a ThreadPool. + + The ThreadPool should properly set the min+max number of the threads to use + in the pool if those limits are valid. + """ + tp = ThreadPool( + server=None, + min=minthreads, + max=maxthreads, + ) + assert tp.min == minthreads + assert tp.max == inited_maxthreads + + +@pytest.mark.parametrize( + ('minthreads', 'maxthreads', 'error'), + ( + (-1, -1, 'min=-1 must be > 0'), + (-1, 0, 'min=-1 must be > 0'), + (-1, 1, 'min=-1 must be > 0'), + (-1, 2, 'min=-1 must be > 0'), + (0, -1, 'min=0 must be > 0'), + (0, 0, 'min=0 must be > 0'), + (0, 1, 'min=0 must be > 0'), + (0, 2, 'min=0 must be > 0'), + (1, 0, 'Expected an integer or the infinity value for the `max` argument but got 0.'), + (1, 0.5, 'Expected an integer or the infinity value for the `max` argument but got 0.5.'), + (2, 0, 'Expected an integer or the infinity value for the `max` argument but got 0.'), + (2, '1', "Expected an integer or the infinity value for the `max` argument but got '1'."), + (2, 1, 'max=1 must be > min=2'), + ), +) +def test_threadpool_invalid_threadrange(minthreads, maxthreads, error): + """Test that a ThreadPool rejects invalid min/max values. + + The ThreadPool should raise an error with the proper message when + initialized with an invalid min+max number of threads. + """ + with pytest.raises((ValueError, TypeError), match=error): + ThreadPool( + server=None, + min=minthreads, + max=maxthreads, + ) + + +def test_threadpool_multistart_validation(monkeypatch): + """Test for ThreadPool multi-start behavior. + + Tests that when calling start() on a ThreadPool multiple times raises a + :exc:`RuntimeError` + """ + # replace _spawn_worker with a function that returns a placeholder to avoid + # actually starting any threads + monkeypatch.setattr( + ThreadPool, + '_spawn_worker', + lambda _: types.SimpleNamespace(ready=True), + ) + + tp = ThreadPool(server=None) + tp.start() + with pytest.raises(RuntimeError, match='Threadpools can only be started once.'): + tp.start() diff --git a/lib/cheroot/test/test_ssl.py b/lib/cheroot/test/test_ssl.py index c55e156f..1900e20d 100644 --- a/lib/cheroot/test/test_ssl.py +++ b/lib/cheroot/test/test_ssl.py @@ -55,17 +55,6 @@ _stdlib_to_openssl_verify = { } -fails_under_py3 = pytest.mark.xfail( - reason='Fails under Python 3+', -) - - -fails_under_py3_in_pypy = pytest.mark.xfail( - IS_PYPY, - reason='Fails under PyPy3', -) - - missing_ipv6 = pytest.mark.skipif( not _probe_ipv6_sock('::1'), reason='' @@ -556,7 +545,6 @@ def test_ssl_env( # noqa: C901 # FIXME # builtin ssl environment generation may use a loopback socket # ensure no ResourceWarning was raised during the test - # NOTE: python 2.7 does not emit ResourceWarning for ssl sockets if IS_PYPY: # NOTE: PyPy doesn't have ResourceWarning # Ref: https://doc.pypy.org/en/latest/cpython_differences.html diff --git a/lib/cheroot/test/webtest.py b/lib/cheroot/test/webtest.py index 1630c8ef..eafa2dd6 100644 --- a/lib/cheroot/test/webtest.py +++ b/lib/cheroot/test/webtest.py @@ -463,16 +463,13 @@ def shb(response): return resp_status_line, response.getheaders(), response.read() -# def openURL(*args, raise_subcls=(), **kwargs): -# py27 compatible signature: -def openURL(*args, **kwargs): +def openURL(*args, raise_subcls=(), **kwargs): """ Open a URL, retrying when it fails. Specify ``raise_subcls`` (class or tuple of classes) to exclude those socket.error subclasses from being suppressed and retried. """ - raise_subcls = kwargs.pop('raise_subcls', ()) opener = functools.partial(_open_url_once, *args, **kwargs) def on_exception(): diff --git a/lib/cheroot/testing.py b/lib/cheroot/testing.py index 169142bf..3e404e59 100644 --- a/lib/cheroot/testing.py +++ b/lib/cheroot/testing.py @@ -119,9 +119,7 @@ def _probe_ipv6_sock(interface): try: with closing(socket.socket(family=socket.AF_INET6)) as sock: sock.bind((interface, 0)) - except (OSError, socket.error) as sock_err: - # In Python 3 socket.error is an alias for OSError - # In Python 2 socket.error is a subclass of IOError + except OSError as sock_err: if sock_err.errno != errno.EADDRNOTAVAIL: raise else: diff --git a/lib/cheroot/workers/threadpool.py b/lib/cheroot/workers/threadpool.py index 2a9878dc..3437d9bd 100644 --- a/lib/cheroot/workers/threadpool.py +++ b/lib/cheroot/workers/threadpool.py @@ -151,12 +151,33 @@ class ThreadPool: server (cheroot.server.HTTPServer): web server object receiving this request min (int): minimum number of worker threads - max (int): maximum number of worker threads + max (int): maximum number of worker threads (-1/inf for no max) accepted_queue_size (int): maximum number of active requests in queue accepted_queue_timeout (int): timeout for putting request into queue + + :raises ValueError: if the min/max values are invalid + :raises TypeError: if the max is not an integer or inf """ + if min < 1: + raise ValueError(f'min={min!s} must be > 0') + + if max == float('inf'): + pass + elif not isinstance(max, int) or max == 0: + raise TypeError( + 'Expected an integer or the infinity value for the `max` ' + f'argument but got {max!r}.', + ) + elif max < 0: + max = float('inf') + + if max < min: + raise ValueError( + f'max={max!s} must be > min={min!s} (or infinity for no max)', + ) + self.server = server self.min = min self.max = max @@ -167,18 +188,13 @@ class ThreadPool: self._pending_shutdowns = collections.deque() def start(self): - """Start the pool of threads.""" - for _ in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.name = ( - 'CP Server {worker_name!s}'. - format(worker_name=worker.name) - ) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) + """Start the pool of threads. + + :raises RuntimeError: if the pool is already started + """ + if self._threads: + raise RuntimeError('Threadpools can only be started once.') + self.grow(self.min) @property def idle(self): # noqa: D401; irrelevant for properties @@ -206,17 +222,13 @@ class ThreadPool: def grow(self, amount): """Spawn new worker threads (not above self.max).""" - if self.max > 0: - budget = max(self.max - len(self._threads), 0) - else: - # self.max <= 0 indicates no maximum - budget = float('inf') - + budget = max(self.max - len(self._threads), 0) n_new = min(amount, budget) workers = [self._spawn_worker() for i in range(n_new)] - while not all(worker.ready for worker in workers): - time.sleep(.1) + for worker in workers: + while not worker.ready: + time.sleep(.1) self._threads.extend(workers) def _spawn_worker(self): diff --git a/lib/cheroot/wsgi.py b/lib/cheroot/wsgi.py index 82faca3e..1dbe10ee 100644 --- a/lib/cheroot/wsgi.py +++ b/lib/cheroot/wsgi.py @@ -43,6 +43,7 @@ class Server(server.HTTPServer): max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, accepted_queue_size=-1, accepted_queue_timeout=10, peercreds_enabled=False, peercreds_resolve_enabled=False, + reuse_port=False, ): """Initialize WSGI Server instance. @@ -69,6 +70,7 @@ class Server(server.HTTPServer): server_name=server_name, peercreds_enabled=peercreds_enabled, peercreds_resolve_enabled=peercreds_resolve_enabled, + reuse_port=reuse_port, ) self.wsgi_app = wsgi_app self.request_queue_size = request_queue_size diff --git a/lib/cheroot/wsgi.pyi b/lib/cheroot/wsgi.pyi index 96075633..f96a18f9 100644 --- a/lib/cheroot/wsgi.pyi +++ b/lib/cheroot/wsgi.pyi @@ -8,7 +8,7 @@ class Server(server.HTTPServer): timeout: Any shutdown_timeout: Any requests: Any - def __init__(self, bind_addr, wsgi_app, numthreads: int = ..., server_name: Any | None = ..., max: int = ..., request_queue_size: int = ..., timeout: int = ..., shutdown_timeout: int = ..., accepted_queue_size: int = ..., accepted_queue_timeout: int = ..., peercreds_enabled: bool = ..., peercreds_resolve_enabled: bool = ...) -> None: ... + def __init__(self, bind_addr, wsgi_app, numthreads: int = ..., server_name: Any | None = ..., max: int = ..., request_queue_size: int = ..., timeout: int = ..., shutdown_timeout: int = ..., accepted_queue_size: int = ..., accepted_queue_timeout: int = ..., peercreds_enabled: bool = ..., peercreds_resolve_enabled: bool = ..., reuse_port: bool = ...) -> None: ... @property def numthreads(self): ... @numthreads.setter diff --git a/requirements.txt b/requirements.txt index e8dbb20d..5abbaaa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ backports.zoneinfo==0.2.1;python_version<"3.9" beautifulsoup4==4.12.2 bleach==6.0.0 certifi==2023.7.22 -cheroot==9.0.0 +cheroot==10.0.0 cherrypy==18.8.0 cloudinary==1.34.0 distro==1.8.0