diff --git a/lib/autocommand/__init__.py b/lib/autocommand/__init__.py new file mode 100644 index 00000000..73fbfca6 --- /dev/null +++ b/lib/autocommand/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2014-2016 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + +# flake8 flags all these imports as unused, hence the NOQAs everywhere. + +from .automain import automain # NOQA +from .autoparse import autoparse, smart_open # NOQA +from .autocommand import autocommand # NOQA + +try: + from .autoasync import autoasync # NOQA +except ImportError: # pragma: no cover + pass diff --git a/lib/autocommand/autoasync.py b/lib/autocommand/autoasync.py new file mode 100644 index 00000000..3c8ebdcf --- /dev/null +++ b/lib/autocommand/autoasync.py @@ -0,0 +1,140 @@ +# Copyright 2014-2015 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + +from asyncio import get_event_loop, iscoroutine +from functools import wraps +from inspect import signature + + +def _launch_forever_coro(coro, args, kwargs, loop): + ''' + This helper function launches an async main function that was tagged with + forever=True. There are two possibilities: + + - The function is a normal function, which handles initializing the event + loop, which is then run forever + - The function is a coroutine, which needs to be scheduled in the event + loop, which is then run forever + - There is also the possibility that the function is a normal function + wrapping a coroutine function + + The function is therefore called unconditionally and scheduled in the event + loop if the return value is a coroutine object. + + The reason this is a separate function is to make absolutely sure that all + the objects created are garbage collected after all is said and done; we + do this to ensure that any exceptions raised in the tasks are collected + ASAP. + ''' + + # Personal note: I consider this an antipattern, as it relies on the use of + # unowned resources. The setup function dumps some stuff into the event + # loop where it just whirls in the ether without a well defined owner or + # lifetime. For this reason, there's a good chance I'll remove the + # forever=True feature from autoasync at some point in the future. + thing = coro(*args, **kwargs) + if iscoroutine(thing): + loop.create_task(thing) + + +def autoasync(coro=None, *, loop=None, forever=False, pass_loop=False): + ''' + Convert an asyncio coroutine into a function which, when called, is + evaluted in an event loop, and the return value returned. This is intented + to make it easy to write entry points into asyncio coroutines, which + otherwise need to be explictly evaluted with an event loop's + run_until_complete. + + If `loop` is given, it is used as the event loop to run the coro in. If it + is None (the default), the loop is retreived using asyncio.get_event_loop. + This call is defered until the decorated function is called, so that + callers can install custom event loops or event loop policies after + @autoasync is applied. + + If `forever` is True, the loop is run forever after the decorated coroutine + is finished. Use this for servers created with asyncio.start_server and the + like. + + If `pass_loop` is True, the event loop object is passed into the coroutine + as the `loop` kwarg when the wrapper function is called. In this case, the + wrapper function's __signature__ is updated to remove this parameter, so + that autoparse can still be used on it without generating a parameter for + `loop`. + + This coroutine can be called with ( @autoasync(...) ) or without + ( @autoasync ) arguments. + + Examples: + + @autoasync + def get_file(host, port): + reader, writer = yield from asyncio.open_connection(host, port) + data = reader.read() + sys.stdout.write(data.decode()) + + get_file(host, port) + + @autoasync(forever=True, pass_loop=True) + def server(host, port, loop): + yield_from loop.create_server(Proto, host, port) + + server('localhost', 8899) + + ''' + if coro is None: + return lambda c: autoasync( + c, loop=loop, + forever=forever, + pass_loop=pass_loop) + + # The old and new signatures are required to correctly bind the loop + # parameter in 100% of cases, even if it's a positional parameter. + # NOTE: A future release will probably require the loop parameter to be + # a kwonly parameter. + if pass_loop: + old_sig = signature(coro) + new_sig = old_sig.replace(parameters=( + param for name, param in old_sig.parameters.items() + if name != "loop")) + + @wraps(coro) + def autoasync_wrapper(*args, **kwargs): + # Defer the call to get_event_loop so that, if a custom policy is + # installed after the autoasync decorator, it is respected at call time + local_loop = get_event_loop() if loop is None else loop + + # Inject the 'loop' argument. We have to use this signature binding to + # ensure it's injected in the correct place (positional, keyword, etc) + if pass_loop: + bound_args = old_sig.bind_partial() + bound_args.arguments.update( + loop=local_loop, + **new_sig.bind(*args, **kwargs).arguments) + args, kwargs = bound_args.args, bound_args.kwargs + + if forever: + _launch_forever_coro(coro, args, kwargs, local_loop) + local_loop.run_forever() + else: + return local_loop.run_until_complete(coro(*args, **kwargs)) + + # Attach the updated signature. This allows 'pass_loop' to be used with + # autoparse + if pass_loop: + autoasync_wrapper.__signature__ = new_sig + + return autoasync_wrapper diff --git a/lib/autocommand/autocommand.py b/lib/autocommand/autocommand.py new file mode 100644 index 00000000..097e86de --- /dev/null +++ b/lib/autocommand/autocommand.py @@ -0,0 +1,70 @@ +# Copyright 2014-2015 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + +from .autoparse import autoparse +from .automain import automain +try: + from .autoasync import autoasync +except ImportError: # pragma: no cover + pass + + +def autocommand( + module, *, + description=None, + epilog=None, + add_nos=False, + parser=None, + loop=None, + forever=False, + pass_loop=False): + + if callable(module): + raise TypeError('autocommand requires a module name argument') + + def autocommand_decorator(func): + # Step 1: if requested, run it all in an asyncio event loop. autoasync + # patches the __signature__ of the decorated function, so that in the + # event that pass_loop is True, the `loop` parameter of the original + # function will *not* be interpreted as a command-line argument by + # autoparse + if loop is not None or forever or pass_loop: + func = autoasync( + func, + loop=None if loop is True else loop, + pass_loop=pass_loop, + forever=forever) + + # Step 2: create parser. We do this second so that the arguments are + # parsed and passed *before* entering the asyncio event loop, if it + # exists. This simplifies the stack trace and ensures errors are + # reported earlier. It also ensures that errors raised during parsing & + # passing are still raised if `forever` is True. + func = autoparse( + func, + description=description, + epilog=epilog, + add_nos=add_nos, + parser=parser) + + # Step 3: call the function automatically if __name__ == '__main__' (or + # if True was provided) + func = automain(module)(func) + + return func + + return autocommand_decorator diff --git a/lib/autocommand/automain.py b/lib/autocommand/automain.py new file mode 100644 index 00000000..6cc45db6 --- /dev/null +++ b/lib/autocommand/automain.py @@ -0,0 +1,59 @@ +# Copyright 2014-2015 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + +import sys +from .errors import AutocommandError + + +class AutomainRequiresModuleError(AutocommandError, TypeError): + pass + + +def automain(module, *, args=(), kwargs=None): + ''' + This decorator automatically invokes a function if the module is being run + as the "__main__" module. Optionally, provide args or kwargs with which to + call the function. If `module` is "__main__", the function is called, and + the program is `sys.exit`ed with the return value. You can also pass `True` + to cause the function to be called unconditionally. If the function is not + called, it is returned unchanged by the decorator. + + Usage: + + @automain(__name__) # Pass __name__ to check __name__=="__main__" + def main(): + ... + + If __name__ is "__main__" here, the main function is called, and then + sys.exit called with the return value. + ''' + + # Check that @automain(...) was called, rather than @automain + if callable(module): + raise AutomainRequiresModuleError(module) + + if module == '__main__' or module is True: + if kwargs is None: + kwargs = {} + + # Use a function definition instead of a lambda for a neater traceback + def automain_decorator(main): + sys.exit(main(*args, **kwargs)) + + return automain_decorator + else: + return lambda main: main diff --git a/lib/autocommand/autoparse.py b/lib/autocommand/autoparse.py new file mode 100644 index 00000000..0276a3fa --- /dev/null +++ b/lib/autocommand/autoparse.py @@ -0,0 +1,333 @@ +# Copyright 2014-2015 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + +import sys +from re import compile as compile_regex +from inspect import signature, getdoc, Parameter +from argparse import ArgumentParser +from contextlib import contextmanager +from functools import wraps +from io import IOBase +from autocommand.errors import AutocommandError + + +_empty = Parameter.empty + + +class AnnotationError(AutocommandError): + '''Annotation error: annotation must be a string, type, or tuple of both''' + + +class PositionalArgError(AutocommandError): + ''' + Postional Arg Error: autocommand can't handle postional-only parameters + ''' + + +class KWArgError(AutocommandError): + '''kwarg Error: autocommand can't handle a **kwargs parameter''' + + +class DocstringError(AutocommandError): + '''Docstring error''' + + +class TooManySplitsError(DocstringError): + ''' + The docstring had too many ---- section splits. Currently we only support + using up to a single split, to split the docstring into description and + epilog parts. + ''' + + +def _get_type_description(annotation): + ''' + Given an annotation, return the (type, description) for the parameter. + If you provide an annotation that is somehow both a string and a callable, + the behavior is undefined. + ''' + if annotation is _empty: + return None, None + elif callable(annotation): + return annotation, None + elif isinstance(annotation, str): + return None, annotation + elif isinstance(annotation, tuple): + try: + arg1, arg2 = annotation + except ValueError as e: + raise AnnotationError(annotation) from e + else: + if callable(arg1) and isinstance(arg2, str): + return arg1, arg2 + elif isinstance(arg1, str) and callable(arg2): + return arg2, arg1 + + raise AnnotationError(annotation) + + +def _add_arguments(param, parser, used_char_args, add_nos): + ''' + Add the argument(s) to an ArgumentParser (using add_argument) for a given + parameter. used_char_args is the set of -short options currently already in + use, and is updated (if necessary) by this function. If add_nos is True, + this will also add an inverse switch for all boolean options. For + instance, for the boolean parameter "verbose", this will create --verbose + and --no-verbose. + ''' + + # Impl note: This function is kept separate from make_parser because it's + # already very long and I wanted to separate out as much as possible into + # its own call scope, to prevent even the possibility of suble mutation + # bugs. + if param.kind is param.POSITIONAL_ONLY: + raise PositionalArgError(param) + elif param.kind is param.VAR_KEYWORD: + raise KWArgError(param) + + # These are the kwargs for the add_argument function. + arg_spec = {} + is_option = False + + # Get the type and default from the annotation. + arg_type, description = _get_type_description(param.annotation) + + # Get the default value + default = param.default + + # If there is no explicit type, and the default is present and not None, + # infer the type from the default. + if arg_type is None and default not in {_empty, None}: + arg_type = type(default) + + # Add default. The presence of a default means this is an option, not an + # argument. + if default is not _empty: + arg_spec['default'] = default + is_option = True + + # Add the type + if arg_type is not None: + # Special case for bool: make it just a --switch + if arg_type is bool: + if not default or default is _empty: + arg_spec['action'] = 'store_true' + else: + arg_spec['action'] = 'store_false' + + # Switches are always options + is_option = True + + # Special case for file types: make it a string type, for filename + elif isinstance(default, IOBase): + arg_spec['type'] = str + + # TODO: special case for list type. + # - How to specificy type of list members? + # - param: [int] + # - param: int =[] + # - action='append' vs nargs='*' + + else: + arg_spec['type'] = arg_type + + # nargs: if the signature includes *args, collect them as trailing CLI + # arguments in a list. *args can't have a default value, so it can never be + # an option. + if param.kind is param.VAR_POSITIONAL: + # TODO: consider depluralizing metavar/name here. + arg_spec['nargs'] = '*' + + # Add description. + if description is not None: + arg_spec['help'] = description + + # Get the --flags + flags = [] + name = param.name + + if is_option: + # Add the first letter as a -short option. + for letter in name[0], name[0].swapcase(): + if letter not in used_char_args: + used_char_args.add(letter) + flags.append('-{}'.format(letter)) + break + + # If the parameter is a --long option, or is a -short option that + # somehow failed to get a flag, add it. + if len(name) > 1 or not flags: + flags.append('--{}'.format(name)) + + arg_spec['dest'] = name + else: + flags.append(name) + + parser.add_argument(*flags, **arg_spec) + + # Create the --no- version for boolean switches + if add_nos and arg_type is bool: + parser.add_argument( + '--no-{}'.format(name), + action='store_const', + dest=name, + const=default if default is not _empty else False) + + +def make_parser(func_sig, description, epilog, add_nos): + ''' + Given the signature of a function, create an ArgumentParser + ''' + parser = ArgumentParser(description=description, epilog=epilog) + + used_char_args = {'h'} + + # Arange the params so that single-character arguments are first. This + # esnures they don't have to get --long versions. sorted is stable, so the + # parameters will otherwise still be in relative order. + params = sorted( + func_sig.parameters.values(), + key=lambda param: len(param.name) > 1) + + for param in params: + _add_arguments(param, parser, used_char_args, add_nos) + + return parser + + +_DOCSTRING_SPLIT = compile_regex(r'\n\s*-{4,}\s*\n') + + +def parse_docstring(docstring): + ''' + Given a docstring, parse it into a description and epilog part + ''' + if docstring is None: + return '', '' + + parts = _DOCSTRING_SPLIT.split(docstring) + + if len(parts) == 1: + return docstring, '' + elif len(parts) == 2: + return parts[0], parts[1] + else: + raise TooManySplitsError() + + +def autoparse( + func=None, *, + description=None, + epilog=None, + add_nos=False, + parser=None): + ''' + This decorator converts a function that takes normal arguments into a + function which takes a single optional argument, argv, parses it using an + argparse.ArgumentParser, and calls the underlying function with the parsed + arguments. If it is not given, sys.argv[1:] is used. This is so that the + function can be used as a setuptools entry point, as well as a normal main + function. sys.argv[1:] is not evaluated until the function is called, to + allow injecting different arguments for testing. + + It uses the argument signature of the function to create an + ArgumentParser. Parameters without defaults become positional parameters, + while parameters *with* defaults become --options. Use annotations to set + the type of the parameter. + + The `desctiption` and `epilog` parameters corrospond to the same respective + argparse parameters. If no description is given, it defaults to the + decorated functions's docstring, if present. + + If add_nos is True, every boolean option (that is, every parameter with a + default of True/False or a type of bool) will have a --no- version created + as well, which inverts the option. For instance, the --verbose option will + have a --no-verbose counterpart. These are not mutually exclusive- + whichever one appears last in the argument list will have precedence. + + If a parser is given, it is used instead of one generated from the function + signature. In this case, no parser is created; instead, the given parser is + used to parse the argv argument. The parser's results' argument names must + match up with the parameter names of the decorated function. + + The decorated function is attached to the result as the `func` attribute, + and the parser is attached as the `parser` attribute. + ''' + + # If @autoparse(...) is used instead of @autoparse + if func is None: + return lambda f: autoparse( + f, description=description, + epilog=epilog, + add_nos=add_nos, + parser=parser) + + func_sig = signature(func) + + docstr_description, docstr_epilog = parse_docstring(getdoc(func)) + + if parser is None: + parser = make_parser( + func_sig, + description or docstr_description, + epilog or docstr_epilog, + add_nos) + + @wraps(func) + def autoparse_wrapper(argv=None): + if argv is None: + argv = sys.argv[1:] + + # Get empty argument binding, to fill with parsed arguments. This + # object does all the heavy lifting of turning named arguments into + # into correctly bound *args and **kwargs. + parsed_args = func_sig.bind_partial() + parsed_args.arguments.update(vars(parser.parse_args(argv))) + + return func(*parsed_args.args, **parsed_args.kwargs) + + # TODO: attach an updated __signature__ to autoparse_wrapper, just in case. + + # Attach the wrapped function and parser, and return the wrapper. + autoparse_wrapper.func = func + autoparse_wrapper.parser = parser + return autoparse_wrapper + + +@contextmanager +def smart_open(filename_or_file, *args, **kwargs): + ''' + This context manager allows you to open a filename, if you want to default + some already-existing file object, like sys.stdout, which shouldn't be + closed at the end of the context. If the filename argument is a str, bytes, + or int, the file object is created via a call to open with the given *args + and **kwargs, sent to the context, and closed at the end of the context, + just like "with open(filename) as f:". If it isn't one of the openable + types, the object simply sent to the context unchanged, and left unclosed + at the end of the context. Example: + + def work_with_file(name=sys.stdout): + with smart_open(name) as f: + # Works correctly if name is a str filename or sys.stdout + print("Some stuff", file=f) + # If it was a filename, f is closed at the end here. + ''' + if isinstance(filename_or_file, (str, bytes, int)): + with open(filename_or_file, *args, **kwargs) as file: + yield file + else: + yield filename_or_file diff --git a/lib/autocommand/errors.py b/lib/autocommand/errors.py new file mode 100644 index 00000000..25706073 --- /dev/null +++ b/lib/autocommand/errors.py @@ -0,0 +1,23 @@ +# Copyright 2014-2016 Nathan West +# +# This file is part of autocommand. +# +# autocommand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# autocommand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with autocommand. If not, see . + + +class AutocommandError(Exception): + '''Base class for autocommand exceptions''' + pass + +# Individual modules will define errors specific to that module. diff --git a/lib/cherrypy/_cpdispatch.py b/lib/cherrypy/_cpdispatch.py index 83eb79cb..5c506e99 100644 --- a/lib/cherrypy/_cpdispatch.py +++ b/lib/cherrypy/_cpdispatch.py @@ -206,12 +206,8 @@ except ImportError: def test_callable_spec(callable, args, kwargs): # noqa: F811 return None else: - getargspec = inspect.getargspec - # Python 3 requires using getfullargspec if - # keyword-only arguments are present - if hasattr(inspect, 'getfullargspec'): - def getargspec(callable): - return inspect.getfullargspec(callable)[:4] + def getargspec(callable): + return inspect.getfullargspec(callable)[:4] class LateParamPageHandler(PageHandler): diff --git a/lib/cherrypy/_cperror.py b/lib/cherrypy/_cperror.py index 4e727682..f6ff2913 100644 --- a/lib/cherrypy/_cperror.py +++ b/lib/cherrypy/_cperror.py @@ -466,7 +466,7 @@ _HTTPErrorTemplate = '''%(traceback)s
- Powered by CherryPy %(version)s + Powered by CherryPy %(version)s
@@ -532,7 +532,8 @@ def get_error_page(status, **kwargs): return result else: # Load the template from this path. - template = io.open(error_page, newline='').read() + with io.open(error_page, newline='') as f: + template = f.read() except Exception: e = _format_exception(*_exc_info())[-1] m = kwargs['message'] diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index 0e608c48..a08f0ed9 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -339,11 +339,8 @@ LoadModule python_module modules/mod_python.so } mpconf = os.path.join(os.path.dirname(__file__), 'cpmodpy.conf') - f = open(mpconf, 'wb') - try: + with open(mpconf, 'wb') as f: f.write(conf_data) - finally: - f.close() response = read_process(self.apache_path, '-k start -f %s' % mpconf) self.ready = True diff --git a/lib/cherrypy/_cprequest.py b/lib/cherrypy/_cprequest.py index 9b86bd67..a661112c 100644 --- a/lib/cherrypy/_cprequest.py +++ b/lib/cherrypy/_cprequest.py @@ -169,7 +169,7 @@ def request_namespace(k, v): def response_namespace(k, v): """Attach response attributes declared in config.""" # Provides config entries to set default response headers - # http://cherrypy.org/ticket/889 + # http://cherrypy.dev/ticket/889 if k[:8] == 'headers.': cherrypy.serving.response.headers[k.split('.', 1)[1]] = v else: @@ -252,7 +252,7 @@ class Request(object): The query component of the Request-URI, a string of information to be interpreted by the resource. The query portion of a URI follows the path component, and is separated by a '?'. For example, the URI - 'http://www.cherrypy.org/wiki?a=3&b=4' has the query component, + 'http://www.cherrypy.dev/wiki?a=3&b=4' has the query component, 'a=3&b=4'.""" query_string_encoding = 'utf8' @@ -742,6 +742,9 @@ class Request(object): if self.protocol >= (1, 1): msg = "HTTP/1.1 requires a 'Host' request header." raise cherrypy.HTTPError(400, msg) + else: + headers['Host'] = httputil.SanitizedHost(dict.get(headers, 'Host')) + host = dict.get(headers, 'Host') if not host: host = self.local.name or self.local.ip diff --git a/lib/cherrypy/lib/auth_digest.py b/lib/cherrypy/lib/auth_digest.py index fbb5df64..981e9a5d 100644 --- a/lib/cherrypy/lib/auth_digest.py +++ b/lib/cherrypy/lib/auth_digest.py @@ -101,13 +101,12 @@ def get_ha1_file_htdigest(filename): """ def get_ha1(realm, username): result = None - f = open(filename, 'r') - for line in f: - u, r, ha1 = line.rstrip().split(':') - if u == username and r == realm: - result = ha1 - break - f.close() + with open(filename, 'r') as f: + for line in f: + u, r, ha1 = line.rstrip().split(':') + if u == username and r == realm: + result = ha1 + break return result return get_ha1 diff --git a/lib/cherrypy/lib/covercp.py b/lib/cherrypy/lib/covercp.py index 3e219713..6c3871fc 100644 --- a/lib/cherrypy/lib/covercp.py +++ b/lib/cherrypy/lib/covercp.py @@ -334,9 +334,10 @@ class CoverStats(object): yield '' def annotated_file(self, filename, statements, excluded, missing): - source = open(filename, 'r') + with open(filename, 'r') as source: + lines = source.readlines() buffer = [] - for lineno, line in enumerate(source.readlines()): + for lineno, line in enumerate(lines): lineno += 1 line = line.strip('\n\r') empty_the_buffer = True diff --git a/lib/cherrypy/lib/httputil.py b/lib/cherrypy/lib/httputil.py index eedf8d89..ced310a0 100644 --- a/lib/cherrypy/lib/httputil.py +++ b/lib/cherrypy/lib/httputil.py @@ -516,3 +516,33 @@ class Host(object): def __repr__(self): return 'httputil.Host(%r, %r, %r)' % (self.ip, self.port, self.name) + + +class SanitizedHost(str): + r""" + Wraps a raw host header received from the network in + a sanitized version that elides dangerous characters. + + >>> SanitizedHost('foo\nbar') + 'foobar' + >>> SanitizedHost('foo\nbar').raw + 'foo\nbar' + + A SanitizedInstance is only returned if sanitization was performed. + + >>> isinstance(SanitizedHost('foobar'), SanitizedHost) + False + """ + dangerous = re.compile(r'[\n\r]') + + def __new__(cls, raw): + sanitized = cls._sanitize(raw) + if sanitized == raw: + return raw + instance = super().__new__(cls, sanitized) + instance.raw = raw + return instance + + @classmethod + def _sanitize(cls, raw): + return cls.dangerous.sub('', raw) diff --git a/lib/cherrypy/lib/reprconf.py b/lib/cherrypy/lib/reprconf.py index 3976652e..76381d7b 100644 --- a/lib/cherrypy/lib/reprconf.py +++ b/lib/cherrypy/lib/reprconf.py @@ -163,11 +163,8 @@ class Parser(configparser.ConfigParser): # fp = open(filename) # except IOError: # continue - fp = open(filename) - try: + with open(filename) as fp: self._read(fp, filename) - finally: - fp.close() def as_dict(self, raw=False, vars=None): """Convert an INI file to a dictionary""" diff --git a/lib/cherrypy/lib/sessions.py b/lib/cherrypy/lib/sessions.py index 5b3328f2..0f56a4fa 100644 --- a/lib/cherrypy/lib/sessions.py +++ b/lib/cherrypy/lib/sessions.py @@ -516,11 +516,8 @@ class FileSession(Session): if path is None: path = self._get_file_path() try: - f = open(path, 'rb') - try: + with open(path, 'rb') as f: return pickle.load(f) - finally: - f.close() except (IOError, EOFError): e = sys.exc_info()[1] if self.debug: @@ -531,11 +528,8 @@ class FileSession(Session): def _save(self, expiration_time): assert self.locked, ('The session was saved without being locked. ' "Check your tools' priority levels.") - f = open(self._get_file_path(), 'wb') - try: + with open(self._get_file_path(), 'wb') as f: pickle.dump((self._data, expiration_time), f, self.pickle_protocol) - finally: - f.close() def _delete(self): assert self.locked, ('The session deletion without being locked. ' diff --git a/lib/cherrypy/process/plugins.py b/lib/cherrypy/process/plugins.py index 2a9952de..e96fb1ce 100644 --- a/lib/cherrypy/process/plugins.py +++ b/lib/cherrypy/process/plugins.py @@ -436,7 +436,8 @@ class PIDFile(SimplePlugin): if self.finalized: self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) else: - open(self.pidfile, 'wb').write(ntob('%s\n' % pid, 'utf8')) + with open(self.pidfile, 'wb') as f: + f.write(ntob('%s\n' % pid, 'utf8')) self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) self.finalized = True start.priority = 70 diff --git a/lib/cherrypy/test/helper.py b/lib/cherrypy/test/helper.py index c1ca4535..cae49533 100644 --- a/lib/cherrypy/test/helper.py +++ b/lib/cherrypy/test/helper.py @@ -505,7 +505,8 @@ server.ssl_private_key: r'%s' def get_pid(self): if self.daemonize: - return int(open(self.pid_file, 'rb').read()) + with open(self.pid_file, 'rb') as f: + return int(f.read()) return self._proc.pid def join(self): diff --git a/lib/cherrypy/test/logtest.py b/lib/cherrypy/test/logtest.py index 344be987..112bdc25 100644 --- a/lib/cherrypy/test/logtest.py +++ b/lib/cherrypy/test/logtest.py @@ -97,7 +97,8 @@ class LogCase(object): def emptyLog(self): """Overwrite self.logfile with 0 bytes.""" - open(self.logfile, 'wb').write('') + with open(self.logfile, 'wb') as f: + f.write('') def markLog(self, key=None): """Insert a marker line into the log and set self.lastmarker.""" @@ -105,10 +106,11 @@ class LogCase(object): key = str(time.time()) self.lastmarker = key - open(self.logfile, 'ab+').write( - b'%s%s\n' - % (self.markerPrefix, key.encode('utf-8')) - ) + with open(self.logfile, 'ab+') as f: + f.write( + b'%s%s\n' + % (self.markerPrefix, key.encode('utf-8')) + ) def _read_marked_region(self, marker=None): """Return lines from self.logfile in the marked region. @@ -122,20 +124,23 @@ class LogCase(object): logfile = self.logfile marker = marker or self.lastmarker if marker is None: - return open(logfile, 'rb').readlines() + with open(logfile, 'rb') as f: + return f.readlines() if isinstance(marker, str): marker = marker.encode('utf-8') data = [] in_region = False - for line in open(logfile, 'rb'): - if in_region: - if line.startswith(self.markerPrefix) and marker not in line: - break - else: - data.append(line) - elif marker in line: - in_region = True + with open(logfile, 'rb') as f: + for line in f: + if in_region: + if (line.startswith(self.markerPrefix) + and marker not in line): + break + else: + data.append(line) + elif marker in line: + in_region = True return data def assertInLog(self, line, marker=None): diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py index 79ec3d18..16927d3c 100644 --- a/lib/cherrypy/test/modfastcgi.py +++ b/lib/cherrypy/test/modfastcgi.py @@ -14,7 +14,7 @@ KNOWN BUGS 1. Apache processes Range headers automatically; CherryPy's truncated output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. + This was worked around in http://www.cherrypy.dev/changeset/1319. 2. Apache does not allow custom HTTP methods like CONNECT as per the spec. See test_core.testHTTPMethods. 3. Max request header and body settings do not work with Apache. @@ -112,15 +112,12 @@ class ModFCGISupervisor(helper.LocalWSGISupervisor): fcgiconf = os.path.join(curdir, fcgiconf) # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: + with open(fcgiconf, 'wb') as f: server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] output = self.template % {'port': self.port, 'root': curdir, 'server': server} output = output.replace('\r\n', '\n') f.write(output) - finally: - f.close() result = read_process(APACHE_PATH, '-k start -f %s' % fcgiconf) if result: diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py index d101bd67..6dd48a94 100644 --- a/lib/cherrypy/test/modfcgid.py +++ b/lib/cherrypy/test/modfcgid.py @@ -14,7 +14,7 @@ KNOWN BUGS 1. Apache processes Range headers automatically; CherryPy's truncated output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. + This was worked around in http://www.cherrypy.dev/changeset/1319. 2. Apache does not allow custom HTTP methods like CONNECT as per the spec. See test_core.testHTTPMethods. 3. Max request header and body settings do not work with Apache. @@ -101,15 +101,12 @@ class ModFCGISupervisor(helper.LocalSupervisor): fcgiconf = os.path.join(curdir, fcgiconf) # Write the Apache conf file. - f = open(fcgiconf, 'wb') - try: + with open(fcgiconf, 'wb') as f: server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1] output = self.template % {'port': self.port, 'root': curdir, 'server': server} output = ntob(output.replace('\r\n', '\n')) f.write(output) - finally: - f.close() result = read_process(APACHE_PATH, '-k start -f %s' % fcgiconf) if result: diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py index 7c288d2c..f4f68523 100644 --- a/lib/cherrypy/test/modpy.py +++ b/lib/cherrypy/test/modpy.py @@ -15,7 +15,7 @@ KNOWN BUGS 1. Apache processes Range headers automatically; CherryPy's truncated output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. + This was worked around in http://www.cherrypy.dev/changeset/1319. 2. Apache does not allow custom HTTP methods like CONNECT as per the spec. See test_core.testHTTPMethods. 3. Max request header and body settings do not work with Apache. @@ -107,13 +107,10 @@ class ModPythonSupervisor(helper.Supervisor): if not os.path.isabs(mpconf): mpconf = os.path.join(curdir, mpconf) - f = open(mpconf, 'wb') - try: + with open(mpconf, 'wb') as f: f.write(self.template % {'port': self.port, 'modulename': modulename, 'host': self.host}) - finally: - f.close() result = read_process(APACHE_PATH, '-k start -f %s' % mpconf) if result: diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py index da7d240b..c3216284 100644 --- a/lib/cherrypy/test/modwsgi.py +++ b/lib/cherrypy/test/modwsgi.py @@ -11,7 +11,7 @@ KNOWN BUGS 1. Apache processes Range headers automatically; CherryPy's truncated output is then truncated again by Apache. See test_core.testRanges. - This was worked around in http://www.cherrypy.org/changeset/1319. + This was worked around in http://www.cherrypy.dev/changeset/1319. 2. Apache does not allow custom HTTP methods like CONNECT as per the spec. See test_core.testHTTPMethods. 3. Max request header and body settings do not work with Apache. @@ -109,14 +109,11 @@ class ModWSGISupervisor(helper.Supervisor): if not os.path.isabs(mpconf): mpconf = os.path.join(curdir, mpconf) - f = open(mpconf, 'wb') - try: + with open(mpconf, 'wb') as f: output = (self.template % {'port': self.port, 'testmod': modulename, 'curdir': curdir}) f.write(output) - finally: - f.close() result = read_process(APACHE_PATH, '-k start -f %s' % mpconf) if result: diff --git a/lib/cherrypy/test/test_auth_basic.py b/lib/cherrypy/test/test_auth_basic.py index d7e69a9b..f178f8f9 100644 --- a/lib/cherrypy/test/test_auth_basic.py +++ b/lib/cherrypy/test/test_auth_basic.py @@ -1,4 +1,4 @@ -# This file is part of CherryPy +# This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 diff --git a/lib/cherrypy/test/test_auth_digest.py b/lib/cherrypy/test/test_auth_digest.py index 745f89e6..4b7b5298 100644 --- a/lib/cherrypy/test/test_auth_digest.py +++ b/lib/cherrypy/test/test_auth_digest.py @@ -1,4 +1,4 @@ -# This file is part of CherryPy +# This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 diff --git a/lib/cherrypy/test/test_core.py b/lib/cherrypy/test/test_core.py index 6fde3a97..42460b3f 100644 --- a/lib/cherrypy/test/test_core.py +++ b/lib/cherrypy/test/test_core.py @@ -586,9 +586,8 @@ class CoreRequestHandlingTest(helper.CPWebCase): def testFavicon(self): # favicon.ico is served by staticfile. icofilename = os.path.join(localDir, '../favicon.ico') - icofile = open(icofilename, 'rb') - data = icofile.read() - icofile.close() + with open(icofilename, 'rb') as icofile: + data = icofile.read() self.getPage('/favicon.ico') self.assertBody(data) diff --git a/lib/cherrypy/test/test_encoding.py b/lib/cherrypy/test/test_encoding.py index 882d7a5b..6075103d 100644 --- a/lib/cherrypy/test/test_encoding.py +++ b/lib/cherrypy/test/test_encoding.py @@ -46,7 +46,7 @@ class EncodingTests(helper.CPWebCase): # any part which is unicode (even ascii), the response # should not fail. cherrypy.response.cookie['candy'] = 'bar' - cherrypy.response.cookie['candy']['domain'] = 'cherrypy.org' + cherrypy.response.cookie['candy']['domain'] = 'cherrypy.dev' cherrypy.response.headers[ 'Some-Header'] = 'My d\xc3\xb6g has fleas' cherrypy.response.headers[ diff --git a/lib/cherrypy/test/test_logging.py b/lib/cherrypy/test/test_logging.py index 5308fb72..2d4aa56f 100644 --- a/lib/cherrypy/test/test_logging.py +++ b/lib/cherrypy/test/test_logging.py @@ -113,7 +113,7 @@ def test_normal_return(log_tracker, server): resp = requests.get( 'http://%s:%s/as_string' % (host, port), headers={ - 'Referer': 'http://www.cherrypy.org/', + 'Referer': 'http://www.cherrypy.dev/', 'User-Agent': 'Mozilla/5.0', }, ) @@ -135,7 +135,7 @@ def test_normal_return(log_tracker, server): log_tracker.assertLog( -1, '] "GET /as_string HTTP/1.1" 200 %s ' - '"http://www.cherrypy.org/" "Mozilla/5.0"' + '"http://www.cherrypy.dev/" "Mozilla/5.0"' % content_length, ) diff --git a/lib/cherrypy/test/test_request_obj.py b/lib/cherrypy/test/test_request_obj.py index 31023e8f..2478aabe 100644 --- a/lib/cherrypy/test/test_request_obj.py +++ b/lib/cherrypy/test/test_request_obj.py @@ -342,7 +342,7 @@ class RequestObjectTests(helper.CPWebCase): self.assertBody('/pathinfo/foo/bar') def testAbsoluteURIPathInfo(self): - # http://cherrypy.org/ticket/1061 + # http://cherrypy.dev/ticket/1061 self.getPage('http://localhost/pathinfo/foo/bar') self.assertBody('/pathinfo/foo/bar') @@ -375,10 +375,10 @@ class RequestObjectTests(helper.CPWebCase): # Make sure that encoded = and & get parsed correctly self.getPage( - '/params/code?url=http%3A//cherrypy.org/index%3Fa%3D1%26b%3D2') + '/params/code?url=http%3A//cherrypy.dev/index%3Fa%3D1%26b%3D2') self.assertBody('args: %s kwargs: %s' % (('code',), - [('url', ntou('http://cherrypy.org/index?a=1&b=2'))])) + [('url', ntou('http://cherrypy.dev/index?a=1&b=2'))])) # Test coordinates sent by self.getPage('/params/ismap?223,114') @@ -756,6 +756,16 @@ class RequestObjectTests(helper.CPWebCase): headers=[('Content-type', 'application/json')]) self.assertBody('application/json') + def test_dangerous_host(self): + """ + Dangerous characters like newlines should be elided. + Ref #1974. + """ + # foo\nbar + encoded = '=?iso-8859-1?q?foo=0Abar?=' + self.getPage('/headers/Host', headers=[('Host', encoded)]) + self.assertBody('foobar') + def test_basic_HTTPMethods(self): helper.webtest.methods_with_bodies = ('POST', 'PUT', 'PROPFIND', 'PATCH') diff --git a/lib/cherrypy/test/test_states.py b/lib/cherrypy/test/test_states.py index 28dd6510..d59a4d87 100644 --- a/lib/cherrypy/test/test_states.py +++ b/lib/cherrypy/test/test_states.py @@ -424,11 +424,12 @@ test_case_name: "test_signal_handler_unsubscribe" p.join() # Assert the old handler ran. - log_lines = list(open(p.error_log, 'rb')) - assert any( - line.endswith(b'I am an old SIGTERM handler.\n') - for line in log_lines - ) + with open(p.error_log, 'rb') as f: + log_lines = list(f) + assert any( + line.endswith(b'I am an old SIGTERM handler.\n') + for line in log_lines + ) def test_safe_wait_INADDR_ANY(): # pylint: disable=invalid-name diff --git a/lib/cherrypy/test/test_tutorials.py b/lib/cherrypy/test/test_tutorials.py index 39ca4d6f..390caac8 100644 --- a/lib/cherrypy/test/test_tutorials.py +++ b/lib/cherrypy/test/test_tutorials.py @@ -78,7 +78,7 @@ class TutorialTest(helper.CPWebCase):

[Return to links page]

''' @@ -166,7 +166,7 @@ class TutorialTest(helper.CPWebCase): self.assertHeader('Content-Disposition', # Make sure the filename is quoted. 'attachment; filename="pdf_file.pdf"') - self.assertEqual(len(self.body), 85698) + self.assertEqual(len(self.body), 11961) def test10HTTPErrors(self): self.setup_tutorial('tut10_http_errors', 'HTTPErrorDemo') diff --git a/lib/cherrypy/tutorial/pdf_file.pdf b/lib/cherrypy/tutorial/pdf_file.pdf index 38b4f15e..226dfe18 100644 Binary files a/lib/cherrypy/tutorial/pdf_file.pdf and b/lib/cherrypy/tutorial/pdf_file.pdf differ diff --git a/lib/cherrypy/tutorial/tut04_complex_site.py b/lib/cherrypy/tutorial/tut04_complex_site.py index 3caa1775..fe04054f 100644 --- a/lib/cherrypy/tutorial/tut04_complex_site.py +++ b/lib/cherrypy/tutorial/tut04_complex_site.py @@ -53,7 +53,7 @@ class LinksPage: