diff --git a/lib/cherrypy/__init__.py b/lib/cherrypy/__init__.py index 6e2f9bdb..8e27c812 100644 --- a/lib/cherrypy/__init__.py +++ b/lib/cherrypy/__init__.py @@ -1,6 +1,5 @@ """CherryPy is a pythonic, object-oriented HTTP framework. - CherryPy consists of not one, but four separate API layers. The APPLICATION LAYER is the simplest. CherryPy applications are written as @@ -53,69 +52,72 @@ with customized or extended components. The core API's are: * Server API * WSGI API -These API's are described in the `CherryPy specification `_. +These API's are described in the `CherryPy specification +`_. """ -__version__ = "5.1.0" - -from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode -from cherrypy._cpcompat import basestring, unicodestr - -from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect -from cherrypy._cperror import NotFound, CherryPyException, TimeoutError - -from cherrypy import _cpdispatch as dispatch - -from cherrypy import _cptools -tools = _cptools.default_toolbox -Tool = _cptools.Tool - -from cherrypy import _cprequest -from cherrypy.lib import httputil as _httputil - -from cherrypy import _cptree -tree = _cptree.Tree() -from cherrypy._cptree import Application -from cherrypy import _cpwsgi as wsgi - -from cherrypy import process try: - from cherrypy.process import win32 + import pkg_resources +except ImportError: + pass + +from threading import local as _local + +from ._cperror import ( + HTTPError, HTTPRedirect, InternalRedirect, + NotFound, CherryPyException, +) + +from . import _cpdispatch as dispatch + +from ._cptools import default_toolbox as tools, Tool +from ._helper import expose, popargs, url + +from . import _cprequest, _cpserver, _cptree, _cplogging, _cpconfig + +import cherrypy.lib.httputil as _httputil + +from ._cptree import Application +from . import _cpwsgi as wsgi + +from . import process +try: + from .process import win32 engine = win32.Win32Bus() engine.console_control_handler = win32.ConsoleCtrlHandler(engine) del win32 except ImportError: engine = process.bus +from . import _cpchecker + +__all__ = ( + 'HTTPError', 'HTTPRedirect', 'InternalRedirect', + 'NotFound', 'CherryPyException', + 'dispatch', 'tools', 'Tool', 'Application', + 'wsgi', 'process', 'tree', 'engine', + 'quickstart', 'serving', 'request', 'response', 'thread_data', + 'log', 'expose', 'popargs', 'url', 'config', +) + + +__import__('cherrypy._cptools') +__import__('cherrypy._cprequest') + + +tree = _cptree.Tree() + + +try: + __version__ = pkg_resources.require('cherrypy')[0].version +except Exception: + __version__ = 'unknown' + -# Timeout monitor. We add two channels to the engine -# to which cherrypy.Application will publish. engine.listeners['before_request'] = set() engine.listeners['after_request'] = set() -class _TimeoutMonitor(process.plugins.Monitor): - - def __init__(self, bus): - self.servings = [] - process.plugins.Monitor.__init__(self, bus, self.run) - - def before_request(self): - self.servings.append((serving.request, serving.response)) - - def after_request(self): - try: - self.servings.remove((serving.request, serving.response)) - except ValueError: - pass - - def run(self): - """Check timeout on all responses. (Internal)""" - for req, resp in self.servings: - resp.check_timeout() -engine.timeout_monitor = _TimeoutMonitor(engine) -engine.timeout_monitor.subscribe() - engine.autoreload = process.plugins.Autoreloader(engine) engine.autoreload.subscribe() @@ -126,29 +128,30 @@ engine.signal_handler = process.plugins.SignalHandler(engine) class _HandleSignalsPlugin(object): + """Handle signals from other processes. - """Handle signals from other processes based on the configured - platform handlers above.""" + Based on the configured platform handlers above. + """ def __init__(self, bus): self.bus = bus def subscribe(self): - """Add the handlers based on the platform""" - if hasattr(self.bus, "signal_handler"): + """Add the handlers based on the platform.""" + if hasattr(self.bus, 'signal_handler'): self.bus.signal_handler.subscribe() - if hasattr(self.bus, "console_control_handler"): + if hasattr(self.bus, 'console_control_handler'): self.bus.console_control_handler.subscribe() + engine.signals = _HandleSignalsPlugin(engine) -from cherrypy import _cpserver server = _cpserver.Server() server.subscribe() -def quickstart(root=None, script_name="", config=None): +def quickstart(root=None, script_name='', config=None): """Mount the given root, start the builtin server (and engine), then block. root: an instance of a "controller class" (a collection of page handler @@ -175,11 +178,7 @@ def quickstart(root=None, script_name="", config=None): engine.block() -from cherrypy._cpcompat import threadlocal as _local - - class _Serving(_local): - """An interface for registering request and response objects. Rather than have a separate "thread local" object for the request and @@ -190,8 +189,8 @@ class _Serving(_local): thread-safe way. """ - request = _cprequest.Request(_httputil.Host("127.0.0.1", 80), - _httputil.Host("127.0.0.1", 1111)) + request = _cprequest.Request(_httputil.Host('127.0.0.1', 80), + _httputil.Host('127.0.0.1', 1111)) """ The request object for the current thread. In the main thread, and any threads which are not receiving HTTP requests, this is None.""" @@ -209,6 +208,7 @@ class _Serving(_local): """Remove all attributes of self.""" self.__dict__.clear() + serving = _Serving() @@ -224,7 +224,7 @@ class _ThreadLocalProxy(object): return getattr(child, name) def __setattr__(self, name, value): - if name in ("__attrname__", ): + if name in ('__attrname__', ): object.__setattr__(self, name, value) else: child = getattr(serving, self.__attrname__) @@ -234,12 +234,12 @@ class _ThreadLocalProxy(object): child = getattr(serving, self.__attrname__) delattr(child, name) - def _get_dict(self): + @property + def __dict__(self): child = getattr(serving, self.__attrname__) d = child.__class__.__dict__.copy() d.update(child.__dict__) return d - __dict__ = property(_get_dict) def __getitem__(self, key): child = getattr(serving, self.__attrname__) @@ -267,6 +267,7 @@ class _ThreadLocalProxy(object): # Python 3 __bool__ = __nonzero__ + # Create request and response object (the same objects will be used # throughout the entire life of the webserver, but will redirect # to the "serving" object) @@ -277,8 +278,9 @@ response = _ThreadLocalProxy('response') class _ThreadData(_local): - """A container for thread-specific data.""" + + thread_data = _ThreadData() @@ -292,6 +294,7 @@ def _cherrypy_pydoc_resolve(thing, forceload=0): thing = getattr(serving, thing.__attrname__) return _pydoc._builtin_resolve(thing, forceload) + try: import pydoc as _pydoc _pydoc._builtin_resolve = _pydoc.resolve @@ -300,11 +303,7 @@ except ImportError: pass -from cherrypy import _cplogging - - class _GlobalLogManager(_cplogging.LogManager): - """A site-wide LogManager; routes to app.log or global log as appropriate. This :class:`LogManager` implements @@ -315,10 +314,13 @@ class _GlobalLogManager(_cplogging.LogManager): """ def __call__(self, *args, **kwargs): - """Log the given message to the app.log or global log as appropriate. + """Log the given message to the app.log or global log. + + Log the given message to the app.log or global + log as appropriate. """ # Do NOT use try/except here. See - # https://bitbucket.org/cherrypy/cherrypy/issue/945 + # https://github.com/cherrypy/cherrypy/issues/945 if hasattr(request, 'app') and hasattr(request.app, 'log'): log = request.app.log else: @@ -326,7 +328,10 @@ class _GlobalLogManager(_cplogging.LogManager): return log.error(*args, **kwargs) def access(self): - """Log an access message to the app.log or global log as appropriate. + """Log an access message to the app.log or global log. + + Log the given message to the app.log or global + log as appropriate. """ try: return request.app.log.access() @@ -342,297 +347,11 @@ log.error_file = '' log.access_file = '' +@engine.subscribe('log') def _buslog(msg, level): log.error(msg, 'ENGINE', severity=level) -engine.subscribe('log', _buslog) - -# Helper functions for CP apps # -def expose(func=None, alias=None): - """Expose the function, optionally providing an alias or set of aliases.""" - def expose_(func): - func.exposed = True - if alias is not None: - if isinstance(alias, basestring): - parents[alias.replace(".", "_")] = func - else: - for a in alias: - parents[a.replace(".", "_")] = func - return func - - import sys - import types - if isinstance(func, (types.FunctionType, types.MethodType)): - if alias is None: - # @expose - func.exposed = True - return func - else: - # func = expose(func, alias) - parents = sys._getframe(1).f_locals - return expose_(func) - elif func is None: - if alias is None: - # @expose() - parents = sys._getframe(1).f_locals - return expose_ - else: - # @expose(alias="alias") or - # @expose(alias=["alias1", "alias2"]) - parents = sys._getframe(1).f_locals - return expose_ - else: - # @expose("alias") or - # @expose(["alias1", "alias2"]) - parents = sys._getframe(1).f_locals - alias = func - return expose_ - - -def popargs(*args, **kwargs): - """A decorator for _cp_dispatch - (cherrypy.dispatch.Dispatcher.dispatch_method_name). - - Optional keyword argument: handler=(Object or Function) - - Provides a _cp_dispatch function that pops off path segments into - cherrypy.request.params under the names specified. The dispatch - is then forwarded on to the next vpath element. - - Note that any existing (and exposed) member function of the class that - popargs is applied to will override that value of the argument. For - instance, if you have a method named "list" on the class decorated with - popargs, then accessing "/list" will call that function instead of popping - it off as the requested parameter. This restriction applies to all - _cp_dispatch functions. The only way around this restriction is to create - a "blank class" whose only function is to provide _cp_dispatch. - - If there are path elements after the arguments, or more arguments - are requested than are available in the vpath, then the 'handler' - keyword argument specifies the next object to handle the parameterized - request. If handler is not specified or is None, then self is used. - If handler is a function rather than an instance, then that function - will be called with the args specified and the return value from that - function used as the next object INSTEAD of adding the parameters to - cherrypy.request.args. - - This decorator may be used in one of two ways: - - As a class decorator: - @cherrypy.popargs('year', 'month', 'day') - class Blog: - def index(self, year=None, month=None, day=None): - #Process the parameters here; any url like - #/, /2009, /2009/12, or /2009/12/31 - #will fill in the appropriate parameters. - - def create(self): - #This link will still be available at /create. Defined functions - #take precedence over arguments. - - Or as a member of a class: - class Blog: - _cp_dispatch = cherrypy.popargs('year', 'month', 'day') - #... - - The handler argument may be used to mix arguments with built in functions. - For instance, the following setup allows different activities at the - day, month, and year level: - - class DayHandler: - def index(self, year, month, day): - #Do something with this day; probably list entries - - def delete(self, year, month, day): - #Delete all entries for this day - - @cherrypy.popargs('day', handler=DayHandler()) - class MonthHandler: - def index(self, year, month): - #Do something with this month; probably list entries - - def delete(self, year, month): - #Delete all entries for this month - - @cherrypy.popargs('month', handler=MonthHandler()) - class YearHandler: - def index(self, year): - #Do something with this year - - #... - - @cherrypy.popargs('year', handler=YearHandler()) - class Root: - def index(self): - #... - - """ - - # Since keyword arg comes after *args, we have to process it ourselves - # for lower versions of python. - - handler = None - handler_call = False - for k, v in kwargs.items(): - if k == 'handler': - handler = v - else: - raise TypeError( - "cherrypy.popargs() got an unexpected keyword argument '{0}'" - .format(k) - ) - - import inspect - - if handler is not None \ - and (hasattr(handler, '__call__') or inspect.isclass(handler)): - handler_call = True - - def decorated(cls_or_self=None, vpath=None): - if inspect.isclass(cls_or_self): - # cherrypy.popargs is a class decorator - cls = cls_or_self - setattr(cls, dispatch.Dispatcher.dispatch_method_name, decorated) - return cls - - # We're in the actual function - self = cls_or_self - parms = {} - for arg in args: - if not vpath: - break - parms[arg] = vpath.pop(0) - - if handler is not None: - if handler_call: - return handler(**parms) - else: - request.params.update(parms) - return handler - - request.params.update(parms) - - # If we are the ultimate handler, then to prevent our _cp_dispatch - # from being called again, we will resolve remaining elements through - # getattr() directly. - if vpath: - return getattr(self, vpath.pop(0), None) - else: - return self - - return decorated - - -def url(path="", qs="", script_name=None, base=None, relative=None): - """Create an absolute URL for the given path. - - If 'path' starts with a slash ('/'), this will return - (base + script_name + path + qs). - If it does not start with a slash, this returns - (base + script_name [+ request.path_info] + path + qs). - - If script_name is None, cherrypy.request will be used - to find a script_name, if available. - - If base is None, cherrypy.request.base will be used (if available). - Note that you can use cherrypy.tools.proxy to change this. - - Finally, note that this function can be used to obtain an absolute URL - for the current request path (minus the querystring) by passing no args. - If you call url(qs=cherrypy.request.query_string), you should get the - original browser URL (assuming no internal redirections). - - If relative is None or not provided, request.app.relative_urls will - be used (if available, else False). If False, the output will be an - absolute URL (including the scheme, host, vhost, and script_name). - If True, the output will instead be a URL that is relative to the - current request path, perhaps including '..' atoms. If relative is - the string 'server', the output will instead be a URL that is - relative to the server root; i.e., it will start with a slash. - """ - if isinstance(qs, (tuple, list, dict)): - qs = _urlencode(qs) - if qs: - qs = '?' + qs - - if request.app: - if not path.startswith("/"): - # Append/remove trailing slash from path_info as needed - # (this is to support mistyped URL's without redirecting; - # if you want to redirect, use tools.trailing_slash). - pi = request.path_info - if request.is_index is True: - if not pi.endswith('/'): - pi = pi + '/' - elif request.is_index is False: - if pi.endswith('/') and pi != '/': - pi = pi[:-1] - - if path == "": - path = pi - else: - path = _urljoin(pi, path) - - if script_name is None: - script_name = request.script_name - if base is None: - base = request.base - - newurl = base + script_name + path + qs - else: - # No request.app (we're being called outside a request). - # We'll have to guess the base from server.* attributes. - # This will produce very different results from the above - # if you're using vhosts or tools.proxy. - if base is None: - base = server.base() - - path = (script_name or "") + path - newurl = base + path + qs - - if './' in newurl: - # Normalize the URL by removing ./ and ../ - atoms = [] - for atom in newurl.split('/'): - if atom == '.': - pass - elif atom == '..': - atoms.pop() - else: - atoms.append(atom) - newurl = '/'.join(atoms) - - # At this point, we should have a fully-qualified absolute URL. - - if relative is None: - relative = getattr(request.app, "relative_urls", False) - - # See http://www.ietf.org/rfc/rfc2396.txt - if relative == 'server': - # "A relative reference beginning with a single slash character is - # termed an absolute-path reference, as defined by ..." - # This is also sometimes called "server-relative". - newurl = '/' + '/'.join(newurl.split('/', 3)[3:]) - elif relative: - # "A relative reference that does not begin with a scheme name - # or a slash character is termed a relative-path reference." - old = url(relative=False).split('/')[:-1] - new = newurl.split('/') - while old and new: - a, b = old[0], new[0] - if a != b: - break - old.pop(0) - new.pop(0) - new = (['..'] * len(old)) + new - newurl = '/'.join(new) - - return newurl - - -# import _cpconfig last so it can reference other top-level objects -from cherrypy import _cpconfig # Use _global_conf_alias so quickstart can use 'config' as an arg # without shadowing cherrypy.config. config = _global_conf_alias = _cpconfig.Config() @@ -642,11 +361,10 @@ config.defaults = { 'tools.trailing_slash.on': True, 'tools.encode.on': True } -config.namespaces["log"] = lambda k, v: setattr(log, k, v) -config.namespaces["checker"] = lambda k, v: setattr(checker, k, v) +config.namespaces['log'] = lambda k, v: setattr(log, k, v) +config.namespaces['checker'] = lambda k, v: setattr(checker, k, v) # Must reset to get our defaults applied. config.reset() -from cherrypy import _cpchecker checker = _cpchecker.Checker() engine.subscribe('start', checker) diff --git a/lib/cherrypy/__main__.py b/lib/cherrypy/__main__.py index b1c9c012..6674f7cb 100644 --- a/lib/cherrypy/__main__.py +++ b/lib/cherrypy/__main__.py @@ -1,4 +1,5 @@ -import cherrypy.daemon +"""CherryPy'd cherryd daemon runner.""" +from cherrypy.daemon import run -if __name__ == '__main__': - cherrypy.daemon.run() + +__name__ == '__main__' and run() diff --git a/lib/cherrypy/_cpchecker.py b/lib/cherrypy/_cpchecker.py index 4ef82597..39b7c972 100644 --- a/lib/cherrypy/_cpchecker.py +++ b/lib/cherrypy/_cpchecker.py @@ -1,12 +1,14 @@ +"""Checker for CherryPy sites and mounted apps.""" import os import warnings +import six +from six.moves import builtins + import cherrypy -from cherrypy._cpcompat import iteritems, copykeys, builtins class Checker(object): - """A checker for CherryPy sites and their mounted applications. When this object is called at engine startup, it executes each @@ -24,6 +26,7 @@ class Checker(object): """If True (the default), run all checks; if False, turn off all checks.""" def __init__(self): + """Initialize Checker instance.""" self._populate_known_types() def __call__(self): @@ -33,7 +36,7 @@ class Checker(object): warnings.formatwarning = self.formatwarning try: for name in dir(self): - if name.startswith("check_"): + if name.startswith('check_'): method = getattr(self, name) if method and hasattr(method, '__call__'): method() @@ -41,15 +44,14 @@ class Checker(object): warnings.formatwarning = oldformatwarning def formatwarning(self, message, category, filename, lineno, line=None): - """Function to format a warning.""" - return "CherryPy Checker:\n%s\n\n" % message + """Format a warning.""" + return 'CherryPy Checker:\n%s\n\n' % message # This value should be set inside _cpconfig. global_config_contained_paths = False def check_app_config_entries_dont_start_with_script_name(self): - """Check for Application config with sections that repeat script_name. - """ + """Check for App config with sections that repeat script_name.""" for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue @@ -57,36 +59,36 @@ class Checker(object): continue if sn == '': continue - sn_atoms = sn.strip("/").split("/") + sn_atoms = sn.strip('/').split('/') for key in app.config.keys(): - key_atoms = key.strip("/").split("/") + key_atoms = key.strip('/').split('/') if key_atoms[:len(sn_atoms)] == sn_atoms: warnings.warn( - "The application mounted at %r has config " - "entries that start with its script name: %r" % (sn, + 'The application mounted at %r has config ' + 'entries that start with its script name: %r' % (sn, key)) def check_site_config_entries_in_app_config(self): """Check for mounted Applications that have site-scoped config.""" - for sn, app in iteritems(cherrypy.tree.apps): + for sn, app in six.iteritems(cherrypy.tree.apps): if not isinstance(app, cherrypy.Application): continue msg = [] - for section, entries in iteritems(app.config): + for section, entries in six.iteritems(app.config): if section.startswith('/'): - for key, value in iteritems(entries): - for n in ("engine.", "server.", "tree.", "checker."): + for key, value in six.iteritems(entries): + for n in ('engine.', 'server.', 'tree.', 'checker.'): if key.startswith(n): - msg.append("[%s] %s = %s" % + msg.append('[%s] %s = %s' % (section, key, value)) if msg: msg.insert(0, - "The application mounted at %r contains the " - "following config entries, which are only allowed " - "in site-wide config. Move them to a [global] " - "section and pass them to cherrypy.config.update() " - "instead of tree.mount()." % sn) + 'The application mounted at %r contains the ' + 'following config entries, which are only allowed ' + 'in site-wide config. Move them to a [global] ' + 'section and pass them to cherrypy.config.update() ' + 'instead of tree.mount().' % sn) warnings.warn(os.linesep.join(msg)) def check_skipped_app_config(self): @@ -95,32 +97,30 @@ class Checker(object): if not isinstance(app, cherrypy.Application): continue if not app.config: - msg = "The Application mounted at %r has an empty config." % sn + msg = 'The Application mounted at %r has an empty config.' % sn if self.global_config_contained_paths: - msg += (" It looks like the config you passed to " - "cherrypy.config.update() contains application-" - "specific sections. You must explicitly pass " - "application config via " - "cherrypy.tree.mount(..., config=app_config)") + msg += (' It looks like the config you passed to ' + 'cherrypy.config.update() contains application-' + 'specific sections. You must explicitly pass ' + 'application config via ' + 'cherrypy.tree.mount(..., config=app_config)') warnings.warn(msg) return def check_app_config_brackets(self): - """Check for Application config with extraneous brackets in section - names. - """ + """Check for App config with extraneous brackets in section names.""" for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue if not app.config: continue for key in app.config.keys(): - if key.startswith("[") or key.endswith("]"): + if key.startswith('[') or key.endswith(']'): warnings.warn( - "The application mounted at %r has config " - "section names with extraneous brackets: %r. " - "Config *files* need brackets; config *dicts* " - "(e.g. passed to tree.mount) do not." % (sn, key)) + 'The application mounted at %r has config ' + 'section names with extraneous brackets: %r. ' + 'Config *files* need brackets; config *dicts* ' + '(e.g. passed to tree.mount) do not.' % (sn, key)) def check_static_paths(self): """Check Application config for incorrect static paths.""" @@ -132,47 +132,47 @@ class Checker(object): request.app = app for section in app.config: # get_resource will populate request.config - request.get_resource(section + "/dummy.html") + request.get_resource(section + '/dummy.html') conf = request.config.get - if conf("tools.staticdir.on", False): - msg = "" - root = conf("tools.staticdir.root") - dir = conf("tools.staticdir.dir") + if conf('tools.staticdir.on', False): + msg = '' + root = conf('tools.staticdir.root') + dir = conf('tools.staticdir.dir') if dir is None: - msg = "tools.staticdir.dir is not set." + msg = 'tools.staticdir.dir is not set.' else: - fulldir = "" + fulldir = '' if os.path.isabs(dir): fulldir = dir if root: - msg = ("dir is an absolute path, even " - "though a root is provided.") + msg = ('dir is an absolute path, even ' + 'though a root is provided.') testdir = os.path.join(root, dir[1:]) if os.path.exists(testdir): msg += ( - "\nIf you meant to serve the " - "filesystem folder at %r, remove the " - "leading slash from dir." % (testdir,)) + '\nIf you meant to serve the ' + 'filesystem folder at %r, remove the ' + 'leading slash from dir.' % (testdir,)) else: if not root: msg = ( - "dir is a relative path and " - "no root provided.") + 'dir is a relative path and ' + 'no root provided.') else: fulldir = os.path.join(root, dir) if not os.path.isabs(fulldir): - msg = ("%r is not an absolute path." % ( + msg = ('%r is not an absolute path.' % ( fulldir,)) if fulldir and not os.path.exists(fulldir): if msg: - msg += "\n" - msg += ("%r (root + dir) is not an existing " - "filesystem path." % fulldir) + msg += '\n' + msg += ('%r (root + dir) is not an existing ' + 'filesystem path.' % fulldir) if msg: - warnings.warn("%s\nsection: [%s]\nroot: %r\ndir: %r" + warnings.warn('%s\nsection: [%s]\nroot: %r\ndir: %r' % (msg, section, root, dir)) # -------------------------- Compatibility -------------------------- # @@ -196,21 +196,21 @@ class Checker(object): """Process config and warn on each obsolete or deprecated entry.""" for section, conf in config.items(): if isinstance(conf, dict): - for k, v in conf.items(): + for k in conf: if k in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead.\n" - "section: [%s]" % + warnings.warn('%r is obsolete. Use %r instead.\n' + 'section: [%s]' % (k, self.obsolete[k], section)) elif k in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead.\n" - "section: [%s]" % + warnings.warn('%r is deprecated. Use %r instead.\n' + 'section: [%s]' % (k, self.deprecated[k], section)) else: if section in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead." + warnings.warn('%r is obsolete. Use %r instead.' % (section, self.obsolete[section])) elif section in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead." + warnings.warn('%r is deprecated. Use %r instead.' % (section, self.deprecated[section])) def check_compatibility(self): @@ -225,40 +225,40 @@ class Checker(object): extra_config_namespaces = [] def _known_ns(self, app): - ns = ["wsgi"] - ns.extend(copykeys(app.toolboxes)) - ns.extend(copykeys(app.namespaces)) - ns.extend(copykeys(app.request_class.namespaces)) - ns.extend(copykeys(cherrypy.config.namespaces)) + ns = ['wsgi'] + ns.extend(app.toolboxes) + ns.extend(app.namespaces) + ns.extend(app.request_class.namespaces) + ns.extend(cherrypy.config.namespaces) ns += self.extra_config_namespaces for section, conf in app.config.items(): - is_path_section = section.startswith("/") + is_path_section = section.startswith('/') if is_path_section and isinstance(conf, dict): - for k, v in conf.items(): - atoms = k.split(".") + for k in conf: + atoms = k.split('.') if len(atoms) > 1: if atoms[0] not in ns: # Spit out a special warning if a known # namespace is preceded by "cherrypy." - if atoms[0] == "cherrypy" and atoms[1] in ns: + if atoms[0] == 'cherrypy' and atoms[1] in ns: msg = ( - "The config entry %r is invalid; " - "try %r instead.\nsection: [%s]" - % (k, ".".join(atoms[1:]), section)) + 'The config entry %r is invalid; ' + 'try %r instead.\nsection: [%s]' + % (k, '.'.join(atoms[1:]), section)) else: msg = ( - "The config entry %r is invalid, " - "because the %r config namespace " - "is unknown.\n" - "section: [%s]" % (k, atoms[0], section)) + 'The config entry %r is invalid, ' + 'because the %r config namespace ' + 'is unknown.\n' + 'section: [%s]' % (k, atoms[0], section)) warnings.warn(msg) - elif atoms[0] == "tools": + elif atoms[0] == 'tools': if atoms[1] not in dir(cherrypy.tools): msg = ( - "The config entry %r may be invalid, " - "because the %r tool was not found.\n" - "section: [%s]" % (k, atoms[1], section)) + 'The config entry %r may be invalid, ' + 'because the %r tool was not found.\n' + 'section: [%s]' % (k, atoms[1], section)) warnings.warn(msg) def check_config_namespaces(self): @@ -282,29 +282,22 @@ class Checker(object): continue vtype = type(getattr(obj, name, None)) if vtype in b: - self.known_config_types[namespace + "." + name] = vtype + self.known_config_types[namespace + '.' + name] = vtype - traverse(cherrypy.request, "request") - traverse(cherrypy.response, "response") - traverse(cherrypy.server, "server") - traverse(cherrypy.engine, "engine") - traverse(cherrypy.log, "log") + traverse(cherrypy.request, 'request') + traverse(cherrypy.response, 'response') + traverse(cherrypy.server, 'server') + traverse(cherrypy.engine, 'engine') + traverse(cherrypy.log, 'log') def _known_types(self, config): - msg = ("The config entry %r in section %r is of type %r, " - "which does not match the expected type %r.") + msg = ('The config entry %r in section %r is of type %r, ' + 'which does not match the expected type %r.') for section, conf in config.items(): - if isinstance(conf, dict): - for k, v in conf.items(): - if v is not None: - expected_type = self.known_config_types.get(k, None) - vtype = type(v) - if expected_type and vtype != expected_type: - warnings.warn(msg % (k, section, vtype.__name__, - expected_type.__name__)) - else: - k, v = section, conf + if not isinstance(conf, dict): + conf = {section: conf} + for k, v in conf.items(): if v is not None: expected_type = self.known_config_types.get(k, None) vtype = type(v) @@ -326,7 +319,7 @@ class Checker(object): for k, v in cherrypy.config.items(): if k == 'server.socket_host' and v == 'localhost': warnings.warn("The use of 'localhost' as a socket host can " - "cause problems on newer systems, since " + 'cause problems on newer systems, since ' "'localhost' can map to either an IPv4 or an " "IPv6 address. You should use '127.0.0.1' " "or '[::1]' instead.") diff --git a/lib/cherrypy/_cpcompat.py b/lib/cherrypy/_cpcompat.py index a73feb0b..f454505c 100644 --- a/lib/cherrypy/_cpcompat.py +++ b/lib/cherrypy/_cpcompat.py @@ -1,32 +1,32 @@ """Compatibility code for using CherryPy with various versions of Python. -CherryPy 3.2 is compatible with Python versions 2.3+. This module provides a +To retain compatibility with older Python versions, this module provides a useful abstraction over the differences between Python versions, sometimes by preferring a newer idiom, sometimes an older one, and sometimes a custom one. In particular, Python 2 uses str and '' for byte strings, while Python 3 uses str and '' for unicode strings. We will call each of these the 'native string' type for each version. Because of this major difference, this module -provides new 'bytestr', 'unicodestr', and 'nativestr' attributes, as well as +provides two functions: 'ntob', which translates native strings (of type 'str') into byte strings regardless of Python version, and 'ntou', which translates native -strings to unicode strings. This also provides a 'BytesIO' name for dealing -specifically with bytes, and a 'StringIO' name for dealing with native strings. -It also provides a 'base64_decode' function with native strings as input and -output. +strings to unicode strings. + +Try not to use the compatibility functions 'ntob', 'ntou', 'tonative'. +They were created with Python 2.3-2.5 compatibility in mind. +Instead, use unicode literals (from __future__) and bytes literals +and their .encode/.decode methods as needed. """ -import os + import re import sys import threading -if sys.version_info >= (3, 0): - py3k = True - bytestr = bytes - unicodestr = str - nativestr = unicodestr - basestring = (bytes, str) +import six +from six.moves import urllib + +if six.PY3: def ntob(n, encoding='ISO-8859-1'): """Return the given native string as a byte string in the given encoding. @@ -49,18 +49,8 @@ if sys.version_info >= (3, 0): if isinstance(n, bytes): return n.decode(encoding) return n - # type("") - from io import StringIO - # bytes: - from io import BytesIO as BytesIO else: # Python 2 - py3k = False - bytestr = str - unicodestr = unicode - nativestr = bytestr - basestring = basestring - def ntob(n, encoding='ISO-8859-1'): """Return the given native string as a byte string in the given encoding. @@ -82,9 +72,9 @@ else: # escapes, but without having to prefix it with u'' for Python 2, # but no prefix for Python 3. if encoding == 'escape': - return unicode( + return six.text_type( # unicode for Python 2 re.sub(r'\\u([0-9a-zA-Z]{4})', - lambda m: unichr(int(m.group(1), 16)), + lambda m: six.unichr(int(m.group(1), 16)), n.decode('ISO-8859-1'))) # Assume it's already in the given encoding, which for ISO-8859-1 # is almost always what was intended. @@ -93,247 +83,58 @@ else: def tonative(n, encoding='ISO-8859-1'): """Return the given string as a native string in the given encoding.""" # In Python 2, the native string type is bytes. - if isinstance(n, unicode): + if isinstance(n, six.text_type): # unicode for Python 2 return n.encode(encoding) return n - try: - # type("") - from cStringIO import StringIO - except ImportError: - # type("") - from StringIO import StringIO - # bytes: - BytesIO = StringIO def assert_native(n): - if not isinstance(n, nativestr): - raise TypeError("n must be a native str (got %s)" % type(n).__name__) + if not isinstance(n, str): + raise TypeError('n must be a native str (got %s)' % type(n).__name__) -try: - # Python 3.1+ - from base64 import decodebytes as _base64_decodebytes -except ImportError: - # Python 3.0- - # since CherryPy claims compability with Python 2.3, we must use - # the legacy API of base64 - from base64 import decodestring as _base64_decodebytes - - -def base64_decode(n, encoding='ISO-8859-1'): - """Return the native string base64-decoded (as a native string).""" - if isinstance(n, unicodestr): - b = n.encode(encoding) - else: - b = n - b = _base64_decodebytes(b) - if nativestr is unicodestr: - return b.decode(encoding) - else: - return b - - -try: - sorted = sorted -except NameError: - def sorted(i): - i = i[:] - i.sort() - return i - -try: - reversed = reversed -except NameError: - def reversed(x): - i = len(x) - while i > 0: - i -= 1 - yield x[i] - -try: - # Python 3 - from urllib.parse import urljoin, urlencode - from urllib.parse import quote, quote_plus - from urllib.request import unquote, urlopen - from urllib.request import parse_http_list, parse_keqv_list -except ImportError: - # Python 2 - from urlparse import urljoin - from urllib import urlencode, urlopen - from urllib import quote, quote_plus - from urllib import unquote - from urllib2 import parse_http_list, parse_keqv_list - -try: - from threading import local as threadlocal -except ImportError: - from cherrypy._cpthreadinglocal import local as threadlocal - -try: - dict.iteritems - # Python 2 - iteritems = lambda d: d.iteritems() - copyitems = lambda d: d.items() -except AttributeError: - # Python 3 - iteritems = lambda d: d.items() - copyitems = lambda d: list(d.items()) - -try: - dict.iterkeys - # Python 2 - iterkeys = lambda d: d.iterkeys() - copykeys = lambda d: d.keys() -except AttributeError: - # Python 3 - iterkeys = lambda d: d.keys() - copykeys = lambda d: list(d.keys()) - -try: - dict.itervalues - # Python 2 - itervalues = lambda d: d.itervalues() - copyvalues = lambda d: d.values() -except AttributeError: - # Python 3 - itervalues = lambda d: d.values() - copyvalues = lambda d: list(d.values()) - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins - -try: - # Python 2. We try Python 2 first clients on Python 2 - # don't try to import the 'http' module from cherrypy.lib - from Cookie import SimpleCookie, CookieError - from httplib import BadStatusLine, HTTPConnection, IncompleteRead - from httplib import NotConnected - from BaseHTTPServer import BaseHTTPRequestHandler -except ImportError: - # Python 3 - from http.cookies import SimpleCookie, CookieError - from http.client import BadStatusLine, HTTPConnection, IncompleteRead - from http.client import NotConnected - from http.server import BaseHTTPRequestHandler # Some platforms don't expose HTTPSConnection, so handle it separately -if py3k: - try: - from http.client import HTTPSConnection - except ImportError: - # Some platforms which don't have SSL don't expose HTTPSConnection - HTTPSConnection = None -else: - try: - from httplib import HTTPSConnection - except ImportError: - HTTPSConnection = None +HTTPSConnection = getattr(six.moves.http_client, 'HTTPSConnection', None) + + +def _unquote_plus_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.unquote_plus(string).decode(encoding, errors) + + +def _unquote_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.unquote(string).decode(encoding, errors) + + +def _quote_compat(string, encoding='utf-8', errors='replace'): + return urllib.parse.quote(string.encode(encoding, errors)) + + +unquote_plus = urllib.parse.unquote_plus if six.PY3 else _unquote_plus_compat +unquote = urllib.parse.unquote if six.PY3 else _unquote_compat +quote = urllib.parse.quote if six.PY3 else _quote_compat try: - # Python 2 - xrange = xrange -except NameError: - # Python 3 - xrange = range - -import threading -if hasattr(threading.Thread, "daemon"): - # Python 2.6+ - def get_daemon(t): - return t.daemon - - def set_daemon(t, val): - t.daemon = val -else: - def get_daemon(t): - return t.isDaemon() - - def set_daemon(t, val): - t.setDaemon(val) - -try: - from email.utils import formatdate - - def HTTPDate(timeval=None): - return formatdate(timeval, usegmt=True) -except ImportError: - from rfc822 import formatdate as HTTPDate - -try: - # Python 3 - from urllib.parse import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote( - atom.replace('+', ' '), - encoding=encoding, - errors=errors) -except ImportError: - # Python 2 - from urllib import unquote as parse_unquote - - def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) - -try: - # Prefer simplejson, which is usually more advanced than the builtin - # module. + # Prefer simplejson import simplejson as json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode except ImportError: - if sys.version_info >= (2, 6): - # Python >=2.6 : json is part of the standard library - import json - json_decode = json.JSONDecoder().decode - _json_encode = json.JSONEncoder().iterencode - else: - json = None - - def json_decode(s): - raise ValueError('No JSON library is available') - - def _json_encode(s): - raise ValueError('No JSON library is available') -finally: - if json and py3k: - # The two Python 3 implementations (simplejson/json) - # outputs str. We need bytes. - def json_encode(value): - for chunk in _json_encode(value): - yield chunk.encode('utf8') - else: - json_encode = _json_encode + import json -try: - import cPickle as pickle -except ImportError: - # In Python 2, pickle is a Python version. - # In Python 3, pickle is the sped-up C version. - import pickle +json_decode = json.JSONDecoder().decode +_json_encode = json.JSONEncoder().iterencode -import binascii -def random20(): - return binascii.hexlify(os.urandom(20)).decode('ascii') +if six.PY3: + # Encode to bytes on Python 3 + def json_encode(value): + for chunk in _json_encode(value): + yield chunk.encode('utf-8') +else: + json_encode = _json_encode -try: - from _thread import get_ident as get_thread_ident -except ImportError: - from thread import get_ident as get_thread_ident -try: - # Python 3 - next = next -except NameError: - # Python 2 - def next(i): - return i.next() +text_or_bytes = six.text_type, bytes + if sys.version_info >= (3, 3): Timer = threading.Timer @@ -343,17 +144,19 @@ else: Timer = threading._Timer Event = threading._Event -# Prior to Python 2.6, the Thread class did not have a .daemon property. -# This mix-in adds that property. +# html module come in 3.2 version +try: + from html import escape +except ImportError: + from cgi import escape -class SetDaemonProperty: +# html module needed the argument quote=False because in cgi the default +# is False. With quote=True the results differ. - def __get_daemon(self): - return self.isDaemon() +def escape_html(s, escape_quote=False): + """Replace special characters "&", "<" and ">" to HTML-safe sequences. - def __set_daemon(self, daemon): - self.setDaemon(daemon) - - if sys.version_info < (2, 6): - daemon = property(__get_daemon, __set_daemon) + When escape_quote=True, escape (') and (") chars. + """ + return escape(s, quote=escape_quote) diff --git a/lib/cherrypy/_cpcompat_subprocess.py b/lib/cherrypy/_cpcompat_subprocess.py deleted file mode 100644 index e3d5109f..00000000 --- a/lib/cherrypy/_cpcompat_subprocess.py +++ /dev/null @@ -1,1544 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* -os.popen* -popen2.* -commands.* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On UNIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On UNIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines some shortcut functions: - -call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - -check_call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete. If the - exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - -check_output(*popenargs, **kwargs): - Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - output = check_output(["ls", "-l", "/dev/null"]) - - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - -check_call() and check_output() will raise CalledProcessError, if the -called process returns a non-zero return code. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional input argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -pid, sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode - else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen("cmd", mode='r', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen("cmd", mode='w', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode, - bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as -the command to execute, in which case arguments will be passed -directly to the program without shell intervention. This usage can be -replaced as follows: - -(child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode, - bufsize) -==> -p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - -Return code handling translates as follows: - -pipe = os.popen("cmd", 'w') -... -rc = pipe.close() -if rc is not None and rc % 256: - print "There were some errors" -==> -process = Popen("cmd", 'w', shell=True, stdin=PIPE) -... -process.stdin.close() -if process.wait() != 0: - print "There were some errors" - - -Replacing popen2.* ------------------- -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -On Unix, popen2 also accepts a sequence as the command to execute, in -which case arguments will be passed directly to the program without -shell intervention. This usage can be replaced as follows: - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, - mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. -""" - -import sys -mswindows = (sys.platform == "win32") - -import os -import types -import traceback -import gc -import signal -import errno - -try: - set -except NameError: - from sets import Set as set - -# Exception classes used by this module. - - -class CalledProcessError(Exception): - - """This exception is raised when a process run by check_call() or - check_output() returns a non-zero exit status. - The exit status will be stored in the returncode attribute; - check_output() will also store the output in the output attribute. - """ - - def __init__(self, returncode, cmd, output=None): - self.returncode = returncode - self.cmd = cmd - self.output = output - - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % ( - self.cmd, self.returncode) - - -if mswindows: - import threading - import msvcrt - import _subprocess - - class STARTUPINFO: - dwFlags = 0 - hStdInput = None - hStdOutput = None - hStdError = None - wShowWindow = 0 - - class pywintypes: - error = IOError -else: - import select - _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - # When select or poll has indicated that the file is writable, - # we can write up to _PIPE_BUF bytes without risk of blocking. - # POSIX defines PIPE_BUF as >= 512. - _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) - - -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", - "check_output", "CalledProcessError"] - -if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, \ - STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, \ - STD_ERROR_HANDLE, SW_HIDE, \ - STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW - - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", - "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", - "STD_ERROR_HANDLE", "SW_HIDE", - "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 - -_active = [] - - -def _cleanup(): - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxint) - if res is not None: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass - -PIPE = -1 -STDOUT = -2 - - -def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except (OSError, IOError), e: - if e.errno == errno.EINTR: - continue - raise - - -def call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return Popen(*popenargs, **kwargs).wait() - - -def check_call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete. If - the exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - """ - retcode = call(*popenargs, **kwargs) - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return 0 - - -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - # or search http://msdn.microsoft.com for - # "Parsing C++ Command-Line Arguments" - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) or not arg - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backslashes. - result.append('\\' * len(bs_buf) * 2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backslashes, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - - -class Popen(object): - - def __init__(self, args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - """Create new Popen instance.""" - _cleanup() - - self._child_created = False - if not isinstance(bufsize, (int, long)): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect " - "stdin/stdout/stderr") - else: - # POSIX - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None - self.stdout = None - self.stderr = None - self.pid = None - self.returncode = None - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - if mswindows: - if p2cwrite is not None: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - - if p2cwrite is not None: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread is not None: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread is not None: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) - - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data - - def __del__(self, _maxint=sys.maxint, _active=_active): - # If __init__ hasn't had a chance to execute (e.g. if it - # was passed an undeclared keyword argument), we don't - # have a _child_created attribute at all. - if not getattr(self, '_child_created', False): - # We didn't get to successfully create a child process. - return - # In case the child hasn't been waited on, check if it's done. - self._internal_poll(_deadstate=_maxint) - if self.returncode is None and _active is not None: - # Child is still running, keep us alive until we can wait on it. - _active.append(self) - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: - stdout = None - stderr = None - if self.stdin: - if input: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE and e.errno != errno.EINVAL: - raise - self.stdin.close() - elif self.stdout: - stdout = _eintr_retry_call(self.stdout.read) - self.stdout.close() - elif self.stderr: - stderr = _eintr_retry_call(self.stderr.read) - self.stderr.close() - self.wait() - return (stdout, stderr) - - return self._communicate(input) - - def poll(self): - return self._internal_poll() - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) - - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - p2cread = _subprocess.GetStdHandle( - _subprocess.STD_INPUT_HANDLE) - if p2cread is None: - p2cread, _ = _subprocess.CreatePipe(None, 0) - elif stdin == PIPE: - p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdin, int): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = _subprocess.GetStdHandle( - _subprocess.STD_OUTPUT_HANDLE) - if c2pwrite is None: - _, c2pwrite = _subprocess.CreatePipe(None, 0) - elif stdout == PIPE: - c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdout, int): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = _subprocess.GetStdHandle( - _subprocess.STD_ERROR_HANDLE) - if errwrite is None: - _, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == PIPE: - errread, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _make_inheritable(self, handle): - """Return a duplicate of handle, which is inheritable""" - return _subprocess.DuplicateHandle( - _subprocess.GetCurrentProcess(), - handle, - _subprocess.GetCurrentProcess(), - 0, - 1, - _subprocess.DUPLICATE_SAME_ACCESS - ) - - def _find_w9xpopen(self): - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join( - os.path.dirname(_subprocess.GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (MS Windows version)""" - - if not isinstance(args, types.StringTypes): - args = list2cmdline(args) - - # Process startup details - if startupinfo is None: - startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = '{0} /c "{1}"'.format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont - # kill children. - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - # Start the process - try: - try: - hp, ht, pid, tid = _subprocess.CreateProcess( - executable, args, - # no special - # security - None, None, - int(not close_fds), - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error, e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or similar), but - # how can this be done from Python? - raise WindowsError(*e.args) - finally: - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread is not None: - p2cread.Close() - if c2pwrite is not None: - c2pwrite.Close() - if errwrite is not None: - errwrite.Close() - - # Retain the process handle, but close the thread handle - self._child_created = True - self._handle = hp - self.pid = pid - ht.Close() - - def _internal_poll( - self, _deadstate=None, - _WaitForSingleObject=_subprocess.WaitForSingleObject, - _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, - _GetExitCodeProcess=_subprocess.GetExitCodeProcess - ): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it can only refer to objects - in its local scope. - - """ - if self.returncode is None: - if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: - self.returncode = _GetExitCodeProcess(self._handle) - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - _subprocess.WaitForSingleObject(self._handle, - _subprocess.INFINITE) - self.returncode = _subprocess.GetExitCodeProcess(self._handle) - return self.returncode - - def _readerthread(self, fh, buffer): - buffer.append(fh.read()) - - def _communicate(self, input): - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) - stderr_thread.start() - - if self.stdin: - if input is not None: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE: - raise - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = stdout[0] - if stderr is not None: - stderr = stderr[0] - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - if sig == signal.SIGTERM: - self.terminate() - elif sig == signal.CTRL_C_EVENT: - os.kill(self.pid, signal.CTRL_C_EVENT) - elif sig == signal.CTRL_BREAK_EVENT: - os.kill(self.pid, signal.CTRL_BREAK_EVENT) - else: - raise ValueError("Unsupported signal: {0}".format(sig)) - - def terminate(self): - """Terminates the process - """ - _subprocess.TerminateProcess(self._handle, 1) - - kill = terminate - - else: - # - # POSIX methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = self.pipe_cloexec() - elif isinstance(stdin, int): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = self.pipe_cloexec() - elif isinstance(stdout, int): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = self.pipe_cloexec() - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - def pipe_cloexec(self): - """Create a pipe with FDs set CLOEXEC.""" - # Pipes' FDs are set CLOEXEC by default because we don't want them - # to be inherited by other subprocesses: the CLOEXEC flag is - # removed from the child's FDs by _dup2(), between fork() and - # exec(). - # This is not atomic: we would need the pipe2() syscall for that. - r, w = os.pipe() - self._set_cloexec_flag(r) - self._set_cloexec_flag(w) - return r, w - - def _close_fds(self, but): - if hasattr(os, 'closerange'): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) - else: - for i in xrange(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except: - pass - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (POSIX version)""" - - if isinstance(args, types.StringTypes): - args = [args] - else: - args = list(args) - - if shell: - args = ["/bin/sh", "-c"] + args - if executable: - args[0] = executable - - if executable is None: - executable = args[0] - - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = self.pipe_cloexec() - try: - try: - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. - # http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = set([None]) - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() - finally: - # be sure the FD is closed no matter what - os.close(errpipe_write) - - if p2cread is not None and p2cwrite is not None: - os.close(p2cread) - if c2pwrite is not None and c2pread is not None: - os.close(c2pwrite) - if errwrite is not None and errread is not None: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising exception - # Exception limited to 1M - data = _eintr_retry_call(os.read, errpipe_read, 1048576) - finally: - # be sure the FD is closed no matter what - os.close(errpipe_read) - - if data != "": - try: - _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - child_exception = pickle.loads(data) - for fd in (p2cwrite, c2pread, errread): - if fd is not None: - os.close(fd) - raise child_exception - - def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, - _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, - _WEXITSTATUS=os.WEXITSTATUS): - # This method is called (indirectly) by __del__, so it cannot - # refer to anything outside of its local scope.""" - if _WIFSIGNALED(sts): - self.returncode = -_WTERMSIG(sts) - elif _WIFEXITED(sts): - self.returncode = _WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, - _WNOHANG=os.WNOHANG, _os_error=os.error): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it cannot reference anything - outside of the local scope (nor can any methods it calls). - - """ - if self.returncode is None: - try: - pid, sts = _waitpid(self.pid, _WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except _os_error: - if _deadstate is not None: - self.returncode = _deadstate - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - def _communicate(self, input): - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if not input: - self.stdin.close() - - if _has_poll: - stdout, stderr = self._communicate_with_poll(input) - else: - stdout, stderr = self._communicate_with_select(input) - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - if stderr is not None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def _communicate_with_poll(self, input): - stdout = None # Return - stderr = None # Return - fd2file = {} - fd2output = {} - - poller = select.poll() - - def register_and_append(file_obj, eventmask): - poller.register(file_obj.fileno(), eventmask) - fd2file[file_obj.fileno()] = file_obj - - def close_unregister_and_remove(fd): - poller.unregister(fd) - fd2file[fd].close() - fd2file.pop(fd) - - if self.stdin and input: - register_and_append(self.stdin, select.POLLOUT) - - select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI - if self.stdout: - register_and_append(self.stdout, select_POLLIN_POLLPRI) - fd2output[self.stdout.fileno()] = stdout = [] - if self.stderr: - register_and_append(self.stderr, select_POLLIN_POLLPRI) - fd2output[self.stderr.fileno()] = stderr = [] - - input_offset = 0 - while fd2file: - try: - ready = poller.poll() - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - for fd, mode in ready: - if mode & select.POLLOUT: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - input_offset += os.write(fd, chunk) - except OSError, e: - if e.errno == errno.EPIPE: - close_unregister_and_remove(fd) - else: - raise - else: - if input_offset >= len(input): - close_unregister_and_remove(fd) - elif mode & select_POLLIN_POLLPRI: - data = os.read(fd, 4096) - if not data: - close_unregister_and_remove(fd) - fd2output[fd].append(data) - else: - # Ignore hang up or errors. - close_unregister_and_remove(fd) - - return (stdout, stderr) - - def _communicate_with_select(self, input): - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin and input: - write_set.append(self.stdin) - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, xlist = select.select( - read_set, write_set, []) - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - if self.stdin in wlist: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - bytes_written = os.write(self.stdin.fileno(), chunk) - except OSError, e: - if e.errno == errno.EPIPE: - self.stdin.close() - write_set.remove(self.stdin) - else: - raise - else: - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if data == "": - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if data == "": - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - os.kill(self.pid, sig) - - def terminate(self): - """Terminate the process with SIGTERM - """ - self.send_signal(signal.SIGTERM) - - def kill(self): - """Kill the process with SIGKILL - """ - self.send_signal(signal.SIGKILL) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print "Looking for 'hda'..." - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 4: Catch execution error - # - print - print "Trying a weird file..." - try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: - if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback - else: - print "Error", e.errno - else: - print >>sys.stderr, "Gosh. No error." - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print "Looking for 'PROMPT' in set output..." - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 2: Simple execution of program - # - print "Executing calc..." - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/lib/cherrypy/_cpconfig.py b/lib/cherrypy/_cpconfig.py index 00207723..8e3fd612 100644 --- a/lib/cherrypy/_cpconfig.py +++ b/lib/cherrypy/_cpconfig.py @@ -46,21 +46,21 @@ To declare global configuration entries, place them in a [global] section. You may also declare config entries directly on the classes and methods (page handlers) that make up your CherryPy application via the ``_cp_config`` -attribute. For example:: +attribute, set with the ``cherrypy.config`` decorator. For example:: + @cherrypy.config(**{'tools.gzip.on': True}) class Demo: - _cp_config = {'tools.gzip.on': True} + @cherrypy.expose + @cherrypy.config(**{'request.show_tracebacks': False}) def index(self): return "Hello world" - index.exposed = True - index._cp_config = {'request.show_tracebacks': False} .. note:: This behavior is only guaranteed for the default dispatcher. Other dispatchers may have different restrictions on where - you can attach _cp_config attributes. + you can attach config attributes. Namespaces @@ -119,11 +119,14 @@ style) context manager. """ import cherrypy -from cherrypy._cpcompat import basestring +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import reprconf -# Deprecated in CherryPy 3.2--remove in 3.3 -NamespaceSet = reprconf.NamespaceSet + +def _if_filename_register_autoreload(ob): + """Register for autoreload if ob is a string (presumed filename).""" + is_filename = isinstance(ob, text_or_bytes) + is_filename and cherrypy.engine.autoreload.files.add(ob) def merge(base, other): @@ -132,67 +135,68 @@ def merge(base, other): If the given config is a filename, it will be appended to the list of files to monitor for "autoreload" changes. """ - if isinstance(other, basestring): - cherrypy.engine.autoreload.files.add(other) + _if_filename_register_autoreload(other) # Load other into base - for section, value_map in reprconf.as_dict(other).items(): + for section, value_map in reprconf.Parser.load(other).items(): if not isinstance(value_map, dict): raise ValueError( - "Application config must include section headers, but the " + 'Application config must include section headers, but the ' "config you tried to merge doesn't have any sections. " - "Wrap your config in another dict with paths as section " + 'Wrap your config in another dict with paths as section ' "headers, for example: {'/': config}.") base.setdefault(section, {}).update(value_map) class Config(reprconf.Config): - """The 'global' configuration data for the entire CherryPy process.""" def update(self, config): """Update self from a dict, file or filename.""" - if isinstance(config, basestring): - # Filename - cherrypy.engine.autoreload.files.add(config) - reprconf.Config.update(self, config) + _if_filename_register_autoreload(config) + super(Config, self).update(config) def _apply(self, config): """Update self from a dict.""" - if isinstance(config.get("global"), dict): + if isinstance(config.get('global'), dict): if len(config) > 1: cherrypy.checker.global_config_contained_paths = True - config = config["global"] + config = config['global'] if 'tools.staticdir.dir' in config: - config['tools.staticdir.section'] = "global" - reprconf.Config._apply(self, config) - - def __call__(self, *args, **kwargs): - """Decorator for page handlers to set _cp_config.""" - if args: - raise TypeError( - "The cherrypy.config decorator does not accept positional " - "arguments; you must use keyword arguments.") + config['tools.staticdir.section'] = 'global' + super(Config, self)._apply(config) + @staticmethod + def __call__(**kwargs): + """Decorate for page handlers to set _cp_config.""" def tool_decorator(f): - if not hasattr(f, "_cp_config"): - f._cp_config = {} - for k, v in kwargs.items(): - f._cp_config[k] = v + _Vars(f).setdefault('_cp_config', {}).update(kwargs) return f return tool_decorator +class _Vars(object): + """Adapter allowing setting a default attribute on a function or class.""" + + def __init__(self, target): + self.target = target + + def setdefault(self, key, default): + if not hasattr(self.target, key): + setattr(self.target, key, default) + return getattr(self.target, key) + + # Sphinx begin config.environments Config.environments = environments = { - "staging": { + 'staging': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, 'request.show_mismatched_params': False, }, - "production": { + 'production': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, @@ -200,7 +204,7 @@ Config.environments = environments = { 'request.show_mismatched_params': False, 'log.screen': False, }, - "embedded": { + 'embedded': { # For use with CherryPy embedded in another deployment stack. 'engine.autoreload.on': False, 'checker.on': False, @@ -211,7 +215,7 @@ Config.environments = environments = { 'engine.SIGHUP': None, 'engine.SIGTERM': None, }, - "test_suite": { + 'test_suite': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, @@ -225,11 +229,11 @@ Config.environments = environments = { def _server_namespace_handler(k, v): """Config handler for the "server" namespace.""" - atoms = k.split(".", 1) + atoms = k.split('.', 1) if len(atoms) > 1: # Special-case config keys of the form 'server.servername.socket_port' # to configure additional HTTP servers. - if not hasattr(cherrypy, "servers"): + if not hasattr(cherrypy, 'servers'): cherrypy.servers = {} servername, k = atoms @@ -248,60 +252,33 @@ def _server_namespace_handler(k, v): setattr(cherrypy.servers[servername], k, v) else: setattr(cherrypy.server, k, v) -Config.namespaces["server"] = _server_namespace_handler + + +Config.namespaces['server'] = _server_namespace_handler def _engine_namespace_handler(k, v): - """Backward compatibility handler for the "engine" namespace.""" + """Config handler for the "engine" namespace.""" engine = cherrypy.engine - deprecated = { - 'autoreload_on': 'autoreload.on', - 'autoreload_frequency': 'autoreload.frequency', - 'autoreload_match': 'autoreload.match', - 'reload_files': 'autoreload.files', - 'deadlock_poll_freq': 'timeout_monitor.frequency' - } + if k in {'SIGHUP', 'SIGTERM'}: + engine.subscribe(k, v) + return - if k in deprecated: - engine.log( - 'WARNING: Use of engine.%s is deprecated and will be removed in a ' - 'future version. Use engine.%s instead.' % (k, deprecated[k])) - - if k == 'autoreload_on': - if v: - engine.autoreload.subscribe() - else: - engine.autoreload.unsubscribe() - elif k == 'autoreload_frequency': - engine.autoreload.frequency = v - elif k == 'autoreload_match': - engine.autoreload.match = v - elif k == 'reload_files': - engine.autoreload.files = set(v) - elif k == 'deadlock_poll_freq': - engine.timeout_monitor.frequency = v - elif k == 'SIGHUP': - engine.listeners['SIGHUP'] = set([v]) - elif k == 'SIGTERM': - engine.listeners['SIGTERM'] = set([v]) - elif "." in k: - plugin, attrname = k.split(".", 1) + if '.' in k: + plugin, attrname = k.split('.', 1) plugin = getattr(engine, plugin) - if attrname == 'on': - if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): - plugin.subscribe() - return - elif ( - (not v) and - hasattr(getattr(plugin, 'unsubscribe', None), '__call__') - ): - plugin.unsubscribe() - return + op = 'subscribe' if v else 'unsubscribe' + sub_unsub = getattr(plugin, op, None) + if attrname == 'on' and callable(sub_unsub): + sub_unsub() + return setattr(plugin, attrname, v) else: setattr(engine, k, v) -Config.namespaces["engine"] = _engine_namespace_handler + + +Config.namespaces['engine'] = _engine_namespace_handler def _tree_namespace_handler(k, v): @@ -309,9 +286,11 @@ def _tree_namespace_handler(k, v): if isinstance(v, dict): for script_name, app in v.items(): cherrypy.tree.graft(app, script_name) - cherrypy.engine.log("Mounted: %s on %s" % - (app, script_name or "/")) + msg = 'Mounted: %s on %s' % (app, script_name or '/') + cherrypy.engine.log(msg) else: cherrypy.tree.graft(v, v.script_name) - cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) -Config.namespaces["tree"] = _tree_namespace_handler + cherrypy.engine.log('Mounted: %s on %s' % (v, v.script_name or '/')) + + +Config.namespaces['tree'] = _tree_namespace_handler diff --git a/lib/cherrypy/_cpdispatch.py b/lib/cherrypy/_cpdispatch.py index 710bb3fd..83eb79cb 100644 --- a/lib/cherrypy/_cpdispatch.py +++ b/lib/cherrypy/_cpdispatch.py @@ -29,32 +29,26 @@ class PageHandler(object): self.args = args self.kwargs = kwargs - def get_args(self): + @property + def args(self): + """The ordered args should be accessible from post dispatch hooks.""" return cherrypy.serving.request.args - def set_args(self, args): + @args.setter + def args(self, args): cherrypy.serving.request.args = args return cherrypy.serving.request.args - args = property( - get_args, - set_args, - doc="The ordered args should be accessible from post dispatch hooks" - ) - - def get_kwargs(self): + @property + def kwargs(self): + """The named kwargs should be accessible from post dispatch hooks.""" return cherrypy.serving.request.kwargs - def set_kwargs(self, kwargs): + @kwargs.setter + def kwargs(self, kwargs): cherrypy.serving.request.kwargs = kwargs return cherrypy.serving.request.kwargs - kwargs = property( - get_kwargs, - set_kwargs, - doc="The named kwargs should be accessible from post dispatch hooks" - ) - def __call__(self): try: return self.callable(*self.args, **self.kwargs) @@ -64,7 +58,7 @@ class PageHandler(object): test_callable_spec(self.callable, self.args, self.kwargs) except cherrypy.HTTPError: raise sys.exc_info()[1] - except: + except Exception: raise x raise @@ -102,7 +96,13 @@ def test_callable_spec(callable, callable_args, callable_kwargs): # the original error raise - if args and args[0] == 'self': + if args and ( + # For callable objects, which have a __call__(self) method + hasattr(callable, '__call__') or + # For normal methods + inspect.ismethod(callable) + ): + # Strip 'self' args = args[1:] arg_usage = dict([(arg, 0,) for arg in args]) @@ -153,7 +153,7 @@ def test_callable_spec(callable, callable_args, callable_kwargs): # arguments it's definitely a 404. message = None if show_mismatched_params: - message = "Missing parameters: %s" % ",".join(missing_args) + message = 'Missing parameters: %s' % ','.join(missing_args) raise cherrypy.HTTPError(404, message=message) # the extra positional arguments come from the path - 404 Not Found @@ -175,8 +175,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): message = None if show_mismatched_params: - message = "Multiple values for parameters: "\ - "%s" % ",".join(multiple_args) + message = 'Multiple values for parameters: '\ + '%s' % ','.join(multiple_args) raise cherrypy.HTTPError(error, message=message) if not varkw and varkw_usage > 0: @@ -186,8 +186,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_qs_params: message = None if show_mismatched_params: - message = "Unexpected query string "\ - "parameters: %s" % ", ".join(extra_qs_params) + message = 'Unexpected query string '\ + 'parameters: %s' % ', '.join(extra_qs_params) raise cherrypy.HTTPError(404, message=message) # If there were any extra body parameters, it's a 400 Not Found @@ -195,18 +195,20 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_body_params: message = None if show_mismatched_params: - message = "Unexpected body parameters: "\ - "%s" % ", ".join(extra_body_params) + message = 'Unexpected body parameters: '\ + '%s' % ', '.join(extra_body_params) raise cherrypy.HTTPError(400, message=message) try: import inspect except ImportError: - test_callable_spec = lambda callable, args, kwargs: None + def test_callable_spec(callable, args, kwargs): # noqa: F811 + return None else: getargspec = inspect.getargspec - # Python 3 requires using getfullargspec if keyword-only arguments are present + # Python 3 requires using getfullargspec if + # keyword-only arguments are present if hasattr(inspect, 'getfullargspec'): def getargspec(callable): return inspect.getfullargspec(callable)[:4] @@ -222,20 +224,19 @@ class LateParamPageHandler(PageHandler): (it's more complicated than that, but that's the effect). """ - def _get_kwargs(self): + @property + def kwargs(self): + """Page handler kwargs (with cherrypy.request.params copied in).""" kwargs = cherrypy.serving.request.params.copy() if self._kwargs: kwargs.update(self._kwargs) return kwargs - def _set_kwargs(self, kwargs): + @kwargs.setter + def kwargs(self, kwargs): cherrypy.serving.request.kwargs = kwargs self._kwargs = kwargs - kwargs = property(_get_kwargs, _set_kwargs, - doc='page handler kwargs (with ' - 'cherrypy.request.params copied in)') - if sys.version_info < (3, 0): punctuation_to_underscores = string.maketrans( @@ -244,14 +245,14 @@ if sys.version_info < (3, 0): def validate_translator(t): if not isinstance(t, str) or len(t) != 256: raise ValueError( - "The translate argument must be a str of len 256.") + 'The translate argument must be a str of len 256.') else: punctuation_to_underscores = str.maketrans( string.punctuation, '_' * len(string.punctuation)) def validate_translator(t): if not isinstance(t, dict): - raise ValueError("The translate argument must be a dict.") + raise ValueError('The translate argument must be a dict.') class Dispatcher(object): @@ -289,7 +290,7 @@ class Dispatcher(object): if func: # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] + vpath = [x.replace('%2F', '/') for x in vpath] request.handler = LateParamPageHandler(func, *vpath) else: request.handler = cherrypy.NotFound() @@ -323,10 +324,10 @@ class Dispatcher(object): fullpath_len = len(fullpath) segleft = fullpath_len nodeconf = {} - if hasattr(root, "_cp_config"): + if hasattr(root, '_cp_config'): nodeconf.update(root._cp_config) - if "/" in app.config: - nodeconf.update(app.config["/"]) + if '/' in app.config: + nodeconf.update(app.config['/']) object_trail = [['root', root, nodeconf, segleft]] node = root @@ -361,9 +362,9 @@ class Dispatcher(object): if segleft > pre_len: # No path segment was removed. Raise an error. raise cherrypy.CherryPyException( - "A vpath segment was added. Custom dispatchers may only " - + "remove elements. While trying to process " - + "{0} in {1}".format(name, fullpath) + 'A vpath segment was added. Custom dispatchers may only ' + 'remove elements. While trying to process ' + '{0} in {1}'.format(name, fullpath) ) elif segleft == pre_len: # Assume that the handler used the current path segment, but @@ -375,7 +376,7 @@ class Dispatcher(object): if node is not None: # Get _cp_config attached to this node. - if hasattr(node, "_cp_config"): + if hasattr(node, '_cp_config'): nodeconf.update(node._cp_config) # Mix in values from app.config for this path. @@ -414,16 +415,16 @@ class Dispatcher(object): continue # Try a "default" method on the current leaf. - if hasattr(candidate, "default"): + if hasattr(candidate, 'default'): defhandler = candidate.default if getattr(defhandler, 'exposed', False): # Insert any extra _cp_config from the default handler. - conf = getattr(defhandler, "_cp_config", {}) + conf = getattr(defhandler, '_cp_config', {}) object_trail.insert( - i + 1, ["default", defhandler, conf, segleft]) + i + 1, ['default', defhandler, conf, segleft]) request.config = set_conf() - # See https://bitbucket.org/cherrypy/cherrypy/issue/613 - request.is_index = path.endswith("/") + # See https://github.com/cherrypy/cherrypy/issues/613 + request.is_index = path.endswith('/') return defhandler, fullpath[fullpath_len - segleft:-1] # Uncomment the next line to restrict positional params to @@ -470,23 +471,23 @@ class MethodDispatcher(Dispatcher): if resource: # Set Allow header avail = [m for m in dir(resource) if m.isupper()] - if "GET" in avail and "HEAD" not in avail: - avail.append("HEAD") + if 'GET' in avail and 'HEAD' not in avail: + avail.append('HEAD') avail.sort() - cherrypy.serving.response.headers['Allow'] = ", ".join(avail) + cherrypy.serving.response.headers['Allow'] = ', '.join(avail) # Find the subhandler meth = request.method.upper() func = getattr(resource, meth, None) - if func is None and meth == "HEAD": - func = getattr(resource, "GET", None) + if func is None and meth == 'HEAD': + func = getattr(resource, 'GET', None) if func: # Grab any _cp_config on the subhandler. - if hasattr(func, "_cp_config"): + if hasattr(func, '_cp_config'): request.config.update(func._cp_config) # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] + vpath = [x.replace('%2F', '/') for x in vpath] request.handler = LateParamPageHandler(func, *vpath) else: request.handler = cherrypy.HTTPError(405) @@ -554,28 +555,28 @@ class RoutesDispatcher(object): # Get config for the root object/path. request.config = base = cherrypy.config.copy() - curpath = "" + curpath = '' def merge(nodeconf): if 'tools.staticdir.dir' in nodeconf: - nodeconf['tools.staticdir.section'] = curpath or "/" + nodeconf['tools.staticdir.section'] = curpath or '/' base.update(nodeconf) app = request.app root = app.root - if hasattr(root, "_cp_config"): + if hasattr(root, '_cp_config'): merge(root._cp_config) - if "/" in app.config: - merge(app.config["/"]) + if '/' in app.config: + merge(app.config['/']) # Mix in values from app.config. - atoms = [x for x in path_info.split("/") if x] + atoms = [x for x in path_info.split('/') if x] if atoms: last = atoms.pop() else: last = None for atom in atoms: - curpath = "/".join((curpath, atom)) + curpath = '/'.join((curpath, atom)) if curpath in app.config: merge(app.config[curpath]) @@ -587,14 +588,14 @@ class RoutesDispatcher(object): if isinstance(controller, classtype): controller = controller() # Get config from the controller. - if hasattr(controller, "_cp_config"): + if hasattr(controller, '_cp_config'): merge(controller._cp_config) action = result.get('action') if action is not None: handler = getattr(controller, action, None) # Get config from the handler - if hasattr(handler, "_cp_config"): + if hasattr(handler, '_cp_config'): merge(handler._cp_config) else: handler = controller @@ -602,7 +603,7 @@ class RoutesDispatcher(object): # Do the last path atom here so it can # override the controller's _cp_config. if last: - curpath = "/".join((curpath, last)) + curpath = '/'.join((curpath, last)) if curpath in app.config: merge(app.config[curpath]) @@ -666,16 +667,16 @@ def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, domain = header('Host', '') if use_x_forwarded_host: - domain = header("X-Forwarded-Host", domain) + domain = header('X-Forwarded-Host', domain) - prefix = domains.get(domain, "") + prefix = domains.get(domain, '') if prefix: path_info = httputil.urljoin(prefix, path_info) result = next_dispatcher(path_info) # Touch up staticdir config. See - # https://bitbucket.org/cherrypy/cherrypy/issue/614. + # https://github.com/cherrypy/cherrypy/issues/614. section = request.config.get('tools.staticdir.section') if section: section = section[len(prefix):] diff --git a/lib/cherrypy/_cperror.py b/lib/cherrypy/_cperror.py index 6256595b..e2a8fad8 100644 --- a/lib/cherrypy/_cperror.py +++ b/lib/cherrypy/_cperror.py @@ -29,8 +29,9 @@ user: 300 Multiple Choices Confirm with the user 301 Moved Permanently Confirm with the user 302 Found (Object moved temporarily) Confirm with the user -303 See Other GET the new URI--no confirmation -304 Not modified (for conditional GET only--POST should not raise this error) +303 See Other GET the new URI; no confirmation +304 Not modified for conditional GET only; + POST should not raise this error 305 Use Proxy Confirm with the user 307 Temporary Redirect Confirm with the user ===== ================================= =========== @@ -58,7 +59,8 @@ The 'error_page' config namespace can be used to provide custom HTML output for expected responses (like 404 Not Found). Supply a filename from which the output will be read. The contents will be interpolated with the values %(status)s, %(message)s, %(traceback)s, and %(version)s using plain old Python -`string formatting `_. +`string formatting +`_. :: @@ -100,26 +102,37 @@ send an e-mail containing the error:: def handle_error(): cherrypy.response.status = 500 cherrypy.response.body = [ - "Sorry, an error occured" + "Sorry, an error occurred" ] sendMail('error@domain.com', 'Error in your web app', _cperror.format_exc()) + @cherrypy.config(**{'request.error_response': handle_error}) class Root: - _cp_config = {'request.error_response': handle_error} - + pass Note that you have to explicitly set :attr:`response.body ` and not simply return an error message as a result. """ -from cgi import escape as _escape +import io +import contextlib from sys import exc_info as _exc_info from traceback import format_exception as _format_exception -from cherrypy._cpcompat import basestring, bytestr, iteritems, ntob -from cherrypy._cpcompat import tonative, urljoin as _urljoin +from xml.sax import saxutils + +import six +from six.moves import urllib + +from more_itertools import always_iterable + +import cherrypy +from cherrypy._cpcompat import escape_html +from cherrypy._cpcompat import ntob +from cherrypy._cpcompat import tonative +from cherrypy._helper import classproperty from cherrypy.lib import httputil as _httputil @@ -129,12 +142,6 @@ class CherryPyException(Exception): pass -class TimeoutError(CherryPyException): - - """Exception raised when Response.timed_out is detected.""" - pass - - class InternalRedirect(CherryPyException): """Exception raised to switch to the handler for a different URL. @@ -145,20 +152,19 @@ class InternalRedirect(CherryPyException): URL. """ - def __init__(self, path, query_string=""): - import cherrypy + def __init__(self, path, query_string=''): self.request = cherrypy.serving.request self.query_string = query_string - if "?" in path: + if '?' in path: # Separate any params included in the path - path, self.query_string = path.split("?", 1) + path, self.query_string = path.split('?', 1) # Note that urljoin will "do the right thing" whether url is: # 1. a URL relative to root (e.g. "/dummy") # 2. a URL relative to the current path # Note that any query string will be discarded. - path = _urljoin(self.request.path_info, path) + path = urllib.parse.urljoin(self.request.path_info, path) # Set a 'path' member attribute so that code which traps this # error can have access to it. @@ -193,9 +199,6 @@ class HTTPRedirect(CherryPyException): See :ref:`redirectingpost` for additional caveats. """ - status = None - """The integer HTTP status code to emit.""" - urls = None """The list of URL's to emit.""" @@ -203,41 +206,46 @@ class HTTPRedirect(CherryPyException): """The encoding when passed urls are not native strings""" def __init__(self, urls, status=None, encoding=None): - import cherrypy - request = cherrypy.serving.request - - if isinstance(urls, basestring): - urls = [urls] - - abs_urls = [] - for url in urls: - url = tonative(url, encoding or self.encoding) - + self.urls = abs_urls = [ # Note that urljoin will "do the right thing" whether url is: # 1. a complete URL with host (e.g. "http://www.example.com/test") # 2. a URL relative to root (e.g. "/dummy") # 3. a URL relative to the current path # Note that any query string in cherrypy.request is discarded. - url = _urljoin(cherrypy.url(), url) - abs_urls.append(url) - self.urls = abs_urls + urllib.parse.urljoin( + cherrypy.url(), + tonative(url, encoding or self.encoding), + ) + for url in always_iterable(urls) + ] - # RFC 2616 indicates a 301 response code fits our goal; however, - # browser support for 301 is quite messy. Do 302/303 instead. See - # http://www.alanflavell.org.uk/www/post-redirect.html - if status is None: - if request.protocol >= (1, 1): - status = 303 - else: - status = 302 - else: - status = int(status) - if status < 300 or status > 399: - raise ValueError("status must be between 300 and 399.") + status = ( + int(status) + if status is not None + else self.default_status + ) + if not 300 <= status <= 399: + raise ValueError('status must be between 300 and 399.') - self.status = status CherryPyException.__init__(self, abs_urls, status) + @classproperty + def default_status(cls): + """ + The default redirect status for the request. + + RFC 2616 indicates a 301 response code fits our goal; however, + browser support for 301 is quite messy. Use 302/303 instead. See + http://www.alanflavell.org.uk/www/post-redirect.html + """ + return 303 if cherrypy.serving.request.protocol >= (1, 1) else 302 + + @property + def status(self): + """The integer HTTP status code to emit.""" + _, status = self.args[:2] + return status + def set_response(self): """Modify cherrypy.response status, headers, and body to represent self. @@ -245,12 +253,11 @@ class HTTPRedirect(CherryPyException): CherryPy uses this internally, but you can also use it to create an HTTPRedirect object and set its output without *raising* the exception. """ - import cherrypy response = cherrypy.serving.response response.status = status = self.status if status in (300, 301, 302, 303, 307): - response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers['Content-Type'] = 'text/html;charset=utf-8' # "The ... URI SHOULD be given by the Location field # in the response." response.headers['Location'] = self.urls[0] @@ -259,16 +266,18 @@ class HTTPRedirect(CherryPyException): # SHOULD contain a short hypertext note with a hyperlink to the # new URI(s)." msg = { - 300: "This resource can be found at ", - 301: "This resource has permanently moved to ", - 302: "This resource resides temporarily at ", - 303: "This resource can be found at ", - 307: "This resource has moved temporarily to ", + 300: 'This resource can be found at ', + 301: 'This resource has permanently moved to ', + 302: 'This resource resides temporarily at ', + 303: 'This resource can be found at ', + 307: 'This resource has moved temporarily to ', }[status] msg += '%s.' - from xml.sax import saxutils - msgs = [msg % (saxutils.quoteattr(u), u) for u in self.urls] - response.body = ntob("
\n".join(msgs), 'utf-8') + msgs = [ + msg % (saxutils.quoteattr(u), escape_html(u)) + for u in self.urls + ] + response.body = ntob('
\n'.join(msgs), 'utf-8') # Previous code may have set C-L, so we have to reset it # (allow finalize to set it). response.headers.pop('Content-Length', None) @@ -293,12 +302,12 @@ class HTTPRedirect(CherryPyException): elif status == 305: # Use Proxy. # self.urls[0] should be the URI of the proxy. - response.headers['Location'] = self.urls[0] + response.headers['Location'] = ntob(self.urls[0], 'utf-8') response.body = None # Previous code may have set C-L, so we have to reset it. response.headers.pop('Content-Length', None) else: - raise ValueError("The %s status code is unknown." % status) + raise ValueError('The %s status code is unknown.' % status) def __call__(self): """Use this exception as a request.handler (raise self).""" @@ -307,16 +316,14 @@ class HTTPRedirect(CherryPyException): def clean_headers(status): """Remove any headers which should not apply to an error response.""" - import cherrypy - response = cherrypy.serving.response # Remove headers which applied to the original content, # but do not apply to the error page. respheaders = response.headers - for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After", - "Vary", "Content-Encoding", "Content-Length", "Expires", - "Content-Location", "Content-MD5", "Last-Modified"]: + for key in ['Accept-Ranges', 'Age', 'ETag', 'Location', 'Retry-After', + 'Vary', 'Content-Encoding', 'Content-Length', 'Expires', + 'Content-Location', 'Content-MD5', 'Last-Modified']: if key in respheaders: del respheaders[key] @@ -327,8 +334,8 @@ def clean_headers(status): # specifies the current length of the selected resource. # A response with status code 206 (Partial Content) MUST NOT # include a Content-Range field with a byte-range- resp-spec of "*". - if "Content-Range" in respheaders: - del respheaders["Content-Range"] + if 'Content-Range' in respheaders: + del respheaders['Content-Range'] class HTTPError(CherryPyException): @@ -368,7 +375,7 @@ class HTTPError(CherryPyException): raise self.__class__(500, _exc_info()[1].args[0]) if self.code < 400 or self.code > 599: - raise ValueError("status must be between 400 and 599.") + raise ValueError('status must be between 400 and 599.') # See http://www.python.org/dev/peps/pep-0352/ # self.message = message @@ -382,8 +389,6 @@ class HTTPError(CherryPyException): CherryPy uses this internally, but you can also use it to create an HTTPError object and set its output without *raising* the exception. """ - import cherrypy - response = cherrypy.serving.response clean_headers(self.code) @@ -410,6 +415,15 @@ class HTTPError(CherryPyException): """Use this exception as a request.handler (raise self).""" raise self + @classmethod + @contextlib.contextmanager + def handle(cls, exception, status=500, message=''): + """Translate exception into an HTTPError.""" + try: + yield + except exception as exc: + raise cls(status, message or str(exc)) + class NotFound(HTTPError): @@ -421,7 +435,6 @@ class NotFound(HTTPError): def __init__(self, path=None): if path is None: - import cherrypy request = cherrypy.serving.request path = request.script_name + request.path_info self.args = (path,) @@ -467,8 +480,6 @@ def get_error_page(status, **kwargs): status should be an int or a str. kwargs will be interpolated into the page template. """ - import cherrypy - try: code, reason, message = _httputil.valid_status(status) except ValueError: @@ -477,7 +488,7 @@ def get_error_page(status, **kwargs): # We can't use setdefault here, because some # callers send None for kwarg values. if kwargs.get('status') is None: - kwargs['status'] = "%s %s" % (code, reason) + kwargs['status'] = '%s %s' % (code, reason) if kwargs.get('message') is None: kwargs['message'] = message if kwargs.get('traceback') is None: @@ -485,11 +496,11 @@ def get_error_page(status, **kwargs): if kwargs.get('version') is None: kwargs['version'] = cherrypy.__version__ - for k, v in iteritems(kwargs): + for k, v in six.iteritems(kwargs): if v is None: - kwargs[k] = "" + kwargs[k] = '' else: - kwargs[k] = _escape(kwargs[k]) + kwargs[k] = escape_html(kwargs[k]) # Use a custom template or callable for the error page? pages = cherrypy.serving.request.error_page @@ -509,33 +520,33 @@ def get_error_page(status, **kwargs): if cherrypy.lib.is_iterator(result): from cherrypy.lib.encoding import UTF8StreamEncoder return UTF8StreamEncoder(result) - elif isinstance(result, cherrypy._cpcompat.unicodestr): + elif isinstance(result, six.text_type): return result.encode('utf-8') else: - if not isinstance(result, cherrypy._cpcompat.bytestr): - raise ValueError('error page function did not ' - 'return a bytestring, unicodestring or an ' + if not isinstance(result, bytes): + raise ValueError( + 'error page function did not ' + 'return a bytestring, six.text_type or an ' 'iterator - returned object of type %s.' % (type(result).__name__)) return result else: # Load the template from this path. - template = tonative(open(error_page, 'rb').read()) - except: + template = io.open(error_page, newline='').read() + except Exception: e = _format_exception(*_exc_info())[-1] m = kwargs['message'] if m: - m += "
" - m += "In addition, the custom error page failed:\n
%s" % e + m += '
' + m += 'In addition, the custom error page failed:\n
%s' % e kwargs['message'] = m response = cherrypy.serving.response - response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers['Content-Type'] = 'text/html;charset=utf-8' result = template % kwargs return result.encode('utf-8') - _ie_friendly_error_sizes = { 400: 512, 403: 256, 404: 512, 405: 256, 406: 512, 408: 512, 409: 512, 410: 256, @@ -544,7 +555,6 @@ _ie_friendly_error_sizes = { def _be_ie_unfriendly(status): - import cherrypy response = cherrypy.serving.response # For some statuses, Internet Explorer 5+ shows "friendly error @@ -558,11 +568,11 @@ def _be_ie_unfriendly(status): # Since we are issuing an HTTP error status, we assume that # the entity is short, and we should just collapse it. content = response.collapse_body() - l = len(content) - if l and l < s: + content_length = len(content) + if content_length and content_length < s: # IN ADDITION: the response must be written to IE # in one chunk or it will still get replaced! Bah. - content = content + (ntob(" ") * (s - l)) + content = content + (b' ' * (s - content_length)) response.body = content response.headers['Content-Length'] = str(len(content)) @@ -573,9 +583,9 @@ def format_exc(exc=None): if exc is None: exc = _exc_info() if exc == (None, None, None): - return "" + return '' import traceback - return "".join(traceback.format_exception(*exc)) + return ''.join(traceback.format_exception(*exc)) finally: del exc @@ -597,13 +607,13 @@ def bare_error(extrabody=None): # it cannot be allowed to fail. Therefore, don't add to it! # In particular, don't call any other CP functions. - body = ntob("Unrecoverable error in the server.") + body = b'Unrecoverable error in the server.' if extrabody is not None: - if not isinstance(extrabody, bytestr): + if not isinstance(extrabody, bytes): extrabody = extrabody.encode('utf-8') - body += ntob("\n") + extrabody + body += b'\n' + extrabody - return (ntob("500 Internal Server Error"), - [(ntob('Content-Type'), ntob('text/plain')), - (ntob('Content-Length'), ntob(str(len(body)), 'ISO-8859-1'))], + return (b'500 Internal Server Error', + [(b'Content-Type', b'text/plain'), + (b'Content-Length', ntob(str(len(body)), 'ISO-8859-1'))], [body]) diff --git a/lib/cherrypy/_cplogging.py b/lib/cherrypy/_cplogging.py index 19d1d91e..53b9addb 100644 --- a/lib/cherrypy/_cplogging.py +++ b/lib/cherrypy/_cplogging.py @@ -59,7 +59,8 @@ tracebacks, if enabled). If you are logging the access log and error log to the same source, then there is a possibility that a specially crafted error message may replicate an access log message as described in CWE-117. In this case it is the application -developer's responsibility to manually escape data before using CherryPy's log() +developer's responsibility to manually escape data before +using CherryPy's log() functionality, or they may create an application that is vulnerable to CWE-117. This would be achieved by using a custom handler escape any special characters, and attached as described below. @@ -109,15 +110,18 @@ the "log.error_file" config entry, for example). import datetime import logging -# Silence the no-handlers "warning" (stderr write!) in stdlib logging -logging.Logger.manager.emittedNoHandlerWarning = 1 -logfmt = logging.Formatter("%(message)s") import os import sys +import six + import cherrypy from cherrypy import _cperror -from cherrypy._cpcompat import ntob, py3k + + +# Silence the no-handlers "warning" (stderr write!) in stdlib logging +logging.Logger.manager.emittedNoHandlerWarning = 1 +logfmt = logging.Formatter('%(message)s') class NullHandler(logging.Handler): @@ -151,12 +155,11 @@ class LogManager(object): access_log = None """The actual :class:`logging.Logger` instance for access messages.""" - if py3k: - access_log_format = \ - '{h} {l} {u} {t} "{r}" {s} {b} "{f}" "{a}"' - else: - access_log_format = \ - '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + access_log_format = ( + '{h} {l} {u} {t} "{r}" {s} {b} "{f}" "{a}"' + if six.PY3 else + '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + ) logger_root = None """The "top-level" logger name. @@ -169,17 +172,17 @@ class LogManager(object): cherrypy.access. """ - def __init__(self, appid=None, logger_root="cherrypy"): + def __init__(self, appid=None, logger_root='cherrypy'): self.logger_root = logger_root self.appid = appid if appid is None: - self.error_log = logging.getLogger("%s.error" % logger_root) - self.access_log = logging.getLogger("%s.access" % logger_root) + self.error_log = logging.getLogger('%s.error' % logger_root) + self.access_log = logging.getLogger('%s.access' % logger_root) else: self.error_log = logging.getLogger( - "%s.error.%s" % (logger_root, appid)) + '%s.error.%s' % (logger_root, appid)) self.access_log = logging.getLogger( - "%s.access.%s" % (logger_root, appid)) + '%s.access.%s' % (logger_root, appid)) self.error_log.setLevel(logging.INFO) self.access_log.setLevel(logging.INFO) @@ -213,7 +216,11 @@ class LogManager(object): if traceback: exc_info = _cperror._exc_info() - self.error_log.log(severity, ' '.join((self.time(), context, msg)), exc_info=exc_info) + self.error_log.log( + severity, + ' '.join((self.time(), context, msg)), + exc_info=exc_info, + ) def __call__(self, *args, **kwargs): """An alias for ``error``.""" @@ -223,7 +230,8 @@ class LogManager(object): """Write to the access log (in Apache/NCSA Combined Log format). See the - `apache documentation `_ + `apache documentation + `_ for format details. CherryPy calls this automatically for you. Note there are no arguments; @@ -243,24 +251,26 @@ class LogManager(object): outheaders = response.headers inheaders = request.headers if response.output_status is None: - status = "-" + status = '-' else: - status = response.output_status.split(ntob(" "), 1)[0] - if py3k: + status = response.output_status.split(b' ', 1)[0] + if six.PY3: status = status.decode('ISO-8859-1') atoms = {'h': remote.name or remote.ip, 'l': '-', - 'u': getattr(request, "login", None) or "-", + 'u': getattr(request, 'login', None) or '-', 't': self.time(), 'r': request.request_line, 's': status, - 'b': dict.get(outheaders, 'Content-Length', '') or "-", + 'b': dict.get(outheaders, 'Content-Length', '') or '-', 'f': dict.get(inheaders, 'Referer', ''), 'a': dict.get(inheaders, 'User-Agent', ''), 'o': dict.get(inheaders, 'Host', '-'), + 'i': request.unique_id, + 'z': LazyRfc3339UtcTime(), } - if py3k: + if six.PY3: for k, v in atoms.items(): if not isinstance(v, str): v = str(v) @@ -280,11 +290,11 @@ class LogManager(object): try: self.access_log.log( logging.INFO, self.access_log_format.format(**atoms)) - except: + except Exception: self(traceback=True) else: for k, v in atoms.items(): - if isinstance(v, unicode): + if isinstance(v, six.text_type): v = v.encode('utf8') elif not isinstance(v, str): v = str(v) @@ -297,7 +307,7 @@ class LogManager(object): try: self.access_log.log( logging.INFO, self.access_log_format % atoms) - except: + except Exception: self(traceback=True) def time(self): @@ -311,48 +321,49 @@ class LogManager(object): def _get_builtin_handler(self, log, key): for h in log.handlers: - if getattr(h, "_cpbuiltin", None) == key: + if getattr(h, '_cpbuiltin', None) == key: return h # ------------------------- Screen handlers ------------------------- # def _set_screen_handler(self, log, enable, stream=None): - h = self._get_builtin_handler(log, "screen") + h = self._get_builtin_handler(log, 'screen') if enable: if not h: if stream is None: stream = sys.stderr h = logging.StreamHandler(stream) h.setFormatter(logfmt) - h._cpbuiltin = "screen" + h._cpbuiltin = 'screen' log.addHandler(h) elif h: log.handlers.remove(h) - def _get_screen(self): - h = self._get_builtin_handler - has_h = h(self.error_log, "screen") or h(self.access_log, "screen") - return bool(has_h) - - def _set_screen(self, newvalue): - self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) - self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) - screen = property(_get_screen, _set_screen, - doc="""Turn stderr/stdout logging on or off. + @property + def screen(self): + """Turn stderr/stdout logging on or off. If you set this to True, it'll add the appropriate StreamHandler for you. If you set it to False, it will remove the handler. - """) + """ + h = self._get_builtin_handler + has_h = h(self.error_log, 'screen') or h(self.access_log, 'screen') + return bool(has_h) + + @screen.setter + def screen(self, newvalue): + self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) + self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) # -------------------------- File handlers -------------------------- # def _add_builtin_file_handler(self, log, fname): h = logging.FileHandler(fname) h.setFormatter(logfmt) - h._cpbuiltin = "file" + h._cpbuiltin = 'file' log.addHandler(h) def _set_file_handler(self, log, filename): - h = self._get_builtin_handler(log, "file") + h = self._get_builtin_handler(log, 'file') if filename: if h: if h.baseFilename != os.path.abspath(filename): @@ -366,62 +377,65 @@ class LogManager(object): h.close() log.handlers.remove(h) - def _get_error_file(self): - h = self._get_builtin_handler(self.error_log, "file") + @property + def error_file(self): + """The filename for self.error_log. + + If you set this to a string, it'll add the appropriate FileHandler for + you. If you set it to ``None`` or ``''``, it will remove the handler. + """ + h = self._get_builtin_handler(self.error_log, 'file') if h: return h.baseFilename return '' - def _set_error_file(self, newvalue): + @error_file.setter + def error_file(self, newvalue): self._set_file_handler(self.error_log, newvalue) - error_file = property(_get_error_file, _set_error_file, - doc="""The filename for self.error_log. + + @property + def access_file(self): + """The filename for self.access_log. If you set this to a string, it'll add the appropriate FileHandler for you. If you set it to ``None`` or ``''``, it will remove the handler. - """) - - def _get_access_file(self): - h = self._get_builtin_handler(self.access_log, "file") + """ + h = self._get_builtin_handler(self.access_log, 'file') if h: return h.baseFilename return '' - def _set_access_file(self, newvalue): + @access_file.setter + def access_file(self, newvalue): self._set_file_handler(self.access_log, newvalue) - access_file = property(_get_access_file, _set_access_file, - doc="""The filename for self.access_log. - - If you set this to a string, it'll add the appropriate FileHandler for - you. If you set it to ``None`` or ``''``, it will remove the handler. - """) # ------------------------- WSGI handlers ------------------------- # def _set_wsgi_handler(self, log, enable): - h = self._get_builtin_handler(log, "wsgi") + h = self._get_builtin_handler(log, 'wsgi') if enable: if not h: h = WSGIErrorHandler() h.setFormatter(logfmt) - h._cpbuiltin = "wsgi" + h._cpbuiltin = 'wsgi' log.addHandler(h) elif h: log.handlers.remove(h) - def _get_wsgi(self): - return bool(self._get_builtin_handler(self.error_log, "wsgi")) - - def _set_wsgi(self, newvalue): - self._set_wsgi_handler(self.error_log, newvalue) - wsgi = property(_get_wsgi, _set_wsgi, - doc="""Write errors to wsgi.errors. + @property + def wsgi(self): + """Write errors to wsgi.errors. If you set this to True, it'll add the appropriate :class:`WSGIErrorHandler` for you (which writes errors to ``wsgi.errors``). If you set it to False, it will remove the handler. - """) + """ + return bool(self._get_builtin_handler(self.error_log, 'wsgi')) + + @wsgi.setter + def wsgi(self, newvalue): + self._set_wsgi_handler(self.error_log, newvalue) class WSGIErrorHandler(logging.Handler): @@ -446,16 +460,23 @@ class WSGIErrorHandler(logging.Handler): else: try: msg = self.format(record) - fs = "%s\n" + fs = '%s\n' import types # if no unicode support... - if not hasattr(types, "UnicodeType"): + if not hasattr(types, 'UnicodeType'): stream.write(fs % msg) else: try: stream.write(fs % msg) except UnicodeError: - stream.write(fs % msg.encode("UTF-8")) + stream.write(fs % msg.encode('UTF-8')) self.flush() - except: + except Exception: self.handleError(record) + + +class LazyRfc3339UtcTime(object): + def __str__(self): + """Return now() in RFC3339 UTC Format.""" + now = datetime.datetime.now() + return now.isoformat('T') + 'Z' diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index 02154d69..ac91e625 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -55,11 +55,17 @@ resides in the global site-package this won't be needed. Then restart apache2 and access http://127.0.0.1:8080 """ +import io import logging +import os +import re import sys +import six + +from more_itertools import always_iterable + import cherrypy -from cherrypy._cpcompat import BytesIO, copyitems, ntob from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil @@ -85,18 +91,19 @@ def setup(req): func() cherrypy.config.update({'log.screen': False, - "tools.ignore_headers.on": True, - "tools.ignore_headers.headers": ['Range'], + 'tools.ignore_headers.on': True, + 'tools.ignore_headers.headers': ['Range'], }) engine = cherrypy.engine - if hasattr(engine, "signal_handler"): + if hasattr(engine, 'signal_handler'): engine.signal_handler.unsubscribe() - if hasattr(engine, "console_control_handler"): + if hasattr(engine, 'console_control_handler'): engine.console_control_handler.unsubscribe() engine.autoreload.unsubscribe() cherrypy.server.unsubscribe() + @engine.subscribe('log') def _log(msg, level): newlevel = apache.APLOG_ERR if logging.DEBUG >= level: @@ -109,7 +116,6 @@ def setup(req): # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html # Also, "When server is not specified...LogLevel does not apply..." apache.log_error(msg, newlevel, req.server) - engine.subscribe('log', _log) engine.start() @@ -146,10 +152,10 @@ def handler(req): # Obtain a Request object from CherryPy local = req.connection.local_addr local = httputil.Host( - local[0], local[1], req.connection.local_host or "") + local[0], local[1], req.connection.local_host or '') remote = req.connection.remote_addr remote = httputil.Host( - remote[0], remote[1], req.connection.remote_host or "") + remote[0], remote[1], req.connection.remote_host or '') scheme = req.parsed_uri[0] or 'http' req.get_basic_auth_pw() @@ -162,7 +168,9 @@ def handler(req): except AttributeError: bad_value = ("You must provide a PythonOption '%s', " "either 'on' or 'off', when running a version " - "of mod_python < 3.1") + 'of mod_python < 3.1') + + options = req.get_options() threaded = options.get('multithread', '').lower() if threaded == 'on': @@ -170,7 +178,7 @@ def handler(req): elif threaded == 'off': threaded = False else: - raise ValueError(bad_value % "multithread") + raise ValueError(bad_value % 'multithread') forked = options.get('multiprocess', '').lower() if forked == 'on': @@ -178,18 +186,18 @@ def handler(req): elif forked == 'off': forked = False else: - raise ValueError(bad_value % "multiprocess") + raise ValueError(bad_value % 'multiprocess') - sn = cherrypy.tree.script_name(req.uri or "/") + sn = cherrypy.tree.script_name(req.uri or '/') if sn is None: send_response(req, '404 Not Found', [], '') else: app = cherrypy.tree.apps[sn] method = req.method path = req.uri - qs = req.args or "" + qs = req.args or '' reqproto = req.protocol - headers = copyitems(req.headers_in) + headers = list(six.iteritems(req.headers_in)) rfile = _ReadOnlyRequest(req) prev = None @@ -197,7 +205,7 @@ def handler(req): redirections = [] while True: request, response = app.get_serving(local, remote, scheme, - "HTTP/1.1") + 'HTTP/1.1') request.login = req.user request.multithread = bool(threaded) request.multiprocess = bool(forked) @@ -216,27 +224,27 @@ def handler(req): if not recursive: if ir.path in redirections: raise RuntimeError( - "InternalRedirector visited the same URL " - "twice: %r" % ir.path) + 'InternalRedirector visited the same URL ' + 'twice: %r' % ir.path) else: # Add the *previous* path_info + qs to # redirections. if qs: - qs = "?" + qs + qs = '?' + qs redirections.append(sn + path + qs) # Munge environment and try again. - method = "GET" + method = 'GET' path = ir.path qs = ir.query_string - rfile = BytesIO() + rfile = io.BytesIO() send_response( req, response.output_status, response.header_list, response.body, response.stream) finally: app.release_serving() - except: + except Exception: tb = format_exc() cherrypy.log(tb, 'MOD_PYTHON', severity=logging.ERROR) s, h, b = bare_error() @@ -249,7 +257,7 @@ def send_response(req, status, headers, body, stream=False): req.status = int(status[:3]) # Set response headers - req.content_type = "text/plain" + req.content_type = 'text/plain' for header, value in headers: if header.lower() == 'content-type': req.content_type = value @@ -261,16 +269,11 @@ def send_response(req, status, headers, body, stream=False): req.flush() # Set response body - if isinstance(body, basestring): - req.write(body) - else: - for seg in body: - req.write(seg) + for seg in always_iterable(body): + req.write(seg) # --------------- Startup tools for CherryPy + mod_python --------------- # -import os -import re try: import subprocess @@ -285,13 +288,13 @@ except ImportError: return pipeout -def read_process(cmd, args=""): - fullcmd = "%s %s" % (cmd, args) +def read_process(cmd, args=''): + fullcmd = '%s %s' % (cmd, args) pipeout = popen(fullcmd) try: firstline = pipeout.readline() cmd_not_found = re.search( - ntob("(not recognized|No such file|not found)"), + b'(not recognized|No such file|not found)', firstline, re.IGNORECASE ) @@ -320,8 +323,8 @@ LoadModule python_module modules/mod_python.so """ - def __init__(self, loc="/", port=80, opts=None, apache_path="apache", - handler="cherrypy._cpmodpy::handler"): + def __init__(self, loc='/', port=80, opts=None, apache_path='apache', + handler='cherrypy._cpmodpy::handler'): self.loc = loc self.port = port self.opts = opts @@ -329,25 +332,25 @@ LoadModule python_module modules/mod_python.so self.handler = handler def start(self): - opts = "".join([" PythonOption %s %s\n" % (k, v) + opts = ''.join([' PythonOption %s %s\n' % (k, v) for k, v in self.opts]) - conf_data = self.template % {"port": self.port, - "loc": self.loc, - "opts": opts, - "handler": self.handler, + conf_data = self.template % {'port': self.port, + 'loc': self.loc, + 'opts': opts, + 'handler': self.handler, } - mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") + mpconf = os.path.join(os.path.dirname(__file__), 'cpmodpy.conf') f = open(mpconf, 'wb') try: f.write(conf_data) finally: f.close() - response = read_process(self.apache_path, "-k start -f %s" % mpconf) + response = read_process(self.apache_path, '-k start -f %s' % mpconf) self.ready = True return response def stop(self): - os.popen("apache -k stop") + os.popen('apache -k stop') self.ready = False diff --git a/lib/cherrypy/_cpnative_server.py b/lib/cherrypy/_cpnative_server.py index e303573d..e9671d28 100644 --- a/lib/cherrypy/_cpnative_server.py +++ b/lib/cherrypy/_cpnative_server.py @@ -2,37 +2,45 @@ import logging import sys +import io + +import cheroot.server import cherrypy -from cherrypy._cpcompat import BytesIO from cherrypy._cperror import format_exc, bare_error from cherrypy.lib import httputil -from cherrypy import wsgiserver +from ._cpcompat import tonative -class NativeGateway(wsgiserver.Gateway): +class NativeGateway(cheroot.server.Gateway): + """Native gateway implementation allowing to bypass WSGI.""" recursive = False def respond(self): + """Obtain response from CherryPy machinery and then send it.""" req = self.req try: # Obtain a Request object from CherryPy - local = req.server.bind_addr - local = httputil.Host(local[0], local[1], "") - remote = req.conn.remote_addr, req.conn.remote_port - remote = httputil.Host(remote[0], remote[1], "") + local = req.server.bind_addr # FIXME: handle UNIX sockets + local = tonative(local[0]), local[1] + local = httputil.Host(local[0], local[1], '') + remote = tonative(req.conn.remote_addr), req.conn.remote_port + remote = httputil.Host(remote[0], remote[1], '') - scheme = req.scheme - sn = cherrypy.tree.script_name(req.uri or "/") + scheme = tonative(req.scheme) + sn = cherrypy.tree.script_name(tonative(req.uri or '/')) if sn is None: self.send_response('404 Not Found', [], ['']) else: app = cherrypy.tree.apps[sn] - method = req.method - path = req.path - qs = req.qs or "" - headers = req.inheaders.items() + method = tonative(req.method) + path = tonative(req.path) + qs = tonative(req.qs or '') + headers = ( + (tonative(h), tonative(v)) + for h, v in req.inheaders.items() + ) rfile = req.rfile prev = None @@ -40,7 +48,7 @@ class NativeGateway(wsgiserver.Gateway): redirections = [] while True: request, response = app.get_serving( - local, remote, scheme, "HTTP/1.1") + local, remote, scheme, 'HTTP/1.1') request.multithread = True request.multiprocess = False request.app = app @@ -49,8 +57,11 @@ class NativeGateway(wsgiserver.Gateway): # Run the CherryPy Request object and obtain the # response try: - request.run(method, path, qs, - req.request_protocol, headers, rfile) + request.run( + method, path, qs, + tonative(req.request_protocol), + headers, rfile, + ) break except cherrypy.InternalRedirect: ir = sys.exc_info()[1] @@ -60,27 +71,27 @@ class NativeGateway(wsgiserver.Gateway): if not self.recursive: if ir.path in redirections: raise RuntimeError( - "InternalRedirector visited the same " - "URL twice: %r" % ir.path) + 'InternalRedirector visited the same ' + 'URL twice: %r' % ir.path) else: # Add the *previous* path_info + qs to # redirections. if qs: - qs = "?" + qs + qs = '?' + qs redirections.append(sn + path + qs) # Munge environment and try again. - method = "GET" + method = 'GET' path = ir.path qs = ir.query_string - rfile = BytesIO() + rfile = io.BytesIO() self.send_response( response.output_status, response.header_list, response.body) finally: app.release_serving() - except: + except Exception: tb = format_exc() # print tb cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) @@ -88,10 +99,11 @@ class NativeGateway(wsgiserver.Gateway): self.send_response(s, h, b) def send_response(self, status, headers, body): + """Send response to HTTP request.""" req = self.req # Set response status - req.status = str(status or "500 Server Error") + req.status = status or b'500 Server Error' # Set response headers for header, value in headers: @@ -105,24 +117,24 @@ class NativeGateway(wsgiserver.Gateway): req.write(seg) -class CPHTTPServer(wsgiserver.HTTPServer): +class CPHTTPServer(cheroot.server.HTTPServer): + """Wrapper for cheroot.server.HTTPServer. - """Wrapper for wsgiserver.HTTPServer. - - wsgiserver has been designed to not reference CherryPy in any way, + cheroot has been designed to not reference CherryPy in any way, so that it can be used in other frameworks and applications. Therefore, we wrap it here, so we can apply some attributes from config -> cherrypy.server -> HTTPServer. """ def __init__(self, server_adapter=cherrypy.server): + """Initialize CPHTTPServer.""" self.server_adapter = server_adapter server_name = (self.server_adapter.socket_host or self.server_adapter.socket_file or None) - wsgiserver.HTTPServer.__init__( + cheroot.server.HTTPServer.__init__( self, server_adapter.bind_addr, NativeGateway, minthreads=server_adapter.thread_pool, maxthreads=server_adapter.thread_pool_max, @@ -140,15 +152,17 @@ class CPHTTPServer(wsgiserver.HTTPServer): ssl_module = self.server_adapter.ssl_module or 'pyopenssl' if self.server_adapter.ssl_context: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + adapter_class = cheroot.server.get_ssl_adapter_class(ssl_module) self.ssl_adapter = adapter_class( self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) + self.server_adapter.ssl_certificate_chain, + self.server_adapter.ssl_ciphers) self.ssl_adapter.context = self.server_adapter.ssl_context elif self.server_adapter.ssl_certificate: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + adapter_class = cheroot.server.get_ssl_adapter_class(ssl_module) self.ssl_adapter = adapter_class( self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) + self.server_adapter.ssl_certificate_chain, + self.server_adapter.ssl_ciphers) diff --git a/lib/cherrypy/_cpreqbody.py b/lib/cherrypy/_cpreqbody.py index d2dbbc92..893fe5f5 100644 --- a/lib/cherrypy/_cpreqbody.py +++ b/lib/cherrypy/_cpreqbody.py @@ -61,7 +61,7 @@ Here's the built-in JSON tool for an example:: def json_in(force=True, debug=False): request = cherrypy.serving.request def json_processor(entity): - \"""Read application/json data into request.json.\""" + '''Read application/json data into request.json.''' if not entity.headers.get("Content-Length", ""): raise cherrypy.HTTPError(411) @@ -120,8 +120,8 @@ try: except ImportError: def unquote_plus(bs): """Bytes version of urllib.parse.unquote_plus.""" - bs = bs.replace(ntob('+'), ntob(' ')) - atoms = bs.split(ntob('%')) + bs = bs.replace(b'+', b' ') + atoms = bs.split(b'%') for i in range(1, len(atoms)): item = atoms[i] try: @@ -129,10 +129,13 @@ except ImportError: atoms[i] = bytes([pct]) + item[2:] except ValueError: pass - return ntob('').join(atoms) + return b''.join(atoms) + +import six +import cheroot.server import cherrypy -from cherrypy._cpcompat import basestring, ntob, ntou +from cherrypy._cpcompat import ntou, unquote from cherrypy.lib import httputil @@ -144,14 +147,14 @@ def process_urlencoded(entity): for charset in entity.attempt_charsets: try: params = {} - for aparam in qs.split(ntob('&')): - for pair in aparam.split(ntob(';')): + for aparam in qs.split(b'&'): + for pair in aparam.split(b';'): if not pair: continue - atoms = pair.split(ntob('='), 1) + atoms = pair.split(b'=', 1) if len(atoms) == 1: - atoms.append(ntob('')) + atoms.append(b'') key = unquote_plus(atoms[0]).decode(charset) value = unquote_plus(atoms[1]).decode(charset) @@ -169,8 +172,8 @@ def process_urlencoded(entity): break else: raise cherrypy.HTTPError( - 400, "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(entity.attempt_charsets)) + 400, 'The request entity could not be decoded. The following ' + 'charsets were attempted: %s' % repr(entity.attempt_charsets)) # Now that all values have been successfully parsed and decoded, # apply them to the entity.params dict. @@ -185,7 +188,7 @@ def process_urlencoded(entity): def process_multipart(entity): """Read all multipart parts into entity.parts.""" - ib = "" + ib = '' if 'boundary' in entity.content_type.params: # http://tools.ietf.org/html/rfc2046#section-5.1.1 # "The grammar for parameters on the Content-type field is such that it @@ -193,7 +196,7 @@ def process_multipart(entity): # on the Content-type line" ib = entity.content_type.params['boundary'].strip('"') - if not re.match("^[ -~]{0,200}[!-~]$", ib): + if not re.match('^[ -~]{0,200}[!-~]$', ib): raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) ib = ('--' + ib).encode('ascii') @@ -315,7 +318,8 @@ class Entity(object): :attr:`request.body.parts`. You can enable it with:: - cherrypy.request.body.processors['multipart'] = _cpreqbody.process_multipart + cherrypy.request.body.processors['multipart'] = \ + _cpreqbody.process_multipart in an ``on_start_resource`` tool. """ @@ -325,14 +329,15 @@ class Entity(object): # absence of a charset parameter, is US-ASCII." # However, many browsers send data in utf-8 with no charset. attempt_charsets = ['utf-8'] - """A list of strings, each of which should be a known encoding. + r"""A list of strings, each of which should be a known encoding. When the Content-Type of the request body warrants it, each of the given encodings will be tried in order. The first one to successfully decode the entity without raising an error is stored as :attr:`entity.charset`. This defaults to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), + `HTTP/1.1 + `_), but ``['us-ascii', 'utf-8']`` for multipart parts. """ @@ -428,7 +433,7 @@ class Entity(object): # Copy the class 'attempt_charsets', prepending any Content-Type # charset - dec = self.content_type.params.get("charset", None) + dec = self.content_type.params.get('charset', None) if dec: self.attempt_charsets = [dec] + [c for c in self.attempt_charsets if c != dec] @@ -465,13 +470,10 @@ class Entity(object): self.filename.endswith('"') ): self.filename = self.filename[1:-1] - - # The 'type' attribute is deprecated in 3.2; remove it in 3.3. - type = property( - lambda self: self.content_type, - doc="A deprecated alias for " - ":attr:`content_type`." - ) + if 'filename*' in disp.params: + # @see https://tools.ietf.org/html/rfc5987 + encoding, lang, filename = disp.params['filename*'].split("'") + self.filename = unquote(str(filename), encoding) def read(self, size=None, fp_out=None): return self.fp.read(size, fp_out) @@ -520,8 +522,26 @@ class Entity(object): self.file.seek(0) else: value = self.value + value = self.decode_entity(value) return value + def decode_entity(self, value): + """Return a given byte encoded value as a string""" + for charset in self.attempt_charsets: + try: + value = value.decode(charset) + except UnicodeDecodeError: + pass + else: + self.charset = charset + return value + else: + raise cherrypy.HTTPError( + 400, + 'The request entity could not be decoded. The following ' + 'charsets were attempted: %s' % repr(self.attempt_charsets) + ) + def process(self): """Execute the best-match processor for the given media type.""" proc = None @@ -556,14 +576,15 @@ class Part(Entity): # "The default character set, which must be assumed in the absence of a # charset parameter, is US-ASCII." attempt_charsets = ['us-ascii', 'utf-8'] - """A list of strings, each of which should be a known encoding. + r"""A list of strings, each of which should be a known encoding. When the Content-Type of the request body warrants it, each of the given encodings will be tried in order. The first one to successfully decode the entity without raising an error is stored as :attr:`entity.charset`. This defaults to ``['utf-8']`` (plus 'ISO-8859-1' for "text/\*" types, as required by - `HTTP/1.1 `_), + `HTTP/1.1 + `_), but ``['us-ascii', 'utf-8']`` for multipart parts. """ @@ -595,40 +616,40 @@ class Part(Entity): self.file = None self.value = None + @classmethod def from_fp(cls, fp, boundary): headers = cls.read_headers(fp) return cls(fp, headers, boundary) - from_fp = classmethod(from_fp) + @classmethod def read_headers(cls, fp): headers = httputil.HeaderMap() while True: line = fp.readline() if not line: # No more data--illegal end of headers - raise EOFError("Illegal end of headers.") + raise EOFError('Illegal end of headers.') - if line == ntob('\r\n'): + if line == b'\r\n': # Normal end of headers break - if not line.endswith(ntob('\r\n')): - raise ValueError("MIME requires CRLF terminators: %r" % line) + if not line.endswith(b'\r\n'): + raise ValueError('MIME requires CRLF terminators: %r' % line) - if line[0] in ntob(' \t'): + if line[0] in b' \t': # It's a continuation line. v = line.strip().decode('ISO-8859-1') else: - k, v = line.split(ntob(":"), 1) + k, v = line.split(b':', 1) k = k.strip().decode('ISO-8859-1') v = v.strip().decode('ISO-8859-1') existing = headers.get(k) if existing: - v = ", ".join((existing, v)) + v = ', '.join((existing, v)) headers[k] = v return headers - read_headers = classmethod(read_headers) def read_lines_to_boundary(self, fp_out=None): """Read bytes from self.fp and return or write them to a file. @@ -640,16 +661,16 @@ class Part(Entity): object that supports the 'write' method; all bytes read will be written to the fp, and that fp is returned. """ - endmarker = self.boundary + ntob("--") - delim = ntob("") + endmarker = self.boundary + b'--' + delim = b'' prev_lf = True lines = [] seen = 0 while True: line = self.fp.readline(1 << 16) if not line: - raise EOFError("Illegal end of multipart body.") - if line.startswith(ntob("--")) and prev_lf: + raise EOFError('Illegal end of multipart body.') + if line.startswith(b'--') and prev_lf: strippedline = line.strip() if strippedline == self.boundary: break @@ -659,16 +680,16 @@ class Part(Entity): line = delim + line - if line.endswith(ntob("\r\n")): - delim = ntob("\r\n") + if line.endswith(b'\r\n'): + delim = b'\r\n' line = line[:-2] prev_lf = True - elif line.endswith(ntob("\n")): - delim = ntob("\n") + elif line.endswith(b'\n'): + delim = b'\n' line = line[:-1] prev_lf = True else: - delim = ntob("") + delim = b'' prev_lf = False if fp_out is None: @@ -682,21 +703,8 @@ class Part(Entity): fp_out.write(line) if fp_out is None: - result = ntob('').join(lines) - for charset in self.attempt_charsets: - try: - result = result.decode(charset) - except UnicodeDecodeError: - pass - else: - self.charset = charset - return result - else: - raise cherrypy.HTTPError( - 400, - "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(self.attempt_charsets) - ) + result = b''.join(lines) + return result else: fp_out.seek(0) return fp_out @@ -710,7 +718,7 @@ class Part(Entity): self.file = self.read_into_file() else: result = self.read_lines_to_boundary() - if isinstance(result, basestring): + if isinstance(result, bytes): self.value = result else: self.file = result @@ -725,31 +733,10 @@ class Part(Entity): self.read_lines_to_boundary(fp_out=fp_out) return fp_out + Entity.part_class = Part -try: - inf = float('inf') -except ValueError: - # Python 2.4 and lower - class Infinity(object): - - def __cmp__(self, other): - return 1 - - def __sub__(self, other): - return self - inf = Infinity() - - -comma_separated_headers = [ - 'Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', - 'Cache-Control', 'Connection', 'Content-Encoding', - 'Content-Language', 'Expect', 'If-Match', - 'If-None-Match', 'Pragma', 'Proxy-Authenticate', - 'Te', 'Trailer', 'Transfer-Encoding', 'Upgrade', - 'Vary', 'Via', 'Warning', 'Www-Authenticate' -] +inf = float('inf') class SizedReader: @@ -760,7 +747,7 @@ class SizedReader: self.fp = fp self.length = length self.maxbytes = maxbytes - self.buffer = ntob('') + self.buffer = b'' self.bufsize = bufsize self.bytes_read = 0 self.done = False @@ -796,7 +783,7 @@ class SizedReader: if remaining == 0: self.finish() if fp_out is None: - return ntob('') + return b'' else: return None @@ -806,7 +793,7 @@ class SizedReader: if self.buffer: if remaining is inf: data = self.buffer - self.buffer = ntob('') + self.buffer = b'' else: data = self.buffer[:remaining] self.buffer = self.buffer[remaining:] @@ -834,7 +821,7 @@ class SizedReader: if e.__class__.__name__ == 'MaxSizeExceeded': # Post data is too big raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) + 413, 'Maximum request length: %r' % e.args[1]) else: raise if not data: @@ -855,7 +842,7 @@ class SizedReader: fp_out.write(data) if fp_out is None: - return ntob('').join(chunks) + return b''.join(chunks) def readline(self, size=None): """Read a line from the request body and return it.""" @@ -867,7 +854,7 @@ class SizedReader: data = self.read(chunksize) if not data: break - pos = data.find(ntob('\n')) + 1 + pos = data.find(b'\n') + 1 if pos: chunks.append(data[:pos]) remainder = data[pos:] @@ -876,7 +863,7 @@ class SizedReader: break else: chunks.append(data) - return ntob('').join(chunks) + return b''.join(chunks) def readlines(self, sizehint=None): """Read lines from the request body and return them.""" @@ -905,28 +892,28 @@ class SizedReader: try: for line in self.fp.read_trailer_lines(): - if line[0] in ntob(' \t'): + if line[0] in b' \t': # It's a continuation line. v = line.strip() else: try: - k, v = line.split(ntob(":"), 1) + k, v = line.split(b':', 1) except ValueError: - raise ValueError("Illegal header line.") + raise ValueError('Illegal header line.') k = k.strip().title() v = v.strip() - if k in comma_separated_headers: - existing = self.trailers.get(envname) + if k in cheroot.server.comma_separated_headers: + existing = self.trailers.get(k) if existing: - v = ntob(", ").join((existing, v)) + v = b', '.join((existing, v)) self.trailers[k] = v except Exception: e = sys.exc_info()[1] if e.__class__.__name__ == 'MaxSizeExceeded': # Post data is too big raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) + 413, 'Maximum request length: %r' % e.args[1]) else: raise @@ -940,7 +927,7 @@ class RequestBody(Entity): # Don't parse the request body at all if the client didn't provide # a Content-Type header. See - # https://bitbucket.org/cherrypy/cherrypy/issue/790 + # https://github.com/cherrypy/cherrypy/issues/790 default_content_type = '' """This defines a default ``Content-Type`` to use if no Content-Type header is given. The empty string is used for RequestBody, which results in the @@ -1002,7 +989,7 @@ class RequestBody(Entity): # Python 2 only: keyword arguments must be byte strings (type # 'str'). if sys.version_info < (3, 0): - if isinstance(key, unicode): + if isinstance(key, six.text_type): key = key.encode('ISO-8859-1') if key in request_params: diff --git a/lib/cherrypy/_cprequest.py b/lib/cherrypy/_cprequest.py index 290bd2eb..3cc0c811 100644 --- a/lib/cherrypy/_cprequest.py +++ b/lib/cherrypy/_cprequest.py @@ -1,15 +1,18 @@ - -import os import sys import time -import warnings + +import uuid + +import six +from six.moves.http_cookies import SimpleCookie, CookieError + +from more_itertools import consume import cherrypy -from cherrypy._cpcompat import basestring, copykeys, ntob, unicodestr -from cherrypy._cpcompat import SimpleCookie, CookieError, py3k -from cherrypy import _cpreqbody, _cpconfig +from cherrypy._cpcompat import ntob +from cherrypy import _cpreqbody from cherrypy._cperror import format_exc, bare_error -from cherrypy.lib import httputil, file_generator +from cherrypy.lib import httputil, reprconf, encoding class Hook(object): @@ -41,33 +44,32 @@ class Hook(object): self.callback = callback if failsafe is None: - failsafe = getattr(callback, "failsafe", False) + failsafe = getattr(callback, 'failsafe', False) self.failsafe = failsafe if priority is None: - priority = getattr(callback, "priority", 50) + priority = getattr(callback, 'priority', 50) self.priority = priority self.kwargs = kwargs def __lt__(self, other): - # Python 3 + """ + Hooks sort by priority, ascending, such that + hooks of lower priority are run first. + """ return self.priority < other.priority - def __cmp__(self, other): - # Python 2 - return cmp(self.priority, other.priority) - def __call__(self): """Run self.callback(**self.kwargs).""" return self.callback(**self.kwargs) def __repr__(self): cls = self.__class__ - return ("%s.%s(callback=%r, failsafe=%r, priority=%r, %s)" + return ('%s.%s(callback=%r, failsafe=%r, priority=%r, %s)' % (cls.__module__, cls.__name__, self.callback, self.failsafe, self.priority, - ", ".join(['%s=%r' % (k, v) + ', '.join(['%s=%r' % (k, v) for k, v in self.kwargs.items()]))) @@ -107,7 +109,7 @@ class HookMap(dict): except (cherrypy.HTTPError, cherrypy.HTTPRedirect, cherrypy.InternalRedirect): exc = sys.exc_info()[1] - except: + except Exception: exc = sys.exc_info()[1] cherrypy.log(traceback=True, severity=40) if exc: @@ -124,10 +126,10 @@ class HookMap(dict): def __repr__(self): cls = self.__class__ - return "%s.%s(points=%r)" % ( + return '%s.%s(points=%r)' % ( cls.__module__, cls.__name__, - copykeys(self) + list(self) ) @@ -138,9 +140,9 @@ def hooks_namespace(k, v): # Use split again to allow multiple hooks for a single # hookpoint per path (e.g. "hooks.before_handler.1"). # Little-known fact you only get from reading source ;) - hookpoint = k.split(".", 1)[0] - if isinstance(v, basestring): - v = cherrypy.lib.attributes(v) + hookpoint = k.split('.', 1)[0] + if isinstance(v, six.string_types): + v = cherrypy.lib.reprconf.attributes(v) if not isinstance(v, Hook): v = Hook(v) cherrypy.serving.request.hooks[hookpoint].append(v) @@ -199,23 +201,23 @@ class Request(object): unless we are processing an InternalRedirect.""" # Conversation/connection attributes - local = httputil.Host("127.0.0.1", 80) - "An httputil.Host(ip, port, hostname) object for the server socket." + local = httputil.Host('127.0.0.1', 80) + 'An httputil.Host(ip, port, hostname) object for the server socket.' - remote = httputil.Host("127.0.0.1", 1111) - "An httputil.Host(ip, port, hostname) object for the client socket." + remote = httputil.Host('127.0.0.1', 1111) + 'An httputil.Host(ip, port, hostname) object for the client socket.' - scheme = "http" + scheme = 'http' """ The protocol used between client and server. In most cases, this will be either 'http' or 'https'.""" - server_protocol = "HTTP/1.1" + server_protocol = 'HTTP/1.1' """ The HTTP version for which the HTTP server is at least conditionally compliant.""" - base = "" + base = '' """The (scheme://host) portion of the requested URL. In some cases (e.g. when proxying via mod_rewrite), this may contain path segments which cherrypy.url uses when constructing url's, but @@ -223,13 +225,13 @@ class Request(object): MUST NOT end in a slash.""" # Request-Line attributes - request_line = "" + request_line = '' """ The complete Request-Line received from the client. This is a single string consisting of the request method, URI, and protocol version (joined by spaces). Any final CRLF is removed.""" - method = "GET" + method = 'GET' """ Indicates the HTTP method to be performed on the resource identified by the Request-URI. Common methods include GET, HEAD, POST, PUT, and @@ -237,7 +239,7 @@ class Request(object): servers and gateways may restrict the set of allowable methods. CherryPy applications SHOULD restrict the set (on a per-URI basis).""" - query_string = "" + query_string = '' """ The query component of the Request-URI, a string of information to be interpreted by the resource. The query portion of a URI follows the @@ -312,7 +314,7 @@ class Request(object): If True, the rfile (if any) is automatically read and parsed, and the result placed into request.params or request.body.""" - methods_with_bodies = ("POST", "PUT") + methods_with_bodies = ('POST', 'PUT', 'PATCH') """ A sequence of HTTP methods for which CherryPy will automatically attempt to read a body from the rfile. If you are going to change @@ -341,7 +343,7 @@ class Request(object): to a hierarchical arrangement of objects, starting at request.app.root. See help(cherrypy.dispatch) for more information.""" - script_name = "" + script_name = '' """ The 'mount point' of the application which is handling this request. @@ -349,7 +351,7 @@ class Request(object): the root of the URI, it MUST be an empty string (not "/"). """ - path_info = "/" + path_info = '/' """ The 'relative path' portion of the Request-URI. This is relative to the script_name ('mount point') of the application which is @@ -467,16 +469,19 @@ class Request(object): A string containing the stage reached in the request-handling process. This is useful when debugging a live server with hung requests.""" - namespaces = _cpconfig.NamespaceSet( - **{"hooks": hooks_namespace, - "request": request_namespace, - "response": response_namespace, - "error_page": error_page_namespace, - "tools": cherrypy.tools, + unique_id = None + """A lazy object generating and memorizing UUID4 on ``str()`` render.""" + + namespaces = reprconf.NamespaceSet( + **{'hooks': hooks_namespace, + 'request': request_namespace, + 'response': response_namespace, + 'error_page': error_page_namespace, + 'tools': cherrypy.tools, }) - def __init__(self, local_host, remote_host, scheme="http", - server_protocol="HTTP/1.1"): + def __init__(self, local_host, remote_host, scheme='http', + server_protocol='HTTP/1.1'): """Populate a new Request object. local_host should be an httputil.Host object with the server info. @@ -498,6 +503,8 @@ class Request(object): self.stage = None + self.unique_id = LazyUUID4() + def close(self): """Run cleanup code. (Core)""" if not self.closed: @@ -544,7 +551,7 @@ class Request(object): self.error_response = cherrypy.HTTPError(500).set_response self.method = method - path = path or "/" + path = path or '/' self.query_string = query_string or '' self.params = {} @@ -590,7 +597,7 @@ class Request(object): except self.throws: raise - except: + except Exception: if self.throw_errors: raise else: @@ -600,95 +607,92 @@ class Request(object): if self.show_tracebacks: body = format_exc() else: - body = "" + body = '' r = bare_error(body) response.output_status, response.header_list, response.body = r - if self.method == "HEAD": + if self.method == 'HEAD': # HEAD requests MUST NOT return a message-body in the response. response.body = [] try: cherrypy.log.access() - except: + except Exception: cherrypy.log.error(traceback=True) - if response.timed_out: - raise cherrypy.TimeoutError() - return response - # Uncomment for stage debugging - # stage = property(lambda self: self._stage, lambda self, v: print(v)) - def respond(self, path_info): """Generate a response for the resource at self.path_info. (Core)""" - response = cherrypy.serving.response try: try: try: - if self.app is None: - raise cherrypy.NotFound() - - # Get the 'Host' header, so we can HTTPRedirect properly. - self.stage = 'process_headers' - self.process_headers() - - # Make a copy of the class hooks - self.hooks = self.__class__.hooks.copy() - self.toolmaps = {} - - self.stage = 'get_resource' - self.get_resource(path_info) - - self.body = _cpreqbody.RequestBody( - self.rfile, self.headers, request_params=self.params) - - self.namespaces(self.config) - - self.stage = 'on_start_resource' - self.hooks.run('on_start_resource') - - # Parse the querystring - self.stage = 'process_query_string' - self.process_query_string() - - # Process the body - if self.process_request_body: - if self.method not in self.methods_with_bodies: - self.process_request_body = False - self.stage = 'before_request_body' - self.hooks.run('before_request_body') - if self.process_request_body: - self.body.process() - - # Run the handler - self.stage = 'before_handler' - self.hooks.run('before_handler') - if self.handler: - self.stage = 'handler' - response.body = self.handler() - - # Finalize - self.stage = 'before_finalize' - self.hooks.run('before_finalize') - response.finalize() + self._do_respond(path_info) except (cherrypy.HTTPRedirect, cherrypy.HTTPError): inst = sys.exc_info()[1] inst.set_response() self.stage = 'before_finalize (HTTPError)' self.hooks.run('before_finalize') - response.finalize() + cherrypy.serving.response.finalize() finally: self.stage = 'on_end_resource' self.hooks.run('on_end_resource') except self.throws: raise - except: + except Exception: if self.throw_errors: raise self.handle_error() + def _do_respond(self, path_info): + response = cherrypy.serving.response + + if self.app is None: + raise cherrypy.NotFound() + + self.hooks = self.__class__.hooks.copy() + self.toolmaps = {} + + # Get the 'Host' header, so we can HTTPRedirect properly. + self.stage = 'process_headers' + self.process_headers() + + self.stage = 'get_resource' + self.get_resource(path_info) + + self.body = _cpreqbody.RequestBody( + self.rfile, self.headers, request_params=self.params) + + self.namespaces(self.config) + + self.stage = 'on_start_resource' + self.hooks.run('on_start_resource') + + # Parse the querystring + self.stage = 'process_query_string' + self.process_query_string() + + # Process the body + if self.process_request_body: + if self.method not in self.methods_with_bodies: + self.process_request_body = False + self.stage = 'before_request_body' + self.hooks.run('before_request_body') + if self.process_request_body: + self.body.process() + + # Run the handler + self.stage = 'before_handler' + self.hooks.run('before_handler') + if self.handler: + self.stage = 'handler' + response.body = self.handler() + + # Finalize + self.stage = 'before_finalize' + self.hooks.run('before_finalize') + response.finalize() + def process_query_string(self): """Parse the query string into Python structures. (Core)""" try: @@ -696,14 +700,14 @@ class Request(object): self.query_string, encoding=self.query_string_encoding) except UnicodeDecodeError: raise cherrypy.HTTPError( - 404, "The given query string could not be processed. Query " - "strings for this resource must be encoded with %r." % + 404, 'The given query string could not be processed. Query ' + 'strings for this resource must be encoded with %r.' % self.query_string_encoding) # Python 2 only: keyword arguments must be byte strings (type 'str'). - if not py3k: + if six.PY2: for key, value in p.items(): - if isinstance(key, unicode): + if isinstance(key, six.text_type): del p[key] p[key.encode(self.query_string_encoding)] = value self.params.update(p) @@ -718,23 +722,16 @@ class Request(object): name = name.title() value = value.strip() - # Warning: if there is more than one header entry for cookies - # (AFAIK, only Konqueror does that), only the last one will - # remain in headers (but they will be correctly stored in - # request.cookie). - if "=?" in value: - dict.__setitem__(headers, name, httputil.decode_TEXT(value)) - else: - dict.__setitem__(headers, name, value) + headers[name] = httputil.decode_TEXT_maybe(value) - # Handle cookies differently because on Konqueror, multiple - # cookies come on different lines with the same key + # Some clients, notably Konquoror, supply multiple + # cookies on different lines with the same key. To + # handle this case, store all cookies in self.cookie. if name == 'Cookie': try: self.cookie.load(value) - except CookieError: - msg = "Illegal cookie name %s" % value.split('=')[0] - raise cherrypy.HTTPError(400, msg) + except CookieError as exc: + raise cherrypy.HTTPError(400, str(exc)) if not dict.__contains__(headers, 'Host'): # All Internet-based HTTP/1.1 servers MUST respond with a 400 @@ -746,7 +743,7 @@ class Request(object): host = dict.get(headers, 'Host') if not host: host = self.local.name or self.local.ip - self.base = "%s://%s" % (self.scheme, host) + self.base = '%s://%s' % (self.scheme, host) def get_resource(self, path): """Call a dispatcher (which sets self.handler and .config). (Core)""" @@ -754,7 +751,7 @@ class Request(object): # dispatchers can only be specified in app.config, not in _cp_config # (since custom dispatchers may not even have an app.root). dispatch = self.app.find_config( - path, "request.dispatch", self.dispatch) + path, 'request.dispatch', self.dispatch) # dispatch() should set self.handler and self.config dispatch(path) @@ -762,46 +759,23 @@ class Request(object): def handle_error(self): """Handle the last unanticipated exception. (Core)""" try: - self.hooks.run("before_error_response") + self.hooks.run('before_error_response') if self.error_response: self.error_response() - self.hooks.run("after_error_response") + self.hooks.run('after_error_response') cherrypy.serving.response.finalize() except cherrypy.HTTPRedirect: inst = sys.exc_info()[1] inst.set_response() cherrypy.serving.response.finalize() - # ------------------------- Properties ------------------------- # - - def _get_body_params(self): - warnings.warn( - "body_params is deprecated in CherryPy 3.2, will be removed in " - "CherryPy 3.3.", - DeprecationWarning - ) - return self.body.params - body_params = property(_get_body_params, - doc=""" - If the request Content-Type is 'application/x-www-form-urlencoded' or - multipart, this will be a dict of the params pulled from the entity - body; that is, it will be the portion of request.params that come - from the message body (sometimes called "POST params", although they - can be sent with various HTTP method verbs). This value is set between - the 'before_request_body' and 'before_handler' hooks (assuming that - process_request_body is True). - - Deprecated in 3.2, will be removed for 3.3 in favor of - :attr:`request.body.params`.""") - class ResponseBody(object): """The body of the HTTP response (the response entity).""" - if py3k: - unicode_err = ("Page handlers MUST return bytes. Use tools.encode " - "if you wish to return unicode.") + unicode_err = ('Page handlers MUST return bytes. Use tools.encode ' + 'if you wish to return unicode.') def __get__(self, obj, objclass=None): if obj is None: @@ -812,37 +786,21 @@ class ResponseBody(object): def __set__(self, obj, value): # Convert the given value to an iterable object. - if py3k and isinstance(value, str): + if isinstance(value, six.text_type): raise ValueError(self.unicode_err) - - if isinstance(value, basestring): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if value: - value = [value] - else: - # [''] doesn't evaluate to False, so replace it with []. - value = [] - elif py3k and isinstance(value, list): + elif isinstance(value, list): # every item in a list must be bytes... - for i, item in enumerate(value): - if isinstance(item, str): - raise ValueError(self.unicode_err) - # Don't use isinstance here; io.IOBase which has an ABC takes - # 1000 times as long as, say, isinstance(value, str) - elif hasattr(value, 'read'): - value = file_generator(value) - elif value is None: - value = [] - obj._body = value + if any(isinstance(item, six.text_type) for item in value): + raise ValueError(self.unicode_err) + + obj._body = encoding.prepare_iter(value) class Response(object): """An HTTP Response, including status, headers, and body.""" - status = "" + status = '' """The HTTP Status-Code and Reason-Phrase.""" header_list = [] @@ -872,14 +830,6 @@ class Response(object): time = None """The value of time.time() when created. Use in HTTP dates.""" - timeout = 300 - """Seconds after which the response will be aborted.""" - - timed_out = False - """ - Flag to indicate the response should be aborted, because it has - exceeded its timeout.""" - stream = False """If False, buffer the response body.""" @@ -893,27 +843,25 @@ class Response(object): # Since we know all our keys are titled strings, we can # bypass HeaderMap.update and get a big speed boost. dict.update(self.headers, { - "Content-Type": 'text/html', - "Server": "CherryPy/" + cherrypy.__version__, - "Date": httputil.HTTPDate(self.time), + 'Content-Type': 'text/html', + 'Server': 'CherryPy/' + cherrypy.__version__, + 'Date': httputil.HTTPDate(self.time), }) self.cookie = SimpleCookie() def collapse_body(self): """Collapse self.body to a single string; replace it and return it.""" - if isinstance(self.body, basestring): - return self.body + new_body = b''.join(self.body) + self.body = new_body + return new_body - newbody = [] - for chunk in self.body: - if py3k and not isinstance(chunk, bytes): - raise TypeError("Chunk %s is not of type 'bytes'." % - repr(chunk)) - newbody.append(chunk) - newbody = ntob('').join(newbody) - - self.body = newbody - return newbody + def _flush_body(self): + """ + Discard self.body but consume any generator such that + any finalization can occur, such as is required by + caching.tee_output(). + """ + consume(iter(self.body)) def finalize(self): """Transform headers (and cookies) into self.header_list. (Core)""" @@ -924,9 +872,9 @@ class Response(object): headers = self.headers - self.status = "%s %s" % (code, reason) + self.status = '%s %s' % (code, reason) self.output_status = ntob(str(code), 'ascii') + \ - ntob(" ") + headers.encode(reason) + b' ' + headers.encode(reason) if self.stream: # The upshot: wsgiserver will chunk the response if @@ -939,7 +887,8 @@ class Response(object): # and 304 (not modified) responses MUST NOT # include a message-body." dict.pop(headers, 'Content-Length', None) - self.body = ntob("") + self._flush_body() + self.body = b'' else: # Responses which are not streamed should have a Content-Length, # but allow user code to set Content-Length if desired. @@ -952,22 +901,30 @@ class Response(object): cookie = self.cookie.output() if cookie: - for line in cookie.split("\n"): - if line.endswith("\r"): - # Python 2.4 emits cookies joined by LF but 2.5+ by CRLF. - line = line[:-1] - name, value = line.split(": ", 1) - if isinstance(name, unicodestr): - name = name.encode("ISO-8859-1") - if isinstance(value, unicodestr): + for line in cookie.split('\r\n'): + name, value = line.split(': ', 1) + if isinstance(name, six.text_type): + name = name.encode('ISO-8859-1') + if isinstance(value, six.text_type): value = headers.encode(value) h.append((name, value)) - def check_timeout(self): - """If now > self.time + self.timeout, set self.timed_out. - This purposefully sets a flag, rather than raising an error, - so that a monitor thread can interrupt the Response thread. +class LazyUUID4(object): + def __str__(self): + """Return UUID4 and keep it for future calls.""" + return str(self.uuid4) + + @property + def uuid4(self): + """Provide unique id on per-request basis using UUID4. + + It's evaluated lazily on render. """ - if time.time() > self.time + self.timeout: - self.timed_out = True + try: + self._uuid4 + except AttributeError: + # evaluate on first access + self._uuid4 = uuid.uuid4() + + return self._uuid4 diff --git a/lib/cherrypy/_cpserver.py b/lib/cherrypy/_cpserver.py index a31e7428..0f60e2c8 100644 --- a/lib/cherrypy/_cpserver.py +++ b/lib/cherrypy/_cpserver.py @@ -1,18 +1,17 @@ """Manage HTTP servers with CherryPy.""" -import warnings +import six import cherrypy -from cherrypy.lib import attributes -from cherrypy._cpcompat import basestring, py3k +from cherrypy.lib.reprconf import attributes +from cherrypy._cpcompat import text_or_bytes +from cherrypy.process.servers import ServerAdapter -# We import * because we want to export check_port -# et al as attributes of this module. -from cherrypy.process.servers import * + +__all__ = ('Server', ) class Server(ServerAdapter): - """An adapter for an HTTP server. You can set attributes (like socket_host and socket_port) @@ -28,26 +27,26 @@ class Server(ServerAdapter): _socket_host = '127.0.0.1' - def _get_socket_host(self): - return self._socket_host - - def _set_socket_host(self, value): - if value == '': - raise ValueError("The empty string ('') is not an allowed value. " - "Use '0.0.0.0' instead to listen on all active " - "interfaces (INADDR_ANY).") - self._socket_host = value - socket_host = property( - _get_socket_host, - _set_socket_host, - doc="""The hostname or IP address on which to listen for connections. + @property + def socket_host(self): # noqa: D401; irrelevant for properties + """The hostname or IP address on which to listen for connections. Host values may be any IPv4 or IPv6 address, or any valid hostname. The string 'localhost' is a synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). The string '0.0.0.0' is a special IPv4 entry meaning "any active interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for IPv6. The empty string or None are - not allowed.""") + not allowed. + """ + return self._socket_host + + @socket_host.setter + def socket_host(self, value): + if value == '': + raise ValueError("The empty string ('') is not an allowed value. " + "Use '0.0.0.0' instead to listen on all active " + 'interfaces (INADDR_ANY).') + self._socket_host = value socket_file = None """If given, the name of the UNIX socket to use instead of TCP/IP. @@ -61,11 +60,11 @@ class Server(ServerAdapter): socket_timeout = 10 """The timeout in seconds for accepted connections (default 10).""" - + accepted_queue_size = -1 """The maximum number of requests which will be queued up before the server refuses to accept it (default -1, meaning no limit).""" - + accepted_queue_timeout = 10 """The timeout in seconds for attempting to add a request to the queue when the queue is full (default 10).""" @@ -96,7 +95,8 @@ class Server(ServerAdapter): instance = None """If not None, this should be an HTTP server instance (such as - CPWSGIServer) which cherrypy.server will control. Use this when you need + cheroot.wsgi.Server) which cherrypy.server will control. + Use this when you need more control over object instantiation than is available in the various configuration options.""" @@ -113,20 +113,23 @@ class Server(ServerAdapter): ssl_private_key = None """The filename of the private key to use with SSL.""" - if py3k: + ssl_ciphers = None + """The ciphers list of SSL.""" + + if six.PY3: ssl_module = 'builtin' """The name of a registered SSL adaptation module to use with the builtin WSGI server. Builtin options are: 'builtin' (to use the SSL library built into recent versions of Python). You may also register your own classes in the - wsgiserver.ssl_adapters dict.""" + cheroot.server.ssl_adapters dict.""" else: ssl_module = 'pyopenssl' """The name of a registered SSL adaptation module to use with the builtin WSGI server. Builtin options are 'builtin' (to use the SSL library built into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL project, which you must install separately). You - may also register your own classes in the wsgiserver.ssl_adapters + may also register your own classes in the cheroot.server.ssl_adapters dict.""" statistics = False @@ -141,9 +144,29 @@ class Server(ServerAdapter): which declares it covers WSGI version 1.0.1 but still mandates the wsgi.version (1, 0)] and ('u', 0), an experimental unicode version. You may create and register your own experimental versions of the WSGI - protocol by adding custom classes to the wsgiserver.wsgi_gateways dict.""" + protocol by adding custom classes to the cheroot.server.wsgi_gateways dict. + """ + + peercreds = False + """If True, peer cred lookup for UNIX domain socket will put to WSGI env. + + This information will then be available through WSGI env vars: + * X_REMOTE_PID + * X_REMOTE_UID + * X_REMOTE_GID + """ + + peercreds_resolve = False + """If True, username/group will be looked up in the OS from peercreds. + + This information will then be available through WSGI env vars: + * REMOTE_USER + * X_REMOTE_USER + * X_REMOTE_GROUP + """ def __init__(self): + """Initialize Server instance.""" self.bus = cherrypy.engine self.httpserver = None self.interrupt = None @@ -156,7 +179,7 @@ class Server(ServerAdapter): if httpserver is None: from cherrypy import _cpwsgi_server httpserver = _cpwsgi_server.CPWSGIServer(self) - if isinstance(httpserver, basestring): + if isinstance(httpserver, text_or_bytes): # Is anyone using this? Can I add an arg? httpserver = attributes(httpserver)(self) return httpserver, self.bind_addr @@ -165,22 +188,28 @@ class Server(ServerAdapter): """Start the HTTP server.""" if not self.httpserver: self.httpserver, self.bind_addr = self.httpserver_from_self() - ServerAdapter.start(self) + super(Server, self).start() start.priority = 75 - def _get_bind_addr(self): + @property + def bind_addr(self): + """Return bind address. + + A (host, port) tuple for TCP sockets or a str for Unix domain sockts. + """ if self.socket_file: return self.socket_file if self.socket_host is None and self.socket_port is None: return None return (self.socket_host, self.socket_port) - def _set_bind_addr(self, value): + @bind_addr.setter + def bind_addr(self, value): if value is None: self.socket_file = None self.socket_host = None self.socket_port = None - elif isinstance(value, basestring): + elif isinstance(value, text_or_bytes): self.socket_file = value self.socket_host = None self.socket_port = None @@ -189,17 +218,14 @@ class Server(ServerAdapter): self.socket_host, self.socket_port = value self.socket_file = None except ValueError: - raise ValueError("bind_addr must be a (host, port) tuple " - "(for TCP sockets) or a string (for Unix " - "domain sockets), not %r" % value) - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc='A (host, port) tuple for TCP sockets or ' - 'a str for Unix domain sockets.') + raise ValueError('bind_addr must be a (host, port) tuple ' + '(for TCP sockets) or a string (for Unix ' + 'domain sockets), not %r' % value) def base(self): - """Return the base (scheme://host[:port] or sock file) for this server. + """Return the base for this server. + + e.i. scheme://host[:port] or sock file """ if self.socket_file: return self.socket_file @@ -215,12 +241,12 @@ class Server(ServerAdapter): port = self.socket_port if self.ssl_certificate: - scheme = "https" + scheme = 'https' if port != 443: - host += ":%s" % port + host += ':%s' % port else: - scheme = "http" + scheme = 'http' if port != 80: - host += ":%s" % port + host += ':%s' % port - return "%s://%s" % (scheme, host) + return '%s://%s' % (scheme, host) diff --git a/lib/cherrypy/_cpthreadinglocal.py b/lib/cherrypy/_cpthreadinglocal.py deleted file mode 100644 index 238c3224..00000000 --- a/lib/cherrypy/_cpthreadinglocal.py +++ /dev/null @@ -1,241 +0,0 @@ -# This is a backport of Python-2.4's threading.local() implementation - -"""Thread-local objects - -(Note that this module provides a Python version of thread - threading.local class. Depending on the version of Python you're - using, there may be a faster one available. You should always import - the local class from threading.) - -Thread-local objects support the management of thread-local data. -If you have data that you want to be local to a thread, simply create -a thread-local object and use its attributes: - - >>> mydata = local() - >>> mydata.number = 42 - >>> mydata.number - 42 - -You can also access the local-object's dictionary: - - >>> mydata.__dict__ - {'number': 42} - >>> mydata.__dict__.setdefault('widgets', []) - [] - >>> mydata.widgets - [] - -What's important about thread-local objects is that their data are -local to a thread. If we access the data in a different thread: - - >>> log = [] - >>> def f(): - ... items = mydata.__dict__.items() - ... items.sort() - ... log.append(items) - ... mydata.number = 11 - ... log.append(mydata.number) - - >>> import threading - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[], 11] - -we get different data. Furthermore, changes made in the other thread -don't affect data seen in this thread: - - >>> mydata.number - 42 - -Of course, values you get from a local object, including a __dict__ -attribute, are for whatever thread was current at the time the -attribute was read. For that reason, you generally don't want to save -these values across threads, as they apply only to the thread they -came from. - -You can create custom local objects by subclassing the local class: - - >>> class MyLocal(local): - ... number = 2 - ... initialized = False - ... def __init__(self, **kw): - ... if self.initialized: - ... raise SystemError('__init__ called too many times') - ... self.initialized = True - ... self.__dict__.update(kw) - ... def squared(self): - ... return self.number ** 2 - -This can be useful to support default values, methods and -initialization. Note that if you define an __init__ method, it will be -called each time the local object is used in a separate thread. This -is necessary to initialize each thread's dictionary. - -Now if we create a local object: - - >>> mydata = MyLocal(color='red') - -Now we have a default number: - - >>> mydata.number - 2 - -an initial color: - - >>> mydata.color - 'red' - >>> del mydata.color - -And a method that operates on the data: - - >>> mydata.squared() - 4 - -As before, we can access the data in a separate thread: - - >>> log = [] - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[('color', 'red'), ('initialized', True)], 11] - -without affecting this thread's data: - - >>> mydata.number - 2 - >>> mydata.color - Traceback (most recent call last): - ... - AttributeError: 'MyLocal' object has no attribute 'color' - -Note that subclasses can define slots, but they are not thread -local. They are shared across threads: - - >>> class MyLocal(local): - ... __slots__ = 'number' - - >>> mydata = MyLocal() - >>> mydata.number = 42 - >>> mydata.color = 'red' - -So, the separate thread: - - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - -affects what we see: - - >>> mydata.number - 11 - ->>> del mydata -""" - -# Threading import is at end - - -class _localbase(object): - __slots__ = '_local__key', '_local__args', '_local__lock' - - def __new__(cls, *args, **kw): - self = object.__new__(cls) - key = 'thread.local.' + str(id(self)) - object.__setattr__(self, '_local__key', key) - object.__setattr__(self, '_local__args', (args, kw)) - object.__setattr__(self, '_local__lock', RLock()) - - if args or kw and (cls.__init__ is object.__init__): - raise TypeError("Initialization arguments are not supported") - - # We need to create the thread dict in anticipation of - # __init__ being called, to make sure we don't call it - # again ourselves. - dict = object.__getattribute__(self, '__dict__') - currentThread().__dict__[key] = dict - - return self - - -def _patch(self): - key = object.__getattribute__(self, '_local__key') - d = currentThread().__dict__.get(key) - if d is None: - d = {} - currentThread().__dict__[key] = d - object.__setattr__(self, '__dict__', d) - - # we have a new instance dict, so call out __init__ if we have - # one - cls = type(self) - if cls.__init__ is not object.__init__: - args, kw = object.__getattribute__(self, '_local__args') - cls.__init__(self, *args, **kw) - else: - object.__setattr__(self, '__dict__', d) - - -class local(_localbase): - - def __getattribute__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__getattribute__(self, name) - finally: - lock.release() - - def __setattr__(self, name, value): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__setattr__(self, name, value) - finally: - lock.release() - - def __delattr__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__delattr__(self, name) - finally: - lock.release() - - def __del__(): - threading_enumerate = enumerate - __getattribute__ = object.__getattribute__ - - def __del__(self): - key = __getattribute__(self, '_local__key') - - try: - threads = list(threading_enumerate()) - except: - # if enumerate fails, as it seems to do during - # shutdown, we'll skip cleanup under the assumption - # that there is nothing to clean up - return - - for thread in threads: - try: - __dict__ = thread.__dict__ - except AttributeError: - # Thread is dying, rest in peace - continue - - if key in __dict__: - try: - del __dict__[key] - except KeyError: - pass # didn't have anything in this thread - - return __del__ - __del__ = __del__() - -from threading import currentThread, enumerate, RLock diff --git a/lib/cherrypy/_cptools.py b/lib/cherrypy/_cptools.py index 06a56e87..57460285 100644 --- a/lib/cherrypy/_cptools.py +++ b/lib/cherrypy/_cptools.py @@ -22,17 +22,22 @@ Tools may be implemented as any object with a namespace. The builtins are generally either modules or instances of the tools.Tool class. """ -import sys -import warnings +import six import cherrypy +from cherrypy._helper import expose + +from cherrypy.lib import cptools, encoding, static, jsontools +from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc +from cherrypy.lib import caching as _caching +from cherrypy.lib import auth_basic, auth_digest def _getargs(func): """Return the names of all static arguments to the given function.""" # Use this instead of importing inspect for less mem overhead. import types - if sys.version_info >= (3, 0): + if six.PY3: if isinstance(func, types.MethodType): func = func.__func__ co = func.__code__ @@ -44,8 +49,8 @@ def _getargs(func): _attr_error = ( - "CherryPy Tools cannot be turned on directly. Instead, turn them " - "on via config, or use them as decorators on your page handlers." + 'CherryPy Tools cannot be turned on directly. Instead, turn them ' + 'on via config, or use them as decorators on your page handlers.' ) @@ -56,7 +61,7 @@ class Tool(object): help(tool.callable) should give you more information about this Tool. """ - namespace = "tools" + namespace = 'tools' def __init__(self, point, callable, name=None, priority=50): self._point = point @@ -66,12 +71,13 @@ class Tool(object): self.__doc__ = self.callable.__doc__ self._setargs() - def _get_on(self): + @property + def on(self): raise AttributeError(_attr_error) - def _set_on(self, value): + @on.setter + def on(self, value): raise AttributeError(_attr_error) - on = property(_get_on, _set_on) def _setargs(self): """Copy func parameter names to obj attributes.""" @@ -79,7 +85,7 @@ class Tool(object): for arg in _getargs(self.callable): setattr(self, arg, None) except (TypeError, AttributeError): - if hasattr(self.callable, "__call__"): + if hasattr(self.callable, '__call__'): for arg in _getargs(self.callable.__call__): setattr(self, arg, None) # IronPython 1.0 raises NotImplementedError because @@ -103,8 +109,8 @@ class Tool(object): if self._name in tm: conf.update(tm[self._name]) - if "on" in conf: - del conf["on"] + if 'on' in conf: + del conf['on'] return conf @@ -113,21 +119,21 @@ class Tool(object): For example:: + @expose @tools.proxy() def whats_my_base(self): return cherrypy.request.base - whats_my_base.exposed = True """ if args: - raise TypeError("The %r Tool does not accept positional " - "arguments; you must use keyword arguments." + raise TypeError('The %r Tool does not accept positional ' + 'arguments; you must use keyword arguments.' % self._name) def tool_decorator(f): - if not hasattr(f, "_cp_config"): + if not hasattr(f, '_cp_config'): f._cp_config = {} - subspace = self.namespace + "." + self._name + "." - f._cp_config[subspace + "on"] = True + subspace = self.namespace + '.' + self._name + '.' + f._cp_config[subspace + 'on'] = True for k, v in kwargs.items(): f._cp_config[subspace + k] = v return f @@ -140,9 +146,9 @@ class Tool(object): method when the tool is "turned on" in config. """ conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) cherrypy.serving.request.hooks.attach(self._point, self.callable, priority=p, **conf) @@ -171,12 +177,12 @@ class HandlerTool(Tool): nav = tools.staticdir.handler(section="/nav", dir="nav", root=absDir) """ + @expose def handle_func(*a, **kw): handled = self.callable(*args, **self._merged_args(kwargs)) if not handled: raise cherrypy.NotFound() return cherrypy.serving.response.body - handle_func.exposed = True return handle_func def _wrapper(self, **kwargs): @@ -190,9 +196,9 @@ class HandlerTool(Tool): method when the tool is "turned on" in config. """ conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) cherrypy.serving.request.hooks.attach(self._point, self._wrapper, priority=p, **conf) @@ -253,11 +259,6 @@ class ErrorTool(Tool): # Builtin tools # -from cherrypy.lib import cptools, encoding, auth, static, jsontools -from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc -from cherrypy.lib import caching as _caching -from cherrypy.lib import auth_basic, auth_digest - class SessionTool(Tool): @@ -271,7 +272,7 @@ class SessionTool(Tool): body. This is off by default for safety reasons; for example, a large upload would block the session, denying an AJAX progress meter - (`issue `_). + (`issue `_). When 'explicit' (or any other value), you need to call cherrypy.session.acquire_lock() yourself before using @@ -295,9 +296,9 @@ class SessionTool(Tool): conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) hooks.attach(self._point, self.callable, priority=p, **conf) @@ -321,9 +322,12 @@ class SessionTool(Tool): sess.regenerate() # Grab cookie-relevant tool args - conf = dict([(k, v) for k, v in self._merged_args().items() - if k in ('path', 'path_header', 'name', 'timeout', - 'domain', 'secure')]) + relevant = 'path', 'path_header', 'name', 'timeout', 'domain', 'secure' + conf = dict( + (k, v) + for k, v in self._merged_args().items() + if k in relevant + ) _sessions.set_response_cookie(**conf) @@ -365,6 +369,7 @@ class XMLRPCController(object): # would be if someone actually disabled the default_toolbox. Meh. _cp_config = {'tools.xmlrpc.on': True} + @expose def default(self, *vpath, **params): rpcparams, rpcmethod = _xmlrpc.process_body() @@ -372,30 +377,25 @@ class XMLRPCController(object): for attr in str(rpcmethod).split('.'): subhandler = getattr(subhandler, attr, None) - if subhandler and getattr(subhandler, "exposed", False): + if subhandler and getattr(subhandler, 'exposed', False): body = subhandler(*(vpath + rpcparams), **params) else: - # https://bitbucket.org/cherrypy/cherrypy/issue/533 + # https://github.com/cherrypy/cherrypy/issues/533 # if a method is not found, an xmlrpclib.Fault should be returned # raising an exception here will do that; see # cherrypy.lib.xmlrpcutil.on_error raise Exception('method "%s" is not supported' % attr) - conf = cherrypy.serving.request.toolmaps['tools'].get("xmlrpc", {}) + conf = cherrypy.serving.request.toolmaps['tools'].get('xmlrpc', {}) _xmlrpc.respond(body, conf.get('encoding', 'utf-8'), conf.get('allow_none', 0)) return cherrypy.serving.response.body - default.exposed = True class SessionAuthTool(HandlerTool): - - def _setargs(self): - for name in dir(cptools.SessionAuth): - if not name.startswith("__"): - setattr(self, name, None) + pass class CachingTool(Tool): @@ -410,14 +410,14 @@ class CachingTool(Tool): if request.cacheable: # Note the devious technique here of adding hooks on the fly request.hooks.attach('before_finalize', _caching.tee_output, - priority=90) - _wrapper.priority = 20 + priority=100) + _wrapper.priority = 90 def _setup(self): """Hook caching into cherrypy.request.""" conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) cherrypy.serving.request.hooks.attach('before_handler', self._wrapper, priority=p, **conf) @@ -446,7 +446,7 @@ class Toolbox(object): cherrypy.serving.request.toolmaps[self.namespace] = map = {} def populate(k, v): - toolname, arg = k.split(".", 1) + toolname, arg = k.split('.', 1) bucket = map.setdefault(toolname, {}) bucket[arg] = v return populate @@ -456,33 +456,24 @@ class Toolbox(object): map = cherrypy.serving.request.toolmaps.get(self.namespace) if map: for name, settings in map.items(): - if settings.get("on", False): + if settings.get('on', False): tool = getattr(self, name) tool._setup() - -class DeprecatedTool(Tool): - - _name = None - warnmsg = "This Tool is deprecated." - - def __init__(self, point, warnmsg=None): - self.point = point - if warnmsg is not None: - self.warnmsg = warnmsg - - def __call__(self, *args, **kwargs): - warnings.warn(self.warnmsg) - - def tool_decorator(f): - return f - return tool_decorator - - def _setup(self): - warnings.warn(self.warnmsg) + def register(self, point, **kwargs): + """ + Return a decorator which registers the function + at the given hook point. + """ + def decorator(func): + attr_name = kwargs.get('name', func.__name__) + tool = Tool(point, func, **kwargs) + setattr(self, attr_name, tool) + return func + return decorator -default_toolbox = _d = Toolbox("tools") +default_toolbox = _d = Toolbox('tools') _d.session_auth = SessionAuthTool(cptools.session_auth) _d.allow = Tool('on_start_resource', cptools.allow) _d.proxy = Tool('before_request_body', cptools.proxy, priority=30) @@ -502,20 +493,8 @@ _d.sessions = SessionTool() _d.xmlrpc = ErrorTool(_xmlrpc.on_error) _d.caching = CachingTool('before_handler', _caching.get, 'caching') _d.expires = Tool('before_finalize', _caching.expires) -_d.tidy = DeprecatedTool( - 'before_finalize', - "The tidy tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") -_d.nsgmls = DeprecatedTool( - 'before_finalize', - "The nsgmls tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") _d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) _d.referer = Tool('before_request_body', cptools.referer) -_d.basic_auth = Tool('on_start_resource', auth.basic_auth) -_d.digest_auth = Tool('on_start_resource', auth.digest_auth) _d.trailing_slash = Tool('before_handler', cptools.trailing_slash, priority=60) _d.flatten = Tool('before_finalize', cptools.flatten) _d.accept = Tool('on_start_resource', cptools.accept) @@ -525,5 +504,6 @@ _d.json_in = Tool('before_request_body', jsontools.json_in, priority=30) _d.json_out = Tool('before_handler', jsontools.json_out, priority=30) _d.auth_basic = Tool('before_handler', auth_basic.basic_auth, priority=1) _d.auth_digest = Tool('before_handler', auth_digest.digest_auth, priority=1) +_d.params = Tool('before_handler', cptools.convert_params, priority=15) -del _d, cptools, encoding, auth, static +del _d, cptools, encoding, static diff --git a/lib/cherrypy/_cptree.py b/lib/cherrypy/_cptree.py index a31b2793..ceb54379 100644 --- a/lib/cherrypy/_cptree.py +++ b/lib/cherrypy/_cptree.py @@ -2,14 +2,15 @@ import os +import six + import cherrypy -from cherrypy._cpcompat import ntou, py3k +from cherrypy._cpcompat import ntou from cherrypy import _cpconfig, _cplogging, _cprequest, _cpwsgi, tools -from cherrypy.lib import httputil +from cherrypy.lib import httputil, reprconf class Application(object): - """A CherryPy Application. Servers and gateways should not instantiate Request objects directly. @@ -30,7 +31,7 @@ class Application(object): """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict of {key: value} pairs.""" - namespaces = _cpconfig.NamespaceSet() + namespaces = reprconf.NamespaceSet() toolboxes = {'tools': cherrypy.tools} log = None @@ -44,22 +45,24 @@ class Application(object): relative_urls = False - def __init__(self, root, script_name="", config=None): + def __init__(self, root, script_name='', config=None): + """Initialize Application with given root.""" self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) self.root = root self.script_name = script_name self.wsgiapp = _cpwsgi.CPWSGIApp(self) self.namespaces = self.namespaces.copy() - self.namespaces["log"] = lambda k, v: setattr(self.log, k, v) - self.namespaces["wsgi"] = self.wsgiapp.namespace_handler + self.namespaces['log'] = lambda k, v: setattr(self.log, k, v) + self.namespaces['wsgi'] = self.wsgiapp.namespace_handler self.config = self.__class__.config.copy() if config: self.merge(config) def __repr__(self): - return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, + """Generate a representation of the Application instance.""" + return '%s.%s(%r, %r)' % (self.__module__, self.__class__.__name__, self.root, self.script_name) script_name_doc = """The URI "mount point" for this app. A mount point @@ -78,42 +81,58 @@ class Application(object): provided for each call from request.wsgi_environ['SCRIPT_NAME']. """ - def _get_script_name(self): + @property + def script_name(self): # noqa: D401; irrelevant for properties + """The URI "mount point" for this app. + + A mount point is that portion of the URI which is constant for all URIs + that are serviced by this application; it does not include scheme, + host, or proxy ("virtual host") portions of the URI. + + For example, if script_name is "/my/cool/app", then the URL + "http://www.example.com/my/cool/app/page1" might be handled by a + "page1" method on the root object. + + The value of script_name MUST NOT end in a slash. If the script_name + refers to the root of the URI, it MUST be an empty string (not "/"). + + If script_name is explicitly set to None, then the script_name will be + provided for each call from request.wsgi_environ['SCRIPT_NAME']. + """ if self._script_name is not None: return self._script_name # A `_script_name` with a value of None signals that the script name # should be pulled from WSGI environ. - return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") + return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip('/') - def _set_script_name(self, value): + @script_name.setter + def script_name(self, value): if value: - value = value.rstrip("/") + value = value.rstrip('/') self._script_name = value - script_name = property(fget=_get_script_name, fset=_set_script_name, - doc=script_name_doc) def merge(self, config): """Merge the given config into self.config.""" _cpconfig.merge(self.config, config) # Handle namespaces specified in config. - self.namespaces(self.config.get("/", {})) + self.namespaces(self.config.get('/', {})) def find_config(self, path, key, default=None): """Return the most-specific value for key along path, or default.""" - trail = path or "/" + trail = path or '/' while trail: nodeconf = self.config.get(trail, {}) if key in nodeconf: return nodeconf[key] - lastslash = trail.rfind("/") + lastslash = trail.rfind('/') if lastslash == -1: break - elif lastslash == 0 and trail != "/": - trail = "/" + elif lastslash == 0 and trail != '/': + trail = '/' else: trail = trail[:lastslash] @@ -142,17 +161,17 @@ class Application(object): try: req.close() - except: + except Exception: cherrypy.log(traceback=True, severity=40) cherrypy.serving.clear() def __call__(self, environ, start_response): + """Call a WSGI-callable.""" return self.wsgiapp(environ, start_response) class Tree(object): - """A registry of CherryPy applications, mounted at diverse points. An instance of this class may also be used as a WSGI callable @@ -168,9 +187,10 @@ class Tree(object): WSGI callable if you happen to be using a WSGI server).""" def __init__(self): + """Initialize registry Tree.""" self.apps = {} - def mount(self, root, script_name="", config=None): + def mount(self, root, script_name='', config=None): """Mount a new app from a root object, script_name, and config. root @@ -195,29 +215,36 @@ class Tree(object): if script_name is None: raise TypeError( "The 'script_name' argument may not be None. Application " - "objects may, however, possess a script_name of None (in " - "order to inpect the WSGI environ for SCRIPT_NAME upon each " - "request). You cannot mount such Applications on this Tree; " - "you must pass them to a WSGI server interface directly.") + 'objects may, however, possess a script_name of None (in ' + 'order to inpect the WSGI environ for SCRIPT_NAME upon each ' + 'request). You cannot mount such Applications on this Tree; ' + 'you must pass them to a WSGI server interface directly.') # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") + script_name = script_name.rstrip('/') if isinstance(root, Application): app = root - if script_name != "" and script_name != app.script_name: + if script_name != '' and script_name != app.script_name: raise ValueError( - "Cannot specify a different script name and pass an " - "Application instance to cherrypy.mount") + 'Cannot specify a different script name and pass an ' + 'Application instance to cherrypy.mount') script_name = app.script_name else: app = Application(root, script_name) # If mounted at "", add favicon.ico - if (script_name == "" and root is not None - and not hasattr(root, "favicon_ico")): - favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), - "favicon.ico") + needs_favicon = ( + script_name == '' + and root is not None + and not hasattr(root, 'favicon_ico') + ) + if needs_favicon: + favicon = os.path.join( + os.getcwd(), + os.path.dirname(__file__), + 'favicon.ico', + ) root.favicon_ico = tools.staticfile.handler(favicon) if config: @@ -227,14 +254,14 @@ class Tree(object): return app - def graft(self, wsgi_callable, script_name=""): + def graft(self, wsgi_callable, script_name=''): """Mount a wsgi callable at the given script_name.""" # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") + script_name = script_name.rstrip('/') self.apps[script_name] = wsgi_callable def script_name(self, path=None): - """The script_name of the app at the given path, or None. + """Return the script_name of the app at the given path, or None. If path is None, cherrypy.request is used. """ @@ -250,22 +277,23 @@ class Tree(object): if path in self.apps: return path - if path == "": + if path == '': return None # Move one node up the tree and try again. - path = path[:path.rfind("/")] + path = path[:path.rfind('/')] def __call__(self, environ, start_response): + """Pre-initialize WSGI env and call WSGI-callable.""" # If you're calling this, then you're probably setting SCRIPT_NAME # to '' (some WSGI servers always set SCRIPT_NAME to ''). # Try to look up the app using the full path. env1x = environ - if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + if six.PY2 and environ.get(ntou('wsgi.version')) == (ntou('u'), 0): env1x = _cpwsgi.downgrade_wsgi_ux_to_1x(environ) path = httputil.urljoin(env1x.get('SCRIPT_NAME', ''), env1x.get('PATH_INFO', '')) - sn = self.script_name(path or "/") + sn = self.script_name(path or '/') if sn is None: start_response('404 Not Found', []) return [] @@ -274,26 +302,12 @@ class Tree(object): # Correct the SCRIPT_NAME and PATH_INFO environ entries. environ = environ.copy() - if not py3k: - if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): - # Python 2/WSGI u.0: all strings MUST be of type unicode - enc = environ[ntou('wsgi.url_encoding')] - environ[ntou('SCRIPT_NAME')] = sn.decode(enc) - environ[ntou('PATH_INFO')] = path[ - len(sn.rstrip("/")):].decode(enc) - else: - # Python 2/WSGI 1.x: all strings MUST be of type str - environ['SCRIPT_NAME'] = sn - environ['PATH_INFO'] = path[len(sn.rstrip("/")):] + if six.PY2 and environ.get(ntou('wsgi.version')) == (ntou('u'), 0): + # Python 2/WSGI u.0: all strings MUST be of type unicode + enc = environ[ntou('wsgi.url_encoding')] + environ[ntou('SCRIPT_NAME')] = sn.decode(enc) + environ[ntou('PATH_INFO')] = path[len(sn.rstrip('/')):].decode(enc) else: - if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): - # Python 3/WSGI u.0: all strings MUST be full unicode - environ['SCRIPT_NAME'] = sn - environ['PATH_INFO'] = path[len(sn.rstrip("/")):] - else: - # Python 3/WSGI 1.x: all strings MUST be ISO-8859-1 str - environ['SCRIPT_NAME'] = sn.encode( - 'utf-8').decode('ISO-8859-1') - environ['PATH_INFO'] = path[ - len(sn.rstrip("/")):].encode('utf-8').decode('ISO-8859-1') + environ['SCRIPT_NAME'] = sn + environ['PATH_INFO'] = path[len(sn.rstrip('/')):] return app(environ, start_response) diff --git a/lib/cherrypy/_cpwsgi.py b/lib/cherrypy/_cpwsgi.py index a8068fb0..0b4942ff 100644 --- a/lib/cherrypy/_cpwsgi.py +++ b/lib/cherrypy/_cpwsgi.py @@ -8,13 +8,17 @@ still be translatable to bytes via the Latin-1 encoding!" """ import sys as _sys +import io + +import six import cherrypy as _cherrypy -from cherrypy._cpcompat import BytesIO, bytestr, ntob, ntou, py3k, unicodestr +from cherrypy._cpcompat import ntou from cherrypy import _cperror from cherrypy.lib import httputil from cherrypy.lib import is_closable_iterator + def downgrade_wsgi_ux_to_1x(environ): """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ. """ @@ -24,7 +28,7 @@ def downgrade_wsgi_ux_to_1x(environ): for k, v in list(environ.items()): if k in [ntou('PATH_INFO'), ntou('SCRIPT_NAME'), ntou('QUERY_STRING')]: v = v.encode(url_encoding) - elif isinstance(v, unicodestr): + elif isinstance(v, six.text_type): v = v.encode('ISO-8859-1') env1x[k.encode('ISO-8859-1')] = v @@ -43,10 +47,13 @@ class VirtualHost(object): Domain2App = cherrypy.Application(root) SecureApp = cherrypy.Application(Secure()) - vhost = cherrypy._cpwsgi.VirtualHost(RootApp, - domains={'www.domain2.example': Domain2App, - 'www.domain2.example:443': SecureApp, - }) + vhost = cherrypy._cpwsgi.VirtualHost( + RootApp, + domains={ + 'www.domain2.example': Domain2App, + 'www.domain2.example:443': SecureApp, + }, + ) cherrypy.tree.graft(vhost) """ @@ -75,7 +82,7 @@ class VirtualHost(object): def __call__(self, environ, start_response): domain = environ.get('HTTP_HOST', '') if self.use_x_forwarded_host: - domain = environ.get("HTTP_X_FORWARDED_HOST", domain) + domain = environ.get('HTTP_X_FORWARDED_HOST', domain) nextapp = self.domains.get(domain) if nextapp is None: @@ -106,7 +113,7 @@ class InternalRedirector(object): # Add the *previous* path_info + qs to redirections. old_uri = sn + path if qs: - old_uri += "?" + qs + old_uri += '?' + qs redirections.append(old_uri) if not self.recursive: @@ -114,18 +121,20 @@ class InternalRedirector(object): # already new_uri = sn + ir.path if ir.query_string: - new_uri += "?" + ir.query_string + new_uri += '?' + ir.query_string if new_uri in redirections: ir.request.close() - raise RuntimeError("InternalRedirector visited the " - "same URL twice: %r" % new_uri) + tmpl = ( + 'InternalRedirector visited the same URL twice: %r' + ) + raise RuntimeError(tmpl % new_uri) # Munge the environment and try again. - environ['REQUEST_METHOD'] = "GET" + environ['REQUEST_METHOD'] = 'GET' environ['PATH_INFO'] = ir.path environ['QUERY_STRING'] = ir.query_string - environ['wsgi.input'] = BytesIO() - environ['CONTENT_LENGTH'] = "0" + environ['wsgi.input'] = io.BytesIO() + environ['CONTENT_LENGTH'] = '0' environ['cherrypy.previous_request'] = ir.request @@ -157,19 +166,20 @@ class _TrappedResponse(object): self.throws = throws self.started_response = False self.response = self.trap( - self.nextapp, self.environ, self.start_response) + self.nextapp, self.environ, self.start_response, + ) self.iter_response = iter(self.response) def __iter__(self): self.started_response = True return self - if py3k: - def __next__(self): - return self.trap(next, self.iter_response) - else: - def next(self): - return self.trap(self.iter_response.next) + def __next__(self): + return self.trap(next, self.iter_response) + + # todo: https://pythonhosted.org/six/#six.Iterator + if six.PY2: + next = __next__ def close(self): if hasattr(self.response, 'close'): @@ -182,18 +192,19 @@ class _TrappedResponse(object): raise except StopIteration: raise - except: + except Exception: tb = _cperror.format_exc() - #print('trapped (started %s):' % self.started_response, tb) _cherrypy.log(tb, severity=40) if not _cherrypy.request.show_tracebacks: - tb = "" + tb = '' s, h, b = _cperror.bare_error(tb) - if py3k: + if six.PY3: # What fun. s = s.decode('ISO-8859-1') - h = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in h] + h = [ + (k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in h + ] if self.started_response: # Empty our iterable (so future calls raise StopIteration) self.iter_response = iter([]) @@ -202,7 +213,7 @@ class _TrappedResponse(object): try: self.start_response(s, h, _sys.exc_info()) - except: + except Exception: # "The application must not trap any exceptions raised by # start_response, if it called start_response with exc_info. # Instead, it should allow such exceptions to propagate @@ -212,7 +223,7 @@ class _TrappedResponse(object): raise if self.started_response: - return ntob("").join(b) + return b''.join(b) else: return b @@ -227,7 +238,7 @@ class AppResponse(object): def __init__(self, environ, start_response, cpapp): self.cpapp = cpapp try: - if not py3k: + if six.PY2: if environ.get(ntou('wsgi.version')) == (ntou('u'), 0): environ = downgrade_wsgi_ux_to_1x(environ) self.environ = environ @@ -236,45 +247,47 @@ class AppResponse(object): r = _cherrypy.serving.response outstatus = r.output_status - if not isinstance(outstatus, bytestr): - raise TypeError("response.output_status is not a byte string.") + if not isinstance(outstatus, bytes): + raise TypeError('response.output_status is not a byte string.') outheaders = [] for k, v in r.header_list: - if not isinstance(k, bytestr): - raise TypeError( - "response.header_list key %r is not a byte string." % - k) - if not isinstance(v, bytestr): - raise TypeError( - "response.header_list value %r is not a byte string." % - v) + if not isinstance(k, bytes): + tmpl = 'response.header_list key %r is not a byte string.' + raise TypeError(tmpl % k) + if not isinstance(v, bytes): + tmpl = ( + 'response.header_list value %r is not a byte string.' + ) + raise TypeError(tmpl % v) outheaders.append((k, v)) - if py3k: + if six.PY3: # According to PEP 3333, when using Python 3, the response # status and headers must be bytes masquerading as unicode; # that is, they must be of type "str" but are restricted to # code points in the "latin-1" set. outstatus = outstatus.decode('ISO-8859-1') - outheaders = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in outheaders] + outheaders = [ + (k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in outheaders + ] self.iter_response = iter(r.body) self.write = start_response(outstatus, outheaders) - except: + except BaseException: self.close() raise def __iter__(self): return self - if py3k: - def __next__(self): - return next(self.iter_response) - else: - def next(self): - return self.iter_response.next() + def __next__(self): + return next(self.iter_response) + + # todo: https://pythonhosted.org/six/#six.Iterator + if six.PY2: + next = __next__ def close(self): """Close and de-reference the current request and response. (Core)""" @@ -296,14 +309,18 @@ class AppResponse(object): """Create a Request object using environ.""" env = self.environ.get - local = httputil.Host('', - int(env('SERVER_PORT', 80) or -1), - env('SERVER_NAME', '')) - remote = httputil.Host(env('REMOTE_ADDR', ''), - int(env('REMOTE_PORT', -1) or -1), - env('REMOTE_HOST', '')) + local = httputil.Host( + '', + int(env('SERVER_PORT', 80) or -1), + env('SERVER_NAME', ''), + ) + remote = httputil.Host( + env('REMOTE_ADDR', ''), + int(env('REMOTE_PORT', -1) or -1), + env('REMOTE_HOST', ''), + ) scheme = env('wsgi.url_scheme') - sproto = env('ACTUAL_SERVER_PROTOCOL', "HTTP/1.1") + sproto = env('ACTUAL_SERVER_PROTOCOL', 'HTTP/1.1') request, resp = self.cpapp.get_serving(local, remote, scheme, sproto) # LOGON_USER is served by IIS, and is the name of the @@ -317,44 +334,54 @@ class AppResponse(object): meth = self.environ['REQUEST_METHOD'] - path = httputil.urljoin(self.environ.get('SCRIPT_NAME', ''), - self.environ.get('PATH_INFO', '')) + path = httputil.urljoin( + self.environ.get('SCRIPT_NAME', ''), + self.environ.get('PATH_INFO', ''), + ) qs = self.environ.get('QUERY_STRING', '') - if py3k: - # This isn't perfect; if the given PATH_INFO is in the - # wrong encoding, it may fail to match the appropriate config - # section URI. But meh. - old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') - new_enc = self.cpapp.find_config(self.environ.get('PATH_INFO', ''), - "request.uri_encoding", 'utf-8') - if new_enc.lower() != old_enc.lower(): - # Even though the path and qs are unicode, the WSGI server - # is required by PEP 3333 to coerce them to ISO-8859-1 - # masquerading as unicode. So we have to encode back to - # bytes and then decode again using the "correct" encoding. - try: - u_path = path.encode(old_enc).decode(new_enc) - u_qs = qs.encode(old_enc).decode(new_enc) - except (UnicodeEncodeError, UnicodeDecodeError): - # Just pass them through without transcoding and hope. - pass - else: - # Only set transcoded values if they both succeed. - path = u_path - qs = u_qs + path, qs = self.recode_path_qs(path, qs) or (path, qs) rproto = self.environ.get('SERVER_PROTOCOL') headers = self.translate_headers(self.environ) rfile = self.environ['wsgi.input'] request.run(meth, path, qs, rproto, headers, rfile) - headerNames = {'HTTP_CGI_AUTHORIZATION': 'Authorization', - 'CONTENT_LENGTH': 'Content-Length', - 'CONTENT_TYPE': 'Content-Type', - 'REMOTE_HOST': 'Remote-Host', - 'REMOTE_ADDR': 'Remote-Addr', - } + headerNames = { + 'HTTP_CGI_AUTHORIZATION': 'Authorization', + 'CONTENT_LENGTH': 'Content-Length', + 'CONTENT_TYPE': 'Content-Type', + 'REMOTE_HOST': 'Remote-Host', + 'REMOTE_ADDR': 'Remote-Addr', + } + + def recode_path_qs(self, path, qs): + if not six.PY3: + return + + # This isn't perfect; if the given PATH_INFO is in the + # wrong encoding, it may fail to match the appropriate config + # section URI. But meh. + old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') + new_enc = self.cpapp.find_config( + self.environ.get('PATH_INFO', ''), + 'request.uri_encoding', 'utf-8', + ) + if new_enc.lower() == old_enc.lower(): + return + + # Even though the path and qs are unicode, the WSGI server + # is required by PEP 3333 to coerce them to ISO-8859-1 + # masquerading as unicode. So we have to encode back to + # bytes and then decode again using the "correct" encoding. + try: + return ( + path.encode(old_enc).decode(new_enc), + qs.encode(old_enc).decode(new_enc), + ) + except (UnicodeEncodeError, UnicodeDecodeError): + # Just pass them through without transcoding and hope. + pass def translate_headers(self, environ): """Translate CGI-environ header names to HTTP header names.""" @@ -362,9 +389,9 @@ class AppResponse(object): # We assume all incoming header keys are uppercase already. if cgiName in self.headerNames: yield self.headerNames[cgiName], environ[cgiName] - elif cgiName[:5] == "HTTP_": + elif cgiName[:5] == 'HTTP_': # Hackish attempt at recovering original header names. - translatedHeader = cgiName[5:].replace("_", "-") + translatedHeader = cgiName[5:].replace('_', '-') yield translatedHeader, environ[cgiName] @@ -372,9 +399,10 @@ class CPWSGIApp(object): """A WSGI application object for a CherryPy Application.""" - pipeline = [('ExceptionTrapper', ExceptionTrapper), - ('InternalRedirector', InternalRedirector), - ] + pipeline = [ + ('ExceptionTrapper', ExceptionTrapper), + ('InternalRedirector', InternalRedirector), + ] """A list of (name, wsgiapp) pairs. Each 'wsgiapp' MUST be a constructor that takes an initial, positional 'nextapp' argument, plus optional keyword arguments, and returns a WSGI application @@ -424,16 +452,16 @@ class CPWSGIApp(object): def namespace_handler(self, k, v): """Config handler for the 'wsgi' namespace.""" - if k == "pipeline": + if k == 'pipeline': # Note this allows multiple 'wsgi.pipeline' config entries # (but each entry will be processed in a 'random' order). # It should also allow developers to set default middleware # in code (passed to self.__init__) that deployers can add to # (but not remove) via config. self.pipeline.extend(v) - elif k == "response_class": + elif k == 'response_class': self.response_class = v else: - name, arg = k.split(".", 1) + name, arg = k.split('.', 1) bucket = self.config.setdefault(name, {}) bucket[arg] = v diff --git a/lib/cherrypy/_cpwsgi_server.py b/lib/cherrypy/_cpwsgi_server.py index 874e2e9f..11dd846a 100644 --- a/lib/cherrypy/_cpwsgi_server.py +++ b/lib/cherrypy/_cpwsgi_server.py @@ -1,23 +1,55 @@ -"""WSGI server interface (see PEP 333). This adds some CP-specific bits to -the framework-agnostic wsgiserver package. +""" +WSGI server interface (see PEP 333). + +This adds some CP-specific bits to the framework-agnostic cheroot package. """ import sys +import cheroot.wsgi +import cheroot.server + import cherrypy -from cherrypy import wsgiserver -class CPWSGIServer(wsgiserver.CherryPyWSGIServer): +class CPWSGIHTTPRequest(cheroot.server.HTTPRequest): + """Wrapper for cheroot.server.HTTPRequest. - """Wrapper for wsgiserver.CherryPyWSGIServer. - - wsgiserver has been designed to not reference CherryPy in any way, - so that it can be used in other frameworks and applications. Therefore, - we wrap it here, so we can set our own mount points from cherrypy.tree - and apply some attributes from config -> cherrypy.server -> wsgiserver. + This is a layer, which preserves URI parsing mode like it which was + before Cheroot v5.8.0. """ + def __init__(self, server, conn): + """Initialize HTTP request container instance. + + Args: + server (cheroot.server.HTTPServer): + web server object receiving this request + conn (cheroot.server.HTTPConnection): + HTTP connection object for this request + """ + super(CPWSGIHTTPRequest, self).__init__( + server, conn, proxy_mode=True + ) + + +class CPWSGIServer(cheroot.wsgi.Server): + """Wrapper for cheroot.wsgi.Server. + + cheroot has been designed to not reference CherryPy in any way, + so that it can be used in other frameworks and applications. Therefore, + we wrap it here, so we can set our own mount points from cherrypy.tree + and apply some attributes from config -> cherrypy.server -> wsgi.Server. + """ + + fmt = 'CherryPy/{cherrypy.__version__} {cheroot.wsgi.Server.version}' + version = fmt.format(**globals()) + def __init__(self, server_adapter=cherrypy.server): + """Initialize CPWSGIServer instance. + + Args: + server_adapter (cherrypy._cpserver.Server): ... + """ self.server_adapter = server_adapter self.max_request_header_size = ( self.server_adapter.max_request_header_size or 0 @@ -31,17 +63,22 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): None) self.wsgi_version = self.server_adapter.wsgi_version - s = wsgiserver.CherryPyWSGIServer - s.__init__(self, server_adapter.bind_addr, cherrypy.tree, - self.server_adapter.thread_pool, - server_name, - max=self.server_adapter.thread_pool_max, - request_queue_size=self.server_adapter.socket_queue_size, - timeout=self.server_adapter.socket_timeout, - shutdown_timeout=self.server_adapter.shutdown_timeout, - accepted_queue_size=self.server_adapter.accepted_queue_size, - accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, - ) + + super(CPWSGIServer, self).__init__( + server_adapter.bind_addr, cherrypy.tree, + self.server_adapter.thread_pool, + server_name, + max=self.server_adapter.thread_pool_max, + request_queue_size=self.server_adapter.socket_queue_size, + timeout=self.server_adapter.socket_timeout, + shutdown_timeout=self.server_adapter.shutdown_timeout, + accepted_queue_size=self.server_adapter.accepted_queue_size, + accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, + peercreds_enabled=self.server_adapter.peercreds, + peercreds_resolve_enabled=self.server_adapter.peercreds_resolve, + ) + self.ConnectionClass.RequestHandlerClass = CPWSGIHTTPRequest + self.protocol = self.server_adapter.protocol_version self.nodelay = self.server_adapter.nodelay @@ -50,21 +87,24 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): else: ssl_module = self.server_adapter.ssl_module or 'pyopenssl' if self.server_adapter.ssl_context: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + adapter_class = cheroot.server.get_ssl_adapter_class(ssl_module) self.ssl_adapter = adapter_class( self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) + self.server_adapter.ssl_certificate_chain, + self.server_adapter.ssl_ciphers) self.ssl_adapter.context = self.server_adapter.ssl_context elif self.server_adapter.ssl_certificate: - adapter_class = wsgiserver.get_ssl_adapter_class(ssl_module) + adapter_class = cheroot.server.get_ssl_adapter_class(ssl_module) self.ssl_adapter = adapter_class( self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, - self.server_adapter.ssl_certificate_chain) + self.server_adapter.ssl_certificate_chain, + self.server_adapter.ssl_ciphers) self.stats['Enabled'] = getattr( self.server_adapter, 'statistics', False) - def error_log(self, msg="", level=20, traceback=False): + def error_log(self, msg='', level=20, traceback=False): + """Write given message to the error log.""" cherrypy.engine.log(msg, level, traceback) diff --git a/lib/cherrypy/_helper.py b/lib/cherrypy/_helper.py new file mode 100644 index 00000000..314550cb --- /dev/null +++ b/lib/cherrypy/_helper.py @@ -0,0 +1,344 @@ +"""Helper functions for CP apps.""" + +import six +from six.moves import urllib + +from cherrypy._cpcompat import text_or_bytes + +import cherrypy + + +def expose(func=None, alias=None): + """Expose the function or class. + + Optionally provide an alias or set of aliases. + """ + def expose_(func): + func.exposed = True + if alias is not None: + if isinstance(alias, text_or_bytes): + parents[alias.replace('.', '_')] = func + else: + for a in alias: + parents[a.replace('.', '_')] = func + return func + + import sys + import types + decoratable_types = types.FunctionType, types.MethodType, type, + if six.PY2: + # Old-style classes are type types.ClassType. + decoratable_types += types.ClassType, + if isinstance(func, decoratable_types): + if alias is None: + # @expose + func.exposed = True + return func + else: + # func = expose(func, alias) + parents = sys._getframe(1).f_locals + return expose_(func) + elif func is None: + if alias is None: + # @expose() + parents = sys._getframe(1).f_locals + return expose_ + else: + # @expose(alias="alias") or + # @expose(alias=["alias1", "alias2"]) + parents = sys._getframe(1).f_locals + return expose_ + else: + # @expose("alias") or + # @expose(["alias1", "alias2"]) + parents = sys._getframe(1).f_locals + alias = func + return expose_ + + +def popargs(*args, **kwargs): + """Decorate _cp_dispatch. + + (cherrypy.dispatch.Dispatcher.dispatch_method_name) + + Optional keyword argument: handler=(Object or Function) + + Provides a _cp_dispatch function that pops off path segments into + cherrypy.request.params under the names specified. The dispatch + is then forwarded on to the next vpath element. + + Note that any existing (and exposed) member function of the class that + popargs is applied to will override that value of the argument. For + instance, if you have a method named "list" on the class decorated with + popargs, then accessing "/list" will call that function instead of popping + it off as the requested parameter. This restriction applies to all + _cp_dispatch functions. The only way around this restriction is to create + a "blank class" whose only function is to provide _cp_dispatch. + + If there are path elements after the arguments, or more arguments + are requested than are available in the vpath, then the 'handler' + keyword argument specifies the next object to handle the parameterized + request. If handler is not specified or is None, then self is used. + If handler is a function rather than an instance, then that function + will be called with the args specified and the return value from that + function used as the next object INSTEAD of adding the parameters to + cherrypy.request.args. + + This decorator may be used in one of two ways: + + As a class decorator: + @cherrypy.popargs('year', 'month', 'day') + class Blog: + def index(self, year=None, month=None, day=None): + #Process the parameters here; any url like + #/, /2009, /2009/12, or /2009/12/31 + #will fill in the appropriate parameters. + + def create(self): + #This link will still be available at /create. Defined functions + #take precedence over arguments. + + Or as a member of a class: + class Blog: + _cp_dispatch = cherrypy.popargs('year', 'month', 'day') + #... + + The handler argument may be used to mix arguments with built in functions. + For instance, the following setup allows different activities at the + day, month, and year level: + + class DayHandler: + def index(self, year, month, day): + #Do something with this day; probably list entries + + def delete(self, year, month, day): + #Delete all entries for this day + + @cherrypy.popargs('day', handler=DayHandler()) + class MonthHandler: + def index(self, year, month): + #Do something with this month; probably list entries + + def delete(self, year, month): + #Delete all entries for this month + + @cherrypy.popargs('month', handler=MonthHandler()) + class YearHandler: + def index(self, year): + #Do something with this year + + #... + + @cherrypy.popargs('year', handler=YearHandler()) + class Root: + def index(self): + #... + + """ + # Since keyword arg comes after *args, we have to process it ourselves + # for lower versions of python. + + handler = None + handler_call = False + for k, v in kwargs.items(): + if k == 'handler': + handler = v + else: + tm = "cherrypy.popargs() got an unexpected keyword argument '{0}'" + raise TypeError(tm.format(k)) + + import inspect + + if handler is not None \ + and (hasattr(handler, '__call__') or inspect.isclass(handler)): + handler_call = True + + def decorated(cls_or_self=None, vpath=None): + if inspect.isclass(cls_or_self): + # cherrypy.popargs is a class decorator + cls = cls_or_self + name = cherrypy.dispatch.Dispatcher.dispatch_method_name + setattr(cls, name, decorated) + return cls + + # We're in the actual function + self = cls_or_self + parms = {} + for arg in args: + if not vpath: + break + parms[arg] = vpath.pop(0) + + if handler is not None: + if handler_call: + return handler(**parms) + else: + cherrypy.request.params.update(parms) + return handler + + cherrypy.request.params.update(parms) + + # If we are the ultimate handler, then to prevent our _cp_dispatch + # from being called again, we will resolve remaining elements through + # getattr() directly. + if vpath: + return getattr(self, vpath.pop(0), None) + else: + return self + + return decorated + + +def url(path='', qs='', script_name=None, base=None, relative=None): + """Create an absolute URL for the given path. + + If 'path' starts with a slash ('/'), this will return + (base + script_name + path + qs). + If it does not start with a slash, this returns + (base + script_name [+ request.path_info] + path + qs). + + If script_name is None, cherrypy.request will be used + to find a script_name, if available. + + If base is None, cherrypy.request.base will be used (if available). + Note that you can use cherrypy.tools.proxy to change this. + + Finally, note that this function can be used to obtain an absolute URL + for the current request path (minus the querystring) by passing no args. + If you call url(qs=cherrypy.request.query_string), you should get the + original browser URL (assuming no internal redirections). + + If relative is None or not provided, request.app.relative_urls will + be used (if available, else False). If False, the output will be an + absolute URL (including the scheme, host, vhost, and script_name). + If True, the output will instead be a URL that is relative to the + current request path, perhaps including '..' atoms. If relative is + the string 'server', the output will instead be a URL that is + relative to the server root; i.e., it will start with a slash. + """ + if isinstance(qs, (tuple, list, dict)): + qs = urllib.parse.urlencode(qs) + if qs: + qs = '?' + qs + + if cherrypy.request.app: + if not path.startswith('/'): + # Append/remove trailing slash from path_info as needed + # (this is to support mistyped URL's without redirecting; + # if you want to redirect, use tools.trailing_slash). + pi = cherrypy.request.path_info + if cherrypy.request.is_index is True: + if not pi.endswith('/'): + pi = pi + '/' + elif cherrypy.request.is_index is False: + if pi.endswith('/') and pi != '/': + pi = pi[:-1] + + if path == '': + path = pi + else: + path = urllib.parse.urljoin(pi, path) + + if script_name is None: + script_name = cherrypy.request.script_name + if base is None: + base = cherrypy.request.base + + newurl = base + script_name + normalize_path(path) + qs + else: + # No request.app (we're being called outside a request). + # We'll have to guess the base from server.* attributes. + # This will produce very different results from the above + # if you're using vhosts or tools.proxy. + if base is None: + base = cherrypy.server.base() + + path = (script_name or '') + path + newurl = base + normalize_path(path) + qs + + # At this point, we should have a fully-qualified absolute URL. + + if relative is None: + relative = getattr(cherrypy.request.app, 'relative_urls', False) + + # See http://www.ietf.org/rfc/rfc2396.txt + if relative == 'server': + # "A relative reference beginning with a single slash character is + # termed an absolute-path reference, as defined by ..." + # This is also sometimes called "server-relative". + newurl = '/' + '/'.join(newurl.split('/', 3)[3:]) + elif relative: + # "A relative reference that does not begin with a scheme name + # or a slash character is termed a relative-path reference." + old = url(relative=False).split('/')[:-1] + new = newurl.split('/') + while old and new: + a, b = old[0], new[0] + if a != b: + break + old.pop(0) + new.pop(0) + new = (['..'] * len(old)) + new + newurl = '/'.join(new) + + return newurl + + +def normalize_path(path): + """Resolve given path from relative into absolute form.""" + if './' not in path: + return path + + # Normalize the URL by removing ./ and ../ + atoms = [] + for atom in path.split('/'): + if atom == '.': + pass + elif atom == '..': + # Don't pop from empty list + # (i.e. ignore redundant '..') + if atoms: + atoms.pop() + elif atom: + atoms.append(atom) + + newpath = '/'.join(atoms) + # Preserve leading '/' + if path.startswith('/'): + newpath = '/' + newpath + + return newpath + + +#### +# Inlined from jaraco.classes 1.4.3 +# Ref #1673 +class _ClassPropertyDescriptor(object): + """Descript for read-only class-based property. + + Turns a classmethod-decorated func into a read-only property of that class + type (means the value cannot be set). + """ + + def __init__(self, fget, fset=None): + """Initialize a class property descriptor. + + Instantiated by ``_helper.classproperty``. + """ + self.fget = fget + self.fset = fset + + def __get__(self, obj, klass=None): + """Return property value.""" + if klass is None: + klass = type(obj) + return self.fget.__get__(obj, klass)() + + +def classproperty(func): # noqa: D401; irrelevant for properties + """Decorator like classmethod to implement a static class property.""" + if not isinstance(func, (classmethod, staticmethod)): + func = classmethod(func) + + return _ClassPropertyDescriptor(func) +#### diff --git a/lib/cherrypy/daemon.py b/lib/cherrypy/daemon.py index 395a2e68..74488c06 100644 --- a/lib/cherrypy/daemon.py +++ b/lib/cherrypy/daemon.py @@ -13,7 +13,7 @@ def start(configfiles=None, daemonize=False, environment=None, """Subscribe all engine plugins and start the engine.""" sys.path = [''] + sys.path for i in imports or []: - exec("import %s" % i) + exec('import %s' % i) for c in configfiles or []: cherrypy.config.update(c) @@ -37,18 +37,18 @@ def start(configfiles=None, daemonize=False, environment=None, if pidfile: plugins.PIDFile(engine, pidfile).subscribe() - if hasattr(engine, "signal_handler"): + if hasattr(engine, 'signal_handler'): engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): + if hasattr(engine, 'console_control_handler'): engine.console_control_handler.subscribe() if (fastcgi and (scgi or cgi)) or (scgi and cgi): - cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " - "scgi options.", 'ENGINE') + cherrypy.log.error('You may only specify one of the cgi, fastcgi, and ' + 'scgi options.', 'ENGINE') sys.exit(1) elif fastcgi or scgi or cgi: # Turn off autoreload when using *cgi. - cherrypy.config.update({'engine.autoreload_on': False}) + cherrypy.config.update({'engine.autoreload.on': False}) # Turn off the default HTTP server (which is subscribed by default). cherrypy.server.unsubscribe() @@ -65,7 +65,7 @@ def start(configfiles=None, daemonize=False, environment=None, # Always start the engine; this will start all other services try: engine.start() - except: + except Exception: # Assume the error has been logged already via bus.log. sys.exit(1) else: @@ -73,28 +73,29 @@ def start(configfiles=None, daemonize=False, environment=None, def run(): + """Run cherryd CLI.""" from optparse import OptionParser p = OptionParser() - p.add_option('-c', '--config', action="append", dest='config', - help="specify config file(s)") - p.add_option('-d', action="store_true", dest='daemonize', - help="run the server as a daemon") + p.add_option('-c', '--config', action='append', dest='config', + help='specify config file(s)') + p.add_option('-d', action='store_true', dest='daemonize', + help='run the server as a daemon') p.add_option('-e', '--environment', dest='environment', default=None, - help="apply the given config environment") - p.add_option('-f', action="store_true", dest='fastcgi', - help="start a fastcgi server instead of the default HTTP " - "server") - p.add_option('-s', action="store_true", dest='scgi', - help="start a scgi server instead of the default HTTP server") - p.add_option('-x', action="store_true", dest='cgi', - help="start a cgi server instead of the default HTTP server") - p.add_option('-i', '--import', action="append", dest='imports', - help="specify modules to import") + help='apply the given config environment') + p.add_option('-f', action='store_true', dest='fastcgi', + help='start a fastcgi server instead of the default HTTP ' + 'server') + p.add_option('-s', action='store_true', dest='scgi', + help='start a scgi server instead of the default HTTP server') + p.add_option('-x', action='store_true', dest='cgi', + help='start a cgi server instead of the default HTTP server') + p.add_option('-i', '--import', action='append', dest='imports', + help='specify modules to import') p.add_option('-p', '--pidfile', dest='pidfile', default=None, - help="store the process id in the given file") - p.add_option('-P', '--Path', action="append", dest='Path', - help="add the given paths to sys.path") + help='store the process id in the given file') + p.add_option('-P', '--Path', action='append', dest='Path', + help='add the given paths to sys.path') options, args = p.parse_args() if options.Path: diff --git a/lib/cherrypy/favicon.ico b/lib/cherrypy/favicon.ico new file mode 100644 index 00000000..f0d7e61b Binary files /dev/null and b/lib/cherrypy/favicon.ico differ diff --git a/lib/cherrypy/lib/__init__.py b/lib/cherrypy/lib/__init__.py index a75a53da..f815f76a 100644 --- a/lib/cherrypy/lib/__init__.py +++ b/lib/cherrypy/lib/__init__.py @@ -1,12 +1,14 @@ -"""CherryPy Library""" +"""CherryPy Library.""" -# Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 -from cherrypy.lib.reprconf import unrepr, modules, attributes def is_iterator(obj): - '''Returns a boolean indicating if the object provided implements - the iterator protocol (i.e. like a generator). This will return - false for objects which iterable, but not iterators themselves.''' + """Detect if the object provided implements the iterator protocol. + + (i.e. like a generator). + + This will return False for objects which are iterable, + but not iterators themselves. + """ from types import GeneratorType if isinstance(obj, GeneratorType): return True @@ -16,22 +18,23 @@ def is_iterator(obj): # Types which implement the protocol must return themselves when # invoking 'iter' upon them. return iter(obj) is obj - + + def is_closable_iterator(obj): - + """Detect if the given object is both closable and iterator.""" # Not an iterator. if not is_iterator(obj): return False - + # A generator - the easiest thing to deal with. import inspect if inspect.isgenerator(obj): return True - + # A custom iterator. Look for a close method... if not (hasattr(obj, 'close') and callable(obj.close)): return False - + # ... which doesn't require any arguments. try: inspect.getcallargs(obj.close) @@ -40,18 +43,24 @@ def is_closable_iterator(obj): else: return True -class file_generator(object): - """Yield the given input (a file object) in chunks (default 64k). (Core)""" +class file_generator(object): + """Yield the given input (a file object) in chunks (default 64k). + + (Core) + """ def __init__(self, input, chunkSize=65536): + """Initialize file_generator with file ``input`` for chunked access.""" self.input = input self.chunkSize = chunkSize def __iter__(self): + """Return iterator.""" return self def __next__(self): + """Return next chunk of file.""" chunk = self.input.read(self.chunkSize) if chunk: return chunk @@ -63,8 +72,10 @@ class file_generator(object): def file_generator_limited(fileobj, count, chunk_size=65536): - """Yield the given file object in chunks, stopping after `count` - bytes has been emitted. Default chunk size is 64kB. (Core) + """Yield the given file object in chunks. + + Stopps after `count` bytes has been emitted. + Default chunk size is 64kB. (Core) """ remaining = count while remaining > 0: @@ -77,9 +88,9 @@ def file_generator_limited(fileobj, count, chunk_size=65536): def set_vary_header(response, header_name): - "Add a Vary header to a response" - varies = response.headers.get("Vary", "") - varies = [x.strip() for x in varies.split(",") if x.strip()] + """Add a Vary header to a response.""" + varies = response.headers.get('Vary', '') + varies = [x.strip() for x in varies.split(',') if x.strip()] if header_name not in varies: varies.append(header_name) - response.headers['Vary'] = ", ".join(varies) + response.headers['Vary'] = ', '.join(varies) diff --git a/lib/cherrypy/lib/auth.py b/lib/cherrypy/lib/auth.py deleted file mode 100644 index 71591aaa..00000000 --- a/lib/cherrypy/lib/auth.py +++ /dev/null @@ -1,97 +0,0 @@ -import cherrypy -from cherrypy.lib import httpauth - - -def check_auth(users, encrypt=None, realm=None): - """If an authorization header contains credentials, return True or False. - """ - request = cherrypy.serving.request - if 'authorization' in request.headers: - # make sure the provided credentials are correctly set - ah = httpauth.parseAuthorization(request.headers['authorization']) - if ah is None: - raise cherrypy.HTTPError(400, 'Bad Request') - - if not encrypt: - encrypt = httpauth.DIGEST_AUTH_ENCODERS[httpauth.MD5] - - if hasattr(users, '__call__'): - try: - # backward compatibility - users = users() # expect it to return a dictionary - - if not isinstance(users, dict): - raise ValueError( - "Authentication users must be a dictionary") - - # fetch the user password - password = users.get(ah["username"], None) - except TypeError: - # returns a password (encrypted or clear text) - password = users(ah["username"]) - else: - if not isinstance(users, dict): - raise ValueError("Authentication users must be a dictionary") - - # fetch the user password - password = users.get(ah["username"], None) - - # validate the authorization by re-computing it here - # and compare it with what the user-agent provided - if httpauth.checkResponse(ah, password, method=request.method, - encrypt=encrypt, realm=realm): - request.login = ah["username"] - return True - - request.login = False - return False - - -def basic_auth(realm, users, encrypt=None, debug=False): - """If auth fails, raise 401 with a basic authentication header. - - realm - A string containing the authentication realm. - - users - A dict of the form: {username: password} or a callable returning - a dict. - - encrypt - callable used to encrypt the password returned from the user-agent. - if None it defaults to a md5 encryption. - - """ - if check_auth(users, encrypt): - if debug: - cherrypy.log('Auth successful', 'TOOLS.BASIC_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.basicAuth(realm) - - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") - - -def digest_auth(realm, users, debug=False): - """If auth fails, raise 401 with a digest authentication header. - - realm - A string containing the authentication realm. - users - A dict of the form: {username: password} or a callable returning - a dict. - """ - if check_auth(users, realm=realm): - if debug: - cherrypy.log('Auth successful', 'TOOLS.DIGEST_AUTH') - return - - # inform the user-agent this path is protected - cherrypy.serving.response.headers[ - 'www-authenticate'] = httpauth.digestAuth(realm) - - raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/auth_basic.py b/lib/cherrypy/lib/auth_basic.py index 5ba16f7f..ad379a26 100644 --- a/lib/cherrypy/lib/auth_basic.py +++ b/lib/cherrypy/lib/auth_basic.py @@ -1,8 +1,9 @@ # This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +"""HTTP Basic Authentication tool. -__doc__ = """This module provides a CherryPy 3.x tool which implements +This module provides a CherryPy 3.x tool which implements the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. @@ -14,18 +15,23 @@ as the credentials store:: basic_auth = {'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'earth', 'tools.auth_basic.checkpassword': checkpassword, + 'tools.auth_basic.accept_charset': 'UTF-8', } app_config = { '/' : basic_auth } """ +import binascii +import unicodedata +import base64 + +import cherrypy +from cherrypy._cpcompat import ntou, tonative + + __author__ = 'visteya' __date__ = 'April 2009' -import binascii -from cherrypy._cpcompat import base64_decode -import cherrypy - def checkpassword_dict(user_password_dict): """Returns a checkpassword function which checks credentials @@ -42,9 +48,10 @@ def checkpassword_dict(user_password_dict): return checkpassword -def basic_auth(realm, checkpassword, debug=False): +def basic_auth(realm, checkpassword, debug=False, accept_charset='utf-8'): """A CherryPy tool which hooks at before_handler to perform - HTTP Basic Access Authentication, as specified in :rfc:`2617`. + HTTP Basic Access Authentication, as specified in :rfc:`2617` + and :rfc:`7617`. If the request has an 'authorization' header with a 'Basic' scheme, this tool attempts to authenticate the credentials supplied in that header. If @@ -64,27 +71,50 @@ def basic_auth(realm, checkpassword, debug=False): """ + fallback_charset = 'ISO-8859-1' + if '"' in realm: raise ValueError('Realm cannot contain the " (quote) character.') request = cherrypy.serving.request auth_header = request.headers.get('authorization') if auth_header is not None: - try: + # split() error, base64.decodestring() error + msg = 'Bad Request' + with cherrypy.HTTPError.handle((ValueError, binascii.Error), 400, msg): scheme, params = auth_header.split(' ', 1) if scheme.lower() == 'basic': - username, password = base64_decode(params).split(':', 1) + charsets = accept_charset, fallback_charset + decoded_params = base64.b64decode(params.encode('ascii')) + decoded_params = _try_decode(decoded_params, charsets) + decoded_params = ntou(decoded_params) + decoded_params = unicodedata.normalize('NFC', decoded_params) + decoded_params = tonative(decoded_params) + username, password = decoded_params.split(':', 1) if checkpassword(realm, username, password): if debug: cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') request.login = username return # successful authentication - # split() error, base64.decodestring() error - except (ValueError, binascii.Error): - raise cherrypy.HTTPError(400, 'Bad Request') + charset = accept_charset.upper() + charset_declaration = ( + (', charset="%s"' % charset) + if charset != fallback_charset + else '' + ) # Respond with 401 status and a WWW-Authenticate header - cherrypy.serving.response.headers[ - 'www-authenticate'] = 'Basic realm="%s"' % realm + cherrypy.serving.response.headers['www-authenticate'] = ( + 'Basic realm="%s"%s' % (realm, charset_declaration) + ) raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') + + +def _try_decode(subject, charsets): + for charset in charsets[:-1]: + try: + return tonative(subject, charset) + except ValueError: + pass + return tonative(subject, charsets[-1]) diff --git a/lib/cherrypy/lib/auth_digest.py b/lib/cherrypy/lib/auth_digest.py index e833ff77..9b4f55c8 100644 --- a/lib/cherrypy/lib/auth_digest.py +++ b/lib/cherrypy/lib/auth_digest.py @@ -1,8 +1,9 @@ # This file is part of CherryPy # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +"""HTTP Digest Authentication tool. -__doc__ = """An implementation of the server-side of HTTP Digest Access +An implementation of the server-side of HTTP Digest Access Authentication, which is described in :rfc:`2617`. Example usage, using the built-in get_ha1_dict_plain function which uses a dict @@ -14,21 +15,28 @@ of plaintext passwords as the credentials store:: 'tools.auth_digest.realm': 'wonderland', 'tools.auth_digest.get_ha1': get_ha1, 'tools.auth_digest.key': 'a565c27146791cfb', + 'tools.auth_digest.accept_charset': 'UTF-8', } app_config = { '/' : digest_auth } """ +import time +import functools +from hashlib import md5 + +from six.moves.urllib.request import parse_http_list, parse_keqv_list + +import cherrypy +from cherrypy._cpcompat import ntob, tonative + + __author__ = 'visteya' __date__ = 'April 2009' -import time -from hashlib import md5 -from cherrypy._cpcompat import parse_http_list, parse_keqv_list +def md5_hex(s): + return md5(ntob(s, 'utf-8')).hexdigest() -import cherrypy -from cherrypy._cpcompat import ntob -md5_hex = lambda s: md5(ntob(s)).hexdigest() qop_auth = 'auth' qop_auth_int = 'auth-int' @@ -36,6 +44,9 @@ valid_qops = (qop_auth, qop_auth_int) valid_algorithms = ('MD5', 'MD5-sess') +FALLBACK_CHARSET = 'ISO-8859-1' +DEFAULT_CHARSET = 'UTF-8' + def TRACE(msg): cherrypy.log(msg, context='TOOLS.AUTH_DIGEST') @@ -130,24 +141,47 @@ def H(s): return md5_hex(s) -class HttpDigestAuthorization (object): +def _try_decode_header(header, charset): + global FALLBACK_CHARSET - """Class to parse a Digest Authorization header and perform re-calculation - of the digest. + for enc in (charset, FALLBACK_CHARSET): + try: + return tonative(ntob(tonative(header, 'latin1'), 'latin1'), enc) + except ValueError as ve: + last_err = ve + else: + raise last_err + + +class HttpDigestAuthorization(object): """ + Parses a Digest Authorization header and performs + re-calculation of the digest. + """ + + scheme = 'digest' def errmsg(self, s): return 'Digest Authorization header: %s' % s - def __init__(self, auth_header, http_method, debug=False): + @classmethod + def matches(cls, header): + scheme, _, _ = header.partition(' ') + return scheme.lower() == cls.scheme + + def __init__( + self, auth_header, http_method, + debug=False, accept_charset=DEFAULT_CHARSET[:], + ): self.http_method = http_method self.debug = debug - scheme, params = auth_header.split(" ", 1) - self.scheme = scheme.lower() - if self.scheme != 'digest': + + if not self.matches(auth_header): raise ValueError('Authorization scheme is not "Digest"') - self.auth_header = auth_header + self.auth_header = _try_decode_header(auth_header, accept_charset) + + scheme, params = self.auth_header.split(' ', 1) # make a dict of the params items = parse_http_list(params) @@ -180,7 +214,7 @@ class HttpDigestAuthorization (object): ) if not has_reqd: raise ValueError( - self.errmsg("Not all required parameters are present.")) + self.errmsg('Not all required parameters are present.')) if self.qop: if self.qop not in valid_qops: @@ -188,13 +222,13 @@ class HttpDigestAuthorization (object): self.errmsg("Unsupported value for qop: '%s'" % self.qop)) if not (self.cnonce and self.nc): raise ValueError( - self.errmsg("If qop is sent then " - "cnonce and nc MUST be present")) + self.errmsg('If qop is sent then ' + 'cnonce and nc MUST be present')) else: if self.cnonce or self.nc: raise ValueError( - self.errmsg("If qop is not sent, " - "neither cnonce nor nc can be present")) + self.errmsg('If qop is not sent, ' + 'neither cnonce nor nc can be present')) def __str__(self): return 'authorization : %s' % self.auth_header @@ -239,7 +273,7 @@ class HttpDigestAuthorization (object): except ValueError: # int() error pass if self.debug: - TRACE("nonce is stale") + TRACE('nonce is stale') return True def HA2(self, entity_body=''): @@ -251,14 +285,14 @@ class HttpDigestAuthorization (object): # # If the "qop" value is "auth-int", then A2 is: # A2 = method ":" digest-uri-value ":" H(entity-body) - if self.qop is None or self.qop == "auth": + if self.qop is None or self.qop == 'auth': a2 = '%s:%s' % (self.http_method, self.uri) - elif self.qop == "auth-int": - a2 = "%s:%s:%s" % (self.http_method, self.uri, H(entity_body)) + elif self.qop == 'auth-int': + a2 = '%s:%s:%s' % (self.http_method, self.uri, H(entity_body)) else: # in theory, this should never happen, since I validate qop in # __init__() - raise ValueError(self.errmsg("Unrecognized value for qop!")) + raise ValueError(self.errmsg('Unrecognized value for qop!')) return H(a2) def request_digest(self, ha1, entity_body=''): @@ -279,10 +313,10 @@ class HttpDigestAuthorization (object): ha2 = self.HA2(entity_body) # Request-Digest -- RFC 2617 3.2.2.1 if self.qop: - req = "%s:%s:%s:%s:%s" % ( + req = '%s:%s:%s:%s:%s' % ( self.nonce, self.nc, self.cnonce, self.qop, ha2) else: - req = "%s:%s" % (self.nonce, ha2) + req = '%s:%s' % (self.nonce, ha2) # RFC 2617 3.2.2.2 # @@ -302,25 +336,44 @@ class HttpDigestAuthorization (object): return digest -def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, - stale=False): +def _get_charset_declaration(charset): + global FALLBACK_CHARSET + charset = charset.upper() + return ( + (', charset="%s"' % charset) + if charset != FALLBACK_CHARSET + else '' + ) + + +def www_authenticate( + realm, key, algorithm='MD5', nonce=None, qop=qop_auth, + stale=False, accept_charset=DEFAULT_CHARSET[:], +): """Constructs a WWW-Authenticate header for Digest authentication.""" if qop not in valid_qops: raise ValueError("Unsupported value for qop: '%s'" % qop) if algorithm not in valid_algorithms: raise ValueError("Unsupported value for algorithm: '%s'" % algorithm) + HEADER_PATTERN = ( + 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"%s%s' + ) + if nonce is None: nonce = synthesize_nonce(realm, key) - s = 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop) - if stale: - s += ', stale="true"' - return s + + stale_param = ', stale="true"' if stale else '' + + charset_declaration = _get_charset_declaration(accept_charset) + + return HEADER_PATTERN % ( + realm, nonce, algorithm, qop, stale_param, charset_declaration, + ) -def digest_auth(realm, get_ha1, key, debug=False): - """A CherryPy tool which hooks at before_handler to perform +def digest_auth(realm, get_ha1, key, debug=False, accept_charset='utf-8'): + """A CherryPy tool that hooks at before_handler to perform HTTP Digest Access Authentication, as specified in :rfc:`2617`. If the request has an 'authorization' header with a 'Digest' scheme, @@ -333,7 +386,7 @@ def digest_auth(realm, get_ha1, key, debug=False): A string containing the authentication realm. get_ha1 - A callable which looks up a username in a credentials store + A callable that looks up a username in a credentials store and returns the HA1 string, which is defined in the RFC to be MD5(username : realm : password). The function's signature is: ``get_ha1(realm, username)`` @@ -349,43 +402,63 @@ def digest_auth(realm, get_ha1, key, debug=False): request = cherrypy.serving.request auth_header = request.headers.get('authorization') - nonce_is_stale = False - if auth_header is not None: - try: - auth = HttpDigestAuthorization( - auth_header, request.method, debug=debug) - except ValueError: - raise cherrypy.HTTPError( - 400, "The Authorization header could not be parsed.") - if debug: - TRACE(str(auth)) + respond_401 = functools.partial( + _respond_401, realm, key, accept_charset, debug) - if auth.validate_nonce(realm, key): - ha1 = get_ha1(realm, auth.username) - if ha1 is not None: - # note that for request.body to be available we need to - # hook in at before_handler, not on_start_resource like - # 3.1.x digest_auth does. - digest = auth.request_digest(ha1, entity_body=request.body) - if digest == auth.response: # authenticated - if debug: - TRACE("digest matches auth.response") - # Now check if nonce is stale. - # The choice of ten minutes' lifetime for nonce is somewhat - # arbitrary - nonce_is_stale = auth.is_nonce_stale(max_age_seconds=600) - if not nonce_is_stale: - request.login = auth.username - if debug: - TRACE("authentication of %s successful" % - auth.username) - return + if not HttpDigestAuthorization.matches(auth_header or ''): + respond_401() - # Respond with 401 status and a WWW-Authenticate header - header = www_authenticate(realm, key, stale=nonce_is_stale) + msg = 'The Authorization header could not be parsed.' + with cherrypy.HTTPError.handle(ValueError, 400, msg): + auth = HttpDigestAuthorization( + auth_header, request.method, + debug=debug, accept_charset=accept_charset, + ) + + if debug: + TRACE(str(auth)) + + if not auth.validate_nonce(realm, key): + respond_401() + + ha1 = get_ha1(realm, auth.username) + + if ha1 is None: + respond_401() + + # note that for request.body to be available we need to + # hook in at before_handler, not on_start_resource like + # 3.1.x digest_auth does. + digest = auth.request_digest(ha1, entity_body=request.body) + if digest != auth.response: + respond_401() + + # authenticated + if debug: + TRACE('digest matches auth.response') + # Now check if nonce is stale. + # The choice of ten minutes' lifetime for nonce is somewhat + # arbitrary + if auth.is_nonce_stale(max_age_seconds=600): + respond_401(stale=True) + + request.login = auth.username + if debug: + TRACE('authentication of %s successful' % auth.username) + + +def _respond_401(realm, key, accept_charset, debug, **kwargs): + """ + Respond with 401 status and a WWW-Authenticate header + """ + header = www_authenticate( + realm, key, + accept_charset=accept_charset, + **kwargs + ) if debug: TRACE(header) cherrypy.serving.response.headers['WWW-Authenticate'] = header raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') diff --git a/lib/cherrypy/lib/caching.py b/lib/cherrypy/lib/caching.py index fab6b569..1673b3c8 100644 --- a/lib/cherrypy/lib/caching.py +++ b/lib/cherrypy/lib/caching.py @@ -37,9 +37,11 @@ import sys import threading import time +import six + import cherrypy from cherrypy.lib import cptools, httputil -from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted, Event +from cherrypy._cpcompat import Event class Cache(object): @@ -48,19 +50,19 @@ class Cache(object): def get(self): """Return the current variant if in the cache, else None.""" - raise NotImplemented + raise NotImplementedError def put(self, obj, size): """Store the current variant in the cache.""" - raise NotImplemented + raise NotImplementedError def delete(self): """Remove ALL cached variants of the current resource.""" - raise NotImplemented + raise NotImplementedError def clear(self): """Reset the cache to its initial, empty state.""" - raise NotImplemented + raise NotImplementedError # ------------------------------ Memory Cache ------------------------------- # @@ -170,7 +172,7 @@ class MemoryCache(Cache): # Run self.expire_cache in a separate daemon thread. t = threading.Thread(target=self.expire_cache, name='expire_cache') self.expiration_thread = t - set_daemon(t, True) + t.daemon = True t.start() def clear(self): @@ -197,7 +199,8 @@ class MemoryCache(Cache): now = time.time() # Must make a copy of expirations so it doesn't change size # during iteration - for expiration_time, objects in copyitems(self.expirations): + items = list(six.iteritems(self.expirations)) + for expiration_time, objects in items: if expiration_time <= now: for obj_size, uri, sel_header_values in objects: try: @@ -265,7 +268,7 @@ class MemoryCache(Cache): self.store.pop(uri, None) -def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): +def get(invalid_methods=('POST', 'PUT', 'DELETE'), debug=False, **kwargs): """Try to obtain cached output. If fresh enough, raise HTTPError(304). If POST, PUT, or DELETE: @@ -291,9 +294,9 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): request = cherrypy.serving.request response = cherrypy.serving.response - if not hasattr(cherrypy, "_cache"): + if not hasattr(cherrypy, '_cache'): # Make a process-wide Cache object. - cherrypy._cache = kwargs.pop("cache_class", MemoryCache)() + cherrypy._cache = kwargs.pop('cache_class', MemoryCache)() # Take all remaining kwargs and set them on the Cache object. for k, v in kwargs.items(): @@ -328,7 +331,7 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): if directive == 'max-age': if len(atoms) != 1 or not atoms[0].isdigit(): raise cherrypy.HTTPError( - 400, "Invalid Cache-Control header") + 400, 'Invalid Cache-Control header') max_age = int(atoms[0]) break elif directive == 'no-cache': @@ -353,13 +356,13 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): return False # Copy the response headers. See - # https://bitbucket.org/cherrypy/cherrypy/issue/721. + # https://github.com/cherrypy/cherrypy/issues/721. response.headers = rh = httputil.HeaderMap() for k in h: dict.__setitem__(rh, k, dict.__getitem__(h, k)) # Add the required Age header - response.headers["Age"] = str(age) + response.headers['Age'] = str(age) try: # Note that validate_since depends on a Last-Modified header; @@ -402,10 +405,19 @@ def tee_output(): output.append(chunk) yield chunk - # save the cache data - body = ntob('').join(output) - cherrypy._cache.put((response.status, response.headers or {}, - body, response.time), len(body)) + # Save the cache data, but only if the body isn't empty. + # e.g. a 304 Not Modified on a static file response will + # have an empty body. + # If the body is empty, delete the cache because it + # contains a stale Threading._Event object that will + # stall all consecutive requests until the _Event times + # out + body = b''.join(output) + if not body: + cherrypy._cache.delete() + else: + cherrypy._cache.put((response.status, response.headers or {}, + body, response.time), len(body)) response = cherrypy.serving.response response.body = tee(response.body) @@ -457,14 +469,14 @@ def expires(secs=0, force=False, debug=False): secs = (86400 * secs.days) + secs.seconds if secs == 0: - if force or ("Pragma" not in headers): - headers["Pragma"] = "no-cache" + if force or ('Pragma' not in headers): + headers['Pragma'] = 'no-cache' if cherrypy.serving.request.protocol >= (1, 1): - if force or "Cache-Control" not in headers: - headers["Cache-Control"] = "no-cache, must-revalidate" + if force or 'Cache-Control' not in headers: + headers['Cache-Control'] = 'no-cache, must-revalidate' # Set an explicit Expires date in the past. expiry = httputil.HTTPDate(1169942400.0) else: expiry = httputil.HTTPDate(response.time + secs) - if force or "Expires" not in headers: - headers["Expires"] = expiry + if force or 'Expires' not in headers: + headers['Expires'] = expiry diff --git a/lib/cherrypy/lib/covercp.py b/lib/cherrypy/lib/covercp.py index a74ec342..0bafca13 100644 --- a/lib/cherrypy/lib/covercp.py +++ b/lib/cherrypy/lib/covercp.py @@ -23,10 +23,15 @@ it will call ``serve()`` for you. import re import sys import cgi -from cherrypy._cpcompat import quote_plus import os import os.path -localFile = os.path.join(os.path.dirname(__file__), "coverage.cache") + +from six.moves import urllib + +import cherrypy + + +localFile = os.path.join(os.path.dirname(__file__), 'coverage.cache') the_coverage = None try: @@ -42,8 +47,8 @@ except ImportError: import warnings warnings.warn( - "No code coverage will be performed; " - "coverage.py could not be imported.") + 'No code coverage will be performed; ' + 'coverage.py could not be imported.') def start(): pass @@ -193,7 +198,7 @@ def _percent(statements, missing): return 0 -def _show_branch(root, base, path, pct=0, showpct=False, exclude="", +def _show_branch(root, base, path, pct=0, showpct=False, exclude='', coverage=the_coverage): # Show the directory name and any of our children @@ -204,11 +209,11 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", if newpath.lower().startswith(base): relpath = newpath[len(base):] - yield "| " * relpath.count(os.sep) + yield '| ' * relpath.count(os.sep) yield ( "%s\n" % - (newpath, quote_plus(exclude), name) + (newpath, urllib.parse.quote_plus(exclude), name) ) for chunk in _show_branch( @@ -225,22 +230,22 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", for name in files: newpath = os.path.join(path, name) - pc_str = "" + pc_str = '' if showpct: try: _, statements, _, missing, _ = coverage.analysis2(newpath) - except: + except Exception: # Yes, we really want to pass on all errors. pass else: pc = _percent(statements, missing) - pc_str = ("%3d%% " % pc).replace(' ', ' ') + pc_str = ('%3d%% ' % pc).replace(' ', ' ') if pc < float(pct) or pc == -1: pc_str = "%s" % pc_str else: pc_str = "%s" % pc_str - yield TEMPLATE_ITEM % ("| " * (relpath.count(os.sep) + 1), + yield TEMPLATE_ITEM % ('| ' * (relpath.count(os.sep) + 1), pc_str, newpath, name) @@ -260,8 +265,8 @@ def _graft(path, tree): break atoms.append(tail) atoms.append(p) - if p != "/": - atoms.append("/") + if p != '/': + atoms.append('/') atoms.reverse() for node in atoms: @@ -286,15 +291,15 @@ class CoverStats(object): if root is None: # Guess initial depth. Files outside this path will not be # reachable from the web interface. - import cherrypy root = os.path.dirname(cherrypy.__file__) self.root = root + @cherrypy.expose def index(self): return TEMPLATE_FRAMESET % self.root.lower() - index.exposed = True - def menu(self, base="/", pct="50", showpct="", + @cherrypy.expose + def menu(self, base='/', pct='50', showpct='', exclude=r'python\d\.\d|test|tut\d|tutorial'): # The coverage module uses all-lower-case names. @@ -305,37 +310,36 @@ class CoverStats(object): # Start by showing links for parent paths yield "
" - path = "" + path = '' atoms = base.split(os.sep) atoms.pop() for atom in atoms: path += atom + os.sep yield ("%s %s" - % (path, quote_plus(exclude), atom, os.sep)) - yield "
" + % (path, urllib.parse.quote_plus(exclude), atom, os.sep)) + yield '' yield "
" # Then display the tree tree = get_tree(base, exclude, self.coverage) if not tree: - yield "

No modules covered.

" + yield '

No modules covered.

' else: - for chunk in _show_branch(tree, base, "/", pct, + for chunk in _show_branch(tree, base, '/', pct, showpct == 'checked', exclude, coverage=self.coverage): yield chunk - yield "
" - yield "" - menu.exposed = True + yield '' + yield '' def annotated_file(self, filename, statements, excluded, missing): source = open(filename, 'r') buffer = [] for lineno, line in enumerate(source.readlines()): lineno += 1 - line = line.strip("\n\r") + line = line.strip('\n\r') empty_the_buffer = True if lineno in excluded: template = TEMPLATE_LOC_EXCLUDED @@ -352,6 +356,7 @@ class CoverStats(object): buffer = [] yield template % (lineno, cgi.escape(line)) + @cherrypy.expose def report(self, name): filename, statements, excluded, missing, _ = self.coverage.analysis2( name) @@ -366,22 +371,21 @@ class CoverStats(object): yield '' yield '' yield '' - report.exposed = True def serve(path=localFile, port=8080, root=None): if coverage is None: - raise ImportError("The coverage module could not be imported.") + raise ImportError('The coverage module could not be imported.') from coverage import coverage cov = coverage(data_file=path) cov.load() - import cherrypy cherrypy.config.update({'server.socket_port': int(port), 'server.thread_pool': 10, - 'environment': "production", + 'environment': 'production', }) cherrypy.quickstart(CoverStats(cov, root)) -if __name__ == "__main__": + +if __name__ == '__main__': serve(*tuple(sys.argv[1:])) diff --git a/lib/cherrypy/lib/cpstats.py b/lib/cherrypy/lib/cpstats.py index 4aeabd7d..ae9f7475 100644 --- a/lib/cherrypy/lib/cpstats.py +++ b/lib/cherrypy/lib/cpstats.py @@ -187,9 +187,19 @@ To format statistics reports:: """ +import logging +import os +import sys +import threading +import time + +import six + +import cherrypy +from cherrypy._cpcompat import json + # ------------------------------- Statistics -------------------------------- # -import logging if not hasattr(logging, 'statistics'): logging.statistics = {} @@ -210,12 +220,6 @@ def extrapolate_statistics(scope): # -------------------- CherryPy Applications Statistics --------------------- # -import sys -import threading -import time - -import cherrypy - appstats = logging.statistics.setdefault('CherryPy Applications', {}) appstats.update({ 'Enabled': True, @@ -246,7 +250,9 @@ appstats.update({ 'Requests': {}, }) -proc_time = lambda s: time.time() - s['Start Time'] + +def proc_time(s): + return time.time() - s['Start Time'] class ByteCountWrapper(object): @@ -292,7 +298,8 @@ class ByteCountWrapper(object): return data -average_uriset_time = lambda s: s['Count'] and (s['Sum'] / s['Count']) or 0 +def average_uriset_time(s): + return s['Count'] and (s['Sum'] / s['Count']) or 0 def _get_threading_ident(): @@ -300,6 +307,7 @@ def _get_threading_ident(): return threading.get_ident() return threading._get_ident() + class StatsTool(cherrypy.Tool): """Record various information about the current request.""" @@ -390,28 +398,22 @@ class StatsTool(cherrypy.Tool): sq.pop(0) -import cherrypy cherrypy.tools.cpstats = StatsTool() # ---------------------- CherryPy Statistics Reporting ---------------------- # -import os thisdir = os.path.abspath(os.path.dirname(__file__)) -try: - import json -except ImportError: - try: - import simplejson as json - except ImportError: - json = None - - missing = object() -locale_date = lambda v: time.strftime('%c', time.gmtime(v)) -iso_format = lambda v: time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) + +def locale_date(v): + return time.strftime('%c', time.gmtime(v)) + + +def iso_format(v): + return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) def pause_resume(ns): @@ -475,6 +477,7 @@ class StatsPage(object): }, } + @cherrypy.expose def index(self): # Transform the raw data into pretty output for HTML yield """ @@ -578,7 +581,6 @@ table.stats2 th { """ - index.exposed = True def get_namespaces(self): """Yield (title, scalars, collections) for each namespace.""" @@ -611,12 +613,7 @@ table.stats2 th { """Return ([headers], [rows]) for the given collection.""" # E.g., the 'Requests' dict. headers = [] - try: - # python2 - vals = v.itervalues() - except AttributeError: - # python3 - vals = v.values() + vals = six.itervalues(v) for record in vals: for k3 in record: format = formatting.get(k3, missing) @@ -678,22 +675,22 @@ table.stats2 th { return headers, subrows if json is not None: + @cherrypy.expose def data(self): s = extrapolate_statistics(logging.statistics) cherrypy.response.headers['Content-Type'] = 'application/json' return json.dumps(s, sort_keys=True, indent=4) - data.exposed = True + @cherrypy.expose def pause(self, namespace): logging.statistics.get(namespace, {})['Enabled'] = False raise cherrypy.HTTPRedirect('./') - pause.exposed = True pause.cp_config = {'tools.allow.on': True, 'tools.allow.methods': ['POST']} + @cherrypy.expose def resume(self, namespace): logging.statistics.get(namespace, {})['Enabled'] = True raise cherrypy.HTTPRedirect('./') - resume.exposed = True resume.cp_config = {'tools.allow.on': True, 'tools.allow.methods': ['POST']} diff --git a/lib/cherrypy/lib/cptools.py b/lib/cherrypy/lib/cptools.py index 4c8991f8..1c079634 100644 --- a/lib/cherrypy/lib/cptools.py +++ b/lib/cherrypy/lib/cptools.py @@ -4,8 +4,11 @@ import logging import re from hashlib import md5 +import six +from six.moves import urllib + import cherrypy -from cherrypy._cpcompat import basestring, unicodestr +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import httputil as _httputil from cherrypy.lib import is_iterator @@ -31,7 +34,7 @@ def validate_etags(autotags=False, debug=False): response = cherrypy.serving.response # Guard against being run twice. - if hasattr(response, "ETag"): + if hasattr(response, 'ETag'): return status, reason, msg = _httputil.valid_status(response.status) @@ -70,24 +73,24 @@ def validate_etags(autotags=False, debug=False): if debug: cherrypy.log('If-Match conditions: %s' % repr(conditions), 'TOOLS.ETAGS') - if conditions and not (conditions == ["*"] or etag in conditions): - raise cherrypy.HTTPError(412, "If-Match failed: ETag %r did " - "not match %r" % (etag, conditions)) + if conditions and not (conditions == ['*'] or etag in conditions): + raise cherrypy.HTTPError(412, 'If-Match failed: ETag %r did ' + 'not match %r' % (etag, conditions)) conditions = request.headers.elements('If-None-Match') or [] conditions = [str(x) for x in conditions] if debug: cherrypy.log('If-None-Match conditions: %s' % repr(conditions), 'TOOLS.ETAGS') - if conditions == ["*"] or etag in conditions: + if conditions == ['*'] or etag in conditions: if debug: cherrypy.log('request.method: %s' % request.method, 'TOOLS.ETAGS') - if request.method in ("GET", "HEAD"): + if request.method in ('GET', 'HEAD'): raise cherrypy.HTTPRedirect([], 304) else: - raise cherrypy.HTTPError(412, "If-None-Match failed: ETag %r " - "matched %r" % (etag, conditions)) + raise cherrypy.HTTPError(412, 'If-None-Match failed: ETag %r ' + 'matched %r' % (etag, conditions)) def validate_since(): @@ -111,7 +114,7 @@ def validate_since(): since = request.headers.get('If-Modified-Since') if since and since == lastmod: if (status >= 200 and status <= 299) or status == 304: - if request.method in ("GET", "HEAD"): + if request.method in ('GET', 'HEAD'): raise cherrypy.HTTPRedirect([], 304) else: raise cherrypy.HTTPError(412) @@ -184,7 +187,7 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', # This is for lighttpd/pound/Mongrel's 'X-Forwarded-Proto: https' scheme = s if not scheme: - scheme = request.base[:request.base.find("://")] + scheme = request.base[:request.base.find('://')] if local: lbase = request.headers.get(local, None) @@ -193,14 +196,12 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', if lbase is not None: base = lbase.split(',')[0] if not base: - base = request.headers.get('Host', '127.0.0.1') - port = request.local.port - if port != 80 and not base.endswith(':%s' % port): - base += ':%s' % port + default = urllib.parse.urlparse(request.base).netloc + base = request.headers.get('Host', default) - if base.find("://") == -1: + if base.find('://') == -1: # add http:// or https:// if needed - base = scheme + "://" + base + base = scheme + '://' + base request.base = base @@ -210,8 +211,8 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') if xff: if remote == 'X-Forwarded-For': - #Bug #1268 - xff = xff.split(',')[0].strip() + # Grab the first IP in a comma-separated list. Ref #1268. + xff = next(ip.strip() for ip in xff.split(',')) request.remote.ip = xff @@ -238,6 +239,8 @@ def response_headers(headers=None, debug=False): 'TOOLS.RESPONSE_HEADERS') for name, value in (headers or []): cherrypy.serving.response.headers[name] = value + + response_headers.failsafe = True @@ -283,7 +286,7 @@ class SessionAuth(object): """Assert that the user is logged in.""" - session_key = "username" + session_key = 'username' debug = False def check_username_and_password(self, username, password): @@ -304,7 +307,7 @@ class SessionAuth(object): def login_screen(self, from_page='..', username='', error_msg='', **kwargs): - return (unicodestr(""" + return (six.text_type(""" Message: %(error_msg)s
Login: @@ -315,7 +318,7 @@ Message: %(error_msg)s
-""") % vars()).encode("utf-8") +""") % vars()).encode('utf-8') def do_login(self, username, password, from_page='..', **kwargs): """Login. May raise redirect, or return True if request handled.""" @@ -324,15 +327,15 @@ Message: %(error_msg)s if error_msg: body = self.login_screen(from_page, username, error_msg) response.body = body - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] return True else: cherrypy.serving.request.login = username cherrypy.session[self.session_key] = username self.on_login(username) - raise cherrypy.HTTPRedirect(from_page or "/") + raise cherrypy.HTTPRedirect(from_page or '/') def do_logout(self, from_page='..', **kwargs): """Logout. May raise redirect, or return True if request handled.""" @@ -362,9 +365,9 @@ Message: %(error_msg)s locals(), ) response.body = self.login_screen(url) - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] return True self._debug_message('Setting request.login to %(username)r', locals()) request.login = username @@ -386,14 +389,14 @@ Message: %(error_msg)s return True elif path.endswith('do_login'): if request.method != 'POST': - response.headers['Allow'] = "POST" + response.headers['Allow'] = 'POST' self._debug_message('do_login requires POST') raise cherrypy.HTTPError(405) self._debug_message('routing %(path)r to do_login', locals()) return self.do_login(**request.params) elif path.endswith('do_logout'): if request.method != 'POST': - response.headers['Allow'] = "POST" + response.headers['Allow'] = 'POST' raise cherrypy.HTTPError(405) self._debug_message('routing %(path)r to do_logout', locals()) return self.do_logout(**request.params) @@ -407,24 +410,28 @@ def session_auth(**kwargs): for k, v in kwargs.items(): setattr(sa, k, v) return sa.run() -session_auth.__doc__ = """Session authentication hook. -Any attribute of the SessionAuth class may be overridden via a keyword arg -to this function: -""" + "\n".join(["%s: %s" % (k, type(getattr(SessionAuth, k)).__name__) - for k in dir(SessionAuth) if not k.startswith("__")]) +session_auth.__doc__ = ( + """Session authentication hook. + + Any attribute of the SessionAuth class may be overridden via a keyword arg + to this function: + + """ + '\n'.join(['%s: %s' % (k, type(getattr(SessionAuth, k)).__name__) + for k in dir(SessionAuth) if not k.startswith('__')]) +) def log_traceback(severity=logging.ERROR, debug=False): """Write the last error's traceback to the cherrypy error log.""" - cherrypy.log("", "HTTP", severity=severity, traceback=True) + cherrypy.log('', 'HTTP', severity=severity, traceback=True) def log_request_headers(debug=False): """Write request headers to the cherrypy error log.""" - h = [" %s: %s" % (k, v) for k, v in cherrypy.serving.request.header_list] - cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP") + h = [' %s: %s' % (k, v) for k, v in cherrypy.serving.request.header_list] + cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), 'HTTP') def log_hooks(debug=False): @@ -440,13 +447,13 @@ def log_hooks(debug=False): points.append(k) for k in points: - msg.append(" %s:" % k) + msg.append(' %s:' % k) v = request.hooks.get(k, []) v.sort() for h in v: - msg.append(" %r" % h) + msg.append(' %r' % h) cherrypy.log('\nRequest Hooks for ' + cherrypy.url() + - ':\n' + '\n'.join(msg), "HTTP") + ':\n' + '\n'.join(msg), 'HTTP') def redirect(url='', internal=True, debug=False): @@ -531,7 +538,7 @@ def accept(media=None, debug=False): """ if not media: return - if isinstance(media, basestring): + if isinstance(media, text_or_bytes): media = [media] request = cherrypy.serving.request @@ -547,12 +554,12 @@ def accept(media=None, debug=False): # Note that 'ranges' is sorted in order of preference for element in ranges: if element.qvalue > 0: - if element.value == "*/*": + if element.value == '*/*': # Matches any type or subtype if debug: cherrypy.log('Match due to */*', 'TOOLS.ACCEPT') return media[0] - elif element.value.endswith("/*"): + elif element.value.endswith('/*'): # Matches any subtype mtype = element.value[:-1] # Keep the slash for m in media: @@ -572,36 +579,23 @@ def accept(media=None, debug=False): # No suitable media-range found. ah = request.headers.get('Accept') if ah is None: - msg = "Your client did not send an Accept header." + msg = 'Your client did not send an Accept header.' else: - msg = "Your client sent this Accept header: %s." % ah - msg += (" But this resource only emits these media types: %s." % - ", ".join(media)) + msg = 'Your client sent this Accept header: %s.' % ah + msg += (' But this resource only emits these media types: %s.' % + ', '.join(media)) raise cherrypy.HTTPError(406, msg) class MonitoredHeaderMap(_httputil.HeaderMap): + def transform_key(self, key): + self.accessed_headers.add(key) + return super(MonitoredHeaderMap, self).transform_key(key) + def __init__(self): self.accessed_headers = set() - - def __getitem__(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.__getitem__(self, key) - - def __contains__(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.__contains__(self, key) - - def get(self, key, default=None): - self.accessed_headers.add(key) - return _httputil.HeaderMap.get(self, key, default=default) - - if hasattr({}, 'has_key'): - # Python 2 - def has_key(self, key): - self.accessed_headers.add(key) - return _httputil.HeaderMap.has_key(self, key) + super(MonitoredHeaderMap, self).__init__() def autovary(ignore=None, debug=False): @@ -628,3 +622,19 @@ def autovary(ignore=None, debug=False): v.sort() resp_h['Vary'] = ', '.join(v) request.hooks.attach('before_finalize', set_response_header, 95) + + +def convert_params(exception=ValueError, error=400): + """Convert request params based on function annotations, with error handling. + + exception + Exception class to catch. + + status + The HTTP error code to return to the client on failure. + """ + request = cherrypy.serving.request + types = request.handler.callable.__annotations__ + with cherrypy.HTTPError.handle(exception, error): + for key in set(types).intersection(request.params): + request.params[key] = types[key](request.params[key]) diff --git a/lib/cherrypy/lib/encoding.py b/lib/cherrypy/lib/encoding.py index fb688f8d..3d001ca6 100644 --- a/lib/cherrypy/lib/encoding.py +++ b/lib/cherrypy/lib/encoding.py @@ -1,8 +1,11 @@ import struct import time +import io + +import six import cherrypy -from cherrypy._cpcompat import basestring, BytesIO, ntob, unicodestr +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import file_generator from cherrypy.lib import is_closable_iterator from cherrypy.lib import set_vary_header @@ -34,6 +37,7 @@ def decode(encoding=None, default_encoding='utf-8'): default_encoding = [default_encoding] body.attempt_charsets = body.attempt_charsets + default_encoding + class UTF8StreamEncoder: def __init__(self, iterator): self._iterator = iterator @@ -46,7 +50,7 @@ class UTF8StreamEncoder: def __next__(self): res = next(self._iterator) - if isinstance(res, unicodestr): + if isinstance(res, six.text_type): res = res.encode('utf-8') return res @@ -63,7 +67,7 @@ class UTF8StreamEncoder: class ResponseEncoder: default_encoding = 'utf-8' - failmsg = "Response body could not be encoded with %r." + failmsg = 'Response body could not be encoded with %r.' encoding = None errors = 'strict' text_only = True @@ -95,7 +99,7 @@ class ResponseEncoder: def encoder(body): for chunk in body: - if isinstance(chunk, unicodestr): + if isinstance(chunk, six.text_type): chunk = chunk.encode(encoding, self.errors) yield chunk self.body = encoder(self.body) @@ -108,7 +112,7 @@ class ResponseEncoder: self.attempted_charsets.add(encoding) body = [] for chunk in self.body: - if isinstance(chunk, unicodestr): + if isinstance(chunk, six.text_type): try: chunk = chunk.encode(encoding, self.errors) except (LookupError, UnicodeError): @@ -128,7 +132,7 @@ class ResponseEncoder: encoder = self.encode_stream else: encoder = self.encode_string - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. # Encoded strings may be of different lengths from their # unicode equivalents, and even from each other. For example: @@ -139,7 +143,7 @@ class ResponseEncoder: # 6 # >>> len(t.encode("utf7")) # 8 - del response.headers["Content-Length"] + del response.headers['Content-Length'] # Parse the Accept-Charset request header, and try to provide one # of the requested charsets (in order of user preference). @@ -154,7 +158,7 @@ class ResponseEncoder: if self.debug: cherrypy.log('Specified encoding %r' % encoding, 'TOOLS.ENCODE') - if (not charsets) or "*" in charsets or encoding in charsets: + if (not charsets) or '*' in charsets or encoding in charsets: if self.debug: cherrypy.log('Attempting encoding %r' % encoding, 'TOOLS.ENCODE') @@ -174,7 +178,7 @@ class ResponseEncoder: else: for element in encs: if element.qvalue > 0: - if element.value == "*": + if element.value == '*': # Matches any charset. Try our default. if self.debug: cherrypy.log('Attempting default encoding due ' @@ -189,7 +193,7 @@ class ResponseEncoder: if encoder(encoding): return encoding - if "*" not in charsets: + if '*' not in charsets: # If no "*" is present in an Accept-Charset field, then all # character sets not explicitly mentioned get a quality # value of 0, except for ISO-8859-1, which gets a quality @@ -205,39 +209,27 @@ class ResponseEncoder: # No suitable encoding found. ac = request.headers.get('Accept-Charset') if ac is None: - msg = "Your client did not send an Accept-Charset header." + msg = 'Your client did not send an Accept-Charset header.' else: - msg = "Your client sent this Accept-Charset header: %s." % ac - _charsets = ", ".join(sorted(self.attempted_charsets)) - msg += " We tried these charsets: %s." % (_charsets,) + msg = 'Your client sent this Accept-Charset header: %s.' % ac + _charsets = ', '.join(sorted(self.attempted_charsets)) + msg += ' We tried these charsets: %s.' % (_charsets,) raise cherrypy.HTTPError(406, msg) def __call__(self, *args, **kwargs): response = cherrypy.serving.response self.body = self.oldhandler(*args, **kwargs) - if isinstance(self.body, basestring): - # strings get wrapped in a list because iterating over a single - # item list is much faster than iterating over every character - # in a long string. - if self.body: - self.body = [self.body] - else: - # [''] doesn't evaluate to False, so replace it with []. - self.body = [] - elif hasattr(self.body, 'read'): - self.body = file_generator(self.body) - elif self.body is None: - self.body = [] + self.body = prepare_iter(self.body) - ct = response.headers.elements("Content-Type") + ct = response.headers.elements('Content-Type') if self.debug: cherrypy.log('Content-Type: %r' % [str(h) for h in ct], 'TOOLS.ENCODE') if ct and self.add_charset: ct = ct[0] if self.text_only: - if ct.value.lower().startswith("text/"): + if ct.value.lower().startswith('text/'): if self.debug: cherrypy.log( 'Content-Type %s starts with "text/"' % ct, @@ -261,10 +253,33 @@ class ResponseEncoder: if self.debug: cherrypy.log('Setting Content-Type %s' % ct, 'TOOLS.ENCODE') - response.headers["Content-Type"] = str(ct) + response.headers['Content-Type'] = str(ct) return self.body + +def prepare_iter(value): + """ + Ensure response body is iterable and resolves to False when empty. + """ + if isinstance(value, text_or_bytes): + # strings get wrapped in a list because iterating over a single + # item list is much faster than iterating over every character + # in a long string. + if value: + value = [value] + else: + # [''] doesn't evaluate to False, so replace it with []. + value = [] + # Don't use isinstance here; io.IOBase which has an ABC takes + # 1000 times as long as, say, isinstance(value, str) + elif hasattr(value, 'read'): + value = file_generator(value) + elif value is None: + value = [] + return value + + # GZIP @@ -273,15 +288,15 @@ def compress(body, compress_level): import zlib # See http://www.gzip.org/zlib/rfc-gzip.html - yield ntob('\x1f\x8b') # ID1 and ID2: gzip marker - yield ntob('\x08') # CM: compression method - yield ntob('\x00') # FLG: none set + yield b'\x1f\x8b' # ID1 and ID2: gzip marker + yield b'\x08' # CM: compression method + yield b'\x00' # FLG: none set # MTIME: 4 bytes - yield struct.pack(" self.maxparents: - return [("[%s referrers]" % len(refs), [])] + return [('[%s referrers]' % len(refs), [])] try: ascendcode = self.ascend.__code__ @@ -72,20 +71,20 @@ class ReferrerTree(object): return self.peek(repr(obj)) if isinstance(obj, dict): - return "{" + ", ".join(["%s: %s" % (self._format(k, descend=False), + return '{' + ', '.join(['%s: %s' % (self._format(k, descend=False), self._format(v, descend=False)) - for k, v in obj.items()]) + "}" + for k, v in obj.items()]) + '}' elif isinstance(obj, list): - return "[" + ", ".join([self._format(item, descend=False) - for item in obj]) + "]" + return '[' + ', '.join([self._format(item, descend=False) + for item in obj]) + ']' elif isinstance(obj, tuple): - return "(" + ", ".join([self._format(item, descend=False) - for item in obj]) + ")" + return '(' + ', '.join([self._format(item, descend=False) + for item in obj]) + ')' r = self.peek(repr(obj)) if isinstance(obj, (str, int, float)): return r - return "%s: %s" % (type(obj), r) + return '%s: %s' % (type(obj), r) def format(self, tree): """Return a list of string reprs from a nested list of referrers.""" @@ -93,7 +92,7 @@ class ReferrerTree(object): def ascend(branch, depth=1): for parent, grandparents in branch: - output.append((" " * depth) + self._format(parent)) + output.append((' ' * depth) + self._format(parent)) if grandparents: ascend(grandparents, depth + 1) ascend(tree) @@ -114,20 +113,22 @@ class RequestCounter(SimplePlugin): def after_request(self): self.count -= 1 + + request_counter = RequestCounter(cherrypy.engine) request_counter.subscribe() def get_context(obj): if isinstance(obj, _cprequest.Request): - return "path=%s;stage=%s" % (obj.path_info, obj.stage) + return 'path=%s;stage=%s' % (obj.path_info, obj.stage) elif isinstance(obj, _cprequest.Response): - return "status=%s" % obj.status + return 'status=%s' % obj.status elif isinstance(obj, _cpwsgi.AppResponse): - return "PATH_INFO=%s" % obj.environ.get('PATH_INFO', '') - elif hasattr(obj, "tb_lineno"): - return "tb_lineno=%s" % obj.tb_lineno - return "" + return 'PATH_INFO=%s' % obj.environ.get('PATH_INFO', '') + elif hasattr(obj, 'tb_lineno'): + return 'tb_lineno=%s' % obj.tb_lineno + return '' class GCRoot(object): @@ -136,26 +137,27 @@ class GCRoot(object): classes = [ (_cprequest.Request, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), + 'Should be 1 in this request thread and 1 in the main thread.'), (_cprequest.Response, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), + 'Should be 1 in this request thread and 1 in the main thread.'), (_cpwsgi.AppResponse, 1, 1, - "Should be 1 in this request thread only."), + 'Should be 1 in this request thread only.'), ] + @cherrypy.expose def index(self): - return "Hello, world!" - index.exposed = True + return 'Hello, world!' + @cherrypy.expose def stats(self): - output = ["Statistics:"] + output = ['Statistics:'] for trial in range(10): if request_counter.count > 0: break time.sleep(0.5) else: - output.append("\nNot all requests closed properly.") + output.append('\nNot all requests closed properly.') # gc_collect isn't perfectly synchronous, because it may # break reference cycles that then take time to fully @@ -173,11 +175,11 @@ class GCRoot(object): for x in gc.garbage: trash[type(x)] = trash.get(type(x), 0) + 1 if trash: - output.insert(0, "\n%s unreachable objects:" % unreachable) + output.insert(0, '\n%s unreachable objects:' % unreachable) trash = [(v, k) for k, v in trash.items()] trash.sort() for pair in trash: - output.append(" " + repr(pair)) + output.append(' ' + repr(pair)) # Check declared classes to verify uncollected instances. # These don't have to be part of a cycle; they can be @@ -193,25 +195,24 @@ class GCRoot(object): if lenobj < minobj or lenobj > maxobj: if minobj == maxobj: output.append( - "\nExpected %s %r references, got %s." % + '\nExpected %s %r references, got %s.' % (minobj, cls, lenobj)) else: output.append( - "\nExpected %s to %s %r references, got %s." % + '\nExpected %s to %s %r references, got %s.' % (minobj, maxobj, cls, lenobj)) for obj in objs: if objgraph is not None: ig = [id(objs), id(inspect.currentframe())] - fname = "graph_%s_%s.png" % (cls.__name__, id(obj)) + fname = 'graph_%s_%s.png' % (cls.__name__, id(obj)) objgraph.show_backrefs( obj, extra_ignore=ig, max_depth=4, too_many=20, filename=fname, extra_info=get_context) - output.append("\nReferrers for %s (refcount=%s):" % + output.append('\nReferrers for %s (refcount=%s):' % (repr(obj), sys.getrefcount(obj))) t = ReferrerTree(ignore=[objs], maxdepth=3) tree = t.ascend(obj) output.extend(t.format(tree)) - return "\n".join(output) - stats.exposed = True + return '\n'.join(output) diff --git a/lib/cherrypy/lib/http.py b/lib/cherrypy/lib/http.py deleted file mode 100644 index 12043ad1..00000000 --- a/lib/cherrypy/lib/http.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -warnings.warn('cherrypy.lib.http has been deprecated and will be removed ' - 'in CherryPy 3.3 use cherrypy.lib.httputil instead.', - DeprecationWarning) - -from cherrypy.lib.httputil import * diff --git a/lib/cherrypy/lib/httpauth.py b/lib/cherrypy/lib/httpauth.py deleted file mode 100644 index 6d519907..00000000 --- a/lib/cherrypy/lib/httpauth.py +++ /dev/null @@ -1,373 +0,0 @@ -""" -This module defines functions to implement HTTP Digest Authentication -(:rfc:`2617`). -This has full compliance with 'Digest' and 'Basic' authentication methods. In -'Digest' it supports both MD5 and MD5-sess algorithms. - -Usage: - First use 'doAuth' to request the client authentication for a - certain resource. You should send an httplib.UNAUTHORIZED response to the - client so he knows he has to authenticate itself. - - Then use 'parseAuthorization' to retrieve the 'auth_map' used in - 'checkResponse'. - - To use 'checkResponse' you must have already verified the password - associated with the 'username' key in 'auth_map' dict. Then you use the - 'checkResponse' function to verify if the password matches the one sent - by the client. - -SUPPORTED_ALGORITHM - list of supported 'Digest' algorithms -SUPPORTED_QOP - list of supported 'Digest' 'qop'. -""" -__version__ = 1, 0, 1 -__author__ = "Tiago Cogumbreiro " -__credits__ = """ - Peter van Kampen for its recipe which implement most of Digest - authentication: - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302378 -""" - -__license__ = """ -Copyright (c) 2005, Tiago Cogumbreiro -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Sylvain Hellegouarch nor the names of his - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -__all__ = ("digestAuth", "basicAuth", "doAuth", "checkResponse", - "parseAuthorization", "SUPPORTED_ALGORITHM", "md5SessionKey", - "calculateNonce", "SUPPORTED_QOP") - -########################################################################## -import time -from hashlib import md5 - -from cherrypy._cpcompat import base64_decode, ntob -from cherrypy._cpcompat import parse_http_list, parse_keqv_list - -MD5 = "MD5" -MD5_SESS = "MD5-sess" -AUTH = "auth" -AUTH_INT = "auth-int" - -SUPPORTED_ALGORITHM = (MD5, MD5_SESS) -SUPPORTED_QOP = (AUTH, AUTH_INT) - -########################################################################## -# doAuth -# -DIGEST_AUTH_ENCODERS = { - MD5: lambda val: md5(ntob(val)).hexdigest(), - MD5_SESS: lambda val: md5(ntob(val)).hexdigest(), - # SHA: lambda val: sha.new(ntob(val)).hexdigest (), -} - - -def calculateNonce(realm, algorithm=MD5): - """This is an auxaliary function that calculates 'nonce' value. It is used - to handle sessions.""" - - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS - assert algorithm in SUPPORTED_ALGORITHM - - try: - encoder = DIGEST_AUTH_ENCODERS[algorithm] - except KeyError: - raise NotImplementedError("The chosen algorithm (%s) does not have " - "an implementation yet" % algorithm) - - return encoder("%d:%s" % (time.time(), realm)) - - -def digestAuth(realm, algorithm=MD5, nonce=None, qop=AUTH): - """Challenges the client for a Digest authentication.""" - global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS, SUPPORTED_QOP - assert algorithm in SUPPORTED_ALGORITHM - assert qop in SUPPORTED_QOP - - if nonce is None: - nonce = calculateNonce(realm, algorithm) - - return 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop - ) - - -def basicAuth(realm): - """Challengenes the client for a Basic authentication.""" - assert '"' not in realm, "Realms cannot contain the \" (quote) character." - - return 'Basic realm="%s"' % realm - - -def doAuth(realm): - """'doAuth' function returns the challenge string b giving priority over - Digest and fallback to Basic authentication when the browser doesn't - support the first one. - - This should be set in the HTTP header under the key 'WWW-Authenticate'.""" - - return digestAuth(realm) + " " + basicAuth(realm) - - -########################################################################## -# Parse authorization parameters -# -def _parseDigestAuthorization(auth_params): - # Convert the auth params to a dict - items = parse_http_list(auth_params) - params = parse_keqv_list(items) - - # Now validate the params - - # Check for required parameters - required = ["username", "realm", "nonce", "uri", "response"] - for k in required: - if k not in params: - return None - - # If qop is sent then cnonce and nc MUST be present - if "qop" in params and not ("cnonce" in params - and "nc" in params): - return None - - # If qop is not sent, neither cnonce nor nc can be present - if ("cnonce" in params or "nc" in params) and \ - "qop" not in params: - return None - - return params - - -def _parseBasicAuthorization(auth_params): - username, password = base64_decode(auth_params).split(":", 1) - return {"username": username, "password": password} - -AUTH_SCHEMES = { - "basic": _parseBasicAuthorization, - "digest": _parseDigestAuthorization, -} - - -def parseAuthorization(credentials): - """parseAuthorization will convert the value of the 'Authorization' key in - the HTTP header to a map itself. If the parsing fails 'None' is returned. - """ - - global AUTH_SCHEMES - - auth_scheme, auth_params = credentials.split(" ", 1) - auth_scheme = auth_scheme.lower() - - parser = AUTH_SCHEMES[auth_scheme] - params = parser(auth_params) - - if params is None: - return - - assert "auth_scheme" not in params - params["auth_scheme"] = auth_scheme - return params - - -########################################################################## -# Check provided response for a valid password -# -def md5SessionKey(params, password): - """ - If the "algorithm" directive's value is "MD5-sess", then A1 - [the session key] is calculated only once - on the first request by the - client following receipt of a WWW-Authenticate challenge from the server. - - This creates a 'session key' for the authentication of subsequent - requests and responses which is different for each "authentication - session", thus limiting the amount of material hashed with any one - key. - - Because the server need only use the hash of the user - credentials in order to create the A1 value, this construction could - be used in conjunction with a third party authentication service so - that the web server would not need the actual password value. The - specification of such a protocol is beyond the scope of this - specification. -""" - - keys = ("username", "realm", "nonce", "cnonce") - params_copy = {} - for key in keys: - params_copy[key] = params[key] - - params_copy["algorithm"] = MD5_SESS - return _A1(params_copy, password) - - -def _A1(params, password): - algorithm = params.get("algorithm", MD5) - H = DIGEST_AUTH_ENCODERS[algorithm] - - if algorithm == MD5: - # If the "algorithm" directive's value is "MD5" or is - # unspecified, then A1 is: - # A1 = unq(username-value) ":" unq(realm-value) ":" passwd - return "%s:%s:%s" % (params["username"], params["realm"], password) - - elif algorithm == MD5_SESS: - - # This is A1 if qop is set - # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) - # ":" unq(nonce-value) ":" unq(cnonce-value) - h_a1 = H("%s:%s:%s" % (params["username"], params["realm"], password)) - return "%s:%s:%s" % (h_a1, params["nonce"], params["cnonce"]) - - -def _A2(params, method, kwargs): - # If the "qop" directive's value is "auth" or is unspecified, then A2 is: - # A2 = Method ":" digest-uri-value - - qop = params.get("qop", "auth") - if qop == "auth": - return method + ":" + params["uri"] - elif qop == "auth-int": - # If the "qop" value is "auth-int", then A2 is: - # A2 = Method ":" digest-uri-value ":" H(entity-body) - entity_body = kwargs.get("entity_body", "") - H = kwargs["H"] - - return "%s:%s:%s" % ( - method, - params["uri"], - H(entity_body) - ) - - else: - raise NotImplementedError("The 'qop' method is unknown: %s" % qop) - - -def _computeDigestResponse(auth_map, password, method="GET", A1=None, - **kwargs): - """ - Generates a response respecting the algorithm defined in RFC 2617 - """ - params = auth_map - - algorithm = params.get("algorithm", MD5) - - H = DIGEST_AUTH_ENCODERS[algorithm] - KD = lambda secret, data: H(secret + ":" + data) - - qop = params.get("qop", None) - - H_A2 = H(_A2(params, method, kwargs)) - - if algorithm == MD5_SESS and A1 is not None: - H_A1 = H(A1) - else: - H_A1 = H(_A1(params, password)) - - if qop in ("auth", "auth-int"): - # If the "qop" value is "auth" or "auth-int": - # request-digest = <"> < KD ( H(A1), unq(nonce-value) - # ":" nc-value - # ":" unq(cnonce-value) - # ":" unq(qop-value) - # ":" H(A2) - # ) <"> - request = "%s:%s:%s:%s:%s" % ( - params["nonce"], - params["nc"], - params["cnonce"], - params["qop"], - H_A2, - ) - elif qop is None: - # If the "qop" directive is not present (this construction is - # for compatibility with RFC 2069): - # request-digest = - # <"> < KD ( H(A1), unq(nonce-value) ":" H(A2) ) > <"> - request = "%s:%s" % (params["nonce"], H_A2) - - return KD(H_A1, request) - - -def _checkDigestResponse(auth_map, password, method="GET", A1=None, **kwargs): - """This function is used to verify the response given by the client when - he tries to authenticate. - Optional arguments: - entity_body - when 'qop' is set to 'auth-int' you MUST provide the - raw data you are going to send to the client (usually the - HTML page. - request_uri - the uri from the request line compared with the 'uri' - directive of the authorization map. They must represent - the same resource (unused at this time). - """ - - if auth_map['realm'] != kwargs.get('realm', None): - return False - - response = _computeDigestResponse( - auth_map, password, method, A1, **kwargs) - - return response == auth_map["response"] - - -def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, - **kwargs): - # Note that the Basic response doesn't provide the realm value so we cannot - # test it - pass_through = lambda password, username=None: password - encrypt = encrypt or pass_through - try: - candidate = encrypt(auth_map["password"], auth_map["username"]) - except TypeError: - # if encrypt only takes one parameter, it's the password - candidate = encrypt(auth_map["password"]) - return candidate == password - -AUTH_RESPONSES = { - "basic": _checkBasicResponse, - "digest": _checkDigestResponse, -} - - -def checkResponse(auth_map, password, method="GET", encrypt=None, **kwargs): - """'checkResponse' compares the auth_map with the password and optionally - other arguments that each implementation might need. - - If the response is of type 'Basic' then the function has the following - signature:: - - checkBasicResponse(auth_map, password) -> bool - - If the response is of type 'Digest' then the function has the following - signature:: - - checkDigestResponse(auth_map, password, method='GET', A1=None) -> bool - - The 'A1' argument is only used in MD5_SESS algorithm based responses. - Check md5SessionKey() for more info. - """ - checker = AUTH_RESPONSES[auth_map["auth_scheme"]] - return checker(auth_map, password, method=method, encrypt=encrypt, - **kwargs) diff --git a/lib/cherrypy/lib/httputil.py b/lib/cherrypy/lib/httputil.py index 69a18d45..59bcc746 100644 --- a/lib/cherrypy/lib/httputil.py +++ b/lib/cherrypy/lib/httputil.py @@ -7,13 +7,24 @@ FuManChu will personally hang you up by your thumbs and submit you to a public caning. """ +import functools +import email.utils +import re from binascii import b2a_base64 -from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou -from cherrypy._cpcompat import basestring, bytestr, iteritems, nativestr -from cherrypy._cpcompat import reversed, sorted, unicodestr, unquote_qs +from cgi import parse_header +from email.header import decode_header + +import six +from six.moves import range, builtins, map +from six.moves.BaseHTTPServer import BaseHTTPRequestHandler + +import cherrypy +from cherrypy._cpcompat import ntob, ntou +from cherrypy._cpcompat import unquote_plus + response_codes = BaseHTTPRequestHandler.responses.copy() -# From https://bitbucket.org/cherrypy/cherrypy/issue/361 +# From https://github.com/cherrypy/cherrypy/issues/361 response_codes[500] = ('Internal Server Error', 'The server encountered an unexpected condition ' 'which prevented it from fulfilling the request.') @@ -22,34 +33,34 @@ response_codes[503] = ('Service Unavailable', 'request due to a temporary overloading or ' 'maintenance of the server.') -import re -import urllib + +HTTPDate = functools.partial(email.utils.formatdate, usegmt=True) def urljoin(*atoms): - """Return the given path \*atoms, joined into a single URL. + r"""Return the given path \*atoms, joined into a single URL. This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. """ - url = "/".join([x for x in atoms if x]) - while "//" in url: - url = url.replace("//", "/") + url = '/'.join([x for x in atoms if x]) + while '//' in url: + url = url.replace('//', '/') # Special-case the final url of "", and return "/" instead. - return url or "/" + return url or '/' def urljoin_bytes(*atoms): - """Return the given path *atoms, joined into a single URL. + """Return the given path `*atoms`, joined into a single URL. This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. """ - url = ntob("/").join([x for x in atoms if x]) - while ntob("//") in url: - url = url.replace(ntob("//"), ntob("/")) + url = b'/'.join([x for x in atoms if x]) + while b'//' in url: + url = url.replace(b'//', b'/') # Special-case the final url of "", and return "/" instead. - return url or ntob("/") + return url or b'/' def protocol_from_http(protocol_str): @@ -72,9 +83,9 @@ def get_ranges(headervalue, content_length): return None result = [] - bytesunit, byteranges = headervalue.split("=", 1) - for brange in byteranges.split(","): - start, stop = [x.strip() for x in brange.split("-", 1)] + bytesunit, byteranges = headervalue.split('=', 1) + for brange in byteranges.split(','): + start, stop = [x.strip() for x in brange.split('-', 1)] if start: if not stop: stop = content_length - 1 @@ -108,9 +119,9 @@ def get_ranges(headervalue, content_length): # If the entity is shorter than the specified suffix-length, # the entire entity-body is used. if int(stop) > content_length: - result.append((0, content_length)) + result.append((0, content_length)) else: - result.append((content_length - int(stop), content_length)) + result.append((content_length - int(stop), content_length)) return result @@ -126,14 +137,14 @@ class HeaderElement(object): self.params = params def __cmp__(self, other): - return cmp(self.value, other.value) + return builtins.cmp(self.value, other.value) def __lt__(self, other): return self.value < other.value def __str__(self): - p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] - return str("%s%s" % (self.value, "".join(p))) + p = [';%s=%s' % (k, v) for k, v in six.iteritems(self.params)] + return str('%s%s' % (self.value, ''.join(p))) def __bytes__(self): return ntob(self.__str__()) @@ -141,32 +152,17 @@ class HeaderElement(object): def __unicode__(self): return ntou(self.__str__()) + @staticmethod def parse(elementstr): """Transform 'token;key=val' to ('token', {'key': 'val'}).""" - # Split the element into a value and parameters. The 'value' may - # be of the form, "token=token", but we don't split that here. - atoms = [x.strip() for x in elementstr.split(";") if x.strip()] - if not atoms: - initial_value = '' - else: - initial_value = atoms.pop(0).strip() - params = {} - for atom in atoms: - atom = [x.strip() for x in atom.split("=", 1) if x.strip()] - key = atom.pop(0) - if atom: - val = atom[0] - else: - val = "" - params[key] = val + initial_value, params = parse_header(elementstr) return initial_value, params - parse = staticmethod(parse) + @classmethod def from_str(cls, elementstr): """Construct an instance from a string of the form 'token;key=val'.""" ival, params = cls.parse(elementstr) return cls(ival, params) - from_str = classmethod(from_str) q_separator = re.compile(r'; *q *=') @@ -183,6 +179,7 @@ class AcceptElement(HeaderElement): have been the other way around, but it's too late to fix now. """ + @classmethod def from_str(cls, elementstr): qvalue = None # The first "q" parameter (if any) separates the initial @@ -196,21 +193,35 @@ class AcceptElement(HeaderElement): media_type, params = cls.parse(media_range) if qvalue is not None: - params["q"] = qvalue + params['q'] = qvalue return cls(media_type, params) - from_str = classmethod(from_str) + @property def qvalue(self): - val = self.params.get("q", "1") + 'The qvalue, or priority, of this value.' + val = self.params.get('q', '1') if isinstance(val, HeaderElement): val = val.value - return float(val) - qvalue = property(qvalue, doc="The qvalue, or priority, of this value.") + try: + return float(val) + except ValueError as val_err: + """Fail client requests with invalid quality value. + + Ref: https://github.com/cherrypy/cherrypy/issues/1370 + """ + six.raise_from( + cherrypy.HTTPError( + 400, + 'Malformed HTTP header: `{}`'. + format(str(self)), + ), + val_err, + ) def __cmp__(self, other): - diff = cmp(self.qvalue, other.qvalue) + diff = builtins.cmp(self.qvalue, other.qvalue) if diff == 0: - diff = cmp(str(self), str(other)) + diff = builtins.cmp(str(self), str(other)) return diff def __lt__(self, other): @@ -219,7 +230,10 @@ class AcceptElement(HeaderElement): else: return self.qvalue < other.qvalue + RE_HEADER_SPLIT = re.compile(',(?=(?:[^"]*"[^"]*")*[^"]*$)') + + def header_elements(fieldname, fieldvalue): """Return a sorted HeaderElement list from a comma-separated header string. """ @@ -228,7 +242,7 @@ def header_elements(fieldname, fieldvalue): result = [] for element in RE_HEADER_SPLIT.split(fieldvalue): - if fieldname.startswith("Accept") or fieldname == 'TE': + if fieldname.startswith('Accept') or fieldname == 'TE': hv = AcceptElement.from_str(element) else: hv = HeaderElement.from_str(element) @@ -238,14 +252,14 @@ def header_elements(fieldname, fieldvalue): def decode_TEXT(value): - r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" - try: - # Python 3 - from email.header import decode_header - except ImportError: - from email.Header import decode_header + r""" + Decode :rfc:`2047` TEXT + + >>> decode_TEXT("=?utf-8?q?f=C3=BCr?=") == b'f\xfcr'.decode('latin-1') + True + """ atoms = decode_header(value) - decodedvalue = "" + decodedvalue = '' for atom, charset in atoms: if charset is not None: atom = atom.decode(charset) @@ -253,41 +267,51 @@ def decode_TEXT(value): return decodedvalue +def decode_TEXT_maybe(value): + """ + Decode the text but only if '=?' appears in it. + """ + return decode_TEXT(value) if '=?' in value else value + + def valid_status(status): """Return legal HTTP status Code, Reason-phrase and Message. - The status arg must be an int, or a str that begins with an int. + The status arg must be an int, a str that begins with an int + or the constant from ``http.client`` stdlib module. - If status is an int, or a str and no reason-phrase is supplied, - a default reason-phrase will be provided. + If status has no reason-phrase is supplied, a default reason- + phrase will be provided. + + >>> from six.moves import http_client + >>> from six.moves.BaseHTTPServer import BaseHTTPRequestHandler + >>> valid_status(http_client.ACCEPTED) == ( + ... int(http_client.ACCEPTED), + ... ) + BaseHTTPRequestHandler.responses[http_client.ACCEPTED] + True """ if not status: status = 200 - status = str(status) - parts = status.split(" ", 1) - if len(parts) == 1: - # No reason supplied. - code, = parts - reason = None - else: - code, reason = parts - reason = reason.strip() + code, reason = status, None + if isinstance(status, six.string_types): + code, _, reason = status.partition(' ') + reason = reason.strip() or None try: code = int(code) - except ValueError: - raise ValueError("Illegal response status from server " - "(%s is non-numeric)." % repr(code)) + except (TypeError, ValueError): + raise ValueError('Illegal response status from server ' + '(%s is non-numeric).' % repr(code)) if code < 100 or code > 599: - raise ValueError("Illegal response status from server " - "(%s is out of range)." % repr(code)) + raise ValueError('Illegal response status from server ' + '(%s is out of range).' % repr(code)) if code not in response_codes: # code is unknown but not illegal - default_reason, message = "", "" + default_reason, message = '', '' else: default_reason, message = response_codes[code] @@ -328,15 +352,15 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): nv = name_value.split('=', 1) if len(nv) != 2: if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) + raise ValueError('bad query field: %r' % (name_value,)) # Handle case of a control-name with no equal sign if keep_blank_values: nv.append('') else: continue if len(nv[1]) or keep_blank_values: - name = unquote_qs(nv[0], encoding) - value = unquote_qs(nv[1], encoding) + name = unquote_plus(nv[0], encoding, errors='strict') + value = unquote_plus(nv[1], encoding, errors='strict') if name in d: if not isinstance(d[name], list): d[name] = [d[name]] @@ -346,7 +370,7 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): return d -image_map_pattern = re.compile(r"[0-9]+,[0-9]+") +image_map_pattern = re.compile(r'[0-9]+,[0-9]+') def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): @@ -359,60 +383,84 @@ def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): if image_map_pattern.match(query_string): # Server-side image map. Map the coords to 'x' and 'y' # (like CGI::Request does). - pm = query_string.split(",") + pm = query_string.split(',') pm = {'x': int(pm[0]), 'y': int(pm[1])} else: pm = _parse_qs(query_string, keep_blank_values, encoding=encoding) return pm -class CaseInsensitiveDict(dict): +#### +# Inlined from jaraco.collections 1.5.2 +# Ref #1673 +class KeyTransformingDict(dict): + """ + A dict subclass that transforms the keys before they're used. + Subclasses may override the default transform_key to customize behavior. + """ + @staticmethod + def transform_key(key): + return key + + def __init__(self, *args, **kargs): + super(KeyTransformingDict, self).__init__() + # build a dictionary using the default constructs + d = dict(*args, **kargs) + # build this dictionary using transformed keys. + for item in d.items(): + self.__setitem__(*item) + + def __setitem__(self, key, val): + key = self.transform_key(key) + super(KeyTransformingDict, self).__setitem__(key, val) + + def __getitem__(self, key): + key = self.transform_key(key) + return super(KeyTransformingDict, self).__getitem__(key) + + def __contains__(self, key): + key = self.transform_key(key) + return super(KeyTransformingDict, self).__contains__(key) + + def __delitem__(self, key): + key = self.transform_key(key) + return super(KeyTransformingDict, self).__delitem__(key) + + def get(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).get(key, *args, **kwargs) + + def setdefault(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).setdefault( + key, *args, **kwargs) + + def pop(self, key, *args, **kwargs): + key = self.transform_key(key) + return super(KeyTransformingDict, self).pop(key, *args, **kwargs) + + def matching_key_for(self, key): + """ + Given a key, return the actual key stored in self that matches. + Raise KeyError if the key isn't found. + """ + try: + return next(e_key for e_key in self.keys() if e_key == key) + except StopIteration: + raise KeyError(key) +#### + + +class CaseInsensitiveDict(KeyTransformingDict): """A case-insensitive dict subclass. Each key is changed on entry to str(key).title(). """ - def __getitem__(self, key): - return dict.__getitem__(self, str(key).title()) - - def __setitem__(self, key, value): - dict.__setitem__(self, str(key).title(), value) - - def __delitem__(self, key): - dict.__delitem__(self, str(key).title()) - - def __contains__(self, key): - return dict.__contains__(self, str(key).title()) - - def get(self, key, default=None): - return dict.get(self, str(key).title(), default) - - if hasattr({}, 'has_key'): - def has_key(self, key): - return str(key).title() in self - - def update(self, E): - for k in E.keys(): - self[str(k).title()] = E[k] - - def fromkeys(cls, seq, value=None): - newdict = cls() - for k in seq: - newdict[str(k).title()] = value - return newdict - fromkeys = classmethod(fromkeys) - - def setdefault(self, key, x=None): - key = str(key).title() - try: - return self[key] - except KeyError: - self[key] = x - return x - - def pop(self, key, default): - return dict.pop(self, str(key).title(), default) + @staticmethod + def transform_key(key): + return str(key).title() # TEXT = @@ -420,10 +468,10 @@ class CaseInsensitiveDict(dict): # A CRLF is allowed in the definition of TEXT only as part of a header # field continuation. It is expected that the folding LWS will be # replaced with a single SP before interpretation of the TEXT value." -if nativestr == bytestr: - header_translate_table = ''.join([chr(i) for i in xrange(256)]) +if str == bytes: + header_translate_table = ''.join([chr(i) for i in range(256)]) header_translate_deletechars = ''.join( - [chr(i) for i in xrange(32)]) + chr(127) + [chr(i) for i in range(32)]) + chr(127) else: header_translate_table = None header_translate_deletechars = bytes(range(32)) + bytes([127]) @@ -440,7 +488,7 @@ class HeaderMap(CaseInsensitiveDict): """ protocol = (1, 1) - encodings = ["ISO-8859-1"] + encodings = ['ISO-8859-1'] # Someday, when http-bis is done, this will probably get dropped # since few servers, clients, or intermediaries do it. But until then, @@ -463,31 +511,30 @@ class HeaderMap(CaseInsensitiveDict): """Transform self into a list of (name, value) tuples.""" return list(self.encode_header_items(self.items())) + @classmethod def encode_header_items(cls, header_items): """ Prepare the sequence of name, value tuples into a form suitable for transmitting on the wire for HTTP. """ for k, v in header_items: - if isinstance(k, unicodestr): - k = cls.encode(k) + if not isinstance(v, six.string_types) and \ + not isinstance(v, six.binary_type): + v = six.text_type(v) - if not isinstance(v, basestring): - v = str(v) + yield tuple(map(cls.encode_header_item, (k, v))) - if isinstance(v, unicodestr): - v = cls.encode(v) + @classmethod + def encode_header_item(cls, item): + if isinstance(item, six.text_type): + item = cls.encode(item) - # See header_translate_* constants above. - # Replace only if you really know what you're doing. - k = k.translate(header_translate_table, - header_translate_deletechars) - v = v.translate(header_translate_table, - header_translate_deletechars) - - yield (k, v) - encode_header_items = classmethod(encode_header_items) + # See header_translate_* constants above. + # Replace only if you really know what you're doing. + return item.translate( + header_translate_table, header_translate_deletechars) + @classmethod def encode(cls, v): """Return the given header name or value, encoded for HTTP output.""" for enc in cls.encodings: @@ -503,12 +550,11 @@ class HeaderMap(CaseInsensitiveDict): # because we never want to fold lines--folding has # been deprecated by the HTTP working group. v = b2a_base64(v.encode('utf-8')) - return (ntob('=?utf-8?b?') + v.strip(ntob('\n')) + ntob('?=')) + return (b'=?utf-8?b?' + v.strip(b'\n') + b'?=') - raise ValueError("Could not encode header part %r using " - "any of the encodings %r." % + raise ValueError('Could not encode header part %r using ' + 'any of the encodings %r.' % (v, cls.encodings)) - encode = classmethod(encode) class Host(object): @@ -521,9 +567,9 @@ class Host(object): """ - ip = "0.0.0.0" + ip = '0.0.0.0' port = 80 - name = "unknown.tld" + name = 'unknown.tld' def __init__(self, ip, port, name=None): self.ip = ip @@ -533,4 +579,4 @@ class Host(object): self.name = name def __repr__(self): - return "httputil.Host(%r, %r, %r)" % (self.ip, self.port, self.name) + return 'httputil.Host(%r, %r, %r)' % (self.ip, self.port, self.name) diff --git a/lib/cherrypy/lib/jsontools.py b/lib/cherrypy/lib/jsontools.py index 90b3ff8a..48683097 100644 --- a/lib/cherrypy/lib/jsontools.py +++ b/lib/cherrypy/lib/jsontools.py @@ -1,17 +1,15 @@ import cherrypy -from cherrypy._cpcompat import basestring, ntou, json_encode, json_decode +from cherrypy._cpcompat import text_or_bytes, ntou, json_encode, json_decode def json_processor(entity): """Read application/json data into request.json.""" - if not entity.headers.get(ntou("Content-Length"), ntou("")): + if not entity.headers.get(ntou('Content-Length'), ntou('')): raise cherrypy.HTTPError(411) body = entity.fp.read() - try: + with cherrypy.HTTPError.handle(ValueError, 400, 'Invalid JSON document'): cherrypy.serving.request.json = json_decode(body.decode('utf-8')) - except ValueError: - raise cherrypy.HTTPError(400, 'Invalid JSON document') def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], @@ -36,12 +34,9 @@ def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], request header, or it will raise "411 Length Required". If for any other reason the request entity cannot be deserialized from JSON, it will raise "400 Bad Request: Invalid JSON document". - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request - if isinstance(content_type, basestring): + if isinstance(content_type, text_or_bytes): content_type = [content_type] if force: @@ -74,9 +69,6 @@ def json_out(content_type='application/json', debug=False, Provide your own handler to use a custom encoder. For example cherrypy.config['tools.json_out.handler'] = , or @json_out(handler=function). - - You must be using Python 2.6 or greater, or have the 'simplejson' - package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request # request.handler may be set to None by e.g. the caching tool diff --git a/lib/cherrypy/lib/lockfile.py b/lib/cherrypy/lib/lockfile.py deleted file mode 100644 index 4cf7b1b6..00000000 --- a/lib/cherrypy/lib/lockfile.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -Platform-independent file locking. Inspired by and modeled after zc.lockfile. -""" - -import os - -try: - import msvcrt -except ImportError: - pass - -try: - import fcntl -except ImportError: - pass - - -class LockError(Exception): - - "Could not obtain a lock" - - msg = "Unable to lock %r" - - def __init__(self, path): - super(LockError, self).__init__(self.msg % path) - - -class UnlockError(LockError): - - "Could not release a lock" - - msg = "Unable to unlock %r" - - -# first, a default, naive locking implementation -class LockFile(object): - - """ - A default, naive locking implementation. Always fails if the file - already exists. - """ - - def __init__(self, path): - self.path = path - try: - fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) - except OSError: - raise LockError(self.path) - os.close(fd) - - def release(self): - os.remove(self.path) - - def remove(self): - pass - - -class SystemLockFile(object): - - """ - An abstract base class for platform-specific locking. - """ - - def __init__(self, path): - self.path = path - - try: - # Open lockfile for writing without truncation: - self.fp = open(path, 'r+') - except IOError: - # If the file doesn't exist, IOError is raised; Use a+ instead. - # Note that there may be a race here. Multiple processes - # could fail on the r+ open and open the file a+, but only - # one will get the the lock and write a pid. - self.fp = open(path, 'a+') - - try: - self._lock_file() - except: - self.fp.seek(1) - self.fp.close() - del self.fp - raise - - self.fp.write(" %s\n" % os.getpid()) - self.fp.truncate() - self.fp.flush() - - def release(self): - if not hasattr(self, 'fp'): - return - self._unlock_file() - self.fp.close() - del self.fp - - def remove(self): - """ - Attempt to remove the file - """ - try: - os.remove(self.path) - except: - pass - - #@abc.abstract_method - # def _lock_file(self): - # """Attempt to obtain the lock on self.fp. Raise LockError if not - # acquired.""" - - def _unlock_file(self): - """Attempt to obtain the lock on self.fp. Raise UnlockError if not - released.""" - - -class WindowsLockFile(SystemLockFile): - - def _lock_file(self): - # Lock just the first byte - try: - msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) - except IOError: - raise LockError(self.fp.name) - - def _unlock_file(self): - try: - self.fp.seek(0) - msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) - except IOError: - raise UnlockError(self.fp.name) - -if 'msvcrt' in globals(): - LockFile = WindowsLockFile - - -class UnixLockFile(SystemLockFile): - - def _lock_file(self): - flags = fcntl.LOCK_EX | fcntl.LOCK_NB - try: - fcntl.flock(self.fp.fileno(), flags) - except IOError: - raise LockError(self.fp.name) - - # no need to implement _unlock_file, it will be unlocked on close() - -if 'fcntl' in globals(): - LockFile = UnixLockFile diff --git a/lib/cherrypy/lib/locking.py b/lib/cherrypy/lib/locking.py index 72dda9b3..317fb58c 100644 --- a/lib/cherrypy/lib/locking.py +++ b/lib/cherrypy/lib/locking.py @@ -11,7 +11,7 @@ class Timer(object): A simple timer that will indicate when an expiration time has passed. """ def __init__(self, expiration): - "Create a timer that expires at `expiration` (UTC datetime)" + 'Create a timer that expires at `expiration` (UTC datetime)' self.expiration = expiration @classmethod @@ -26,7 +26,7 @@ class Timer(object): class LockTimeout(Exception): - "An exception when a lock could not be acquired before a timeout period" + 'An exception when a lock could not be acquired before a timeout period' class LockChecker(object): @@ -43,5 +43,5 @@ class LockChecker(object): def expired(self): if self.timer.expired(): raise LockTimeout( - "Timeout acquiring lock for %(session_id)s" % vars(self)) + 'Timeout acquiring lock for %(session_id)s' % vars(self)) return False diff --git a/lib/cherrypy/lib/profiler.py b/lib/cherrypy/lib/profiler.py index a3477454..fccf2eb8 100644 --- a/lib/cherrypy/lib/profiler.py +++ b/lib/cherrypy/lib/profiler.py @@ -10,9 +10,9 @@ You can profile any of your pages as follows:: class Root: p = profiler.Profiler("/path/to/profile/dir") + @cherrypy.expose def index(self): self.p.run(self._index) - index.exposed = True def _index(self): return "Hello, world!" @@ -33,29 +33,36 @@ module from the command line, it will call ``serve()`` for you. """ - -def new_func_strip_path(func_name): - """Make profiler output more readable by adding `__init__` modules' parents - """ - filename, line, name = func_name - if filename.endswith("__init__.py"): - return os.path.basename(filename[:-12]) + filename[-12:], line, name - return os.path.basename(filename), line, name - -try: - import profile - import pstats - pstats.func_strip_path = new_func_strip_path -except ImportError: - profile = None - pstats = None - +import io import os import os.path import sys import warnings -from cherrypy._cpcompat import StringIO +import cherrypy + + +try: + import profile + import pstats + + def new_func_strip_path(func_name): + """Make profiler output more readable by adding `__init__` modules' parents + """ + filename, line, name = func_name + if filename.endswith('__init__.py'): + return ( + os.path.basename(filename[:-12]) + filename[-12:], + line, + name, + ) + return os.path.basename(filename), line, name + + pstats.func_strip_path = new_func_strip_path +except ImportError: + profile = None + pstats = None + _count = 0 @@ -64,7 +71,7 @@ class Profiler(object): def __init__(self, path=None): if not path: - path = os.path.join(os.path.dirname(__file__), "profile") + path = os.path.join(os.path.dirname(__file__), 'profile') self.path = path if not os.path.exists(path): os.makedirs(path) @@ -73,7 +80,7 @@ class Profiler(object): """Dump profile data into self.path.""" global _count c = _count = _count + 1 - path = os.path.join(self.path, "cp_%04d.prof" % c) + path = os.path.join(self.path, 'cp_%04d.prof' % c) prof = profile.Profile() result = prof.runcall(func, *args, **params) prof.dump_stats(path) @@ -83,12 +90,12 @@ class Profiler(object): """:rtype: list of available profiles. """ return [f for f in os.listdir(self.path) - if f.startswith("cp_") and f.endswith(".prof")] + if f.startswith('cp_') and f.endswith('.prof')] def stats(self, filename, sortby='cumulative'): """:rtype stats(index): output of print_stats() for the given profile. """ - sio = StringIO() + sio = io.StringIO() if sys.version_info >= (2, 5): s = pstats.Stats(os.path.join(self.path, filename), stream=sio) s.strip_dirs() @@ -110,6 +117,7 @@ class Profiler(object): sio.close() return response + @cherrypy.expose def index(self): return """ CherryPy profile data @@ -119,23 +127,21 @@ class Profiler(object): """ - index.exposed = True + @cherrypy.expose def menu(self): - yield "

Profiling runs

" - yield "

Click on one of the runs below to see profiling data.

" + yield '

Profiling runs

' + yield '

Click on one of the runs below to see profiling data.

' runs = self.statfiles() runs.sort() for i in runs: yield "%s
" % ( i, i) - menu.exposed = True + @cherrypy.expose def report(self, filename): - import cherrypy cherrypy.response.headers['Content-Type'] = 'text/plain' return self.stats(filename) - report.exposed = True class ProfileAggregator(Profiler): @@ -147,7 +153,7 @@ class ProfileAggregator(Profiler): self.profiler = profile.Profile() def run(self, func, *args, **params): - path = os.path.join(self.path, "cp_%04d.prof" % self.count) + path = os.path.join(self.path, 'cp_%04d.prof' % self.count) result = self.profiler.runcall(func, *args, **params) self.profiler.dump_stats(path) return result @@ -172,11 +178,11 @@ class make_app: """ if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile " + msg = ('Your installation of Python does not have a profile ' "module. If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") + '`sudo apt-get install python-profiler`. ' + 'See http://www.cherrypy.org/wiki/ProfilingOnDebian ' + 'for details.') warnings.warn(msg) self.nextapp = nextapp @@ -197,20 +203,19 @@ class make_app: def serve(path=None, port=8080): if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile module. " + msg = ('Your installation of Python does not have a profile module. ' "If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") + '`sudo apt-get install python-profiler`. ' + 'See http://www.cherrypy.org/wiki/ProfilingOnDebian ' + 'for details.') warnings.warn(msg) - import cherrypy cherrypy.config.update({'server.socket_port': int(port), 'server.thread_pool': 10, - 'environment': "production", + 'environment': 'production', }) cherrypy.quickstart(Profiler(path)) -if __name__ == "__main__": +if __name__ == '__main__': serve(*tuple(sys.argv[1:])) diff --git a/lib/cherrypy/lib/reprconf.py b/lib/cherrypy/lib/reprconf.py index 8af1f777..fc758490 100644 --- a/lib/cherrypy/lib/reprconf.py +++ b/lib/cherrypy/lib/reprconf.py @@ -18,42 +18,14 @@ by adding a named handler to Config.namespaces. The name can be any string, and the handler must be either a callable or a context manager. """ -try: - # Python 3.0+ - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser +from cherrypy._cpcompat import text_or_bytes +from six.moves import configparser +from six.moves import builtins -try: - set -except NameError: - from sets import Set as set - -try: - basestring -except NameError: - basestring = str - -try: - # Python 3 - import builtins -except ImportError: - # Python 2 - import __builtin__ as builtins - -import operator as _operator +import operator import sys -def as_dict(config): - """Return a dict from 'config' whether it is a dict, file, or filename.""" - if isinstance(config, basestring): - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - config = Parser().dict_from_file(config) - return config - - class NamespaceSet(dict): """A dict of config namespace names and handlers. @@ -83,19 +55,19 @@ class NamespaceSet(dict): # Separate the given config into namespaces ns_confs = {} for k in config: - if "." in k: - ns, name = k.split(".", 1) + if '.' in k: + ns, name = k.split('.', 1) bucket = ns_confs.setdefault(ns, {}) bucket[name] = config[k] # I chose __enter__ and __exit__ so someday this could be # rewritten using Python 2.5's 'with' statement: - # for ns, handler in self.iteritems(): + # for ns, handler in six.iteritems(self): # with handler as callable: - # for k, v in ns_confs.get(ns, {}).iteritems(): + # for k, v in six.iteritems(ns_confs.get(ns, {})): # callable(k, v) for ns, handler in self.items(): - exit = getattr(handler, "__exit__", None) + exit = getattr(handler, '__exit__', None) if exit: callable = handler.__enter__() no_exc = True @@ -103,7 +75,7 @@ class NamespaceSet(dict): try: for k, v in ns_confs.get(ns, {}).items(): callable(k, v) - except: + except Exception: # The exceptional case is handled here no_exc = False if exit is None: @@ -120,7 +92,7 @@ class NamespaceSet(dict): handler(k, v) def __repr__(self): - return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, + return '%s.%s(%s)' % (self.__module__, self.__class__.__name__, dict.__repr__(self)) def __copy__(self): @@ -154,16 +126,8 @@ class Config(dict): dict.update(self, self.defaults) def update(self, config): - """Update self from a dict, file or filename.""" - if isinstance(config, basestring): - # Filename - config = Parser().dict_from_file(config) - elif hasattr(config, 'read'): - # Open file object - config = Parser().dict_from_file(config) - else: - config = config.copy() - self._apply(config) + """Update self from a dict, file, or filename.""" + self._apply(Parser.load(config)) def _apply(self, config): """Update self from a dict.""" @@ -182,7 +146,7 @@ class Config(dict): self.namespaces({k: v}) -class Parser(ConfigParser): +class Parser(configparser.ConfigParser): """Sub-class of ConfigParser that keeps the case of options and that raises an exception if the file cannot be read. @@ -192,7 +156,7 @@ class Parser(ConfigParser): return optionstr def read(self, filenames): - if isinstance(filenames, basestring): + if isinstance(filenames, text_or_bytes): filenames = [filenames] for filename in filenames: # try: @@ -218,8 +182,8 @@ class Parser(ConfigParser): value = unrepr(value) except Exception: x = sys.exc_info()[1] - msg = ("Config error in section: %r, option: %r, " - "value: %r. Config values must be valid Python." % + msg = ('Config error in section: %r, option: %r, ' + 'value: %r. Config values must be valid Python.' % (section, option, value)) raise ValueError(msg, x.__class__.__name__, x.args) result[section][option] = value @@ -232,6 +196,17 @@ class Parser(ConfigParser): self.read(file) return self.as_dict() + @classmethod + def load(self, input): + """Resolve 'input' to dict from a dict, file, or filename.""" + is_file = ( + # Filename + isinstance(input, text_or_bytes) + # Open file object + or hasattr(input, 'read') + ) + return Parser().dict_from_file(input) if is_file else input.copy() + # public domain "unrepr" implementation, found on the web and then improved. @@ -241,7 +216,7 @@ class _Builder2: def build(self, o): m = getattr(self, 'build_' + o.__class__.__name__, None) if m is None: - raise TypeError("unrepr does not recognize %s" % + raise TypeError('unrepr does not recognize %s' % repr(o.__class__.__name__)) return m(o) @@ -254,7 +229,7 @@ class _Builder2: # e.g. IronPython 1.0. return eval(s) - p = compiler.parse("__tempvalue__ = " + s) + p = compiler.parse('__tempvalue__ = ' + s) return p.getChildren()[1].getChildren()[0].getChildren()[1] def build_Subscript(self, o): @@ -279,7 +254,7 @@ class _Builder2: if class_name == 'Keyword': kwargs.update(self.build(child)) # Everything else becomes args - else : + else: args.append(self.build(child)) return callee(*args, **kwargs) @@ -327,7 +302,7 @@ class _Builder2: except AttributeError: pass - raise TypeError("unrepr could not resolve the name %s" % repr(name)) + raise TypeError('unrepr could not resolve the name %s' % repr(name)) def build_Add(self, o): left, right = map(self.build, o.getChildren()) @@ -356,7 +331,7 @@ class _Builder3: def build(self, o): m = getattr(self, 'build_' + o.__class__.__name__, None) if m is None: - raise TypeError("unrepr does not recognize %s" % + raise TypeError('unrepr does not recognize %s' % repr(o.__class__.__name__)) return m(o) @@ -369,7 +344,7 @@ class _Builder3: # e.g. IronPython 1.0. return eval(s) - p = ast.parse("__tempvalue__ = " + s) + p = ast.parse('__tempvalue__ = ' + s) return p.body[0].value def build_Subscript(self, o): @@ -394,16 +369,16 @@ class _Builder3: args.append(self.build(a)) kwargs = {} for kw in o.keywords: - if kw.arg is None: # double asterix `**` + if kw.arg is None: # double asterix `**` rst = self.build(kw.value) if not isinstance(rst, dict): - raise TypeError("Invalid argument for call." - "Must be a mapping object.") + raise TypeError('Invalid argument for call.' + 'Must be a mapping object.') # give preference to the keys set directly from arg=value for k, v in rst.items(): if k not in kwargs: kwargs[k] = v - else: # defined on the call as: arg=value + else: # defined on the call as: arg=value kwargs[kw.arg] = self.build(kw.value) return callee(*args, **kwargs) @@ -427,7 +402,7 @@ class _Builder3: kwargs = {} else: kwargs = self.build(o.kwargs) - if o.keywords is not None: # direct a=b keywords + if o.keywords is not None: # direct a=b keywords for kw in o.keywords: # preference because is a direct keyword against **kwargs kwargs[kw.arg] = self.build(kw.value) @@ -471,11 +446,13 @@ class _Builder3: except AttributeError: pass - raise TypeError("unrepr could not resolve the name %s" % repr(name)) + raise TypeError('unrepr could not resolve the name %s' % repr(name)) def build_NameConstant(self, o): return o.value + build_Constant = build_NameConstant # Python 3.8 change + def build_UnaryOp(self, o): op, operand = map(self.build, [o.op, o.operand]) return op(operand) @@ -485,13 +462,13 @@ class _Builder3: return op(left, right) def build_Add(self, o): - return _operator.add + return operator.add def build_Mult(self, o): - return _operator.mul + return operator.mul def build_USub(self, o): - return _operator.neg + return operator.neg def build_Attribute(self, o): parent = self.build(o.value) @@ -523,7 +500,7 @@ def attributes(full_attribute_name): """Load a module and retrieve an attribute of that module.""" # Parse out the path, module, and attribute - last_dot = full_attribute_name.rfind(".") + last_dot = full_attribute_name.rfind('.') attr_name = full_attribute_name[last_dot + 1:] mod_path = full_attribute_name[:last_dot] diff --git a/lib/cherrypy/lib/sessions.py b/lib/cherrypy/lib/sessions.py index 37556363..5b49ee13 100644 --- a/lib/cherrypy/lib/sessions.py +++ b/lib/cherrypy/lib/sessions.py @@ -4,13 +4,13 @@ You need to edit your config file to use sessions. Here's an example:: [/] tools.sessions.on = True - tools.sessions.storage_type = "file" + tools.sessions.storage_class = cherrypy.lib.sessions.FileSession tools.sessions.storage_path = "/home/site/sessions" tools.sessions.timeout = 60 This sets the session to be stored in files in the directory /home/site/sessions, and the session timeout to 60 minutes. If you omit -``storage_type`` the sessions will be saved in RAM. +``storage_class``, the sessions will be saved in RAM. ``tools.sessions.on`` is the only required line for working sessions, the rest are optional. @@ -57,6 +57,17 @@ However, CherryPy "recognizes" a session id by looking up the saved session data for that id. Therefore, if you never save any session data, **you will get a new session id for every request**. +A side effect of CherryPy overwriting unrecognised session ids is that if you +have multiple, separate CherryPy applications running on a single domain (e.g. +on different ports), each app will overwrite the other's session id because by +default they use the same cookie name (``"session_id"``) but do not recognise +each others sessions. It is therefore a good idea to use a different name for +each, for example:: + + [/] + ... + tools.sessions.name = "my_app_session_id" + ================ Sharing Sessions ================ @@ -94,15 +105,24 @@ import datetime import os import time import threading -import types +import binascii + +import six +from six.moves import cPickle as pickle +import contextlib2 + +import zc.lockfile import cherrypy -from cherrypy._cpcompat import copyitems, pickle, random20, unicodestr from cherrypy.lib import httputil -from cherrypy.lib import lockfile from cherrypy.lib import locking from cherrypy.lib import is_iterator + +if six.PY2: + FileNotFoundError = OSError + + missing = object() @@ -115,17 +135,19 @@ class Session(object): id_observers = None "A list of callbacks to which to pass new id's." - def _get_id(self): + @property + def id(self): + """Return the current session id.""" return self._id - def _set_id(self, value): + @id.setter + def id(self, value): self._id = value for o in self.id_observers: o(value) - id = property(_get_id, _set_id, doc="The current session ID.") timeout = 60 - "Number of minutes after which to delete session data." + 'Number of minutes after which to delete session data.' locked = False """ @@ -138,16 +160,16 @@ class Session(object): automatically on the first attempt to access session data.""" clean_thread = None - "Class-level Monitor which calls self.clean_up." + 'Class-level Monitor which calls self.clean_up.' clean_freq = 5 - "The poll rate for expired session cleanup in minutes." + 'The poll rate for expired session cleanup in minutes.' originalid = None - "The session id passed by the client. May be missing or unsafe." + 'The session id passed by the client. May be missing or unsafe.' missing = False - "True if the session requested by the client did not exist." + 'True if the session requested by the client did not exist.' regenerated = False """ @@ -155,7 +177,7 @@ class Session(object): internal calls to regenerate the session id.""" debug = False - "If True, log debug information." + 'If True, log debug information.' # --------------------- Session management methods --------------------- # @@ -182,7 +204,7 @@ class Session(object): cherrypy.log('Expired or malicious session %r; ' 'making a new one' % id, 'TOOLS.SESSIONS') # Expired or malicious session. Make a new one. - # See https://bitbucket.org/cherrypy/cherrypy/issue/709. + # See https://github.com/cherrypy/cherrypy/issues/709. self.id = None self.missing = True self._regenerate() @@ -236,7 +258,7 @@ class Session(object): def generate_id(self): """Return a new session id.""" - return random20() + return binascii.hexlify(os.urandom(20)).decode('ascii') def save(self): """Save session data.""" @@ -335,13 +357,6 @@ class Session(object): self.load() return key in self._data - if hasattr({}, 'has_key'): - def has_key(self, key): - """D.has_key(k) -> True if D has a key k, else False.""" - if not self.loaded: - self.load() - return key in self._data - def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" if not self.loaded: @@ -395,7 +410,7 @@ class RamSession(Session): """Clean up expired sessions.""" now = self.now() - for _id, (data, expiration_time) in copyitems(self.cache): + for _id, (data, expiration_time) in list(six.iteritems(self.cache)): if expiration_time <= now: try: del self.cache[_id] @@ -410,7 +425,11 @@ class RamSession(Session): # added to remove obsolete lock objects for _id in list(self.locks): - if _id not in self.cache and self.locks[_id].acquire(blocking=False): + locked = ( + _id not in self.cache + and self.locks[_id].acquire(blocking=False) + ) + if locked: lock = self.locks.pop(_id) lock.release() @@ -471,9 +490,11 @@ class FileSession(Session): if isinstance(self.lock_timeout, (int, float)): self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): - raise ValueError("Lock timeout must be numeric seconds or " - "a timedelta instance.") + raise ValueError( + 'Lock timeout must be numeric seconds or a timedelta instance.' + ) + @classmethod def setup(cls, **kwargs): """Set up the storage system for file-based sessions. @@ -485,12 +506,11 @@ class FileSession(Session): for k, v in kwargs.items(): setattr(cls, k, v) - setup = classmethod(setup) def _get_file_path(self): f = os.path.join(self.storage_path, self.SESSION_PREFIX + self.id) if not os.path.abspath(f).startswith(self.storage_path): - raise cherrypy.HTTPError(400, "Invalid session id in cookie.") + raise cherrypy.HTTPError(400, 'Invalid session id in cookie.') return f def _exists(self): @@ -498,12 +518,12 @@ class FileSession(Session): return os.path.exists(path) def _load(self, path=None): - assert self.locked, ("The session load without being locked. " + assert self.locked, ('The session load without being locked. ' "Check your tools' priority levels.") if path is None: path = self._get_file_path() try: - f = open(path, "rb") + f = open(path, 'rb') try: return pickle.load(f) finally: @@ -511,21 +531,21 @@ class FileSession(Session): except (IOError, EOFError): e = sys.exc_info()[1] if self.debug: - cherrypy.log("Error loading the session pickle: %s" % + cherrypy.log('Error loading the session pickle: %s' % e, 'TOOLS.SESSIONS') return None def _save(self, expiration_time): - assert self.locked, ("The session was saved without being locked. " + assert self.locked, ('The session was saved without being locked. ' "Check your tools' priority levels.") - f = open(self._get_file_path(), "wb") + f = open(self._get_file_path(), 'wb') try: pickle.dump((self._data, expiration_time), f, self.pickle_protocol) finally: f.close() def _delete(self): - assert self.locked, ("The session deletion without being locked. " + assert self.locked, ('The session deletion without being locked. ' "Check your tools' priority levels.") try: os.unlink(self._get_file_path()) @@ -540,8 +560,8 @@ class FileSession(Session): checker = locking.LockChecker(self.id, self.lock_timeout) while not checker.expired(): try: - self.lock = lockfile.LockFile(path) - except lockfile.LockError: + self.lock = zc.lockfile.LockFile(path) + except zc.lockfile.LockError: time.sleep(0.1) else: break @@ -551,8 +571,9 @@ class FileSession(Session): def release_lock(self, path=None): """Release the lock on the currently-loaded session data.""" - self.lock.release() - self.lock.remove() + self.lock.close() + with contextlib2.suppress(FileNotFoundError): + os.remove(self.lock._path) self.locked = False def clean_up(self): @@ -560,8 +581,11 @@ class FileSession(Session): now = self.now() # Iterate over all session files in self.storage_path for fname in os.listdir(self.storage_path): - if (fname.startswith(self.SESSION_PREFIX) - and not fname.endswith(self.LOCK_SUFFIX)): + have_session = ( + fname.startswith(self.SESSION_PREFIX) + and not fname.endswith(self.LOCK_SUFFIX) + ) + if have_session: # We have a session file: lock and load it and check # if it's expired. If it fails, nevermind. path = os.path.join(self.storage_path, fname) @@ -587,95 +611,8 @@ class FileSession(Session): def __len__(self): """Return the number of active sessions.""" return len([fname for fname in os.listdir(self.storage_path) - if (fname.startswith(self.SESSION_PREFIX) - and not fname.endswith(self.LOCK_SUFFIX))]) - - -class PostgresqlSession(Session): - - """ Implementation of the PostgreSQL backend for sessions. It assumes - a table like this:: - - create table session ( - id varchar(40), - data text, - expiration_time timestamp - ) - - You must provide your own get_db function. - """ - - pickle_protocol = pickle.HIGHEST_PROTOCOL - - def __init__(self, id=None, **kwargs): - Session.__init__(self, id, **kwargs) - self.cursor = self.db.cursor() - - def setup(cls, **kwargs): - """Set up the storage system for Postgres-based sessions. - - This should only be called once per process; this will be done - automatically when using sessions.init (as the built-in Tool does). - """ - for k, v in kwargs.items(): - setattr(cls, k, v) - - self.db = self.get_db() - setup = classmethod(setup) - - def __del__(self): - if self.cursor: - self.cursor.close() - self.db.commit() - - def _exists(self): - # Select session data from table - self.cursor.execute('select data, expiration_time from session ' - 'where id=%s', (self.id,)) - rows = self.cursor.fetchall() - return bool(rows) - - def _load(self): - # Select session data from table - self.cursor.execute('select data, expiration_time from session ' - 'where id=%s', (self.id,)) - rows = self.cursor.fetchall() - if not rows: - return None - - pickled_data, expiration_time = rows[0] - data = pickle.loads(pickled_data) - return data, expiration_time - - def _save(self, expiration_time): - pickled_data = pickle.dumps(self._data, self.pickle_protocol) - self.cursor.execute('update session set data = %s, ' - 'expiration_time = %s where id = %s', - (pickled_data, expiration_time, self.id)) - - def _delete(self): - self.cursor.execute('delete from session where id=%s', (self.id,)) - - def acquire_lock(self): - """Acquire an exclusive lock on the currently-loaded session data.""" - # We use the "for update" clause to lock the row - self.locked = True - self.cursor.execute('select id from session where id=%s for update', - (self.id,)) - if self.debug: - cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') - - def release_lock(self): - """Release the lock on the currently-loaded session data.""" - # We just close the cursor and that will remove the lock - # introduced by the "for update" clause - self.cursor.close() - self.locked = False - - def clean_up(self): - """Clean up expired sessions.""" - self.cursor.execute('delete from session where expiration_time < %s', - (self.now(),)) + if (fname.startswith(self.SESSION_PREFIX) and + not fname.endswith(self.LOCK_SUFFIX))]) class MemcachedSession(Session): @@ -684,11 +621,12 @@ class MemcachedSession(Session): # Wrap all .get and .set operations in a single lock. mc_lock = threading.RLock() - # This is a seperate set of locks per session id. + # This is a separate set of locks per session id. locks = {} servers = ['127.0.0.1:11211'] + @classmethod def setup(cls, **kwargs): """Set up the storage system for memcached-based sessions. @@ -700,21 +638,6 @@ class MemcachedSession(Session): import memcache cls.cache = memcache.Client(cls.servers) - setup = classmethod(setup) - - def _get_id(self): - return self._id - - def _set_id(self, value): - # This encode() call is where we differ from the superclass. - # Memcache keys MUST be byte strings, not unicode. - if isinstance(value, unicodestr): - value = value.encode('utf-8') - - self._id = value - for o in self.id_observers: - o(value) - id = property(_get_id, _set_id, doc="The current session ID.") def _exists(self): self.mc_lock.acquire() @@ -737,7 +660,7 @@ class MemcachedSession(Session): try: if not self.cache.set(self.id, (self._data, expiration_time), td): raise AssertionError( - "Session data for id %r not set." % self.id) + 'Session data for id %r not set.' % self.id) finally: self.mc_lock.release() @@ -766,13 +689,13 @@ class MemcachedSession(Session): def save(): """Save any changed session data.""" - if not hasattr(cherrypy.serving, "session"): + if not hasattr(cherrypy.serving, 'session'): return request = cherrypy.serving.request response = cherrypy.serving.response # Guard against running twice - if hasattr(request, "_sessionsaved"): + if hasattr(request, '_sessionsaved'): return request._sessionsaved = True @@ -786,28 +709,39 @@ def save(): if is_iterator(response.body): response.collapse_body() cherrypy.session.save() + + save.failsafe = True def close(): """Close the session object for this request.""" - sess = getattr(cherrypy.serving, "session", None) - if getattr(sess, "locked", False): + sess = getattr(cherrypy.serving, 'session', None) + if getattr(sess, 'locked', False): # If the session is still locked we release the lock sess.release_lock() if sess.debug: cherrypy.log('Lock released on close.', 'TOOLS.SESSIONS') + + close.failsafe = True close.priority = 90 -def init(storage_type='ram', path=None, path_header=None, name='session_id', +def init(storage_type=None, path=None, path_header=None, name='session_id', timeout=60, domain=None, secure=False, clean_freq=5, - persistent=True, httponly=False, debug=False, **kwargs): + persistent=True, httponly=False, debug=False, + # Py27 compat + # *, storage_class=RamSession, + **kwargs): """Initialize session object (using cookies). + storage_class + The Session subclass to use. Defaults to RamSession. + storage_type - One of 'ram', 'file', 'postgresql', 'memcached'. This will be + (deprecated) + One of 'ram', 'file', memcached'. This will be used to look up the corresponding class in cherrypy.lib.sessions globals. For example, 'file' will use the FileSession class. @@ -851,10 +785,13 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', you're using for more information. """ + # Py27 compat + storage_class = kwargs.pop('storage_class', RamSession) + request = cherrypy.serving.request # Guard against running twice - if hasattr(request, "_session_init_flag"): + if hasattr(request, '_session_init_flag'): return request._session_init_flag = True @@ -866,11 +803,18 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', cherrypy.log('ID obtained from request.cookie: %r' % id, 'TOOLS.SESSIONS') - # Find the storage class and call setup (first time only). - storage_class = storage_type.title() + 'Session' - storage_class = globals()[storage_class] - if not hasattr(cherrypy, "session"): - if hasattr(storage_class, "setup"): + first_time = not hasattr(cherrypy, 'session') + + if storage_type: + if first_time: + msg = 'storage_type is deprecated. Supply storage_class instead' + cherrypy.log(msg) + storage_class = storage_type.title() + 'Session' + storage_class = globals()[storage_class] + + # call setup first time only + if first_time: + if hasattr(storage_class, 'setup'): storage_class.setup(**kwargs) # Create and attach a new Session instance to cherrypy.serving. @@ -887,7 +831,7 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', sess.id_observers.append(update_cookie) # Create cherrypy.session which will proxy to cherrypy.serving.session - if not hasattr(cherrypy, "session"): + if not hasattr(cherrypy, 'session'): cherrypy.session = cherrypy._ThreadLocalProxy('session') if persistent: @@ -941,24 +885,30 @@ def set_response_cookie(path=None, path_header=None, name='session_id', '/' ) - # We'd like to use the "max-age" param as indicated in - # http://www.faqs.org/rfcs/rfc2109.html but IE doesn't - # save it to disk and the session is lost if people close - # the browser. So we have to use the old "expires" ... sigh ... -## cookie[name]['max-age'] = timeout * 60 if timeout: - e = time.time() + (timeout * 60) - cookie[name]['expires'] = httputil.HTTPDate(e) + cookie[name]['max-age'] = timeout * 60 + _add_MSIE_max_age_workaround(cookie[name], timeout) if domain is not None: cookie[name]['domain'] = domain if secure: cookie[name]['secure'] = 1 if httponly: if not cookie[name].isReservedKey('httponly'): - raise ValueError("The httponly cookie token is not supported.") + raise ValueError('The httponly cookie token is not supported.') cookie[name]['httponly'] = 1 +def _add_MSIE_max_age_workaround(cookie, timeout): + """ + We'd like to use the "max-age" param as indicated in + http://www.faqs.org/rfcs/rfc2109.html but IE doesn't + save it to disk and the session is lost if people close + the browser. So we have to use the old "expires" ... sigh ... + """ + expires = time.time() + timeout * 60 + cookie['expires'] = httputil.HTTPDate(expires) + + def expire(): """Expire the current session cookie.""" name = cherrypy.serving.request.config.get( @@ -966,3 +916,4 @@ def expire(): one_year = 60 * 60 * 24 * 365 e = time.time() - one_year cherrypy.serving.response.cookie[name]['expires'] = httputil.HTTPDate(e) + cherrypy.serving.response.cookie[name].pop('max-age', None) diff --git a/lib/cherrypy/lib/static.py b/lib/cherrypy/lib/static.py index 6a78fc13..da9d9373 100644 --- a/lib/cherrypy/lib/static.py +++ b/lib/cherrypy/lib/static.py @@ -1,23 +1,32 @@ +"""Module with helpers for serving static files.""" + import os +import platform import re import stat import mimetypes -try: - from io import UnsupportedOperation -except ImportError: - UnsupportedOperation = object() +from email.generator import _make_boundary as make_boundary +from io import UnsupportedOperation + +from six.moves import urllib import cherrypy -from cherrypy._cpcompat import ntob, unquote +from cherrypy._cpcompat import ntob from cherrypy.lib import cptools, httputil, file_generator_limited -mimetypes.init() -mimetypes.types_map['.dwg'] = 'image/x-dwg' -mimetypes.types_map['.ico'] = 'image/x-icon' -mimetypes.types_map['.bz2'] = 'application/x-bzip2' -mimetypes.types_map['.gz'] = 'application/x-gzip' +def _setup_mimetypes(): + """Pre-initialize global mimetype map.""" + if not mimetypes.inited: + mimetypes.init() + mimetypes.types_map['.dwg'] = 'image/x-dwg' + mimetypes.types_map['.ico'] = 'image/x-icon' + mimetypes.types_map['.bz2'] = 'application/x-bzip2' + mimetypes.types_map['.gz'] = 'application/x-gzip' + + +_setup_mimetypes() def serve_file(path, content_type=None, disposition=None, name=None, @@ -33,7 +42,6 @@ def serve_file(path, content_type=None, disposition=None, name=None, to the basename of path. If disposition is None, no Content-Disposition header will be written. """ - response = cherrypy.serving.response # If path is relative, users should fix it by making path absolute. @@ -71,7 +79,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, if content_type is None: # Set content-type based on filename extension - ext = "" + ext = '' i = path.rfind('.') if i != -1: ext = path[i:].lower() @@ -86,7 +94,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, if name is None: name = os.path.basename(path) cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd + response.headers['Content-Disposition'] = cd if debug: cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') @@ -115,7 +123,6 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, serve_fileobj(), expecting that the data would be served starting from that position. """ - response = cherrypy.serving.response try: @@ -144,7 +151,7 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, cd = disposition else: cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd + response.headers['Content-Disposition'] = cd if debug: cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') @@ -158,12 +165,12 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code request = cherrypy.serving.request if request.protocol >= (1, 1): - response.headers["Accept-Ranges"] = "bytes" + response.headers['Accept-Ranges'] = 'bytes' r = httputil.get_ranges(request.headers.get('Range'), content_length) if r == []: - response.headers['Content-Range'] = "bytes */%s" % content_length - message = ("Invalid Range (first-byte-pos greater than " - "Content-Length)") + response.headers['Content-Range'] = 'bytes */%s' % content_length + message = ('Invalid Range (first-byte-pos greater than ' + 'Content-Length)') if debug: cherrypy.log(message, 'TOOLS.STATIC') raise cherrypy.HTTPError(416, message) @@ -179,31 +186,25 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): cherrypy.log( 'Single part; start: %r, stop: %r' % (start, stop), 'TOOLS.STATIC') - response.status = "206 Partial Content" + response.status = '206 Partial Content' response.headers['Content-Range'] = ( - "bytes %s-%s/%s" % (start, stop - 1, content_length)) + 'bytes %s-%s/%s' % (start, stop - 1, content_length)) response.headers['Content-Length'] = r_len fileobj.seek(start) response.body = file_generator_limited(fileobj, r_len) else: # Return a multipart/byteranges response. - response.status = "206 Partial Content" - try: - # Python 3 - from email.generator import _make_boundary as make_boundary - except ImportError: - # Python 2 - from mimetools import choose_boundary as make_boundary + response.status = '206 Partial Content' boundary = make_boundary() - ct = "multipart/byteranges; boundary=%s" % boundary + ct = 'multipart/byteranges; boundary=%s' % boundary response.headers['Content-Type'] = ct - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] def file_ranges(): # Apache compatibility: - yield ntob("\r\n") + yield b'\r\n' for start, stop in r: if debug: @@ -211,23 +212,23 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): 'Multipart; start: %r, stop: %r' % ( start, stop), 'TOOLS.STATIC') - yield ntob("--" + boundary, 'ascii') - yield ntob("\r\nContent-type: %s" % content_type, + yield ntob('--' + boundary, 'ascii') + yield ntob('\r\nContent-type: %s' % content_type, 'ascii') yield ntob( - "\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % ( + '\r\nContent-range: bytes %s-%s/%s\r\n\r\n' % ( start, stop - 1, content_length), 'ascii') fileobj.seek(start) gen = file_generator_limited(fileobj, stop - start) for chunk in gen: yield chunk - yield ntob("\r\n") + yield b'\r\n' # Final boundary - yield ntob("--" + boundary + "--", 'ascii') + yield ntob('--' + boundary + '--', 'ascii') # Apache compatibility: - yield ntob("\r\n") + yield b'\r\n' response.body = file_ranges() return response.body else: @@ -244,7 +245,7 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): def serve_download(path, name=None): """Serve 'path' as an application/x-download attachment.""" # This is such a common idiom I felt it deserved its own wrapper. - return serve_file(path, "application/x-download", "attachment", name) + return serve_file(path, 'application/x-download', 'attachment', name) def _attempt(filename, content_types, debug=False): @@ -268,7 +269,7 @@ def _attempt(filename, content_types, debug=False): return False -def staticdir(section, dir, root="", match="", content_types=None, index="", +def staticdir(section, dir, root='', match='', content_types=None, index='', debug=False): """Serve a static resource from the given (root +) dir. @@ -306,7 +307,7 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", # If dir is relative, make absolute using "root". if not os.path.isabs(dir): if not root: - msg = "Static dir requires an absolute dir (or root)." + msg = 'Static dir requires an absolute dir (or root).' if debug: cherrypy.log(msg, 'TOOLS.STATICDIR') raise ValueError(msg) @@ -315,10 +316,18 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", # Determine where we are in the object tree relative to 'section' # (where the static tool was defined). if section == 'global': - section = "/" - section = section.rstrip(r"\/") + section = '/' + section = section.rstrip(r'\/') branch = request.path_info[len(section) + 1:] - branch = unquote(branch.lstrip(r"\/")) + branch = urllib.parse.unquote(branch.lstrip(r'\/')) + + # Requesting a file in sub-dir of the staticdir results + # in mixing of delimiter styles, e.g. C:\static\js/script.js. + # Windows accepts this form except not when the path is + # supplied in extended-path notation, e.g. \\?\C:\static\js/script.js. + # http://bit.ly/1vdioCX + if platform.system() == 'Windows': + branch = branch.replace('/', '\\') # If branch is "", filename will end in a slash filename = os.path.join(dir, branch) @@ -338,11 +347,11 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", if index: handled = _attempt(os.path.join(filename, index), content_types) if handled: - request.is_index = filename[-1] in (r"\/") + request.is_index = filename[-1] in (r'\/') return handled -def staticfile(filename, root=None, match="", content_types=None, debug=False): +def staticfile(filename, root=None, match='', content_types=None, debug=False): """Serve a static resource from the given (root +) filename. match diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index 9fc9564f..ddaac86a 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -1,21 +1,19 @@ +"""XML-RPC tool helpers.""" import sys +from six.moves.xmlrpc_client import ( + loads as xmlrpc_loads, dumps as xmlrpc_dumps, + Fault as XMLRPCFault +) + import cherrypy from cherrypy._cpcompat import ntob -def get_xmlrpclib(): - try: - import xmlrpc.client as x - except ImportError: - import xmlrpclib as x - return x - - def process_body(): """Return (params, method) from request body.""" try: - return get_xmlrpclib().loads(cherrypy.request.body.read()) + return xmlrpc_loads(cherrypy.request.body.read()) except Exception: return ('ERROR PARAMS', ), 'ERRORMETHOD' @@ -31,9 +29,10 @@ def patched_path(path): def _set_response(body): + """Set up HTTP status, headers and body within CherryPy.""" # The XML-RPC spec (http://www.xmlrpc.com/spec) says: # "Unless there's a lower-level error, always return 200 OK." - # Since Python's xmlrpclib interprets a non-200 response + # Since Python's xmlrpc_client interprets a non-200 response # as a "Protocol Error", we'll just return 200 every time. response = cherrypy.response response.status = '200 OK' @@ -43,15 +42,20 @@ def _set_response(body): def respond(body, encoding='utf-8', allow_none=0): - xmlrpclib = get_xmlrpclib() - if not isinstance(body, xmlrpclib.Fault): + """Construct HTTP response body.""" + if not isinstance(body, XMLRPCFault): body = (body,) - _set_response(xmlrpclib.dumps(body, methodresponse=1, - encoding=encoding, - allow_none=allow_none)) + + _set_response( + xmlrpc_dumps( + body, methodresponse=1, + encoding=encoding, + allow_none=allow_none + ) + ) def on_error(*args, **kwargs): + """Construct HTTP response body for an error response.""" body = str(sys.exc_info()[1]) - xmlrpclib = get_xmlrpclib() - _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) + _set_response(xmlrpc_dumps(XMLRPCFault(1, body))) diff --git a/lib/cherrypy/process/__init__.py b/lib/cherrypy/process/__init__.py index f15b1237..f242d226 100644 --- a/lib/cherrypy/process/__init__.py +++ b/lib/cherrypy/process/__init__.py @@ -10,5 +10,8 @@ use with the bus. Some use tool-specific channels; see the documentation for each class. """ -from cherrypy.process.wspbus import bus -from cherrypy.process import plugins, servers +from .wspbus import bus +from . import plugins, servers + + +__all__ = ('bus', 'plugins', 'servers') diff --git a/lib/cherrypy/process/plugins.py b/lib/cherrypy/process/plugins.py index 0ec585c0..8c246c81 100644 --- a/lib/cherrypy/process/plugins.py +++ b/lib/cherrypy/process/plugins.py @@ -7,8 +7,10 @@ import sys import time import threading -from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident -from cherrypy._cpcompat import ntob, Timer, SetDaemonProperty +from six.moves import _thread + +from cherrypy._cpcompat import text_or_bytes +from cherrypy._cpcompat import ntob, Timer # _module__file__base is used by Autoreload to make # absolute any filenames retrieved from sys.modules which are not @@ -104,15 +106,14 @@ class SignalHandler(object): if sys.platform[:4] == 'java': del self.handlers['SIGUSR1'] self.handlers['SIGUSR2'] = self.bus.graceful - self.bus.log("SIGUSR1 cannot be set on the JVM platform. " - "Using SIGUSR2 instead.") + self.bus.log('SIGUSR1 cannot be set on the JVM platform. ' + 'Using SIGUSR2 instead.') self.handlers['SIGINT'] = self._jython_SIGINT_handler self._previous_handlers = {} # used to determine is the process is a daemon in `self._is_daemonized` self._original_pid = os.getpid() - def _jython_SIGINT_handler(self, signum=None, frame=None): # See http://bugs.jython.org/issue1313 self.bus.log('Keyboard Interrupt: shutting down bus') @@ -131,12 +132,10 @@ class SignalHandler(object): is executing inside other process like in a CI tool (Buildbot, Jenkins). """ - if (self._original_pid != os.getpid() and - not os.isatty(sys.stdin.fileno())): - return True - else: - return False - + return ( + self._original_pid != os.getpid() and + not os.isatty(sys.stdin.fileno()) + ) def subscribe(self): """Subscribe self.handlers to signals.""" @@ -152,19 +151,19 @@ class SignalHandler(object): signame = self.signals[signum] if handler is None: - self.bus.log("Restoring %s handler to SIG_DFL." % signame) + self.bus.log('Restoring %s handler to SIG_DFL.' % signame) handler = _signal.SIG_DFL else: - self.bus.log("Restoring %s handler %r." % (signame, handler)) + self.bus.log('Restoring %s handler %r.' % (signame, handler)) try: our_handler = _signal.signal(signum, handler) if our_handler is None: - self.bus.log("Restored old %s handler %r, but our " - "handler was not registered." % + self.bus.log('Restored old %s handler %r, but our ' + 'handler was not registered.' % (signame, handler), level=30) except ValueError: - self.bus.log("Unable to restore %s handler %r." % + self.bus.log('Unable to restore %s handler %r.' % (signame, handler), level=40, traceback=True) def set_handler(self, signal, listener=None): @@ -176,39 +175,39 @@ class SignalHandler(object): If the given signal name or number is not available on the current platform, ValueError is raised. """ - if isinstance(signal, basestring): + if isinstance(signal, text_or_bytes): signum = getattr(_signal, signal, None) if signum is None: - raise ValueError("No such signal: %r" % signal) + raise ValueError('No such signal: %r' % signal) signame = signal else: try: signame = self.signals[signal] except KeyError: - raise ValueError("No such signal: %r" % signal) + raise ValueError('No such signal: %r' % signal) signum = signal prev = _signal.signal(signum, self._handle_signal) self._previous_handlers[signum] = prev if listener is not None: - self.bus.log("Listening for %s." % signame) + self.bus.log('Listening for %s.' % signame) self.bus.subscribe(signame, listener) def _handle_signal(self, signum=None, frame=None): """Python signal handler (self.set_handler subscribes it for you).""" signame = self.signals[signum] - self.bus.log("Caught signal %s." % signame) + self.bus.log('Caught signal %s.' % signame) self.bus.publish(signame) def handle_SIGHUP(self): """Restart if daemonized, else exit.""" if self._is_daemonized(): - self.bus.log("SIGHUP caught while daemonized. Restarting.") + self.bus.log('SIGHUP caught while daemonized. Restarting.') self.bus.restart() else: # not daemonized (may be foreground or background) - self.bus.log("SIGHUP caught but not daemonized. Exiting.") + self.bus.log('SIGHUP caught but not daemonized. Exiting.') self.bus.exit() @@ -223,7 +222,8 @@ class DropPrivileges(SimplePlugin): """Drop privileges. uid/gid arguments not available on Windows. - Special thanks to `Gavin Baker `_ + Special thanks to `Gavin Baker + `_ """ def __init__(self, bus, umask=None, uid=None, gid=None): @@ -233,57 +233,57 @@ class DropPrivileges(SimplePlugin): self.gid = gid self.umask = umask - def _get_uid(self): + @property + def uid(self): + """The uid under which to run. Availability: Unix.""" return self._uid - def _set_uid(self, val): + @uid.setter + def uid(self, val): if val is not None: if pwd is None: - self.bus.log("pwd module not available; ignoring uid.", + self.bus.log('pwd module not available; ignoring uid.', level=30) val = None - elif isinstance(val, basestring): + elif isinstance(val, text_or_bytes): val = pwd.getpwnam(val)[2] self._uid = val - uid = property(_get_uid, _set_uid, - doc="The uid under which to run. Availability: Unix.") - def _get_gid(self): + @property + def gid(self): + """The gid under which to run. Availability: Unix.""" return self._gid - def _set_gid(self, val): + @gid.setter + def gid(self, val): if val is not None: if grp is None: - self.bus.log("grp module not available; ignoring gid.", + self.bus.log('grp module not available; ignoring gid.', level=30) val = None - elif isinstance(val, basestring): + elif isinstance(val, text_or_bytes): val = grp.getgrnam(val)[2] self._gid = val - gid = property(_get_gid, _set_gid, - doc="The gid under which to run. Availability: Unix.") - def _get_umask(self): + @property + def umask(self): + """The default permission mode for newly created files and directories. + + Usually expressed in octal format, for example, ``0644``. + Availability: Unix, Windows. + """ return self._umask - def _set_umask(self, val): + @umask.setter + def umask(self, val): if val is not None: try: os.umask except AttributeError: - self.bus.log("umask function not available; ignoring umask.", + self.bus.log('umask function not available; ignoring umask.', level=30) val = None self._umask = val - umask = property( - _get_umask, - _set_umask, - doc="""The default permission mode for newly created files and - directories. - - Usually expressed in octal format, for example, ``0644``. - Availability: Unix, Windows. - """) def start(self): # uid/gid @@ -347,7 +347,7 @@ class Daemonizer(SimplePlugin): process still return proper exit codes. Therefore, if you use this plugin to daemonize, don't use the return code as an accurate indicator of whether the process fully started. In fact, that return code only - indicates if the process succesfully finished the first fork. + indicates if the process successfully finished the first fork. """ def __init__(self, bus, stdin='/dev/null', stdout='/dev/null', @@ -372,6 +372,15 @@ class Daemonizer(SimplePlugin): 'Daemonizing now may cause strange failures.' % threading.enumerate(), level=30) + self.daemonize(self.stdin, self.stdout, self.stderr, self.bus.log) + + self.finalized = True + start.priority = 65 + + @staticmethod + def daemonize( + stdin='/dev/null', stdout='/dev/null', stderr='/dev/null', + logger=lambda msg: None): # See http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 # (or http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7) # and http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012 @@ -380,41 +389,29 @@ class Daemonizer(SimplePlugin): sys.stdout.flush() sys.stderr.flush() - # Do first fork. - try: - pid = os.fork() - if pid == 0: - # This is the child process. Continue. - pass - else: - # This is the first parent. Exit, now that we've forked. - self.bus.log('Forking once.') - os._exit(0) - except OSError: - # Python raises OSError rather than returning negative numbers. - exc = sys.exc_info()[1] - sys.exit("%s: fork #1 failed: (%d) %s\n" - % (sys.argv[0], exc.errno, exc.strerror)) + error_tmpl = ( + '{sys.argv[0]}: fork #{n} failed: ({exc.errno}) {exc.strerror}\n' + ) - os.setsid() + for fork in range(2): + msg = ['Forking once.', 'Forking twice.'][fork] + try: + pid = os.fork() + if pid > 0: + # This is the parent; exit. + logger(msg) + os._exit(0) + except OSError as exc: + # Python raises OSError rather than returning negative numbers. + sys.exit(error_tmpl.format(sys=sys, exc=exc, n=fork + 1)) + if fork == 0: + os.setsid() - # Do second fork - try: - pid = os.fork() - if pid > 0: - self.bus.log('Forking twice.') - os._exit(0) # Exit second parent - except OSError: - exc = sys.exc_info()[1] - sys.exit("%s: fork #2 failed: (%d) %s\n" - % (sys.argv[0], exc.errno, exc.strerror)) - - os.chdir("/") os.umask(0) - si = open(self.stdin, "r") - so = open(self.stdout, "a+") - se = open(self.stderr, "a+") + si = open(stdin, 'r') + so = open(stdout, 'a+') + se = open(stderr, 'a+') # os.dup2(fd, fd2) will close fd2 if necessary, # so we don't explicitly close stdin/out/err. @@ -423,9 +420,7 @@ class Daemonizer(SimplePlugin): os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) - self.bus.log('Daemonized to PID: %s' % os.getpid()) - self.finalized = True - start.priority = 65 + logger('Daemonized to PID: %s' % os.getpid()) class PIDFile(SimplePlugin): @@ -442,7 +437,7 @@ class PIDFile(SimplePlugin): if self.finalized: self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) else: - open(self.pidfile, "wb").write(ntob("%s\n" % pid, 'utf8')) + open(self.pidfile, 'wb').write(ntob('%s\n' % pid, 'utf8')) self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) self.finalized = True start.priority = 70 @@ -453,7 +448,7 @@ class PIDFile(SimplePlugin): self.bus.log('PID file removed: %r.' % self.pidfile) except (KeyboardInterrupt, SystemExit): raise - except: + except Exception: pass @@ -481,13 +476,13 @@ class PerpetualTimer(Timer): except Exception: if self.bus: self.bus.log( - "Error in perpetual timer thread function %r." % + 'Error in perpetual timer thread function %r.' % self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise -class BackgroundTask(SetDaemonProperty, threading.Thread): +class BackgroundTask(threading.Thread): """A subclass of threading.Thread whose run() method repeats. @@ -499,7 +494,7 @@ class BackgroundTask(SetDaemonProperty, threading.Thread): """ def __init__(self, interval, function, args=[], kwargs={}, bus=None): - threading.Thread.__init__(self) + super(BackgroundTask, self).__init__() self.interval = interval self.function = function self.args = args @@ -523,7 +518,7 @@ class BackgroundTask(SetDaemonProperty, threading.Thread): self.function(*self.args, **self.kwargs) except Exception: if self.bus: - self.bus.log("Error in background task thread function %r." + self.bus.log('Error in background task thread function %r.' % self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise @@ -560,24 +555,24 @@ class Monitor(SimplePlugin): bus=self.bus) self.thread.setName(threadname) self.thread.start() - self.bus.log("Started monitor thread %r." % threadname) + self.bus.log('Started monitor thread %r.' % threadname) else: - self.bus.log("Monitor thread %r already started." % threadname) + self.bus.log('Monitor thread %r already started.' % threadname) start.priority = 70 def stop(self): """Stop our callback's background task thread.""" if self.thread is None: - self.bus.log("No thread running for %s." % + self.bus.log('No thread running for %s.' % self.name or self.__class__.__name__) else: if self.thread is not threading.currentThread(): name = self.thread.getName() self.thread.cancel() - if not get_daemon(self.thread): - self.bus.log("Joining %r" % name) + if not self.thread.daemon: + self.bus.log('Joining %r' % name) self.thread.join() - self.bus.log("Stopped thread %r." % name) + self.bus.log('Stopped thread %r.' % name) self.thread = None def graceful(self): @@ -632,23 +627,40 @@ class Autoreloader(Monitor): def sysfiles(self): """Return a Set of sys.modules filenames to monitor.""" - files = set() - for k, m in list(sys.modules.items()): - if re.match(self.match, k): - if ( - hasattr(m, '__loader__') and - hasattr(m.__loader__, 'archive') - ): - f = m.__loader__.archive - else: - f = getattr(m, '__file__', None) - if f is not None and not os.path.isabs(f): - # ensure absolute paths so a os.chdir() in the app - # doesn't break me - f = os.path.normpath( - os.path.join(_module__file__base, f)) - files.add(f) - return files + search_mod_names = filter(re.compile(self.match).match, sys.modules) + mods = map(sys.modules.get, search_mod_names) + return set(filter(None, map(self._file_for_module, mods))) + + @classmethod + def _file_for_module(cls, module): + """Return the relevant file for the module.""" + return ( + cls._archive_for_zip_module(module) + or cls._file_for_file_module(module) + ) + + @staticmethod + def _archive_for_zip_module(module): + """Return the archive filename for the module if relevant.""" + try: + return module.__loader__.archive + except AttributeError: + pass + + @classmethod + def _file_for_file_module(cls, module): + """Return the file for the module.""" + try: + return module.__file__ and cls._make_absolute(module.__file__) + except AttributeError: + pass + + @staticmethod + def _make_absolute(filename): + """Ensure filename is absolute to avoid effect of os.chdir.""" + return filename if os.path.isabs(filename) else ( + os.path.normpath(os.path.join(_module__file__base, filename)) + ) def run(self): """Reload the process if registered files have been modified.""" @@ -674,10 +686,10 @@ class Autoreloader(Monitor): else: if mtime is None or mtime > oldtime: # The file has been deleted or modified. - self.bus.log("Restarting because %s changed." % + self.bus.log('Restarting because %s changed.' % filename) self.thread.cancel() - self.bus.log("Stopped thread %r." % + self.bus.log('Stopped thread %r.' % self.thread.getName()) self.bus.restart() return @@ -717,7 +729,7 @@ class ThreadManager(SimplePlugin): If the current thread has already been seen, any 'start_thread' listeners will not be run again. """ - thread_ident = get_thread_ident() + thread_ident = _thread.get_ident() if thread_ident not in self.threads: # We can't just use get_ident as the thread ID # because some platforms reuse thread ID's. @@ -727,7 +739,7 @@ class ThreadManager(SimplePlugin): def release_thread(self): """Release the current thread and run 'stop_thread' listeners.""" - thread_ident = get_thread_ident() + thread_ident = _thread.get_ident() i = self.threads.pop(thread_ident, None) if i is not None: self.bus.publish('stop_thread', i) diff --git a/lib/cherrypy/process/servers.py b/lib/cherrypy/process/servers.py index 91ebf604..dcb34de6 100644 --- a/lib/cherrypy/process/servers.py +++ b/lib/cherrypy/process/servers.py @@ -1,4 +1,4 @@ -""" +r""" Starting in CherryPy 3.1, cherrypy.server is implemented as an :ref:`Engine Plugin`. It's an instance of :class:`cherrypy._cpserver.Server`, which is a subclass of @@ -12,10 +12,14 @@ If you need to start more than one HTTP server (to serve on multiple ports, or protocols, etc.), you can manually register each one and then start them all with engine.start:: - s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80)) - s2 = ServerAdapter(cherrypy.engine, - another.HTTPServer(host='127.0.0.1', - SSL=True)) + s1 = ServerAdapter( + cherrypy.engine, + MyWSGIServer(host='0.0.0.0', port=80) + ) + s2 = ServerAdapter( + cherrypy.engine, + another.HTTPServer(host='127.0.0.1', SSL=True) + ) s1.subscribe() s2.subscribe() cherrypy.engine.start() @@ -58,10 +62,10 @@ hello.py:: import cherrypy class HelloWorld: - \"""Sample request handler class.\""" + '''Sample request handler class.''' + @cherrypy.expose def index(self): return "Hello world!" - index.exposed = True cherrypy.tree.mount(HelloWorld()) # CherryPy autoreload must be disabled for the flup server to work @@ -113,9 +117,18 @@ Please see `Lighttpd FastCGI Docs an explanation of the possible configuration options. """ +import os import sys import time import warnings +import contextlib + +import portend + + +class Timeouts: + occupied = 5 + free = 1 class ServerAdapter(object): @@ -150,49 +163,56 @@ class ServerAdapter(object): def start(self): """Start the HTTP server.""" - if self.bind_addr is None: - on_what = "unknown interface (dynamic?)" - elif isinstance(self.bind_addr, tuple): - on_what = self._get_base() - else: - on_what = "socket file: %s" % self.bind_addr - if self.running: - self.bus.log("Already serving on %s" % on_what) + self.bus.log('Already serving on %s' % self.description) return self.interrupt = None if not self.httpserver: - raise ValueError("No HTTP server has been created.") + raise ValueError('No HTTP server has been created.') - # Start the httpserver in a new thread. - if isinstance(self.bind_addr, tuple): - wait_for_free_port(*self.bind_addr) + if not os.environ.get('LISTEN_PID', None): + # Start the httpserver in a new thread. + if isinstance(self.bind_addr, tuple): + portend.free(*self.bind_addr, timeout=Timeouts.free) import threading t = threading.Thread(target=self._start_http_thread) - t.setName("HTTPServer " + t.getName()) + t.setName('HTTPServer ' + t.getName()) t.start() self.wait() self.running = True - self.bus.log("Serving on %s" % on_what) + self.bus.log('Serving on %s' % self.description) start.priority = 75 + @property + def description(self): + """ + A description about where this server is bound. + """ + if self.bind_addr is None: + on_what = 'unknown interface (dynamic?)' + elif isinstance(self.bind_addr, tuple): + on_what = self._get_base() + else: + on_what = 'socket file: %s' % self.bind_addr + return on_what + def _get_base(self): if not self.httpserver: return '' - host, port = self.bind_addr + host, port = self.bound_addr if getattr(self.httpserver, 'ssl_adapter', None): - scheme = "https" + scheme = 'https' if port != 443: - host += ":%s" % port + host += ':%s' % port else: - scheme = "http" + scheme = 'http' if port != 80: - host += ":%s" % port + host += ':%s' % port - return "%s://%s" % (scheme, host) + return '%s://%s' % (scheme, host) def _start_http_thread(self): """HTTP servers MUST be running in new threads, so that the @@ -204,32 +224,52 @@ class ServerAdapter(object): try: self.httpserver.start() except KeyboardInterrupt: - self.bus.log(" hit: shutting down HTTP server") + self.bus.log(' hit: shutting down HTTP server') self.interrupt = sys.exc_info()[1] self.bus.exit() except SystemExit: - self.bus.log("SystemExit raised: shutting down HTTP server") + self.bus.log('SystemExit raised: shutting down HTTP server') self.interrupt = sys.exc_info()[1] self.bus.exit() raise - except: + except Exception: self.interrupt = sys.exc_info()[1] - self.bus.log("Error in HTTP server: shutting down", + self.bus.log('Error in HTTP server: shutting down', traceback=True, level=40) self.bus.exit() raise def wait(self): """Wait until the HTTP server is ready to receive requests.""" - while not getattr(self.httpserver, "ready", False): + while not getattr(self.httpserver, 'ready', False): if self.interrupt: raise self.interrupt time.sleep(.1) - # Wait for port to be occupied - if isinstance(self.bind_addr, tuple): - host, port = self.bind_addr - wait_for_occupied_port(host, port) + # bypass check when LISTEN_PID is set + if os.environ.get('LISTEN_PID', None): + return + + # bypass check when running via socket-activation + # (for socket-activation the port will be managed by systemd) + if not isinstance(self.bind_addr, tuple): + return + + # wait for port to be occupied + with _safe_wait(*self.bound_addr): + portend.occupied(*self.bound_addr, timeout=Timeouts.occupied) + + @property + def bound_addr(self): + """ + The bind address, or if it's an ephemeral port and the + socket has been bound, return the actual port bound. + """ + host, port = self.bind_addr + if port == 0 and self.httpserver.socket: + # Bound to ephemeral port. Get the actual port allocated. + port = self.httpserver.socket.getsockname()[1] + return host, port def stop(self): """Stop the HTTP server.""" @@ -238,11 +278,11 @@ class ServerAdapter(object): self.httpserver.stop() # Wait for the socket to be truly freed. if isinstance(self.bind_addr, tuple): - wait_for_free_port(*self.bind_addr) + portend.free(*self.bound_addr, timeout=Timeouts.free) self.running = False - self.bus.log("HTTP Server %s shut down" % self.httpserver) + self.bus.log('HTTP Server %s shut down' % self.httpserver) else: - self.bus.log("HTTP Server %s already shut down" % self.httpserver) + self.bus.log('HTTP Server %s already shut down' % self.httpserver) stop.priority = 25 def restart(self): @@ -359,107 +399,18 @@ class FlupSCGIServer(object): self.scgiserver._threadPool.maxSpare = 0 -def client_host(server_host): - """Return the host on which a client can connect to the given listener.""" - if server_host == '0.0.0.0': - # 0.0.0.0 is INADDR_ANY, which should answer on localhost. - return '127.0.0.1' - if server_host in ('::', '::0', '::0.0.0.0'): - # :: is IN6ADDR_ANY, which should answer on localhost. - # ::0 and ::0.0.0.0 are non-canonical but common - # ways to write IN6ADDR_ANY. - return '::1' - return server_host - - -def check_port(host, port, timeout=1.0): - """Raise an error if the given port is not free on the given host.""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - host = client_host(host) - port = int(port) - - import socket - - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) +@contextlib.contextmanager +def _safe_wait(host, port): + """ + On systems where a loopback interface is not available and the + server is bound to all interfaces, it's difficult to determine + whether the server is in fact occupying the port. In this case, + just issue a warning and move on. See issue #1100. + """ try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM) - except socket.gaierror: - if ':' in host: - info = [( - socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0) - )] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] - - for res in info: - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(timeout) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - else: - raise IOError("Port %s is in use on %s; perhaps the previous " - "httpserver did not shut down properly." % - (repr(port), repr(host))) - - -# Feel free to increase these defaults on slow systems: -free_port_timeout = 0.1 -occupied_port_timeout = 1.0 - - -def wait_for_free_port(host, port, timeout=None): - """Wait for the specified port to become free (drop requests).""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - if timeout is None: - timeout = free_port_timeout - - for trial in range(50): - try: - # we are expecting a free port, so reduce the timeout - check_port(host, port, timeout=timeout) - except IOError: - # Give the old server thread time to free the port. - time.sleep(timeout) - else: - return - - raise IOError("Port %r not free on %r" % (port, host)) - - -def wait_for_occupied_port(host, port, timeout=None): - """Wait for the specified port to become active (receive requests).""" - if not host: - raise ValueError("Host values of '' or None are not allowed.") - if timeout is None: - timeout = occupied_port_timeout - - for trial in range(50): - try: - check_port(host, port, timeout=timeout) - except IOError: - # port is occupied - return - else: - time.sleep(timeout) - - if host == client_host(host): - raise IOError("Port %r not bound on %r" % (port, host)) - - # On systems where a loopback interface is not available and the - # server is bound to all interfaces, it's difficult to determine - # whether the server is in fact occupying the port. In this case, - # just issue a warning and move on. See issue #1100. - msg = "Unable to verify that the server is bound on %r" % port - warnings.warn(msg) + yield + except portend.Timeout: + if host == portend.client_host(host): + raise + msg = 'Unable to verify that the server is bound on %r' % port + warnings.warn(msg) diff --git a/lib/cherrypy/process/win32.py b/lib/cherrypy/process/win32.py index 4afd3f14..096b0278 100644 --- a/lib/cherrypy/process/win32.py +++ b/lib/cherrypy/process/win32.py @@ -85,19 +85,20 @@ class Win32Bus(wspbus.Bus): return self.events[state] except KeyError: event = win32event.CreateEvent(None, 0, 0, - "WSPBus %s Event (pid=%r)" % + 'WSPBus %s Event (pid=%r)' % (state.name, os.getpid())) self.events[state] = event return event - def _get_state(self): + @property + def state(self): return self._state - def _set_state(self, value): + @state.setter + def state(self, value): self._state = value event = self._get_state_event(value) win32event.PulseEvent(event) - state = property(_get_state, _set_state) def wait(self, state, interval=0.1, channel=None): """Wait for the given state(s), KeyboardInterrupt or SystemExit. @@ -135,7 +136,8 @@ class _ControlCodes(dict): for key, val in self.items(): if val is obj: return key - raise ValueError("The given object could not be found: %r" % obj) + raise ValueError('The given object could not be found: %r' % obj) + control_codes = _ControlCodes({'graceful': 138}) @@ -153,14 +155,14 @@ class PyWebService(win32serviceutil.ServiceFramework): """Python Web Service.""" - _svc_name_ = "Python Web Service" - _svc_display_name_ = "Python Web Service" + _svc_name_ = 'Python Web Service' + _svc_display_name_ = 'Python Web Service' _svc_deps_ = None # sequence of service names on which this depends - _exe_name_ = "pywebsvc" + _exe_name_ = 'pywebsvc' _exe_args_ = None # Default to no arguments # Only exists on Windows 2000 or later, ignored on windows NT - _svc_description_ = "Python Web Service" + _svc_description_ = 'Python Web Service' def SvcDoRun(self): from cherrypy import process @@ -173,6 +175,7 @@ class PyWebService(win32serviceutil.ServiceFramework): process.bus.exit() def SvcOther(self, control): + from cherrypy import process process.bus.publish(control_codes.key_for(control)) diff --git a/lib/cherrypy/process/wspbus.py b/lib/cherrypy/process/wspbus.py index c9de3511..d91dba48 100644 --- a/lib/cherrypy/process/wspbus.py +++ b/lib/cherrypy/process/wspbus.py @@ -1,4 +1,4 @@ -"""An implementation of the Web Site Process Bus. +r"""An implementation of the Web Site Process Bus. This module is completely standalone, depending only on the stdlib. @@ -61,12 +61,28 @@ the new state.:: """ import atexit + +try: + import ctypes +except ImportError: + """Google AppEngine is shipped without ctypes + + :seealso: http://stackoverflow.com/a/6523777/70170 + """ + ctypes = None + +import operator import os import sys import threading import time import traceback as _traceback import warnings +import subprocess +import functools + +import six + # Here I save the value of os.getcwd(), which, if I am imported early enough, # will be the directory from which the startup script was run. This is needed @@ -78,15 +94,13 @@ _startup_cwd = os.getcwd() class ChannelFailures(Exception): + """Exception raised during errors on Bus.publish().""" - """Exception raised when errors occur in a listener during Bus.publish(). - """ delimiter = '\n' def __init__(self, *args, **kwargs): - # Don't use 'super' here; Exceptions are old-style in Py2.4 - # See https://bitbucket.org/cherrypy/cherrypy/issue/959 - Exception.__init__(self, *args, **kwargs) + """Initialize ChannelFailures errors wrapper.""" + super(ChannelFailures, self).__init__(*args, **kwargs) self._exceptions = list() def handle_exception(self): @@ -98,12 +112,14 @@ class ChannelFailures(Exception): return self._exceptions[:] def __str__(self): + """Render the list of errors, which happened in channel.""" exception_strings = map(repr, self.get_instances()) return self.delimiter.join(exception_strings) __repr__ = __str__ def __bool__(self): + """Determine whether any error happened in channel.""" return bool(self._exceptions) __nonzero__ = __bool__ @@ -116,12 +132,14 @@ class _StateEnum(object): name = None def __repr__(self): - return "states.%s" % self.name + return 'states.%s' % self.name def __setattr__(self, key, value): if isinstance(value, self.State): value.name = key object.__setattr__(self, key, value) + + states = _StateEnum() states.STOPPED = states.State() states.STARTING = states.State() @@ -142,7 +160,6 @@ else: class Bus(object): - """Process state-machine and messenger for HTTP site deployment. All listeners for a given channel are guaranteed to be called even @@ -158,18 +175,31 @@ class Bus(object): max_cloexec_files = max_files def __init__(self): + """Initialize pub/sub bus.""" self.execv = False self.state = states.STOPPED + channels = 'start', 'stop', 'exit', 'graceful', 'log', 'main' self.listeners = dict( - [(channel, set()) for channel - in ('start', 'stop', 'exit', 'graceful', 'log', 'main')]) + (channel, set()) + for channel in channels + ) self._priorities = {} - def subscribe(self, channel, callback, priority=None): - """Add the given callback at the given channel (if not present).""" - if channel not in self.listeners: - self.listeners[channel] = set() - self.listeners[channel].add(callback) + def subscribe(self, channel, callback=None, priority=None): + """Add the given callback at the given channel (if not present). + + If callback is None, return a partial suitable for decorating + the callback. + """ + if callback is None: + return functools.partial( + self.subscribe, + channel, + priority=priority, + ) + + ch_listeners = self.listeners.setdefault(channel, set()) + ch_listeners.add(callback) if priority is None: priority = getattr(callback, 'priority', 50) @@ -190,14 +220,11 @@ class Bus(object): exc = ChannelFailures() output = [] - items = [(self._priorities[(channel, listener)], listener) - for listener in self.listeners[channel]] - try: - items.sort(key=lambda item: item[0]) - except TypeError: - # Python 2.3 had no 'key' arg, but that doesn't matter - # since it could sort dissimilar types just fine. - items.sort() + raw_items = ( + (self._priorities[(channel, listener)], listener) + for listener in self.listeners[channel] + ) + items = sorted(raw_items, key=operator.itemgetter(0)) for priority, listener in items: try: output.append(listener(*args, **kwargs)) @@ -209,26 +236,26 @@ class Bus(object): if exc and e.code == 0: e.code = 1 raise - except: + except Exception: exc.handle_exception() if channel == 'log': # Assume any further messages to 'log' will fail. pass else: - self.log("Error in %r listener %r" % (channel, listener), + self.log('Error in %r listener %r' % (channel, listener), level=40, traceback=True) if exc: raise exc return output def _clean_exit(self): - """An atexit handler which asserts the Bus is not running.""" + """Assert that the Bus is not running in atexit handler callback.""" if self.state != states.EXITING: warnings.warn( - "The main thread is exiting, but the Bus is in the %r state; " - "shutting it down automatically now. You must either call " - "bus.block() after start(), or call bus.exit() before the " - "main thread exits." % self.state, RuntimeWarning) + 'The main thread is exiting, but the Bus is in the %r state; ' + 'shutting it down automatically now. You must either call ' + 'bus.block() after start(), or call bus.exit() before the ' + 'main thread exits.' % self.state, RuntimeWarning) self.exit() def start(self): @@ -243,13 +270,13 @@ class Bus(object): self.log('Bus STARTED') except (KeyboardInterrupt, SystemExit): raise - except: - self.log("Shutting down due to error in start listener:", + except Exception: + self.log('Shutting down due to error in start listener:', level=40, traceback=True) e_info = sys.exc_info()[1] try: self.exit() - except: + except Exception: # Any stop/exit errors will be logged inside publish(). pass # Re-raise the original error @@ -258,6 +285,7 @@ class Bus(object): def exit(self): """Stop all services and prepare to exit the process.""" exitstate = self.state + EX_SOFTWARE = 70 try: self.stop() @@ -267,19 +295,19 @@ class Bus(object): # This isn't strictly necessary, but it's better than seeing # "Waiting for child threads to terminate..." and then nothing. self.log('Bus EXITED') - except: + except Exception: # This method is often called asynchronously (whether thread, # signal handler, console handler, or atexit handler), so we # can't just let exceptions propagate out unhandled. # Assume it's been logged and just die. - os._exit(70) # EX_SOFTWARE + os._exit(EX_SOFTWARE) if exitstate == states.STARTING: # exit() was called before start() finished, possibly due to # Ctrl-C because a start listener got stuck. In this case, # we could get stuck in a loop where Ctrl-C never exits the # process, so we just call os.exit here. - os._exit(70) # EX_SOFTWARE + os._exit(EX_SOFTWARE) def restart(self): """Restart the process (may close connections). @@ -317,11 +345,11 @@ class Bus(object): raise # Waiting for ALL child threads to finish is necessary on OS X. - # See https://bitbucket.org/cherrypy/cherrypy/issue/581. + # See https://github.com/cherrypy/cherrypy/issues/581. # It's also good to let them all shut down before allowing # the main thread to call atexit handlers. - # See https://bitbucket.org/cherrypy/cherrypy/issue/751. - self.log("Waiting for child threads to terminate...") + # See https://github.com/cherrypy/cherrypy/issues/751. + self.log('Waiting for child threads to terminate...') for t in threading.enumerate(): # Validate the we're not trying to join the MainThread # that will cause a deadlock and the case exist when @@ -329,18 +357,13 @@ class Bus(object): # that another thread executes cherrypy.engine.exit() if ( t != threading.currentThread() and - t.isAlive() and - not isinstance(t, threading._MainThread) + not isinstance(t, threading._MainThread) and + # Note that any dummy (external) threads are + # always daemonic. + not t.daemon ): - # Note that any dummy (external) threads are always daemonic. - if hasattr(threading.Thread, "daemon"): - # Python 2.6+ - d = t.daemon - else: - d = t.isDaemon() - if not d: - self.log("Waiting for thread %s." % t.getName()) - t.join() + self.log('Waiting for thread %s.' % t.getName()) + t.join() if self.execv: self._do_execv() @@ -352,23 +375,9 @@ class Bus(object): else: states = [state] - def _wait(): - while self.state not in states: - time.sleep(interval) - self.publish(channel) - - # From http://psyco.sourceforge.net/psycoguide/bugs.html: - # "The compiled machine code does not include the regular polling - # done by Python, meaning that a KeyboardInterrupt will not be - # detected before execution comes back to the regular Python - # interpreter. Your program cannot be interrupted if caught - # into an infinite Psyco-compiled loop." - try: - sys.modules['psyco'].cannotcompile(_wait) - except (KeyError, AttributeError): - pass - - _wait() + while self.state not in states: + time.sleep(interval) + self.publish(channel) def _do_execv(self): """Re-execute the current process. @@ -376,14 +385,20 @@ class Bus(object): This must be called from the main thread, because certain platforms (OS X) don't allow execv to be called in a child thread very well. """ - args = sys.argv[:] + try: + args = self._get_true_argv() + except NotImplementedError: + """It's probably win32 or GAE""" + args = [sys.executable] + self._get_interpreter_argv() + sys.argv + self.log('Re-spawning %s' % ' '.join(args)) + self._extend_pythonpath(os.environ) + if sys.platform[:4] == 'java': from _systemrestart import SystemRestart raise SystemRestart else: - args.insert(0, sys.executable) if sys.platform == 'win32': args = ['"%s"' % arg for arg in args] @@ -392,6 +407,134 @@ class Bus(object): self._set_cloexec() os.execv(sys.executable, args) + @staticmethod + def _get_interpreter_argv(): + """Retrieve current Python interpreter's arguments. + + Returns empty tuple in case of frozen mode, uses built-in arguments + reproduction function otherwise. + + Frozen mode is possible for the app has been packaged into a binary + executable using py2exe. In this case the interpreter's arguments are + already built-in into that executable. + + :seealso: https://github.com/cherrypy/cherrypy/issues/1526 + Ref: https://pythonhosted.org/PyInstaller/runtime-information.html + """ + return ([] + if getattr(sys, 'frozen', False) + else subprocess._args_from_interpreter_flags()) + + @staticmethod + def _get_true_argv(): + """Retrieve all real arguments of the python interpreter. + + ...even those not listed in ``sys.argv`` + + :seealso: http://stackoverflow.com/a/28338254/595220 + :seealso: http://stackoverflow.com/a/6683222/595220 + :seealso: http://stackoverflow.com/a/28414807/595220 + """ + try: + char_p = ctypes.c_char_p if six.PY2 else ctypes.c_wchar_p + + argv = ctypes.POINTER(char_p)() + argc = ctypes.c_int() + + ctypes.pythonapi.Py_GetArgcArgv( + ctypes.byref(argc), + ctypes.byref(argv), + ) + + _argv = argv[:argc.value] + + # The code below is trying to correctly handle special cases. + # `-c`'s argument interpreted by Python itself becomes `-c` as + # well. Same applies to `-m`. This snippet is trying to survive + # at least the case with `-m` + # Ref: https://github.com/cherrypy/cherrypy/issues/1545 + # Ref: python/cpython@418baf9 + argv_len, is_command, is_module = len(_argv), False, False + + try: + m_ind = _argv.index('-m') + if m_ind < argv_len - 1 and _argv[m_ind + 1] in ('-c', '-m'): + """ + In some older Python versions `-m`'s argument may be + substituted with `-c`, not `-m` + """ + is_module = True + except (IndexError, ValueError): + m_ind = None + + try: + c_ind = _argv.index('-c') + if c_ind < argv_len - 1 and _argv[c_ind + 1] == '-c': + is_command = True + except (IndexError, ValueError): + c_ind = None + + if is_module: + """It's containing `-m -m` sequence of arguments""" + if is_command and c_ind < m_ind: + """There's `-c -c` before `-m`""" + raise RuntimeError( + "Cannot reconstruct command from '-c'. Ref: " + 'https://github.com/cherrypy/cherrypy/issues/1545') + # Survive module argument here + original_module = sys.argv[0] + if not os.access(original_module, os.R_OK): + """There's no such module exist""" + raise AttributeError( + "{} doesn't seem to be a module " + 'accessible by current user'.format(original_module)) + del _argv[m_ind:m_ind + 2] # remove `-m -m` + # ... and substitute it with the original module path: + _argv.insert(m_ind, original_module) + elif is_command: + """It's containing just `-c -c` sequence of arguments""" + raise RuntimeError( + "Cannot reconstruct command from '-c'. " + 'Ref: https://github.com/cherrypy/cherrypy/issues/1545') + except AttributeError: + """It looks Py_GetArgcArgv is completely absent in some environments + + It is known, that there's no Py_GetArgcArgv in MS Windows and + ``ctypes`` module is completely absent in Google AppEngine + + :seealso: https://github.com/cherrypy/cherrypy/issues/1506 + :seealso: https://github.com/cherrypy/cherrypy/issues/1512 + :ref: http://bit.ly/2gK6bXK + """ + raise NotImplementedError + else: + return _argv + + @staticmethod + def _extend_pythonpath(env): + """Prepend current working dir to PATH environment variable if needed. + + If sys.path[0] is an empty string, the interpreter was likely + invoked with -m and the effective path is about to change on + re-exec. Add the current directory to $PYTHONPATH to ensure + that the new process sees the same path. + + This issue cannot be addressed in the general case because + Python cannot reliably reconstruct the + original command line (http://bugs.python.org/issue14208). + + (This idea filched from tornado.autoreload) + """ + path_prefix = '.' + os.pathsep + existing_path = env.get('PYTHONPATH', '') + needs_patch = ( + sys.path[0] == '' and + not existing_path.startswith(path_prefix) + ) + + if needs_patch: + env['PYTHONPATH'] = path_prefix + existing_path + def _set_cloexec(self): """Set the CLOEXEC flag on all open files (except stdin/out/err). @@ -437,10 +580,11 @@ class Bus(object): return t - def log(self, msg="", level=20, traceback=False): + def log(self, msg='', level=20, traceback=False): """Log the given message. Append the last traceback if requested.""" if traceback: - msg += "\n" + "".join(_traceback.format_exception(*sys.exc_info())) + msg += '\n' + ''.join(_traceback.format_exception(*sys.exc_info())) self.publish('log', msg, level) + bus = Bus() diff --git a/lib/cherrypy/scaffold/__init__.py b/lib/cherrypy/scaffold/__init__.py index 50de34bb..bcddba2d 100644 --- a/lib/cherrypy/scaffold/__init__.py +++ b/lib/cherrypy/scaffold/__init__.py @@ -8,7 +8,7 @@ then tweak as desired. Even before any tweaking, this should serve a few demonstration pages. Change to this directory and run: - ../cherryd -c site.conf + cherryd -c site.conf """ @@ -19,36 +19,38 @@ import os local_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) +@cherrypy.config(**{'tools.log_tracebacks.on': True}) class Root: + """Declaration of the CherryPy app URI structure.""" - _cp_config = {'tools.log_tracebacks.on': True, - } - + @cherrypy.expose def index(self): + """Render HTML-template at the root path of the web-app.""" return """ Try some other path, or a default path.
Or, just look at the pretty picture:
-""" % (url("other"), url("else"), - url("files/made_with_cherrypy_small.png")) - index.exposed = True +""" % (url('other'), url('else'), + url('files/made_with_cherrypy_small.png')) + @cherrypy.expose def default(self, *args, **kwargs): - return "args: %s kwargs: %s" % (args, kwargs) - default.exposed = True + """Render catch-all args and kwargs.""" + return 'args: %s kwargs: %s' % (args, kwargs) + @cherrypy.expose def other(self, a=2, b='bananas', c=None): + """Render number of fruits based on third argument.""" cherrypy.response.headers['Content-Type'] = 'text/plain' if c is None: - return "Have %d %s." % (int(a), b) + return 'Have %d %s.' % (int(a), b) else: - return "Have %d %s, %s." % (int(a), b, c) - other.exposed = True + return 'Have %d %s, %s.' % (int(a), b, c) - files = cherrypy.tools.staticdir.handler( - section="/files", - dir=os.path.join(local_dir, "static"), + files = tools.staticdir.handler( + section='/files', + dir=os.path.join(local_dir, 'static'), # Ignore .php files, etc. match=r'\.(css|gif|html?|ico|jpe?g|js|png|swf|xml)$', ) @@ -57,5 +59,5 @@ Or, just look at the pretty picture:
root = Root() # Uncomment the following to use your own favicon instead of CP's default. -#favicon_path = os.path.join(local_dir, "favicon.ico") -#root.favicon_ico = tools.staticfile.handler(filename=favicon_path) +# favicon_path = os.path.join(local_dir, "favicon.ico") +# root.favicon_ico = tools.staticfile.handler(filename=favicon_path) diff --git a/lib/cherrypy/scaffold/apache-fcgi.conf b/lib/cherrypy/scaffold/apache-fcgi.conf index 922398ea..6e4f144c 100644 --- a/lib/cherrypy/scaffold/apache-fcgi.conf +++ b/lib/cherrypy/scaffold/apache-fcgi.conf @@ -19,4 +19,4 @@ RewriteRule ^(.*)$ /fastcgi.pyc [L] # If filename does not begin with a slash (/) then it is assumed to be relative to the ServerRoot. # The filename does not have to exist in the local filesystem. URIs that Apache resolves to this # filename will be handled by this external FastCGI application. -FastCgiExternalServer "C:/fastcgi.pyc" -host 127.0.0.1:8088 \ No newline at end of file +FastCgiExternalServer "C:/fastcgi.pyc" -host 127.0.0.1:8088 diff --git a/lib/cherrypy/scaffold/example.conf b/lib/cherrypy/scaffold/example.conf index 93a6e53c..63250fe3 100644 --- a/lib/cherrypy/scaffold/example.conf +++ b/lib/cherrypy/scaffold/example.conf @@ -1,3 +1,3 @@ [/] log.error_file: "error.log" -log.access_file: "access.log" \ No newline at end of file +log.access_file: "access.log" diff --git a/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png b/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png index c3aafeed..724f9d72 100644 Binary files a/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png and b/lib/cherrypy/scaffold/static/made_with_cherrypy_small.png differ diff --git a/lib/cherrypy/test/__init__.py b/lib/cherrypy/test/__init__.py new file mode 100644 index 00000000..068382be --- /dev/null +++ b/lib/cherrypy/test/__init__.py @@ -0,0 +1,24 @@ +""" +Regression test suite for CherryPy. +""" + +import os +import sys + + +def newexit(): + os._exit(1) + + +def setup(): + # We want to monkey patch sys.exit so that we can get some + # information about where exit is being called. + newexit._old = sys.exit + sys.exit = newexit + + +def teardown(): + try: + sys.exit = sys.exit._old + except AttributeError: + sys.exit = sys._exit diff --git a/lib/cherrypy/test/_test_decorators.py b/lib/cherrypy/test/_test_decorators.py new file mode 100644 index 00000000..74832e40 --- /dev/null +++ b/lib/cherrypy/test/_test_decorators.py @@ -0,0 +1,39 @@ +"""Test module for the @-decorator syntax, which is version-specific""" + +import cherrypy +from cherrypy import expose, tools + + +class ExposeExamples(object): + + @expose + def no_call(self): + return 'Mr E. R. Bradshaw' + + @expose() + def call_empty(self): + return 'Mrs. B.J. Smegma' + + @expose('call_alias') + def nesbitt(self): + return 'Mr Nesbitt' + + @expose(['alias1', 'alias2']) + def andrews(self): + return 'Mr Ken Andrews' + + @expose(alias='alias3') + def watson(self): + return 'Mr. and Mrs. Watson' + + +class ToolExamples(object): + + @expose + # This is here to demonstrate that using the config decorator + # does not overwrite other config attributes added by the Tool + # decorator (in this case response_headers). + @cherrypy.config(**{'response.stream': True}) + @tools.response_headers(headers=[('Content-Type', 'application/data')]) + def blah(self): + yield b'blah' diff --git a/lib/cherrypy/test/_test_states_demo.py b/lib/cherrypy/test/_test_states_demo.py new file mode 100644 index 00000000..a49407ba --- /dev/null +++ b/lib/cherrypy/test/_test_states_demo.py @@ -0,0 +1,69 @@ +import os +import sys +import time + +import cherrypy + +starttime = time.time() + + +class Root: + + @cherrypy.expose + def index(self): + return 'Hello World' + + @cherrypy.expose + def mtimes(self): + return repr(cherrypy.engine.publish('Autoreloader', 'mtimes')) + + @cherrypy.expose + def pid(self): + return str(os.getpid()) + + @cherrypy.expose + def start(self): + return repr(starttime) + + @cherrypy.expose + def exit(self): + # This handler might be called before the engine is STARTED if an + # HTTP worker thread handles it before the HTTP server returns + # control to engine.start. We avoid that race condition here + # by waiting for the Bus to be STARTED. + cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) + cherrypy.engine.exit() + + +@cherrypy.engine.subscribe('start', priority=100) +def unsub_sig(): + cherrypy.log('unsubsig: %s' % cherrypy.config.get('unsubsig', False)) + if cherrypy.config.get('unsubsig', False): + cherrypy.log('Unsubscribing the default cherrypy signal handler') + cherrypy.engine.signal_handler.unsubscribe() + try: + from signal import signal, SIGTERM + except ImportError: + pass + else: + def old_term_handler(signum=None, frame=None): + cherrypy.log('I am an old SIGTERM handler.') + sys.exit(0) + cherrypy.log('Subscribing the new one.') + signal(SIGTERM, old_term_handler) + + +@cherrypy.engine.subscribe('start', priority=6) +def starterror(): + if cherrypy.config.get('starterror', False): + 1 / 0 + + +@cherrypy.engine.subscribe('start', priority=6) +def log_test_case_name(): + if cherrypy.config.get('test_case_name', False): + cherrypy.log('STARTED FROM: %s' % + cherrypy.config.get('test_case_name')) + + +cherrypy.tree.mount(Root(), '/', {'/': {}}) diff --git a/lib/cherrypy/test/benchmark.py b/lib/cherrypy/test/benchmark.py new file mode 100644 index 00000000..44dfeff1 --- /dev/null +++ b/lib/cherrypy/test/benchmark.py @@ -0,0 +1,425 @@ +"""CherryPy Benchmark Tool + + Usage: + benchmark.py [options] + + --null: use a null Request object (to bench the HTTP server only) + --notests: start the server but do not run the tests; this allows + you to check the tested pages with a browser + --help: show this help message + --cpmodpy: run tests via apache on 54583 (with the builtin _cpmodpy) + --modpython: run tests via apache on 54583 (with modpython_gateway) + --ab=path: Use the ab script/executable at 'path' (see below) + --apache=path: Use the apache script/exe at 'path' (see below) + + To run the benchmarks, the Apache Benchmark tool "ab" must either be on + your system path, or specified via the --ab=path option. + + To run the modpython tests, the "apache" executable or script must be + on your system path, or provided via the --apache=path option. On some + platforms, "apache" may be called "apachectl" or "apache2ctl"--create + a symlink to them if needed. +""" + +import getopt +import os +import re +import sys +import time + +import cherrypy +from cherrypy import _cperror, _cpmodpy +from cherrypy.lib import httputil + + +curdir = os.path.join(os.getcwd(), os.path.dirname(__file__)) + +AB_PATH = '' +APACHE_PATH = 'apache' +SCRIPT_NAME = '/cpbench/users/rdelon/apps/blog' + +__all__ = ['ABSession', 'Root', 'print_report', + 'run_standard_benchmarks', 'safe_threads', + 'size_report', 'thread_report', + ] + +size_cache = {} + + +class Root: + + @cherrypy.expose + def index(self): + return """ + + CherryPy Benchmark + + + + +""" + + @cherrypy.expose + def hello(self): + return 'Hello, world\r\n' + + @cherrypy.expose + def sizer(self, size): + resp = size_cache.get(size, None) + if resp is None: + size_cache[size] = resp = 'X' * int(size) + return resp + + +def init(): + + cherrypy.config.update({ + 'log.error.file': '', + 'environment': 'production', + 'server.socket_host': '127.0.0.1', + 'server.socket_port': 54583, + 'server.max_request_header_size': 0, + 'server.max_request_body_size': 0, + }) + + # Cheat mode on ;) + del cherrypy.config['tools.log_tracebacks.on'] + del cherrypy.config['tools.log_headers.on'] + del cherrypy.config['tools.trailing_slash.on'] + + appconf = { + '/static': { + 'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static', + 'tools.staticdir.root': curdir, + }, + } + globals().update( + app=cherrypy.tree.mount(Root(), SCRIPT_NAME, appconf), + ) + + +class NullRequest: + + """A null HTTP request class, returning 200 and an empty body.""" + + def __init__(self, local, remote, scheme='http'): + pass + + def close(self): + pass + + def run(self, method, path, query_string, protocol, headers, rfile): + cherrypy.response.status = '200 OK' + cherrypy.response.header_list = [('Content-Type', 'text/html'), + ('Server', 'Null CherryPy'), + ('Date', httputil.HTTPDate()), + ('Content-Length', '0'), + ] + cherrypy.response.body = [''] + return cherrypy.response + + +class NullResponse: + pass + + +class ABSession: + + """A session of 'ab', the Apache HTTP server benchmarking tool. + +Example output from ab: + +This is ApacheBench, Version 2.0.40-dev <$Revision: 1.121.2.1 $> apache-2.0 +Copyright (c) 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ +Copyright (c) 1998-2002 The Apache Software Foundation, http://www.apache.org/ + +Benchmarking 127.0.0.1 (be patient) +Completed 100 requests +Completed 200 requests +Completed 300 requests +Completed 400 requests +Completed 500 requests +Completed 600 requests +Completed 700 requests +Completed 800 requests +Completed 900 requests + + +Server Software: CherryPy/3.1beta +Server Hostname: 127.0.0.1 +Server Port: 54583 + +Document Path: /static/index.html +Document Length: 14 bytes + +Concurrency Level: 10 +Time taken for tests: 9.643867 seconds +Complete requests: 1000 +Failed requests: 0 +Write errors: 0 +Total transferred: 189000 bytes +HTML transferred: 14000 bytes +Requests per second: 103.69 [#/sec] (mean) +Time per request: 96.439 [ms] (mean) +Time per request: 9.644 [ms] (mean, across all concurrent requests) +Transfer rate: 19.08 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 2.9 0 10 +Processing: 20 94 7.3 90 130 +Waiting: 0 43 28.1 40 100 +Total: 20 95 7.3 100 130 + +Percentage of the requests served within a certain time (ms) + 50% 100 + 66% 100 + 75% 100 + 80% 100 + 90% 100 + 95% 100 + 98% 100 + 99% 110 + 100% 130 (longest request) +Finished 1000 requests +""" + + parse_patterns = [ + ('complete_requests', 'Completed', + br'^Complete requests:\s*(\d+)'), + ('failed_requests', 'Failed', + br'^Failed requests:\s*(\d+)'), + ('requests_per_second', 'req/sec', + br'^Requests per second:\s*([0-9.]+)'), + ('time_per_request_concurrent', 'msec/req', + br'^Time per request:\s*([0-9.]+).*concurrent requests\)$'), + ('transfer_rate', 'KB/sec', + br'^Transfer rate:\s*([0-9.]+)') + ] + + def __init__(self, path=SCRIPT_NAME + '/hello', requests=1000, + concurrency=10): + self.path = path + self.requests = requests + self.concurrency = concurrency + + def args(self): + port = cherrypy.server.socket_port + assert self.concurrency > 0 + assert self.requests > 0 + # Don't use "localhost". + # Cf + # http://mail.python.org/pipermail/python-win32/2008-March/007050.html + return ('-k -n %s -c %s http://127.0.0.1:%s%s' % + (self.requests, self.concurrency, port, self.path)) + + def run(self): + # Parse output of ab, setting attributes on self + try: + self.output = _cpmodpy.read_process(AB_PATH or 'ab', self.args()) + except Exception: + print(_cperror.format_exc()) + raise + + for attr, name, pattern in self.parse_patterns: + val = re.search(pattern, self.output, re.MULTILINE) + if val: + val = val.group(1) + setattr(self, attr, val) + else: + setattr(self, attr, None) + + +safe_threads = (25, 50, 100, 200, 400) +if sys.platform in ('win32',): + # For some reason, ab crashes with > 50 threads on my Win2k laptop. + safe_threads = (10, 20, 30, 40, 50) + + +def thread_report(path=SCRIPT_NAME + '/hello', concurrency=safe_threads): + sess = ABSession(path) + attrs, names, patterns = list(zip(*sess.parse_patterns)) + avg = dict.fromkeys(attrs, 0.0) + + yield ('threads',) + names + for c in concurrency: + sess.concurrency = c + sess.run() + row = [c] + for attr in attrs: + val = getattr(sess, attr) + if val is None: + print(sess.output) + row = None + break + val = float(val) + avg[attr] += float(val) + row.append(val) + if row: + yield row + + # Add a row of averages. + yield ['Average'] + [str(avg[attr] / len(concurrency)) for attr in attrs] + + +def size_report(sizes=(10, 100, 1000, 10000, 100000, 100000000), + concurrency=50): + sess = ABSession(concurrency=concurrency) + attrs, names, patterns = list(zip(*sess.parse_patterns)) + yield ('bytes',) + names + for sz in sizes: + sess.path = '%s/sizer?size=%s' % (SCRIPT_NAME, sz) + sess.run() + yield [sz] + [getattr(sess, attr) for attr in attrs] + + +def print_report(rows): + for row in rows: + print('') + for val in row: + sys.stdout.write(str(val).rjust(10) + ' | ') + print('') + + +def run_standard_benchmarks(): + print('') + print('Client Thread Report (1000 requests, 14 byte response body, ' + '%s server threads):' % cherrypy.server.thread_pool) + print_report(thread_report()) + + print('') + print('Client Thread Report (1000 requests, 14 bytes via staticdir, ' + '%s server threads):' % cherrypy.server.thread_pool) + print_report(thread_report('%s/static/index.html' % SCRIPT_NAME)) + + print('') + print('Size Report (1000 requests, 50 client threads, ' + '%s server threads):' % cherrypy.server.thread_pool) + print_report(size_report()) + + +# modpython and other WSGI # + +def startup_modpython(req=None): + """Start the CherryPy app server in 'serverless' mode (for modpython/WSGI). + """ + if cherrypy.engine.state == cherrypy._cpengine.STOPPED: + if req: + if 'nullreq' in req.get_options(): + cherrypy.engine.request_class = NullRequest + cherrypy.engine.response_class = NullResponse + ab_opt = req.get_options().get('ab', '') + if ab_opt: + global AB_PATH + AB_PATH = ab_opt + cherrypy.engine.start() + if cherrypy.engine.state == cherrypy._cpengine.STARTING: + cherrypy.engine.wait() + return 0 # apache.OK + + +def run_modpython(use_wsgi=False): + print('Starting mod_python...') + pyopts = [] + + # Pass the null and ab=path options through Apache + if '--null' in opts: + pyopts.append(('nullreq', '')) + + if '--ab' in opts: + pyopts.append(('ab', opts['--ab'])) + + s = _cpmodpy.ModPythonServer + if use_wsgi: + pyopts.append(('wsgi.application', 'cherrypy::tree')) + pyopts.append( + ('wsgi.startup', 'cherrypy.test.benchmark::startup_modpython')) + handler = 'modpython_gateway::handler' + s = s(port=54583, opts=pyopts, + apache_path=APACHE_PATH, handler=handler) + else: + pyopts.append( + ('cherrypy.setup', 'cherrypy.test.benchmark::startup_modpython')) + s = s(port=54583, opts=pyopts, apache_path=APACHE_PATH) + + try: + s.start() + run() + finally: + s.stop() + + +if __name__ == '__main__': + init() + + longopts = ['cpmodpy', 'modpython', 'null', 'notests', + 'help', 'ab=', 'apache='] + try: + switches, args = getopt.getopt(sys.argv[1:], '', longopts) + opts = dict(switches) + except getopt.GetoptError: + print(__doc__) + sys.exit(2) + + if '--help' in opts: + print(__doc__) + sys.exit(0) + + if '--ab' in opts: + AB_PATH = opts['--ab'] + + if '--notests' in opts: + # Return without stopping the server, so that the pages + # can be tested from a standard web browser. + def run(): + port = cherrypy.server.socket_port + print('You may now open http://127.0.0.1:%s%s/' % + (port, SCRIPT_NAME)) + + if '--null' in opts: + print('Using null Request object') + else: + def run(): + end = time.time() - start + print('Started in %s seconds' % end) + if '--null' in opts: + print('\nUsing null Request object') + try: + try: + run_standard_benchmarks() + except Exception: + print(_cperror.format_exc()) + raise + finally: + cherrypy.engine.exit() + + print('Starting CherryPy app server...') + + class NullWriter(object): + + """Suppresses the printing of socket errors.""" + + def write(self, data): + pass + sys.stderr = NullWriter() + + start = time.time() + + if '--cpmodpy' in opts: + run_modpython() + elif '--modpython' in opts: + run_modpython(use_wsgi=True) + else: + if '--null' in opts: + cherrypy.server.request_class = NullRequest + cherrypy.server.response_class = NullResponse + + cherrypy.engine.start_with_callback(run) + cherrypy.engine.block() diff --git a/lib/cherrypy/test/checkerdemo.py b/lib/cherrypy/test/checkerdemo.py new file mode 100644 index 00000000..3438bd0c --- /dev/null +++ b/lib/cherrypy/test/checkerdemo.py @@ -0,0 +1,49 @@ +"""Demonstration app for cherrypy.checker. + +This application is intentionally broken and badly designed. +To demonstrate the output of the CherryPy Checker, simply execute +this module. +""" + +import os +import cherrypy +thisdir = os.path.dirname(os.path.abspath(__file__)) + + +class Root: + pass + + +if __name__ == '__main__': + conf = {'/base': {'tools.staticdir.root': thisdir, + # Obsolete key. + 'throw_errors': True, + }, + # This entry should be OK. + '/base/static': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static'}, + # Warn on missing folder. + '/base/js': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'js'}, + # Warn on dir with an abs path even though we provide root. + '/base/static2': {'tools.staticdir.on': True, + 'tools.staticdir.dir': '/static'}, + # Warn on dir with a relative path with no root. + '/static3': {'tools.staticdir.on': True, + 'tools.staticdir.dir': 'static'}, + # Warn on unknown namespace + '/unknown': {'toobles.gzip.on': True}, + # Warn special on cherrypy..* + '/cpknown': {'cherrypy.tools.encode.on': True}, + # Warn on mismatched types + '/conftype': {'request.show_tracebacks': 14}, + # Warn on unknown tool. + '/web': {'tools.unknown.on': True}, + # Warn on server.* in app config. + '/app1': {'server.socket_host': '0.0.0.0'}, + # Warn on 'localhost' + 'global': {'server.socket_host': 'localhost'}, + # Warn on '[name]' + '[/extra_brackets]': {}, + } + cherrypy.quickstart(Root(), config=conf) diff --git a/lib/cherrypy/test/fastcgi.conf b/lib/cherrypy/test/fastcgi.conf new file mode 100644 index 00000000..e5c5163c --- /dev/null +++ b/lib/cherrypy/test/fastcgi.conf @@ -0,0 +1,18 @@ + +# Apache2 server conf file for testing CherryPy with mod_fastcgi. +# fumanchu: I had to hard-code paths due to crazy Debian layouts :( +ServerRoot /usr/lib/apache2 +User #1000 +ErrorLog /usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test/mod_fastcgi.error.log + +DocumentRoot "/usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test" +ServerName 127.0.0.1 +Listen 8080 +LoadModule fastcgi_module modules/mod_fastcgi.so +LoadModule rewrite_module modules/mod_rewrite.so + +Options +ExecCGI +SetHandler fastcgi-script +RewriteEngine On +RewriteRule ^(.*)$ /fastcgi.pyc [L] +FastCgiExternalServer "/usr/lib/python2.5/site-packages/cproot/trunk/cherrypy/test/fastcgi.pyc" -host 127.0.0.1:4000 diff --git a/lib/cherrypy/test/fcgi.conf b/lib/cherrypy/test/fcgi.conf new file mode 100644 index 00000000..3062eb35 --- /dev/null +++ b/lib/cherrypy/test/fcgi.conf @@ -0,0 +1,14 @@ + +# Apache2 server conf file for testing CherryPy with mod_fcgid. + +DocumentRoot "/usr/lib/python2.6/site-packages/cproot/trunk/cherrypy/test" +ServerName 127.0.0.1 +Listen 8080 +LoadModule fastcgi_module modules/mod_fastcgi.dll +LoadModule rewrite_module modules/mod_rewrite.so + +Options ExecCGI +SetHandler fastcgi-script +RewriteEngine On +RewriteRule ^(.*)$ /fastcgi.pyc [L] +FastCgiExternalServer "/usr/lib/python2.6/site-packages/cproot/trunk/cherrypy/test/fastcgi.pyc" -host 127.0.0.1:4000 diff --git a/lib/cherrypy/test/helper.py b/lib/cherrypy/test/helper.py new file mode 100644 index 00000000..01c5a0c0 --- /dev/null +++ b/lib/cherrypy/test/helper.py @@ -0,0 +1,542 @@ +"""A library of helper functions for the CherryPy test suite.""" + +import datetime +import io +import logging +import os +import re +import subprocess +import sys +import time +import unittest +import warnings + +import portend +import pytest +import six + +from cheroot.test import webtest + +import cherrypy +from cherrypy._cpcompat import text_or_bytes, HTTPSConnection, ntob +from cherrypy.lib import httputil +from cherrypy.lib import gctools + +log = logging.getLogger(__name__) +thisdir = os.path.abspath(os.path.dirname(__file__)) +serverpem = os.path.join(os.getcwd(), thisdir, 'test.pem') + + +class Supervisor(object): + + """Base class for modeling and controlling servers during testing.""" + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + if k == 'port': + setattr(self, k, int(v)) + setattr(self, k, v) + + +def log_to_stderr(msg, level): + return sys.stderr.write(msg + os.linesep) + + +class LocalSupervisor(Supervisor): + + """Base class for modeling/controlling servers which run in the same + process. + + When the server side runs in a different process, start/stop can dump all + state between each test module easily. When the server side runs in the + same process as the client, however, we have to do a bit more work to + ensure config and mounted apps are reset between tests. + """ + + using_apache = False + using_wsgi = False + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + cherrypy.server.httpserver = self.httpserver_class + + # This is perhaps the wrong place for this call but this is the only + # place that i've found so far that I KNOW is early enough to set this. + cherrypy.config.update({'log.screen': False}) + engine = cherrypy.engine + if hasattr(engine, 'signal_handler'): + engine.signal_handler.subscribe() + if hasattr(engine, 'console_control_handler'): + engine.console_control_handler.subscribe() + + def start(self, modulename=None): + """Load and start the HTTP server.""" + if modulename: + # Unhook httpserver so cherrypy.server.start() creates a new + # one (with config from setup_server, if declared). + cherrypy.server.httpserver = None + + cherrypy.engine.start() + + self.sync_apps() + + def sync_apps(self): + """Tell the server about any apps which the setup functions mounted.""" + pass + + def stop(self): + td = getattr(self, 'teardown', None) + if td: + td() + + cherrypy.engine.exit() + + servers_copy = list(six.iteritems(getattr(cherrypy, 'servers', {}))) + for name, server in servers_copy: + server.unsubscribe() + del cherrypy.servers[name] + + +class NativeServerSupervisor(LocalSupervisor): + + """Server supervisor for the builtin HTTP server.""" + + httpserver_class = 'cherrypy._cpnative_server.CPHTTPServer' + using_apache = False + using_wsgi = False + + def __str__(self): + return 'Builtin HTTP Server on %s:%s' % (self.host, self.port) + + +class LocalWSGISupervisor(LocalSupervisor): + + """Server supervisor for the builtin WSGI server.""" + + httpserver_class = 'cherrypy._cpwsgi_server.CPWSGIServer' + using_apache = False + using_wsgi = True + + def __str__(self): + return 'Builtin WSGI Server on %s:%s' % (self.host, self.port) + + def sync_apps(self): + """Hook a new WSGI app into the origin server.""" + cherrypy.server.httpserver.wsgi_app = self.get_app() + + def get_app(self, app=None): + """Obtain a new (decorated) WSGI app to hook into the origin server.""" + if app is None: + app = cherrypy.tree + + if self.validate: + try: + from wsgiref import validate + except ImportError: + warnings.warn( + 'Error importing wsgiref. The validator will not run.') + else: + # wraps the app in the validator + app = validate.validator(app) + + return app + + +def get_cpmodpy_supervisor(**options): + from cherrypy.test import modpy + sup = modpy.ModPythonSupervisor(**options) + sup.template = modpy.conf_cpmodpy + return sup + + +def get_modpygw_supervisor(**options): + from cherrypy.test import modpy + sup = modpy.ModPythonSupervisor(**options) + sup.template = modpy.conf_modpython_gateway + sup.using_wsgi = True + return sup + + +def get_modwsgi_supervisor(**options): + from cherrypy.test import modwsgi + return modwsgi.ModWSGISupervisor(**options) + + +def get_modfcgid_supervisor(**options): + from cherrypy.test import modfcgid + return modfcgid.ModFCGISupervisor(**options) + + +def get_modfastcgi_supervisor(**options): + from cherrypy.test import modfastcgi + return modfastcgi.ModFCGISupervisor(**options) + + +def get_wsgi_u_supervisor(**options): + cherrypy.server.wsgi_version = ('u', 0) + return LocalWSGISupervisor(**options) + + +class CPWebCase(webtest.WebCase): + + script_name = '' + scheme = 'http' + + available_servers = {'wsgi': LocalWSGISupervisor, + 'wsgi_u': get_wsgi_u_supervisor, + 'native': NativeServerSupervisor, + 'cpmodpy': get_cpmodpy_supervisor, + 'modpygw': get_modpygw_supervisor, + 'modwsgi': get_modwsgi_supervisor, + 'modfcgid': get_modfcgid_supervisor, + 'modfastcgi': get_modfastcgi_supervisor, + } + default_server = 'wsgi' + + @classmethod + def _setup_server(cls, supervisor, conf): + v = sys.version.split()[0] + log.info('Python version used to run this test script: %s' % v) + log.info('CherryPy version: %s' % cherrypy.__version__) + if supervisor.scheme == 'https': + ssl = ' (ssl)' + else: + ssl = '' + log.info('HTTP server version: %s%s' % (supervisor.protocol, ssl)) + log.info('PID: %s' % os.getpid()) + + cherrypy.server.using_apache = supervisor.using_apache + cherrypy.server.using_wsgi = supervisor.using_wsgi + + if sys.platform[:4] == 'java': + cherrypy.config.update({'server.nodelay': False}) + + if isinstance(conf, text_or_bytes): + parser = cherrypy.lib.reprconf.Parser() + conf = parser.dict_from_file(conf).get('global', {}) + else: + conf = conf or {} + baseconf = conf.copy() + baseconf.update({'server.socket_host': supervisor.host, + 'server.socket_port': supervisor.port, + 'server.protocol_version': supervisor.protocol, + 'environment': 'test_suite', + }) + if supervisor.scheme == 'https': + # baseconf['server.ssl_module'] = 'builtin' + baseconf['server.ssl_certificate'] = serverpem + baseconf['server.ssl_private_key'] = serverpem + + # helper must be imported lazily so the coverage tool + # can run against module-level statements within cherrypy. + # Also, we have to do "from cherrypy.test import helper", + # exactly like each test module does, because a relative import + # would stick a second instance of webtest in sys.modules, + # and we wouldn't be able to globally override the port anymore. + if supervisor.scheme == 'https': + webtest.WebCase.HTTP_CONN = HTTPSConnection + return baseconf + + @classmethod + def setup_class(cls): + '' + # Creates a server + conf = { + 'scheme': 'http', + 'protocol': 'HTTP/1.1', + 'port': 54583, + 'host': '127.0.0.1', + 'validate': False, + 'server': 'wsgi', + } + supervisor_factory = cls.available_servers.get( + conf.get('server', 'wsgi')) + if supervisor_factory is None: + raise RuntimeError('Unknown server in config: %s' % conf['server']) + supervisor = supervisor_factory(**conf) + + # Copied from "run_test_suite" + cherrypy.config.reset() + baseconf = cls._setup_server(supervisor, conf) + cherrypy.config.update(baseconf) + setup_client() + + if hasattr(cls, 'setup_server'): + # Clear the cherrypy tree and clear the wsgi server so that + # it can be updated with the new root + cherrypy.tree = cherrypy._cptree.Tree() + cherrypy.server.httpserver = None + cls.setup_server() + # Add a resource for verifying there are no refleaks + # to *every* test class. + cherrypy.tree.mount(gctools.GCRoot(), '/gc') + cls.do_gc_test = True + supervisor.start(cls.__module__) + + cls.supervisor = supervisor + + @classmethod + def teardown_class(cls): + '' + if hasattr(cls, 'setup_server'): + cls.supervisor.stop() + + do_gc_test = False + + def test_gc(self): + if not self.do_gc_test: + return + + self.getPage('/gc/stats') + try: + self.assertBody('Statistics:') + except Exception: + 'Failures occur intermittently. See #1420' + + def prefix(self): + return self.script_name.rstrip('/') + + def base(self): + if ((self.scheme == 'http' and self.PORT == 80) or + (self.scheme == 'https' and self.PORT == 443)): + port = '' + else: + port = ':%s' % self.PORT + + return '%s://%s%s%s' % (self.scheme, self.HOST, port, + self.script_name.rstrip('/')) + + def exit(self): + sys.exit() + + def getPage(self, url, headers=None, method='GET', body=None, + protocol=None, raise_subcls=None): + """Open the url. Return status, headers, body. + + `raise_subcls` must be a tuple with the exceptions classes + or a single exception class that are not going to be considered + a socket.error regardless that they were are subclass of a + socket.error and therefore not considered for a connection retry. + """ + if self.script_name: + url = httputil.urljoin(self.script_name, url) + return webtest.WebCase.getPage(self, url, headers, method, body, + protocol, raise_subcls) + + def skip(self, msg='skipped '): + pytest.skip(msg) + + def assertErrorPage(self, status, message=None, pattern=''): + """Compare the response body with a built in error page. + + The function will optionally look for the regexp pattern, + within the exception embedded in the error page.""" + + # This will never contain a traceback + page = cherrypy._cperror.get_error_page(status, message=message) + + # First, test the response body without checking the traceback. + # Stick a match-all group (.*) in to grab the traceback. + def esc(text): + return re.escape(ntob(text)) + epage = re.escape(page) + epage = epage.replace( + esc('
'),
+            esc('
') + b'(.*)' + esc('
'))
+        m = re.match(epage, self.body, re.DOTALL)
+        if not m:
+            self._handlewebError(
+                'Error page does not match; expected:\n' + page)
+            return
+
+        # Now test the pattern against the traceback
+        if pattern is None:
+            # Special-case None to mean that there should be *no* traceback.
+            if m and m.group(1):
+                self._handlewebError('Error page contains traceback')
+        else:
+            if (m is None) or (
+                not re.search(ntob(re.escape(pattern), self.encoding),
+                              m.group(1))):
+                msg = 'Error page does not contain %s in traceback'
+                self._handlewebError(msg % repr(pattern))
+
+    date_tolerance = 2
+
+    def assertEqualDates(self, dt1, dt2, seconds=None):
+        """Assert abs(dt1 - dt2) is within Y seconds."""
+        if seconds is None:
+            seconds = self.date_tolerance
+
+        if dt1 > dt2:
+            diff = dt1 - dt2
+        else:
+            diff = dt2 - dt1
+        if not diff < datetime.timedelta(seconds=seconds):
+            raise AssertionError('%r and %r are not within %r seconds.' %
+                                 (dt1, dt2, seconds))
+
+
+def _test_method_sorter(_, x, y):
+    """Monkeypatch the test sorter to always run test_gc last in each suite."""
+    if x == 'test_gc':
+        return 1
+    if y == 'test_gc':
+        return -1
+    if x > y:
+        return 1
+    if x < y:
+        return -1
+    return 0
+
+
+unittest.TestLoader.sortTestMethodsUsing = _test_method_sorter
+
+
+def setup_client():
+    """Set up the WebCase classes to match the server's socket settings."""
+    webtest.WebCase.PORT = cherrypy.server.socket_port
+    webtest.WebCase.HOST = cherrypy.server.socket_host
+    if cherrypy.server.ssl_certificate:
+        CPWebCase.scheme = 'https'
+
+# --------------------------- Spawning helpers --------------------------- #
+
+
+class CPProcess(object):
+
+    pid_file = os.path.join(thisdir, 'test.pid')
+    config_file = os.path.join(thisdir, 'test.conf')
+    config_template = """[global]
+server.socket_host: '%(host)s'
+server.socket_port: %(port)s
+checker.on: False
+log.screen: False
+log.error_file: r'%(error_log)s'
+log.access_file: r'%(access_log)s'
+%(ssl)s
+%(extra)s
+"""
+    error_log = os.path.join(thisdir, 'test.error.log')
+    access_log = os.path.join(thisdir, 'test.access.log')
+
+    def __init__(self, wait=False, daemonize=False, ssl=False,
+                 socket_host=None, socket_port=None):
+        self.wait = wait
+        self.daemonize = daemonize
+        self.ssl = ssl
+        self.host = socket_host or cherrypy.server.socket_host
+        self.port = socket_port or cherrypy.server.socket_port
+
+    def write_conf(self, extra=''):
+        if self.ssl:
+            serverpem = os.path.join(thisdir, 'test.pem')
+            ssl = """
+server.ssl_certificate: r'%s'
+server.ssl_private_key: r'%s'
+""" % (serverpem, serverpem)
+        else:
+            ssl = ''
+
+        conf = self.config_template % {
+            'host': self.host,
+            'port': self.port,
+            'error_log': self.error_log,
+            'access_log': self.access_log,
+            'ssl': ssl,
+            'extra': extra,
+        }
+        with io.open(self.config_file, 'w', encoding='utf-8') as f:
+            f.write(six.text_type(conf))
+
+    def start(self, imports=None):
+        """Start cherryd in a subprocess."""
+        portend.free(self.host, self.port, timeout=1)
+
+        args = [
+            '-m',
+            'cherrypy',
+            '-c', self.config_file,
+            '-p', self.pid_file,
+        ]
+        r"""
+        Command for running cherryd server with autoreload enabled
+
+        Using
+
+        ```
+        ['-c',
+         "__requires__ = 'CherryPy'; \
+         import pkg_resources, re, sys; \
+         sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]); \
+         sys.exit(\
+            pkg_resources.load_entry_point(\
+                'CherryPy', 'console_scripts', 'cherryd')())"]
+        ```
+
+        doesn't work as it's impossible to reconstruct the `-c`'s contents.
+        Ref: https://github.com/cherrypy/cherrypy/issues/1545
+        """
+
+        if not isinstance(imports, (list, tuple)):
+            imports = [imports]
+        for i in imports:
+            if i:
+                args.append('-i')
+                args.append(i)
+
+        if self.daemonize:
+            args.append('-d')
+
+        env = os.environ.copy()
+        # Make sure we import the cherrypy package in which this module is
+        # defined.
+        grandparentdir = os.path.abspath(os.path.join(thisdir, '..', '..'))
+        if env.get('PYTHONPATH', ''):
+            env['PYTHONPATH'] = os.pathsep.join(
+                (grandparentdir, env['PYTHONPATH']))
+        else:
+            env['PYTHONPATH'] = grandparentdir
+        self._proc = subprocess.Popen([sys.executable] + args, env=env)
+        if self.wait:
+            self.exit_code = self._proc.wait()
+        else:
+            portend.occupied(self.host, self.port, timeout=5)
+
+        # Give the engine a wee bit more time to finish STARTING
+        if self.daemonize:
+            time.sleep(2)
+        else:
+            time.sleep(1)
+
+    def get_pid(self):
+        if self.daemonize:
+            return int(open(self.pid_file, 'rb').read())
+        return self._proc.pid
+
+    def join(self):
+        """Wait for the process to exit."""
+        if self.daemonize:
+            return self._join_daemon()
+        self._proc.wait()
+
+    def _join_daemon(self):
+        try:
+            try:
+                # Mac, UNIX
+                os.wait()
+            except AttributeError:
+                # Windows
+                try:
+                    pid = self.get_pid()
+                except IOError:
+                    # Assume the subprocess deleted the pidfile on shutdown.
+                    pass
+                else:
+                    os.waitpid(pid, 0)
+        except OSError:
+            x = sys.exc_info()[1]
+            if x.args != (10, 'No child processes'):
+                raise
diff --git a/lib/cherrypy/test/logtest.py b/lib/cherrypy/test/logtest.py
new file mode 100644
index 00000000..ed8f1540
--- /dev/null
+++ b/lib/cherrypy/test/logtest.py
@@ -0,0 +1,228 @@
+"""logtest, a unittest.TestCase helper for testing log output."""
+
+import sys
+import time
+from uuid import UUID
+
+import six
+
+from cherrypy._cpcompat import text_or_bytes, ntob
+
+
+try:
+    # On Windows, msvcrt.getch reads a single char without output.
+    import msvcrt
+
+    def getchar():
+        return msvcrt.getch()
+except ImportError:
+    # Unix getchr
+    import tty
+    import termios
+
+    def getchar():
+        fd = sys.stdin.fileno()
+        old_settings = termios.tcgetattr(fd)
+        try:
+            tty.setraw(sys.stdin.fileno())
+            ch = sys.stdin.read(1)
+        finally:
+            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        return ch
+
+
+class LogCase(object):
+
+    """unittest.TestCase mixin for testing log messages.
+
+    logfile: a filename for the desired log. Yes, I know modes are evil,
+        but it makes the test functions so much cleaner to set this once.
+
+    lastmarker: the last marker in the log. This can be used to search for
+        messages since the last marker.
+
+    markerPrefix: a string with which to prefix log markers. This should be
+        unique enough from normal log output to use for marker identification.
+    """
+
+    logfile = None
+    lastmarker = None
+    markerPrefix = b'test suite marker: '
+
+    def _handleLogError(self, msg, data, marker, pattern):
+        print('')
+        print('    ERROR: %s' % msg)
+
+        if not self.interactive:
+            raise self.failureException(msg)
+
+        p = ('    Show: '
+             '[L]og [M]arker [P]attern; '
+             '[I]gnore, [R]aise, or sys.e[X]it >> ')
+        sys.stdout.write(p + ' ')
+        # ARGH
+        sys.stdout.flush()
+        while True:
+            i = getchar().upper()
+            if i not in 'MPLIRX':
+                continue
+            print(i.upper())  # Also prints new line
+            if i == 'L':
+                for x, line in enumerate(data):
+                    if (x + 1) % self.console_height == 0:
+                        # The \r and comma should make the next line overwrite
+                        sys.stdout.write('<-- More -->\r ')
+                        m = getchar().lower()
+                        # Erase our "More" prompt
+                        sys.stdout.write('            \r ')
+                        if m == 'q':
+                            break
+                    print(line.rstrip())
+            elif i == 'M':
+                print(repr(marker or self.lastmarker))
+            elif i == 'P':
+                print(repr(pattern))
+            elif i == 'I':
+                # return without raising the normal exception
+                return
+            elif i == 'R':
+                raise self.failureException(msg)
+            elif i == 'X':
+                self.exit()
+            sys.stdout.write(p + ' ')
+
+    def exit(self):
+        sys.exit()
+
+    def emptyLog(self):
+        """Overwrite self.logfile with 0 bytes."""
+        open(self.logfile, 'wb').write('')
+
+    def markLog(self, key=None):
+        """Insert a marker line into the log and set self.lastmarker."""
+        if key is None:
+            key = str(time.time())
+        self.lastmarker = key
+
+        open(self.logfile, 'ab+').write(
+            ntob('%s%s\n' % (self.markerPrefix, key), 'utf-8'))
+
+    def _read_marked_region(self, marker=None):
+        """Return lines from self.logfile in the marked region.
+
+        If marker is None, self.lastmarker is used. If the log hasn't
+        been marked (using self.markLog), the entire log will be returned.
+        """
+# Give the logger time to finish writing?
+# time.sleep(0.5)
+
+        logfile = self.logfile
+        marker = marker or self.lastmarker
+        if marker is None:
+            return open(logfile, 'rb').readlines()
+
+        if isinstance(marker, six.text_type):
+            marker = marker.encode('utf-8')
+        data = []
+        in_region = False
+        for line in open(logfile, 'rb'):
+            if in_region:
+                if line.startswith(self.markerPrefix) and marker not in line:
+                    break
+                else:
+                    data.append(line)
+            elif marker in line:
+                in_region = True
+        return data
+
+    def assertInLog(self, line, marker=None):
+        """Fail if the given (partial) line is not in the log.
+
+        The log will be searched from the given marker to the next marker.
+        If marker is None, self.lastmarker is used. If the log hasn't
+        been marked (using self.markLog), the entire log will be searched.
+        """
+        data = self._read_marked_region(marker)
+        for logline in data:
+            if line in logline:
+                return
+        msg = '%r not found in log' % line
+        self._handleLogError(msg, data, marker, line)
+
+    def assertNotInLog(self, line, marker=None):
+        """Fail if the given (partial) line is in the log.
+
+        The log will be searched from the given marker to the next marker.
+        If marker is None, self.lastmarker is used. If the log hasn't
+        been marked (using self.markLog), the entire log will be searched.
+        """
+        data = self._read_marked_region(marker)
+        for logline in data:
+            if line in logline:
+                msg = '%r found in log' % line
+                self._handleLogError(msg, data, marker, line)
+
+    def assertValidUUIDv4(self, marker=None):
+        """Fail if the given UUIDv4 is not valid.
+
+        The log will be searched from the given marker to the next marker.
+        If marker is None, self.lastmarker is used. If the log hasn't
+        been marked (using self.markLog), the entire log will be searched.
+        """
+        data = self._read_marked_region(marker)
+        data = [
+            chunk.decode('utf-8').rstrip('\n').rstrip('\r')
+            for chunk in data
+        ]
+        for log_chunk in data:
+            try:
+                uuid_log = data[-1]
+                uuid_obj = UUID(uuid_log, version=4)
+            except (TypeError, ValueError):
+                pass  # it might be in other chunk
+            else:
+                if str(uuid_obj) == uuid_log:
+                    return
+                msg = '%r is not a valid UUIDv4' % uuid_log
+                self._handleLogError(msg, data, marker, log_chunk)
+
+        msg = 'UUIDv4 not found in log'
+        self._handleLogError(msg, data, marker, log_chunk)
+
+    def assertLog(self, sliceargs, lines, marker=None):
+        """Fail if log.readlines()[sliceargs] is not contained in 'lines'.
+
+        The log will be searched from the given marker to the next marker.
+        If marker is None, self.lastmarker is used. If the log hasn't
+        been marked (using self.markLog), the entire log will be searched.
+        """
+        data = self._read_marked_region(marker)
+        if isinstance(sliceargs, int):
+            # Single arg. Use __getitem__ and allow lines to be str or list.
+            if isinstance(lines, (tuple, list)):
+                lines = lines[0]
+            if isinstance(lines, six.text_type):
+                lines = lines.encode('utf-8')
+            if lines not in data[sliceargs]:
+                msg = '%r not found on log line %r' % (lines, sliceargs)
+                self._handleLogError(
+                    msg,
+                    [data[sliceargs], '--EXTRA CONTEXT--'] + data[
+                        sliceargs + 1:sliceargs + 6],
+                    marker,
+                    lines)
+        else:
+            # Multiple args. Use __getslice__ and require lines to be list.
+            if isinstance(lines, tuple):
+                lines = list(lines)
+            elif isinstance(lines, text_or_bytes):
+                raise TypeError("The 'lines' arg must be a list when "
+                                "'sliceargs' is a tuple.")
+
+            start, stop = sliceargs
+            for line, logline in zip(lines, data[start:stop]):
+                if isinstance(line, six.text_type):
+                    line = line.encode('utf-8')
+                if line not in logline:
+                    msg = '%r not found in log' % line
+                    self._handleLogError(msg, data[start:stop], marker, line)
diff --git a/lib/cherrypy/test/modfastcgi.py b/lib/cherrypy/test/modfastcgi.py
new file mode 100644
index 00000000..79ec3d18
--- /dev/null
+++ b/lib/cherrypy/test/modfastcgi.py
@@ -0,0 +1,136 @@
+"""Wrapper for mod_fastcgi, for use as a CherryPy HTTP server when testing.
+
+To autostart fastcgi, the "apache" executable or script must be
+on your system path, or you must override the global APACHE_PATH.
+On some platforms, "apache" may be called "apachectl", "apache2ctl",
+or "httpd"--create a symlink to them if needed.
+
+You'll also need the WSGIServer from flup.servers.
+See http://projects.amor.org/misc/wiki/ModPythonGateway
+
+
+KNOWN BUGS
+==========
+
+1. Apache processes Range headers automatically; CherryPy's truncated
+    output is then truncated again by Apache. See test_core.testRanges.
+    This was worked around in http://www.cherrypy.org/changeset/1319.
+2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
+    See test_core.testHTTPMethods.
+3. Max request header and body settings do not work with Apache.
+4. Apache replaces status "reason phrases" automatically. For example,
+    CherryPy may set "304 Not modified" but Apache will write out
+    "304 Not Modified" (capital "M").
+5. Apache does not allow custom error codes as per the spec.
+6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
+    Request-URI too early.
+7. mod_python will not read request bodies which use the "chunked"
+    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
+    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
+    mod_python's requestobject.c).
+8. Apache will output a "Content-Length: 0" response header even if there's
+    no response entity body. This isn't really a bug; it just differs from
+    the CherryPy default.
+"""
+
+import os
+import re
+
+import cherrypy
+from cherrypy.process import servers
+from cherrypy.test import helper
+
+curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+
+def read_process(cmd, args=''):
+    pipein, pipeout = os.popen4('%s %s' % (cmd, args))
+    try:
+        firstline = pipeout.readline()
+        if (re.search(r'(not recognized|No such file|not found)', firstline,
+                      re.IGNORECASE)):
+            raise IOError('%s must be on your system path.' % cmd)
+        output = firstline + pipeout.read()
+    finally:
+        pipeout.close()
+    return output
+
+
+APACHE_PATH = 'apache2ctl'
+CONF_PATH = 'fastcgi.conf'
+
+conf_fastcgi = """
+# Apache2 server conf file for testing CherryPy with mod_fastcgi.
+# fumanchu: I had to hard-code paths due to crazy Debian layouts :(
+ServerRoot /usr/lib/apache2
+User #1000
+ErrorLog %(root)s/mod_fastcgi.error.log
+
+DocumentRoot "%(root)s"
+ServerName 127.0.0.1
+Listen %(port)s
+LoadModule fastcgi_module modules/mod_fastcgi.so
+LoadModule rewrite_module modules/mod_rewrite.so
+
+Options +ExecCGI
+SetHandler fastcgi-script
+RewriteEngine On
+RewriteRule ^(.*)$ /fastcgi.pyc [L]
+FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000
+"""
+
+
+def erase_script_name(environ, start_response):
+    environ['SCRIPT_NAME'] = ''
+    return cherrypy.tree(environ, start_response)
+
+
+class ModFCGISupervisor(helper.LocalWSGISupervisor):
+
+    httpserver_class = 'cherrypy.process.servers.FlupFCGIServer'
+    using_apache = True
+    using_wsgi = True
+    template = conf_fastcgi
+
+    def __str__(self):
+        return 'FCGI Server on %s:%s' % (self.host, self.port)
+
+    def start(self, modulename):
+        cherrypy.server.httpserver = servers.FlupFCGIServer(
+            application=erase_script_name, bindAddress=('127.0.0.1', 4000))
+        cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000)
+        cherrypy.server.socket_port = 4000
+        # For FCGI, we both start apache...
+        self.start_apache()
+        # ...and our local server
+        cherrypy.engine.start()
+        self.sync_apps()
+
+    def start_apache(self):
+        fcgiconf = CONF_PATH
+        if not os.path.isabs(fcgiconf):
+            fcgiconf = os.path.join(curdir, fcgiconf)
+
+        # Write the Apache conf file.
+        f = open(fcgiconf, 'wb')
+        try:
+            server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1]
+            output = self.template % {'port': self.port, 'root': curdir,
+                                      'server': server}
+            output = output.replace('\r\n', '\n')
+            f.write(output)
+        finally:
+            f.close()
+
+        result = read_process(APACHE_PATH, '-k start -f %s' % fcgiconf)
+        if result:
+            print(result)
+
+    def stop(self):
+        """Gracefully shutdown a server that is serving forever."""
+        read_process(APACHE_PATH, '-k stop')
+        helper.LocalWSGISupervisor.stop(self)
+
+    def sync_apps(self):
+        cherrypy.server.httpserver.fcgiserver.application = self.get_app(
+            erase_script_name)
diff --git a/lib/cherrypy/test/modfcgid.py b/lib/cherrypy/test/modfcgid.py
new file mode 100644
index 00000000..d101bd67
--- /dev/null
+++ b/lib/cherrypy/test/modfcgid.py
@@ -0,0 +1,124 @@
+"""Wrapper for mod_fcgid, for use as a CherryPy HTTP server when testing.
+
+To autostart fcgid, the "apache" executable or script must be
+on your system path, or you must override the global APACHE_PATH.
+On some platforms, "apache" may be called "apachectl", "apache2ctl",
+or "httpd"--create a symlink to them if needed.
+
+You'll also need the WSGIServer from flup.servers.
+See http://projects.amor.org/misc/wiki/ModPythonGateway
+
+
+KNOWN BUGS
+==========
+
+1. Apache processes Range headers automatically; CherryPy's truncated
+    output is then truncated again by Apache. See test_core.testRanges.
+    This was worked around in http://www.cherrypy.org/changeset/1319.
+2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
+    See test_core.testHTTPMethods.
+3. Max request header and body settings do not work with Apache.
+4. Apache replaces status "reason phrases" automatically. For example,
+    CherryPy may set "304 Not modified" but Apache will write out
+    "304 Not Modified" (capital "M").
+5. Apache does not allow custom error codes as per the spec.
+6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
+    Request-URI too early.
+7. mod_python will not read request bodies which use the "chunked"
+    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
+    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
+    mod_python's requestobject.c).
+8. Apache will output a "Content-Length: 0" response header even if there's
+    no response entity body. This isn't really a bug; it just differs from
+    the CherryPy default.
+"""
+
+import os
+import re
+
+import cherrypy
+from cherrypy._cpcompat import ntob
+from cherrypy.process import servers
+from cherrypy.test import helper
+
+curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+
+def read_process(cmd, args=''):
+    pipein, pipeout = os.popen4('%s %s' % (cmd, args))
+    try:
+        firstline = pipeout.readline()
+        if (re.search(r'(not recognized|No such file|not found)', firstline,
+                      re.IGNORECASE)):
+            raise IOError('%s must be on your system path.' % cmd)
+        output = firstline + pipeout.read()
+    finally:
+        pipeout.close()
+    return output
+
+
+APACHE_PATH = 'httpd'
+CONF_PATH = 'fcgi.conf'
+
+conf_fcgid = """
+# Apache2 server conf file for testing CherryPy with mod_fcgid.
+
+DocumentRoot "%(root)s"
+ServerName 127.0.0.1
+Listen %(port)s
+LoadModule fastcgi_module modules/mod_fastcgi.dll
+LoadModule rewrite_module modules/mod_rewrite.so
+
+Options ExecCGI
+SetHandler fastcgi-script
+RewriteEngine On
+RewriteRule ^(.*)$ /fastcgi.pyc [L]
+FastCgiExternalServer "%(server)s" -host 127.0.0.1:4000
+"""
+
+
+class ModFCGISupervisor(helper.LocalSupervisor):
+
+    using_apache = True
+    using_wsgi = True
+    template = conf_fcgid
+
+    def __str__(self):
+        return 'FCGI Server on %s:%s' % (self.host, self.port)
+
+    def start(self, modulename):
+        cherrypy.server.httpserver = servers.FlupFCGIServer(
+            application=cherrypy.tree, bindAddress=('127.0.0.1', 4000))
+        cherrypy.server.httpserver.bind_addr = ('127.0.0.1', 4000)
+        # For FCGI, we both start apache...
+        self.start_apache()
+        # ...and our local server
+        helper.LocalServer.start(self, modulename)
+
+    def start_apache(self):
+        fcgiconf = CONF_PATH
+        if not os.path.isabs(fcgiconf):
+            fcgiconf = os.path.join(curdir, fcgiconf)
+
+        # Write the Apache conf file.
+        f = open(fcgiconf, 'wb')
+        try:
+            server = repr(os.path.join(curdir, 'fastcgi.pyc'))[1:-1]
+            output = self.template % {'port': self.port, 'root': curdir,
+                                      'server': server}
+            output = ntob(output.replace('\r\n', '\n'))
+            f.write(output)
+        finally:
+            f.close()
+
+        result = read_process(APACHE_PATH, '-k start -f %s' % fcgiconf)
+        if result:
+            print(result)
+
+    def stop(self):
+        """Gracefully shutdown a server that is serving forever."""
+        read_process(APACHE_PATH, '-k stop')
+        helper.LocalServer.stop(self)
+
+    def sync_apps(self):
+        cherrypy.server.httpserver.fcgiserver.application = self.get_app()
diff --git a/lib/cherrypy/test/modpy.py b/lib/cherrypy/test/modpy.py
new file mode 100644
index 00000000..7c288d2c
--- /dev/null
+++ b/lib/cherrypy/test/modpy.py
@@ -0,0 +1,164 @@
+"""Wrapper for mod_python, for use as a CherryPy HTTP server when testing.
+
+To autostart modpython, the "apache" executable or script must be
+on your system path, or you must override the global APACHE_PATH.
+On some platforms, "apache" may be called "apachectl" or "apache2ctl"--
+create a symlink to them if needed.
+
+If you wish to test the WSGI interface instead of our _cpmodpy interface,
+you also need the 'modpython_gateway' module at:
+http://projects.amor.org/misc/wiki/ModPythonGateway
+
+
+KNOWN BUGS
+==========
+
+1. Apache processes Range headers automatically; CherryPy's truncated
+    output is then truncated again by Apache. See test_core.testRanges.
+    This was worked around in http://www.cherrypy.org/changeset/1319.
+2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
+    See test_core.testHTTPMethods.
+3. Max request header and body settings do not work with Apache.
+4. Apache replaces status "reason phrases" automatically. For example,
+    CherryPy may set "304 Not modified" but Apache will write out
+    "304 Not Modified" (capital "M").
+5. Apache does not allow custom error codes as per the spec.
+6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
+    Request-URI too early.
+7. mod_python will not read request bodies which use the "chunked"
+    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
+    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
+    mod_python's requestobject.c).
+8. Apache will output a "Content-Length: 0" response header even if there's
+    no response entity body. This isn't really a bug; it just differs from
+    the CherryPy default.
+"""
+
+import os
+import re
+
+import cherrypy
+from cherrypy.test import helper
+
+curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+
+def read_process(cmd, args=''):
+    pipein, pipeout = os.popen4('%s %s' % (cmd, args))
+    try:
+        firstline = pipeout.readline()
+        if (re.search(r'(not recognized|No such file|not found)', firstline,
+                      re.IGNORECASE)):
+            raise IOError('%s must be on your system path.' % cmd)
+        output = firstline + pipeout.read()
+    finally:
+        pipeout.close()
+    return output
+
+
+APACHE_PATH = 'httpd'
+CONF_PATH = 'test_mp.conf'
+
+conf_modpython_gateway = """
+# Apache2 server conf file for testing CherryPy with modpython_gateway.
+
+ServerName 127.0.0.1
+DocumentRoot "/"
+Listen %(port)s
+LoadModule python_module modules/mod_python.so
+
+SetHandler python-program
+PythonFixupHandler cherrypy.test.modpy::wsgisetup
+PythonOption testmod %(modulename)s
+PythonHandler modpython_gateway::handler
+PythonOption wsgi.application cherrypy::tree
+PythonOption socket_host %(host)s
+PythonDebug On
+"""
+
+conf_cpmodpy = """
+# Apache2 server conf file for testing CherryPy with _cpmodpy.
+
+ServerName 127.0.0.1
+DocumentRoot "/"
+Listen %(port)s
+LoadModule python_module modules/mod_python.so
+
+SetHandler python-program
+PythonFixupHandler cherrypy.test.modpy::cpmodpysetup
+PythonHandler cherrypy._cpmodpy::handler
+PythonOption cherrypy.setup cherrypy.test.%(modulename)s::setup_server
+PythonOption socket_host %(host)s
+PythonDebug On
+"""
+
+
+class ModPythonSupervisor(helper.Supervisor):
+
+    using_apache = True
+    using_wsgi = False
+    template = None
+
+    def __str__(self):
+        return 'ModPython Server on %s:%s' % (self.host, self.port)
+
+    def start(self, modulename):
+        mpconf = CONF_PATH
+        if not os.path.isabs(mpconf):
+            mpconf = os.path.join(curdir, mpconf)
+
+        f = open(mpconf, 'wb')
+        try:
+            f.write(self.template %
+                    {'port': self.port, 'modulename': modulename,
+                     'host': self.host})
+        finally:
+            f.close()
+
+        result = read_process(APACHE_PATH, '-k start -f %s' % mpconf)
+        if result:
+            print(result)
+
+    def stop(self):
+        """Gracefully shutdown a server that is serving forever."""
+        read_process(APACHE_PATH, '-k stop')
+
+
+loaded = False
+
+
+def wsgisetup(req):
+    global loaded
+    if not loaded:
+        loaded = True
+        options = req.get_options()
+
+        cherrypy.config.update({
+            'log.error_file': os.path.join(curdir, 'test.log'),
+            'environment': 'test_suite',
+            'server.socket_host': options['socket_host'],
+        })
+
+        modname = options['testmod']
+        mod = __import__(modname, globals(), locals(), [''])
+        mod.setup_server()
+
+        cherrypy.server.unsubscribe()
+        cherrypy.engine.start()
+    from mod_python import apache
+    return apache.OK
+
+
+def cpmodpysetup(req):
+    global loaded
+    if not loaded:
+        loaded = True
+        options = req.get_options()
+
+        cherrypy.config.update({
+            'log.error_file': os.path.join(curdir, 'test.log'),
+            'environment': 'test_suite',
+            'server.socket_host': options['socket_host'],
+        })
+    from mod_python import apache
+    return apache.OK
diff --git a/lib/cherrypy/test/modwsgi.py b/lib/cherrypy/test/modwsgi.py
new file mode 100644
index 00000000..f558e223
--- /dev/null
+++ b/lib/cherrypy/test/modwsgi.py
@@ -0,0 +1,154 @@
+"""Wrapper for mod_wsgi, for use as a CherryPy HTTP server.
+
+To autostart modwsgi, the "apache" executable or script must be
+on your system path, or you must override the global APACHE_PATH.
+On some platforms, "apache" may be called "apachectl" or "apache2ctl"--
+create a symlink to them if needed.
+
+
+KNOWN BUGS
+==========
+
+##1. Apache processes Range headers automatically; CherryPy's truncated
+##    output is then truncated again by Apache. See test_core.testRanges.
+##    This was worked around in http://www.cherrypy.org/changeset/1319.
+2. Apache does not allow custom HTTP methods like CONNECT as per the spec.
+    See test_core.testHTTPMethods.
+3. Max request header and body settings do not work with Apache.
+##4. Apache replaces status "reason phrases" automatically. For example,
+##    CherryPy may set "304 Not modified" but Apache will write out
+##    "304 Not Modified" (capital "M").
+##5. Apache does not allow custom error codes as per the spec.
+##6. Apache (or perhaps modpython, or modpython_gateway) unquotes %xx in the
+##    Request-URI too early.
+7. mod_wsgi will not read request bodies which use the "chunked"
+    transfer-coding (it passes REQUEST_CHUNKED_ERROR to ap_setup_client_block
+    instead of REQUEST_CHUNKED_DECHUNK, see Apache2's http_protocol.c and
+    mod_python's requestobject.c).
+8. When responding with 204 No Content, mod_wsgi adds a Content-Length
+    header for you.
+9. When an error is raised, mod_wsgi has no facility for printing a
+    traceback as the response content (it's sent to the Apache log instead).
+10. Startup and shutdown of Apache when running mod_wsgi seems slow.
+"""
+
+import os
+import re
+import sys
+import time
+
+import portend
+
+from cheroot.test import webtest
+
+import cherrypy
+from cherrypy.test import helper
+
+curdir = os.path.abspath(os.path.dirname(__file__))
+
+
+def read_process(cmd, args=''):
+    pipein, pipeout = os.popen4('%s %s' % (cmd, args))
+    try:
+        firstline = pipeout.readline()
+        if (re.search(r'(not recognized|No such file|not found)', firstline,
+                      re.IGNORECASE)):
+            raise IOError('%s must be on your system path.' % cmd)
+        output = firstline + pipeout.read()
+    finally:
+        pipeout.close()
+    return output
+
+
+if sys.platform == 'win32':
+    APACHE_PATH = 'httpd'
+else:
+    APACHE_PATH = 'apache'
+
+CONF_PATH = 'test_mw.conf'
+
+conf_modwsgi = r"""
+# Apache2 server conf file for testing CherryPy with modpython_gateway.
+
+ServerName 127.0.0.1
+DocumentRoot "/"
+Listen %(port)s
+
+AllowEncodedSlashes On
+LoadModule rewrite_module modules/mod_rewrite.so
+RewriteEngine on
+RewriteMap escaping int:escape
+
+LoadModule log_config_module modules/mod_log_config.so
+LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-agent}i\"" combined
+CustomLog "%(curdir)s/apache.access.log" combined
+ErrorLog "%(curdir)s/apache.error.log"
+LogLevel debug
+
+LoadModule wsgi_module modules/mod_wsgi.so
+LoadModule env_module modules/mod_env.so
+
+WSGIScriptAlias / "%(curdir)s/modwsgi.py"
+SetEnv testmod %(testmod)s
+""" # noqa E501
+
+
+class ModWSGISupervisor(helper.Supervisor):
+
+    """Server Controller for ModWSGI and CherryPy."""
+
+    using_apache = True
+    using_wsgi = True
+    template = conf_modwsgi
+
+    def __str__(self):
+        return 'ModWSGI Server on %s:%s' % (self.host, self.port)
+
+    def start(self, modulename):
+        mpconf = CONF_PATH
+        if not os.path.isabs(mpconf):
+            mpconf = os.path.join(curdir, mpconf)
+
+        f = open(mpconf, 'wb')
+        try:
+            output = (self.template %
+                      {'port': self.port, 'testmod': modulename,
+                       'curdir': curdir})
+            f.write(output)
+        finally:
+            f.close()
+
+        result = read_process(APACHE_PATH, '-k start -f %s' % mpconf)
+        if result:
+            print(result)
+
+        # Make a request so mod_wsgi starts up our app.
+        # If we don't, concurrent initial requests will 404.
+        portend.occupied('127.0.0.1', self.port, timeout=5)
+        webtest.openURL('/ihopetheresnodefault', port=self.port)
+        time.sleep(1)
+
+    def stop(self):
+        """Gracefully shutdown a server that is serving forever."""
+        read_process(APACHE_PATH, '-k stop')
+
+
+loaded = False
+
+
+def application(environ, start_response):
+    global loaded
+    if not loaded:
+        loaded = True
+        modname = 'cherrypy.test.' + environ['testmod']
+        mod = __import__(modname, globals(), locals(), [''])
+        mod.setup_server()
+
+        cherrypy.config.update({
+            'log.error_file': os.path.join(curdir, 'test.error.log'),
+            'log.access_file': os.path.join(curdir, 'test.access.log'),
+            'environment': 'test_suite',
+            'engine.SIGHUP': None,
+            'engine.SIGTERM': None,
+        })
+    return cherrypy.tree(environ, start_response)
diff --git a/lib/cherrypy/test/sessiondemo.py b/lib/cherrypy/test/sessiondemo.py
new file mode 100644
index 00000000..8226c1b9
--- /dev/null
+++ b/lib/cherrypy/test/sessiondemo.py
@@ -0,0 +1,161 @@
+#!/usr/bin/python
+"""A session demonstration app."""
+
+import calendar
+from datetime import datetime
+import sys
+
+import six
+
+import cherrypy
+from cherrypy.lib import sessions
+
+
+page = """
+
+
+
+
+
+
+
+
Session Demo

+
Reload this page. The session ID should not change from one reload to the next

+
Index | Expire | Regenerate

+
+    
+    
+    
+    
+    
+    
+    
+    
+
Session ID:%(sessionid)s
%(changemsg)s
Request Cookie%(reqcookie)s
Response Cookie%(respcookie)s
Session Data%(sessiondata)s
Server Time%(servertime)s (Unix time: %(serverunixtime)s)
Browser Time 
Cherrypy Version:%(cpversion)s
Python Version:%(pyversion)s

+
+"""  # noqa E501
+
+
+class Root(object):
+
+    def page(self):
+        changemsg = []
+        if cherrypy.session.id != cherrypy.session.originalid:
+            if cherrypy.session.originalid is None:
+                changemsg.append(
+                    'Created new session because no session id was given.')
+            if cherrypy.session.missing:
+                changemsg.append(
+                    'Created new session due to missing '
+                    '(expired or malicious) session.')
+            if cherrypy.session.regenerated:
+                changemsg.append('Application generated a new session.')
+
+        try:
+            expires = cherrypy.response.cookie['session_id']['expires']
+        except KeyError:
+            expires = ''
+
+        return page % {
+            'sessionid': cherrypy.session.id,
+            'changemsg': '
'.join(changemsg),
+            'respcookie': cherrypy.response.cookie.output(),
+            'reqcookie': cherrypy.request.cookie.output(),
+            'sessiondata': list(six.iteritems(cherrypy.session)),
+            'servertime': (
+                datetime.utcnow().strftime('%Y/%m/%d %H:%M') + ' UTC'
+            ),
+            'serverunixtime': calendar.timegm(datetime.utcnow().timetuple()),
+            'cpversion': cherrypy.__version__,
+            'pyversion': sys.version,
+            'expires': expires,
+        }
+
+    @cherrypy.expose
+    def index(self):
+        # Must modify data or the session will not be saved.
+        cherrypy.session['color'] = 'green'
+        return self.page()
+
+    @cherrypy.expose
+    def expire(self):
+        sessions.expire()
+        return self.page()
+
+    @cherrypy.expose
+    def regen(self):
+        cherrypy.session.regenerate()
+        # Must modify data or the session will not be saved.
+        cherrypy.session['color'] = 'yellow'
+        return self.page()
+
+
+if __name__ == '__main__':
+    cherrypy.config.update({
+        # 'environment': 'production',
+        'log.screen': True,
+        'tools.sessions.on': True,
+    })
+    cherrypy.quickstart(Root())
diff --git a/lib/cherrypy/test/static/404.html b/lib/cherrypy/test/static/404.html
new file mode 100644
index 00000000..01b17b09
--- /dev/null
+++ b/lib/cherrypy/test/static/404.html
@@ -0,0 +1,5 @@
+
+ 
+  
I couldn't find that thing you were looking for!

+ 
+
diff --git a/lib/cherrypy/test/static/dirback.jpg b/lib/cherrypy/test/static/dirback.jpg
new file mode 100644
index 00000000..80403dc2
Binary files /dev/null and b/lib/cherrypy/test/static/dirback.jpg differ
diff --git a/lib/cherrypy/test/static/index.html b/lib/cherrypy/test/static/index.html
new file mode 100644
index 00000000..a5c19667
--- /dev/null
+++ b/lib/cherrypy/test/static/index.html
@@ -0,0 +1 @@
+Hello, world
diff --git a/lib/cherrypy/test/style.css b/lib/cherrypy/test/style.css
new file mode 100644
index 00000000..b266e93d
--- /dev/null
+++ b/lib/cherrypy/test/style.css
@@ -0,0 +1 @@
+Dummy stylesheet
diff --git a/lib/cherrypy/test/test.pem b/lib/cherrypy/test/test.pem
new file mode 100644
index 00000000..47a47042
--- /dev/null
+++ b/lib/cherrypy/test/test.pem
@@ -0,0 +1,38 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQDBKo554mzIMY+AByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZ
+R9L4WtImEew05FY3Izerfm3MN3+MC0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Kn
+da+O6xldVSosu8Ev3z9VZ94iC/ZgKzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQAB
+AoGAWOCF0ZrWxn3XMucWq2LNwPKqlvVGwbIwX3cDmX22zmnM4Fy6arXbYh4XlyCj
+9+ofqRrxIFz5k/7tFriTmZ0xag5+Jdx+Kwg0/twiP7XCNKipFogwe1Hznw8OFAoT
+enKBdj2+/n2o0Bvo/tDB59m9L/538d46JGQUmJlzMyqYikECQQDyoq+8CtMNvE18
+8VgHcR/KtApxWAjj4HpaHYL637ATjThetUZkW92mgDgowyplthusxdNqhHWyv7E8
+tWNdYErZAkEAy85ShTR0M5aWmrE7o0r0SpWInAkNBH9aXQRRARFYsdBtNfRu6I0i
+0lvU9wiu3eF57FMEC86yViZ5UBnQfTu7vQJAVesj/Zt7pwaCDfdMa740OsxMUlyR
+MVhhGx4OLpYdPJ8qUecxGQKq13XZ7R1HGyNEY4bd2X80Smq08UFuATfC6QJAH8UB
+yBHtKz2GLIcELOg6PIYizW/7v3+6rlVF60yw7sb2vzpjL40QqIn4IKoR2DSVtOkb
+8FtAIX3N21aq0VrGYQJBAIPiaEc2AZ8Bq2GC4F3wOz/BxJ/izvnkiotR12QK4fh5
+yjZMhTjWCas5zwHR5PDjlD88AWGDMsZ1PicD4348xJQ=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDxTCCAy6gAwIBAgIJAI18BD7eQxlGMA0GCSqGSIb3DQEBBAUAMIGeMQswCQYD
+VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJU2FuIERpZWdv
+MRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0MREwDwYDVQQLEwhkZXYtdGVzdDEW
+MBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4GCSqGSIb3DQEJARYRcmVtaUBjaGVy
+cnlweS5vcmcwHhcNMDYwOTA5MTkyMDIwWhcNMzQwMTI0MTkyMDIwWjCBnjELMAkG
+A1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVNhbiBEaWVn
+bzEZMBcGA1UEChMQQ2hlcnJ5UHkgUHJvamVjdDERMA8GA1UECxMIZGV2LXRlc3Qx
+FjAUBgNVBAMTDUNoZXJyeVB5IFRlYW0xIDAeBgkqhkiG9w0BCQEWEXJlbWlAY2hl
+cnJ5cHkub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDBKo554mzIMY+A
+ByUNpaUOP9bJnQ7ZLQe9XgHwoLJR4VzpyZZZR9L4WtImEew05FY3Izerfm3MN3+M
+C0tJ6yQU9sOiU3vBW6RrLIMlfKsnRwBRZ0Knda+O6xldVSosu8Ev3z9VZ94iC/Zg
+KzrH7Mjj/U8/MQO7RBS/LAqee8bFNQIDAQABo4IBBzCCAQMwHQYDVR0OBBYEFDIQ
+2feb71tVZCWpU0qJ/Tw+wdtoMIHTBgNVHSMEgcswgciAFDIQ2feb71tVZCWpU0qJ
+/Tw+wdtooYGkpIGhMIGeMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTESMBAGA1UEBxMJU2FuIERpZWdvMRkwFwYDVQQKExBDaGVycnlQeSBQcm9qZWN0
+MREwDwYDVQQLEwhkZXYtdGVzdDEWMBQGA1UEAxMNQ2hlcnJ5UHkgVGVhbTEgMB4G
+CSqGSIb3DQEJARYRcmVtaUBjaGVycnlweS5vcmeCCQCNfAQ+3kMZRjAMBgNVHRME
+BTADAQH/MA0GCSqGSIb3DQEBBAUAA4GBAL7AAQz7IePV48ZTAFHKr88ntPALsL5S
+8vHCZPNMevNkLTj3DYUw2BcnENxMjm1kou2F2BkvheBPNZKIhc6z4hAml3ed1xa2
+D7w6e6OTcstdK/+KrPDDHeOP1dhMWNs2JE1bNlfF1LiXzYKSXpe88eCKjCXsCT/T
+NluCaWQys3MS
+-----END CERTIFICATE-----
diff --git a/lib/cherrypy/test/test_auth_basic.py b/lib/cherrypy/test/test_auth_basic.py
new file mode 100644
index 00000000..d7e69a9b
--- /dev/null
+++ b/lib/cherrypy/test/test_auth_basic.py
@@ -0,0 +1,135 @@
+# This file is part of CherryPy 
+# -*- coding: utf-8 -*-
+# vim:ts=4:sw=4:expandtab:fileencoding=utf-8
+
+from hashlib import md5
+
+import cherrypy
+from cherrypy._cpcompat import ntob
+from cherrypy.lib import auth_basic
+from cherrypy.test import helper
+
+
+class BasicAuthTest(helper.CPWebCase):
+
+    @staticmethod
+    def setup_server():
+        class Root:
+
+            @cherrypy.expose
+            def index(self):
+                return 'This is public.'
+
+        class BasicProtected:
+
+            @cherrypy.expose
+            def index(self):
+                return "Hello %s, you've been authorized." % (
+                    cherrypy.request.login)
+
+        class BasicProtected2:
+
+            @cherrypy.expose
+            def index(self):
+                return "Hello %s, you've been authorized." % (
+                    cherrypy.request.login)
+
+        class BasicProtected2_u:
+
+            @cherrypy.expose
+            def index(self):
+                return "Hello %s, you've been authorized." % (
+                    cherrypy.request.login)
+
+        userpassdict = {'xuser': 'xpassword'}
+        userhashdict = {'xuser': md5(b'xpassword').hexdigest()}
+        userhashdict_u = {'xюзер': md5(ntob('їжа', 'utf-8')).hexdigest()}
+
+        def checkpasshash(realm, user, password):
+            p = userhashdict.get(user)
+            return p and p == md5(ntob(password)).hexdigest() or False
+
+        def checkpasshash_u(realm, user, password):
+            p = userhashdict_u.get(user)
+            return p and p == md5(ntob(password, 'utf-8')).hexdigest() or False
+
+        basic_checkpassword_dict = auth_basic.checkpassword_dict(userpassdict)
+        conf = {
+            '/basic': {
+                'tools.auth_basic.on': True,
+                'tools.auth_basic.realm': 'wonderland',
+                'tools.auth_basic.checkpassword': basic_checkpassword_dict
+            },
+            '/basic2': {
+                'tools.auth_basic.on': True,
+                'tools.auth_basic.realm': 'wonderland',
+                'tools.auth_basic.checkpassword': checkpasshash,
+                'tools.auth_basic.accept_charset': 'ISO-8859-1',
+            },
+            '/basic2_u': {
+                'tools.auth_basic.on': True,
+                'tools.auth_basic.realm': 'wonderland',
+                'tools.auth_basic.checkpassword': checkpasshash_u,
+                'tools.auth_basic.accept_charset': 'UTF-8',
+            },
+        }
+
+        root = Root()
+        root.basic = BasicProtected()
+        root.basic2 = BasicProtected2()
+        root.basic2_u = BasicProtected2_u()
+        cherrypy.tree.mount(root, config=conf)
+
+    def testPublic(self):
+        self.getPage('/')
+        self.assertStatus('200 OK')
+        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
+        self.assertBody('This is public.')
+
+    def testBasic(self):
+        self.getPage('/basic/')
+        self.assertStatus(401)
+        self.assertHeader(
+            'WWW-Authenticate',
+            'Basic realm="wonderland", charset="UTF-8"'
+        )
+
+        self.getPage('/basic/',
+                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')])
+        self.assertStatus(401)
+
+        self.getPage('/basic/',
+                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')])
+        self.assertStatus('200 OK')
+        self.assertBody("Hello xuser, you've been authorized.")
+
+    def testBasic2(self):
+        self.getPage('/basic2/')
+        self.assertStatus(401)
+        self.assertHeader('WWW-Authenticate', 'Basic realm="wonderland"')
+
+        self.getPage('/basic2/',
+                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3JX')])
+        self.assertStatus(401)
+
+        self.getPage('/basic2/',
+                     [('Authorization', 'Basic eHVzZXI6eHBhc3N3b3Jk')])
+        self.assertStatus('200 OK')
+        self.assertBody("Hello xuser, you've been authorized.")
+
+    def testBasic2_u(self):
+        self.getPage('/basic2_u/')
+        self.assertStatus(401)
+        self.assertHeader(
+            'WWW-Authenticate',
+            'Basic realm="wonderland", charset="UTF-8"'
+        )
+
+        self.getPage('/basic2_u/',
+                     [('Authorization', 'Basic eNGO0LfQtdGAOtGX0LbRgw==')])
+        self.assertStatus(401)
+
+        self.getPage('/basic2_u/',
+                     [('Authorization', 'Basic eNGO0LfQtdGAOtGX0LbQsA==')])
+        self.assertStatus('200 OK')
+        self.assertBody("Hello xюзер, you've been authorized.")
diff --git a/lib/cherrypy/test/test_auth_digest.py b/lib/cherrypy/test/test_auth_digest.py
new file mode 100644
index 00000000..512e39a5
--- /dev/null
+++ b/lib/cherrypy/test/test_auth_digest.py
@@ -0,0 +1,134 @@
+# This file is part of CherryPy 
+# -*- coding: utf-8 -*-
+# vim:ts=4:sw=4:expandtab:fileencoding=utf-8
+
+import six
+
+
+import cherrypy
+from cherrypy.lib import auth_digest
+from cherrypy._cpcompat import ntob
+
+from cherrypy.test import helper
+
+
+def _fetch_users():
+    return {'test': 'test', '☃йюзер': 'їпароль'}
+
+
+get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(_fetch_users())
+
+
+class DigestAuthTest(helper.CPWebCase):
+
+    @staticmethod
+    def setup_server():
+        class Root:
+
+            @cherrypy.expose
+            def index(self):
+                return 'This is public.'
+
+        class DigestProtected:
+
+            @cherrypy.expose
+            def index(self, *args, **kwargs):
+                return "Hello %s, you've been authorized." % (
+                    cherrypy.request.login)
+
+        conf = {'/digest': {'tools.auth_digest.on': True,
+                            'tools.auth_digest.realm': 'localhost',
+                            'tools.auth_digest.get_ha1': get_ha1,
+                            'tools.auth_digest.key': 'a565c27146791cfb',
+                            'tools.auth_digest.debug': True,
+                            'tools.auth_digest.accept_charset': 'UTF-8'}}
+
+        root = Root()
+        root.digest = DigestProtected()
+        cherrypy.tree.mount(root, config=conf)
+
+    def testPublic(self):
+        self.getPage('/')
+        assert self.status == '200 OK'
+        self.assertHeader('Content-Type', 'text/html;charset=utf-8')
+        assert self.body == b'This is public.'
+
+    def _test_parametric_digest(self, username, realm):
+        test_uri = '/digest/?@/=%2F%40&%f0%9f%99%88=path'
+
+        self.getPage(test_uri)
+        assert self.status_code == 401
+
+        msg = 'Digest authentification scheme was not found'
+        www_auth_digest = tuple(filter(
+            lambda kv: kv[0].lower() == 'www-authenticate'
+            and kv[1].startswith('Digest '),
+            self.headers,
+        ))
+        assert len(www_auth_digest) == 1, msg
+
+        items = www_auth_digest[0][-1][7:].split(', ')
+        tokens = {}
+        for item in items:
+            key, value = item.split('=')
+            tokens[key.lower()] = value
+
+        assert tokens['realm'] == '"localhost"'
+        assert tokens['algorithm'] == '"MD5"'
+        assert tokens['qop'] == '"auth"'
+        assert tokens['charset'] == '"UTF-8"'
+
+        nonce = tokens['nonce'].strip('"')
+
+        # Test user agent response with a wrong value for 'realm'
+        base_auth = ('Digest username="%s", '
+                     'realm="%s", '
+                     'nonce="%s", '
+                     'uri="%s", '
+                     'algorithm=MD5, '
+                     'response="%s", '
+                     'qop=auth, '
+                     'nc=%s, '
+                     'cnonce="1522e61005789929"')
+
+        encoded_user = username
+        if six.PY3:
+            encoded_user = encoded_user.encode('utf-8')
+        encoded_user = encoded_user.decode('latin1')
+        auth_header = base_auth % (
+            encoded_user, realm, nonce, test_uri,
+            '11111111111111111111111111111111', '00000001',
+        )
+        auth = auth_digest.HttpDigestAuthorization(auth_header, 'GET')
+        # calculate the response digest
+        ha1 = get_ha1(auth.realm, auth.username)
+        response = auth.request_digest(ha1)
+        auth_header = base_auth % (
+            encoded_user, realm, nonce, test_uri,
+            response, '00000001',
+        )
+        self.getPage(test_uri, [('Authorization', auth_header)])
+
+    def test_wrong_realm(self):
+        # send response with correct response digest, but wrong realm
+        self._test_parametric_digest(username='test', realm='wrong realm')
+        assert self.status_code == 401
+
+    def test_ascii_user(self):
+        self._test_parametric_digest(username='test', realm='localhost')
+        assert self.status == '200 OK'
+        assert self.body == b"Hello test, you've been authorized."
+
+    def test_unicode_user(self):
+        self._test_parametric_digest(username='☃йюзер', realm='localhost')
+        assert self.status == '200 OK'
+        assert self.body == ntob(
+            "Hello ☃йюзер, you've been authorized.", 'utf-8',
+        )
+
+    def test_wrong_scheme(self):
+        basic_auth = {
+            'Authorization': 'Basic foo:bar',
+        }
+        self.getPage('/digest/', headers=list(basic_auth.items()))
+        assert self.status_code == 401
diff --git a/lib/cherrypy/test/test_bus.py b/lib/cherrypy/test/test_bus.py
new file mode 100644
index 00000000..6026b47e
--- /dev/null
+++ b/lib/cherrypy/test/test_bus.py
@@ -0,0 +1,274 @@
+import threading
+import time
+import unittest
+
+from cherrypy.process import wspbus
+
+
+msg = 'Listener %d on channel %s: %s.'
+
+
+class PublishSubscribeTests(unittest.TestCase):
+
+    def get_listener(self, channel, index):
+        def listener(arg=None):
+            self.responses.append(msg % (index, channel, arg))
+        return listener
+
+    def test_builtin_channels(self):
+        b = wspbus.Bus()
+
+        self.responses, expected = [], []
+
+        for channel in b.listeners:
+            for index, priority in enumerate([100, 50, 0, 51]):
+                b.subscribe(channel,
+                            self.get_listener(channel, index), priority)
+
+        for channel in b.listeners:
+            b.publish(channel)
+            expected.extend([msg % (i, channel, None) for i in (2, 1, 3, 0)])
+            b.publish(channel, arg=79347)
+            expected.extend([msg % (i, channel, 79347) for i in (2, 1, 3, 0)])
+
+        self.assertEqual(self.responses, expected)
+
+    def test_custom_channels(self):
+        b = wspbus.Bus()
+
+        self.responses, expected = [], []
+
+        custom_listeners = ('hugh', 'louis', 'dewey')
+        for channel in custom_listeners:
+            for index, priority in enumerate([None, 10, 60, 40]):
+                b.subscribe(channel,
+                            self.get_listener(channel, index), priority)
+
+        for channel in custom_listeners:
+            b.publish(channel, 'ah so')
+            expected.extend([msg % (i, channel, 'ah so')
+                            for i in (1, 3, 0, 2)])
+            b.publish(channel)
+            expected.extend([msg % (i, channel, None) for i in (1, 3, 0, 2)])
+
+        self.assertEqual(self.responses, expected)
+
+    def test_listener_errors(self):
+        b = wspbus.Bus()
+
+        self.responses, expected = [], []
+        channels = [c for c in b.listeners if c != 'log']
+
+        for channel in channels:
+            b.subscribe(channel, self.get_listener(channel, 1))
+            # This will break since the lambda takes no args.
+            b.subscribe(channel, lambda: None, priority=20)
+
+        for channel in channels:
+            self.assertRaises(wspbus.ChannelFailures, b.publish, channel, 123)
+            expected.append(msg % (1, channel, 123))
+
+        self.assertEqual(self.responses, expected)
+
+
+class BusMethodTests(unittest.TestCase):
+
+    def log(self, bus):
+        self._log_entries = []
+
+        def logit(msg, level):
+            self._log_entries.append(msg)
+        bus.subscribe('log', logit)
+
+    def assertLog(self, entries):
+        self.assertEqual(self._log_entries, entries)
+
+    def get_listener(self, channel, index):
+        def listener(arg=None):
+            self.responses.append(msg % (index, channel, arg))
+        return listener
+
+    def test_start(self):
+        b = wspbus.Bus()
+        self.log(b)
+
+        self.responses = []
+        num = 3
+        for index in range(num):
+            b.subscribe('start', self.get_listener('start', index))
+
+        b.start()
+        try:
+            # The start method MUST call all 'start' listeners.
+            self.assertEqual(
+                set(self.responses),
+                set([msg % (i, 'start', None) for i in range(num)]))
+            # The start method MUST move the state to STARTED
+            # (or EXITING, if errors occur)
+            self.assertEqual(b.state, b.states.STARTED)
+            # The start method MUST log its states.
+            self.assertLog(['Bus STARTING', 'Bus STARTED'])
+        finally:
+            # Exit so the atexit handler doesn't complain.
+            b.exit()
+
+    def test_stop(self):
+        b = wspbus.Bus()
+        self.log(b)
+
+        self.responses = []
+        num = 3
+        for index in range(num):
+            b.subscribe('stop', self.get_listener('stop', index))
+
+        b.stop()
+
+        # The stop method MUST call all 'stop' listeners.
+        self.assertEqual(set(self.responses),
+                         set([msg % (i, 'stop', None) for i in range(num)]))
+        # The stop method MUST move the state to STOPPED
+        self.assertEqual(b.state, b.states.STOPPED)
+        # The stop method MUST log its states.
+        self.assertLog(['Bus STOPPING', 'Bus STOPPED'])
+
+    def test_graceful(self):
+        b = wspbus.Bus()
+        self.log(b)
+
+        self.responses = []
+        num = 3
+        for index in range(num):
+            b.subscribe('graceful', self.get_listener('graceful', index))
+
+        b.graceful()
+
+        # The graceful method MUST call all 'graceful' listeners.
+        self.assertEqual(
+            set(self.responses),
+            set([msg % (i, 'graceful', None) for i in range(num)]))
+        # The graceful method MUST log its states.
+        self.assertLog(['Bus graceful'])
+
+    def test_exit(self):
+        b = wspbus.Bus()
+        self.log(b)
+
+        self.responses = []
+        num = 3
+        for index in range(num):
+            b.subscribe('stop', self.get_listener('stop', index))
+            b.subscribe('exit', self.get_listener('exit', index))
+
+        b.exit()
+
+        # The exit method MUST call all 'stop' listeners,
+        # and then all 'exit' listeners.
+        self.assertEqual(set(self.responses),
+                         set([msg % (i, 'stop', None) for i in range(num)] +
+                             [msg % (i, 'exit', None) for i in range(num)]))
+        # The exit method MUST move the state to EXITING
+        self.assertEqual(b.state, b.states.EXITING)
+        # The exit method MUST log its states.
+        self.assertLog(
+            ['Bus STOPPING', 'Bus STOPPED', 'Bus EXITING', 'Bus EXITED'])
+
+    def test_wait(self):
+        b = wspbus.Bus()
+
+        def f(method):
+            time.sleep(0.2)
+            getattr(b, method)()
+
+        for method, states in [('start', [b.states.STARTED]),
+                               ('stop', [b.states.STOPPED]),
+                               ('start',
+                                [b.states.STARTING, b.states.STARTED]),
+                               ('exit', [b.states.EXITING]),
+                               ]:
+            threading.Thread(target=f, args=(method,)).start()
+            b.wait(states)
+
+            # The wait method MUST wait for the given state(s).
+            if b.state not in states:
+                self.fail('State %r not in %r' % (b.state, states))
+
+    def test_block(self):
+        b = wspbus.Bus()
+        self.log(b)
+
+        def f():
+            time.sleep(0.2)
+            b.exit()
+
+        def g():
+            time.sleep(0.4)
+        threading.Thread(target=f).start()
+        threading.Thread(target=g).start()
+        threads = [t for t in threading.enumerate() if not t.daemon]
+        self.assertEqual(len(threads), 3)
+
+        b.block()
+
+        # The block method MUST wait for the EXITING state.
+        self.assertEqual(b.state, b.states.EXITING)
+        # The block method MUST wait for ALL non-main, non-daemon threads to
+        # finish.
+        threads = [t for t in threading.enumerate() if not t.daemon]
+        self.assertEqual(len(threads), 1)
+        # The last message will mention an indeterminable thread name; ignore
+        # it
+        self.assertEqual(self._log_entries[:-1],
+                         ['Bus STOPPING', 'Bus STOPPED',
+                          'Bus EXITING', 'Bus EXITED',
+                          'Waiting for child threads to terminate...'])
+
+    def test_start_with_callback(self):
+        b = wspbus.Bus()
+        self.log(b)
+        try:
+            events = []
+
+            def f(*args, **kwargs):
+                events.append(('f', args, kwargs))
+
+            def g():
+                events.append('g')
+            b.subscribe('start', g)
+            b.start_with_callback(f, (1, 3, 5), {'foo': 'bar'})
+            # Give wait() time to run f()
+            time.sleep(0.2)
+
+            # The callback method MUST wait for the STARTED state.
+            self.assertEqual(b.state, b.states.STARTED)
+            # The callback method MUST run after all start methods.
+            self.assertEqual(events, ['g', ('f', (1, 3, 5), {'foo': 'bar'})])
+        finally:
+            b.exit()
+
+    def test_log(self):
+        b = wspbus.Bus()
+        self.log(b)
+        self.assertLog([])
+
+        # Try a normal message.
+        expected = []
+        for msg in ["O mah darlin'"] * 3 + ['Clementiiiiiiiine']:
+            b.log(msg)
+            expected.append(msg)
+            self.assertLog(expected)
+
+        # Try an error message
+        try:
+            foo
+        except NameError:
+            b.log('You are lost and gone forever', traceback=True)
+            lastmsg = self._log_entries[-1]
+            if 'Traceback' not in lastmsg or 'NameError' not in lastmsg:
+                self.fail('Last log message %r did not contain '
+                          'the expected traceback.' % lastmsg)
+        else:
+            self.fail('NameError was not raised as expected.')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/lib/cherrypy/test/test_caching.py b/lib/cherrypy/test/test_caching.py
new file mode 100644
index 00000000..1a6ed4f2
--- /dev/null
+++ b/lib/cherrypy/test/test_caching.py
@@ -0,0 +1,392 @@
+import datetime
+from itertools import count
+import os
+import threading
+import time
+
+from six.moves import range
+from six.moves import urllib
+
+import pytest
+
+import cherrypy
+from cherrypy.lib import httputil
+
+from cherrypy.test import helper
+
+
+curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+gif_bytes = (
+    b'GIF89a\x01\x00\x01\x00\x82\x00\x01\x99"\x1e\x00\x00\x00\x00\x00'
+    b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+    b'\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x02\x03\x02\x08\t\x00;'
+)
+
+
+class CacheTest(helper.CPWebCase):
+
+    @staticmethod
+    def setup_server():
+
+        @cherrypy.config(**{'tools.caching.on': True})
+        class Root:
+
+            def __init__(self):
+                self.counter = 0
+                self.control_counter = 0
+                self.longlock = threading.Lock()
+
+            @cherrypy.expose
+            def index(self):
+                self.counter += 1
+                msg = 'visit #%s' % self.counter
+                return msg
+
+            @cherrypy.expose
+            def control(self):
+                self.control_counter += 1
+                return 'visit #%s' % self.control_counter
+
+            @cherrypy.expose
+            def a_gif(self):
+                cherrypy.response.headers[
+                    'Last-Modified'] = httputil.HTTPDate()
+                return gif_bytes
+
+            @cherrypy.expose
+            def long_process(self, seconds='1'):
+                try:
+                    self.longlock.acquire()
+                    time.sleep(float(seconds))
+                finally:
+                    self.longlock.release()
+                return 'success!'
+
+            @cherrypy.expose
+            def clear_cache(self, path):
+                cherrypy._cache.store[cherrypy.request.base + path].clear()
+
+        @cherrypy.config(**{
+            'tools.caching.on': True,
+            'tools.response_headers.on': True,
+            'tools.response_headers.headers': [
+                ('Vary', 'Our-Varying-Header')
+            ],
+        })
+        class VaryHeaderCachingServer(object):
+
+            def __init__(self):
+                self.counter = count(1)
+
+            @cherrypy.expose
+            def index(self):
+                return 'visit #%s' % next(self.counter)
+
+        @cherrypy.config(**{
+            'tools.expires.on': True,
+            'tools.expires.secs': 60,
+            'tools.staticdir.on': True,
+            'tools.staticdir.dir': 'static',
+            'tools.staticdir.root': curdir,
+        })
+        class UnCached(object):
+
+            @cherrypy.expose
+            @cherrypy.config(**{'tools.expires.secs': 0})
+            def force(self):
+                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
+                self._cp_config['tools.expires.force'] = True
+                self._cp_config['tools.expires.secs'] = 0
+                return 'being forceful'
+
+            @cherrypy.expose
+            def dynamic(self):
+                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
+                cherrypy.response.headers['Cache-Control'] = 'private'
+                return 'D-d-d-dynamic!'
+
+            @cherrypy.expose
+            def cacheable(self):
+                cherrypy.response.headers['Etag'] = 'bibbitybobbityboo'
+                return "Hi, I'm cacheable."
+
+            @cherrypy.expose
+            @cherrypy.config(**{'tools.expires.secs': 86400})
+            def specific(self):
+                cherrypy.response.headers[
+                    'Etag'] = 'need_this_to_make_me_cacheable'
+                return 'I am being specific'
+
+            class Foo(object):
+                pass
+
+            @cherrypy.expose
+            @cherrypy.config(**{'tools.expires.secs': Foo()})
+            def wrongtype(self):
+                cherrypy.response.headers[
+                    'Etag'] = 'need_this_to_make_me_cacheable'
+                return 'Woops'
+
+        @cherrypy.config(**{
+            'tools.gzip.mime_types': ['text/*', 'image/*'],
+            'tools.caching.on': True,
+            'tools.staticdir.on': True,
+            'tools.staticdir.dir': 'static',
+            'tools.staticdir.root': curdir
+        })
+        class GzipStaticCache(object):
+            pass
+
+        cherrypy.tree.mount(Root())
+        cherrypy.tree.mount(UnCached(), '/expires')
+        cherrypy.tree.mount(VaryHeaderCachingServer(), '/varying_headers')
+        cherrypy.tree.mount(GzipStaticCache(), '/gzip_static_cache')
+        cherrypy.config.update({'tools.gzip.on': True})
+
+    def testCaching(self):
+        elapsed = 0.0
+        for trial in range(10):
+            self.getPage('/')
+            # The response should be the same every time,
+            # except for the Age response header.
+            self.assertBody('visit #1')
+            if trial != 0:
+                age = int(self.assertHeader('Age'))
+                self.assert_(age >= elapsed)
+                elapsed = age
+
+        # POST, PUT, DELETE should not be cached.
+        self.getPage('/', method='POST')
+        self.assertBody('visit #2')
+        # Because gzip is turned on, the Vary header should always Vary for
+        # content-encoding
+        self.assertHeader('Vary', 'Accept-Encoding')
+        # The previous request should have invalidated the cache,
+        # so this request will recalc the response.
+        self.getPage('/', method='GET')
+        self.assertBody('visit #3')
+        # ...but this request should get the cached copy.
+        self.getPage('/', method='GET')
+        self.assertBody('visit #3')
+        self.getPage('/', method='DELETE')
+        self.assertBody('visit #4')
+
+        # The previous request should have invalidated the cache,
+        # so this request will recalc the response.
+        self.getPage('/', method='GET', headers=[('Accept-Encoding', 'gzip')])
+        self.assertHeader('Content-Encoding', 'gzip')
+        self.assertHeader('Vary')
+        self.assertEqual(
+            cherrypy.lib.encoding.decompress(self.body), b'visit #5')
+
+        # Now check that a second request gets the gzip header and gzipped body
+        # This also tests a bug in 3.0 to 3.0.2 whereby the cached, gzipped
+        # response body was being gzipped a second time.
+        self.getPage('/', method='GET', headers=[('Accept-Encoding', 'gzip')])
+        self.assertHeader('Content-Encoding', 'gzip')
+        self.assertEqual(
+            cherrypy.lib.encoding.decompress(self.body), b'visit #5')
+
+        # Now check that a third request that doesn't accept gzip
+        # skips the cache (because the 'Vary' header denies it).
+        self.getPage('/', method='GET')
+        self.assertNoHeader('Content-Encoding')
+        self.assertBody('visit #6')
+
+    def testVaryHeader(self):
+        self.getPage('/varying_headers/')
+        self.assertStatus('200 OK')
+        self.assertHeaderItemValue('Vary', 'Our-Varying-Header')
+        self.assertBody('visit #1')
+
+        # Now check that different 'Vary'-fields don't evict each other.
+        # This test creates 2 requests with different 'Our-Varying-Header'
+        # and then tests if the first one still exists.
+        self.getPage('/varying_headers/',
+                     headers=[('Our-Varying-Header', 'request 2')])
+        self.assertStatus('200 OK')
+        self.assertBody('visit #2')
+
+        self.getPage('/varying_headers/',
+                     headers=[('Our-Varying-Header', 'request 2')])
+        self.assertStatus('200 OK')
+        self.assertBody('visit #2')
+
+        self.getPage('/varying_headers/')
+        self.assertStatus('200 OK')
+        self.assertBody('visit #1')
+
+    def testExpiresTool(self):
+        # test setting an expires header
+        self.getPage('/expires/specific')
+        self.assertStatus('200 OK')
+        self.assertHeader('Expires')
+
+        # test exceptions for bad time values
+        self.getPage('/expires/wrongtype')
+        self.assertStatus(500)
+        self.assertInBody('TypeError')
+
+        # static content should not have "cache prevention" headers
+        self.getPage('/expires/index.html')
+        self.assertStatus('200 OK')
+        self.assertNoHeader('Pragma')
+        self.assertNoHeader('Cache-Control')
+        self.assertHeader('Expires')
+
+        # dynamic content that sets indicators should not have
+        # "cache prevention" headers
+        self.getPage('/expires/cacheable')
+        self.assertStatus('200 OK')
+        self.assertNoHeader('Pragma')
+        self.assertNoHeader('Cache-Control')
+        self.assertHeader('Expires')
+
+        self.getPage('/expires/dynamic')
+        self.assertBody('D-d-d-dynamic!')
+        # the Cache-Control header should be untouched
+        self.assertHeader('Cache-Control', 'private')
+        self.assertHeader('Expires')
+
+        # configure the tool to ignore indicators and replace existing headers
+        self.getPage('/expires/force')
+        self.assertStatus('200 OK')
+        # This also gives us a chance to test 0 expiry with no other headers
+        self.assertHeader('Pragma', 'no-cache')
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.assertHeader('Cache-Control', 'no-cache, must-revalidate')
+        self.assertHeader('Expires', 'Sun, 28 Jan 2007 00:00:00 GMT')
+
+        # static content should now have "cache prevention" headers
+        self.getPage('/expires/index.html')
+        self.assertStatus('200 OK')
+        self.assertHeader('Pragma', 'no-cache')
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.assertHeader('Cache-Control', 'no-cache, must-revalidate')
+        self.assertHeader('Expires', 'Sun, 28 Jan 2007 00:00:00 GMT')
+
+        # the cacheable handler should now have "cache prevention" headers
+        self.getPage('/expires/cacheable')
+        self.assertStatus('200 OK')
+        self.assertHeader('Pragma', 'no-cache')
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.assertHeader('Cache-Control', 'no-cache, must-revalidate')
+        self.assertHeader('Expires', 'Sun, 28 Jan 2007 00:00:00 GMT')
+
+        self.getPage('/expires/dynamic')
+        self.assertBody('D-d-d-dynamic!')
+        # dynamic sets Cache-Control to private but it should  be
+        # overwritten here ...
+        self.assertHeader('Pragma', 'no-cache')
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.assertHeader('Cache-Control', 'no-cache, must-revalidate')
+        self.assertHeader('Expires', 'Sun, 28 Jan 2007 00:00:00 GMT')
+
+    def _assert_resp_len_and_enc_for_gzip(self, uri):
+        """
+        Test that after querying gzipped content it's remains valid in
+        cache and available non-gzipped as well.
+        """
+        ACCEPT_GZIP_HEADERS = [('Accept-Encoding', 'gzip')]
+        content_len = None
+
+        for _ in range(3):
+            self.getPage(uri, method='GET', headers=ACCEPT_GZIP_HEADERS)
+
+            if content_len is not None:
+                # all requests should get the same length
+                self.assertHeader('Content-Length', content_len)
+                self.assertHeader('Content-Encoding', 'gzip')
+
+            content_len = dict(self.headers)['Content-Length']
+
+        # check that we can still get non-gzipped version
+        self.getPage(uri, method='GET')
+        self.assertNoHeader('Content-Encoding')
+        # non-gzipped version should have a different content length
+        self.assertNoHeaderItemValue('Content-Length', content_len)
+
+    def testGzipStaticCache(self):
+        """Test that cache and gzip tools play well together when both enabled.
+
+        Ref GitHub issue #1190.
+        """
+        GZIP_STATIC_CACHE_TMPL = '/gzip_static_cache/{}'
+        resource_files = ('index.html', 'dirback.jpg')
+
+        for f in resource_files:
+            uri = GZIP_STATIC_CACHE_TMPL.format(f)
+            self._assert_resp_len_and_enc_for_gzip(uri)
+
+    def testLastModified(self):
+        self.getPage('/a.gif')
+        self.assertStatus(200)
+        self.assertBody(gif_bytes)
+        lm1 = self.assertHeader('Last-Modified')
+
+        # this request should get the cached copy.
+        self.getPage('/a.gif')
+        self.assertStatus(200)
+        self.assertBody(gif_bytes)
+        self.assertHeader('Age')
+        lm2 = self.assertHeader('Last-Modified')
+        self.assertEqual(lm1, lm2)
+
+        # this request should match the cached copy, but raise 304.
+        self.getPage('/a.gif', [('If-Modified-Since', lm1)])
+        self.assertStatus(304)
+        self.assertNoHeader('Last-Modified')
+        if not getattr(cherrypy.server, 'using_apache', False):
+            self.assertHeader('Age')
+
+    @pytest.mark.xfail(reason='#1536')
+    def test_antistampede(self):
+        SECONDS = 4
+        slow_url = '/long_process?seconds={SECONDS}'.format(**locals())
+        # We MUST make an initial synchronous request in order to create the
+        # AntiStampedeCache object, and populate its selecting_headers,
+        # before the actual stampede.
+        self.getPage(slow_url)
+        self.assertBody('success!')
+        path = urllib.parse.quote(slow_url, safe='')
+        self.getPage('/clear_cache?path=' + path)
+        self.assertStatus(200)
+
+        start = datetime.datetime.now()
+
+        def run():
+            self.getPage(slow_url)
+            # The response should be the same every time
+            self.assertBody('success!')
+        ts = [threading.Thread(target=run) for i in range(100)]
+        for t in ts:
+            t.start()
+        for t in ts:
+            t.join()
+        finish = datetime.datetime.now()
+        # Allow for overhead, two seconds for slow hosts
+        allowance = SECONDS + 2
+        self.assertEqualDates(start, finish, seconds=allowance)
+
+    def test_cache_control(self):
+        self.getPage('/control')
+        self.assertBody('visit #1')
+        self.getPage('/control')
+        self.assertBody('visit #1')
+
+        self.getPage('/control', headers=[('Cache-Control', 'no-cache')])
+        self.assertBody('visit #2')
+        self.getPage('/control')
+        self.assertBody('visit #2')
+
+        self.getPage('/control', headers=[('Pragma', 'no-cache')])
+        self.assertBody('visit #3')
+        self.getPage('/control')
+        self.assertBody('visit #3')
+
+        time.sleep(1)
+        self.getPage('/control', headers=[('Cache-Control', 'max-age=0')])
+        self.assertBody('visit #4')
+        self.getPage('/control')
+        self.assertBody('visit #4')
diff --git a/lib/cherrypy/test/test_compat.py b/lib/cherrypy/test/test_compat.py
new file mode 100644
index 00000000..44a9fa31
--- /dev/null
+++ b/lib/cherrypy/test/test_compat.py
@@ -0,0 +1,34 @@
+"""Test Python 2/3 compatibility module."""
+from __future__ import unicode_literals
+
+import unittest
+
+import pytest
+import six
+
+from cherrypy import _cpcompat as compat
+
+
+class StringTester(unittest.TestCase):
+    """Tests for string conversion."""
+
+    @pytest.mark.skipif(six.PY3, reason='Only useful on Python 2')
+    def test_ntob_non_native(self):
+        """ntob should raise an Exception on unicode.
+
+        (Python 2 only)
+
+        See #1132 for discussion.
+        """
+        self.assertRaises(TypeError, compat.ntob, 'fight')
+
+
+class EscapeTester(unittest.TestCase):
+    """Class to test escape_html function from _cpcompat."""
+
+    def test_escape_quote(self):
+        """test_escape_quote - Verify the output for &<>"' chars."""
+        self.assertEqual(
+            """xx&<>"aa'""",
+            compat.escape_html("""xx&<>"aa'"""),
+        )
diff --git a/lib/cherrypy/test/test_config.py b/lib/cherrypy/test/test_config.py
new file mode 100644
index 00000000..be17df90
--- /dev/null
+++ b/lib/cherrypy/test/test_config.py
@@ -0,0 +1,303 @@
+"""Tests for the CherryPy configuration system."""
+
+import io
+import os
+import sys
+import unittest
+
+import six
+
+import cherrypy
+
+from cherrypy.test import helper
+
+
+localDir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+
+def StringIOFromNative(x):
+    return io.StringIO(six.text_type(x))
+
+
+def setup_server():
+
+    @cherrypy.config(foo='this', bar='that')
+    class Root:
+
+        def __init__(self):
+            cherrypy.config.namespaces['db'] = self.db_namespace
+
+        def db_namespace(self, k, v):
+            if k == 'scheme':
+                self.db = v
+
+        @cherrypy.expose(alias=('global_', 'xyz'))
+        def index(self, key):
+            return cherrypy.request.config.get(key, 'None')
+
+        @cherrypy.expose
+        def repr(self, key):
+            return repr(cherrypy.request.config.get(key, None))
+
+        @cherrypy.expose
+        def dbscheme(self):
+            return self.db
+
+        @cherrypy.expose
+        @cherrypy.config(**{'request.body.attempt_charsets': ['utf-16']})
+        def plain(self, x):
+            return x
+
+        favicon_ico = cherrypy.tools.staticfile.handler(
+            filename=os.path.join(localDir, '../favicon.ico'))
+
+    @cherrypy.config(foo='this2', baz='that2')
+    class Foo:
+
+        @cherrypy.expose
+        def index(self, key):
+            return cherrypy.request.config.get(key, 'None')
+        nex = index
+
+        @cherrypy.expose
+        @cherrypy.config(**{'response.headers.X-silly': 'sillyval'})
+        def silly(self):
+            return 'Hello world'
+
+        # Test the expose and config decorators
+        @cherrypy.config(foo='this3', **{'bax': 'this4'})
+        @cherrypy.expose
+        def bar(self, key):
+            return repr(cherrypy.request.config.get(key, None))
+
+    class Another:
+
+        @cherrypy.expose
+        def index(self, key):
+            return str(cherrypy.request.config.get(key, 'None'))
+
+    def raw_namespace(key, value):
+        if key == 'input.map':
+            handler = cherrypy.request.handler
+
+            def wrapper():
+                params = cherrypy.request.params
+                for name, coercer in list(value.items()):
+                    try:
+                        params[name] = coercer(params[name])
+                    except KeyError:
+                        pass
+                return handler()
+            cherrypy.request.handler = wrapper
+        elif key == 'output':
+            handler = cherrypy.request.handler
+
+            def wrapper():
+                # 'value' is a type (like int or str).
+                return value(handler())
+            cherrypy.request.handler = wrapper
+
+    @cherrypy.config(**{'raw.output': repr})
+    class Raw:
+
+        @cherrypy.expose
+        @cherrypy.config(**{'raw.input.map': {'num': int}})
+        def incr(self, num):
+            return num + 1
+
+    if not six.PY3:
+        thing3 = "thing3: unicode('test', errors='ignore')"
+    else:
+        thing3 = ''
+
+    ioconf = StringIOFromNative("""
+[/]
+neg: -1234
+filename: os.path.join(sys.prefix, "hello.py")
+thing1: cherrypy.lib.httputil.response_codes[404]
+thing2: __import__('cherrypy.tutorial', globals(), locals(), ['']).thing2
+%s
+complex: 3+2j
+mul: 6*3
+ones: "11"
+twos: "22"
+stradd: %%(ones)s + %%(twos)s + "33"
+
+[/favicon.ico]
+tools.staticfile.filename = %r
+""" % (thing3, os.path.join(localDir, 'static/dirback.jpg')))
+
+    root = Root()
+    root.foo = Foo()
+    root.raw = Raw()
+    app = cherrypy.tree.mount(root, config=ioconf)
+    app.request_class.namespaces['raw'] = raw_namespace
+
+    cherrypy.tree.mount(Another(), '/another')
+    cherrypy.config.update({'luxuryyacht': 'throatwobblermangrove',
+                            'db.scheme': r'sqlite///memory',
+                            })
+
+
+#                             Client-side code                             #
+
+
+class ConfigTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_server)
+
+    def testConfig(self):
+        tests = [
+            ('/', 'nex', 'None'),
+            ('/', 'foo', 'this'),
+            ('/', 'bar', 'that'),
+            ('/xyz', 'foo', 'this'),
+            ('/foo/', 'foo', 'this2'),
+            ('/foo/', 'bar', 'that'),
+            ('/foo/', 'bax', 'None'),
+            ('/foo/bar', 'baz', "'that2'"),
+            ('/foo/nex', 'baz', 'that2'),
+            # If 'foo' == 'this', then the mount point '/another' leaks into
+            # '/'.
+            ('/another/', 'foo', 'None'),
+        ]
+        for path, key, expected in tests:
+            self.getPage(path + '?key=' + key)
+            self.assertBody(expected)
+
+        expectedconf = {
+            # From CP defaults
+            'tools.log_headers.on': False,
+            'tools.log_tracebacks.on': True,
+            'request.show_tracebacks': True,
+            'log.screen': False,
+            'environment': 'test_suite',
+            'engine.autoreload.on': False,
+            # From global config
+            'luxuryyacht': 'throatwobblermangrove',
+            # From Root._cp_config
+            'bar': 'that',
+            # From Foo._cp_config
+            'baz': 'that2',
+            # From Foo.bar._cp_config
+            'foo': 'this3',
+            'bax': 'this4',
+        }
+        for key, expected in expectedconf.items():
+            self.getPage('/foo/bar?key=' + key)
+            self.assertBody(repr(expected))
+
+    def testUnrepr(self):
+        self.getPage('/repr?key=neg')
+        self.assertBody('-1234')
+
+        self.getPage('/repr?key=filename')
+        self.assertBody(repr(os.path.join(sys.prefix, 'hello.py')))
+
+        self.getPage('/repr?key=thing1')
+        self.assertBody(repr(cherrypy.lib.httputil.response_codes[404]))
+
+        if not getattr(cherrypy.server, 'using_apache', False):
+            # The object ID's won't match up when using Apache, since the
+            # server and client are running in different processes.
+            self.getPage('/repr?key=thing2')
+            from cherrypy.tutorial import thing2
+            self.assertBody(repr(thing2))
+
+        if not six.PY3:
+            self.getPage('/repr?key=thing3')
+            self.assertBody(repr(six.text_type('test')))
+
+        self.getPage('/repr?key=complex')
+        self.assertBody('(3+2j)')
+
+        self.getPage('/repr?key=mul')
+        self.assertBody('18')
+
+        self.getPage('/repr?key=stradd')
+        self.assertBody(repr('112233'))
+
+    def testRespNamespaces(self):
+        self.getPage('/foo/silly')
+        self.assertHeader('X-silly', 'sillyval')
+        self.assertBody('Hello world')
+
+    def testCustomNamespaces(self):
+        self.getPage('/raw/incr?num=12')
+        self.assertBody('13')
+
+        self.getPage('/dbscheme')
+        self.assertBody(r'sqlite///memory')
+
+    def testHandlerToolConfigOverride(self):
+        # Assert that config overrides tool constructor args. Above, we set
+        # the favicon in the page handler to be '../favicon.ico',
+        # but then overrode it in config to be './static/dirback.jpg'.
+        self.getPage('/favicon.ico')
+        self.assertBody(open(os.path.join(localDir, 'static/dirback.jpg'),
+                             'rb').read())
+
+    def test_request_body_namespace(self):
+        self.getPage('/plain', method='POST', headers=[
+            ('Content-Type', 'application/x-www-form-urlencoded'),
+            ('Content-Length', '13')],
+            body=b'\xff\xfex\x00=\xff\xfea\x00b\x00c\x00')
+        self.assertBody('abc')
+
+
+class VariableSubstitutionTests(unittest.TestCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_config(self):
+        from textwrap import dedent
+
+        # variable substitution with [DEFAULT]
+        conf = dedent("""
+        [DEFAULT]
+        dir = "/some/dir"
+        my.dir = %(dir)s + "/sub"
+
+        [my]
+        my.dir = %(dir)s + "/my/dir"
+        my.dir2 = %(my.dir)s + '/dir2'
+
+        """)
+
+        fp = StringIOFromNative(conf)
+
+        cherrypy.config.update(fp)
+        self.assertEqual(cherrypy.config['my']['my.dir'], '/some/dir/my/dir')
+        self.assertEqual(cherrypy.config['my']
+                         ['my.dir2'], '/some/dir/my/dir/dir2')
+
+
+class CallablesInConfigTest(unittest.TestCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_call_with_literal_dict(self):
+        from textwrap import dedent
+        conf = dedent("""
+        [my]
+        value = dict(**{'foo': 'bar'})
+        """)
+        fp = StringIOFromNative(conf)
+        cherrypy.config.update(fp)
+        self.assertEqual(cherrypy.config['my']['value'], {'foo': 'bar'})
+
+    def test_call_with_kwargs(self):
+        from textwrap import dedent
+        conf = dedent("""
+        [my]
+        value = dict(foo="buzz", **cherrypy._test_dict)
+        """)
+        test_dict = {
+            'foo': 'bar',
+            'bar': 'foo',
+            'fizz': 'buzz'
+        }
+        cherrypy._test_dict = test_dict
+        fp = StringIOFromNative(conf)
+        cherrypy.config.update(fp)
+        test_dict['foo'] = 'buzz'
+        self.assertEqual(cherrypy.config['my']['value']['foo'], 'buzz')
+        self.assertEqual(cherrypy.config['my']['value'], test_dict)
+        del cherrypy._test_dict
diff --git a/lib/cherrypy/test/test_config_server.py b/lib/cherrypy/test/test_config_server.py
new file mode 100644
index 00000000..7b183530
--- /dev/null
+++ b/lib/cherrypy/test/test_config_server.py
@@ -0,0 +1,126 @@
+"""Tests for the CherryPy configuration system."""
+
+import os
+
+import cherrypy
+from cherrypy.test import helper
+
+
+localDir = os.path.join(os.getcwd(), os.path.dirname(__file__))
+
+
+#                             Client-side code                             #
+
+
+class ServerConfigTests(helper.CPWebCase):
+
+    @staticmethod
+    def setup_server():
+
+        class Root:
+
+            @cherrypy.expose
+            def index(self):
+                return cherrypy.request.wsgi_environ['SERVER_PORT']
+
+            @cherrypy.expose
+            def upload(self, file):
+                return 'Size: %s' % len(file.file.read())
+
+            @cherrypy.expose
+            @cherrypy.config(**{'request.body.maxbytes': 100})
+            def tinyupload(self):
+                return cherrypy.request.body.read()
+
+        cherrypy.tree.mount(Root())
+
+        cherrypy.config.update({
+            'server.socket_host': '0.0.0.0',
+            'server.socket_port': 9876,
+            'server.max_request_body_size': 200,
+            'server.max_request_header_size': 500,
+            'server.socket_timeout': 0.5,
+
+            # Test explicit server.instance
+            'server.2.instance': 'cherrypy._cpwsgi_server.CPWSGIServer',
+            'server.2.socket_port': 9877,
+
+            # Test non-numeric 
+            # Also test default server.instance = builtin server
+            'server.yetanother.socket_port': 9878,
+        })
+
+    PORT = 9876
+
+    def testBasicConfig(self):
+        self.getPage('/')
+        self.assertBody(str(self.PORT))
+
+    def testAdditionalServers(self):
+        if self.scheme == 'https':
+            return self.skip('not available under ssl')
+        self.PORT = 9877
+        self.getPage('/')
+        self.assertBody(str(self.PORT))
+        self.PORT = 9878
+        self.getPage('/')
+        self.assertBody(str(self.PORT))
+
+    def testMaxRequestSizePerHandler(self):
+        if getattr(cherrypy.server, 'using_apache', False):
+            return self.skip('skipped due to known Apache differences... ')
+
+        self.getPage('/tinyupload', method='POST',
+                     headers=[('Content-Type', 'text/plain'),
+                              ('Content-Length', '100')],
+                     body='x' * 100)
+        self.assertStatus(200)
+        self.assertBody('x' * 100)
+
+        self.getPage('/tinyupload', method='POST',
+                     headers=[('Content-Type', 'text/plain'),
+                              ('Content-Length', '101')],
+                     body='x' * 101)
+        self.assertStatus(413)
+
+    def testMaxRequestSize(self):
+        if getattr(cherrypy.server, 'using_apache', False):
+            return self.skip('skipped due to known Apache differences... ')
+
+        for size in (500, 5000, 50000):
+            self.getPage('/', headers=[('From', 'x' * 500)])
+            self.assertStatus(413)
+
+        # Test for https://github.com/cherrypy/cherrypy/issues/421
+        # (Incorrect border condition in readline of SizeCheckWrapper).
+        # This hangs in rev 891 and earlier.
+        lines256 = 'x' * 248
+        self.getPage('/',
+                     headers=[('Host', '%s:%s' % (self.HOST, self.PORT)),
+                              ('From', lines256)])
+
+        # Test upload
+        cd = (
+            'Content-Disposition: form-data; '
+            'name="file"; '
+            'filename="hello.txt"'
+        )
+        body = '\r\n'.join([
+            '--x',
+            cd,
+            'Content-Type: text/plain',
+            '',
+            '%s',
+            '--x--'])
+        partlen = 200 - len(body)
+        b = body % ('x' * partlen)
+        h = [('Content-type', 'multipart/form-data; boundary=x'),
+             ('Content-Length', '%s' % len(b))]
+        self.getPage('/upload', h, 'POST', b)
+        self.assertBody('Size: %d' % partlen)
+
+        b = body % ('x' * 200)
+        h = [('Content-type', 'multipart/form-data; boundary=x'),
+             ('Content-Length', '%s' % len(b))]
+        self.getPage('/upload', h, 'POST', b)
+        self.assertStatus(413)
diff --git a/lib/cherrypy/test/test_conn.py b/lib/cherrypy/test/test_conn.py
new file mode 100644
index 00000000..7d60c6fb
--- /dev/null
+++ b/lib/cherrypy/test/test_conn.py
@@ -0,0 +1,873 @@
+"""Tests for TCP connection handling, including proper and timely close."""
+
+import errno
+import socket
+import sys
+import time
+
+import six
+from six.moves import urllib
+from six.moves.http_client import BadStatusLine, HTTPConnection, NotConnected
+
+import pytest
+
+from cheroot.test import webtest
+
+import cherrypy
+from cherrypy._cpcompat import HTTPSConnection, ntob, tonative
+from cherrypy.test import helper
+
+
+timeout = 1
+pov = 'pPeErRsSiIsStTeEnNcCeE oOfF vViIsSiIoOnN'
+
+
+def setup_server():
+
+    def raise500():
+        raise cherrypy.HTTPError(500)
+
+    class Root:
+
+        @cherrypy.expose
+        def index(self):
+            return pov
+        page1 = index
+        page2 = index
+        page3 = index
+
+        @cherrypy.expose
+        def hello(self):
+            return 'Hello, world!'
+
+        @cherrypy.expose
+        def timeout(self, t):
+            return str(cherrypy.server.httpserver.timeout)
+
+        @cherrypy.expose
+        @cherrypy.config(**{'response.stream': True})
+        def stream(self, set_cl=False):
+            if set_cl:
+                cherrypy.response.headers['Content-Length'] = 10
+
+            def content():
+                for x in range(10):
+                    yield str(x)
+
+            return content()
+
+        @cherrypy.expose
+        def error(self, code=500):
+            raise cherrypy.HTTPError(code)
+
+        @cherrypy.expose
+        def upload(self):
+            if not cherrypy.request.method == 'POST':
+                raise AssertionError("'POST' != request.method %r" %
+                                     cherrypy.request.method)
+            return "thanks for '%s'" % cherrypy.request.body.read()
+
+        @cherrypy.expose
+        def custom(self, response_code):
+            cherrypy.response.status = response_code
+            return 'Code = %s' % response_code
+
+        @cherrypy.expose
+        @cherrypy.config(**{'hooks.on_start_resource': raise500})
+        def err_before_read(self):
+            return 'ok'
+
+        @cherrypy.expose
+        def one_megabyte_of_a(self):
+            return ['a' * 1024] * 1024
+
+        @cherrypy.expose
+        # Turn off the encoding tool so it doens't collapse
+        # our response body and reclaculate the Content-Length.
+        @cherrypy.config(**{'tools.encode.on': False})
+        def custom_cl(self, body, cl):
+            cherrypy.response.headers['Content-Length'] = cl
+            if not isinstance(body, list):
+                body = [body]
+            newbody = []
+            for chunk in body:
+                if isinstance(chunk, six.text_type):
+                    chunk = chunk.encode('ISO-8859-1')
+                newbody.append(chunk)
+            return newbody
+
+    cherrypy.tree.mount(Root())
+    cherrypy.config.update({
+        'server.max_request_body_size': 1001,
+        'server.socket_timeout': timeout,
+    })
+
+
+class ConnectionCloseTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_HTTP11(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        self.persistent = True
+
+        # Make the first request and assert there's no "Connection: close".
+        self.getPage('/')
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        self.assertNoHeader('Connection')
+
+        # Make another request on the same connection.
+        self.getPage('/page1')
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        self.assertNoHeader('Connection')
+
+        # Test client-side close.
+        self.getPage('/page2', headers=[('Connection', 'close')])
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        self.assertHeader('Connection', 'close')
+
+        # Make another request on the same connection, which should error.
+        self.assertRaises(NotConnected, self.getPage, '/')
+
+    def test_Streaming_no_len(self):
+        try:
+            self._streaming(set_cl=False)
+        finally:
+            try:
+                self.HTTP_CONN.close()
+            except (TypeError, AttributeError):
+                pass
+
+    def test_Streaming_with_len(self):
+        try:
+            self._streaming(set_cl=True)
+        finally:
+            try:
+                self.HTTP_CONN.close()
+            except (TypeError, AttributeError):
+                pass
+
+    def _streaming(self, set_cl):
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.PROTOCOL = 'HTTP/1.1'
+
+            self.persistent = True
+
+            # Make the first request and assert there's no "Connection: close".
+            self.getPage('/')
+            self.assertStatus('200 OK')
+            self.assertBody(pov)
+            self.assertNoHeader('Connection')
+
+            # Make another, streamed request on the same connection.
+            if set_cl:
+                # When a Content-Length is provided, the content should stream
+                # without closing the connection.
+                self.getPage('/stream?set_cl=Yes')
+                self.assertHeader('Content-Length')
+                self.assertNoHeader('Connection', 'close')
+                self.assertNoHeader('Transfer-Encoding')
+
+                self.assertStatus('200 OK')
+                self.assertBody('0123456789')
+            else:
+                # When no Content-Length response header is provided,
+                # streamed output will either close the connection, or use
+                # chunked encoding, to determine transfer-length.
+                self.getPage('/stream')
+                self.assertNoHeader('Content-Length')
+                self.assertStatus('200 OK')
+                self.assertBody('0123456789')
+
+                chunked_response = False
+                for k, v in self.headers:
+                    if k.lower() == 'transfer-encoding':
+                        if str(v) == 'chunked':
+                            chunked_response = True
+
+                if chunked_response:
+                    self.assertNoHeader('Connection', 'close')
+                else:
+                    self.assertHeader('Connection', 'close')
+
+                    # Make another request on the same connection, which should
+                    # error.
+                    self.assertRaises(NotConnected, self.getPage, '/')
+
+                # Try HEAD. See
+                # https://github.com/cherrypy/cherrypy/issues/864.
+                self.getPage('/stream', method='HEAD')
+                self.assertStatus('200 OK')
+                self.assertBody('')
+                self.assertNoHeader('Transfer-Encoding')
+        else:
+            self.PROTOCOL = 'HTTP/1.0'
+
+            self.persistent = True
+
+            # Make the first request and assert Keep-Alive.
+            self.getPage('/', headers=[('Connection', 'Keep-Alive')])
+            self.assertStatus('200 OK')
+            self.assertBody(pov)
+            self.assertHeader('Connection', 'Keep-Alive')
+
+            # Make another, streamed request on the same connection.
+            if set_cl:
+                # When a Content-Length is provided, the content should
+                # stream without closing the connection.
+                self.getPage('/stream?set_cl=Yes',
+                             headers=[('Connection', 'Keep-Alive')])
+                self.assertHeader('Content-Length')
+                self.assertHeader('Connection', 'Keep-Alive')
+                self.assertNoHeader('Transfer-Encoding')
+                self.assertStatus('200 OK')
+                self.assertBody('0123456789')
+            else:
+                # When a Content-Length is not provided,
+                # the server should close the connection.
+                self.getPage('/stream', headers=[('Connection', 'Keep-Alive')])
+                self.assertStatus('200 OK')
+                self.assertBody('0123456789')
+
+                self.assertNoHeader('Content-Length')
+                self.assertNoHeader('Connection', 'Keep-Alive')
+                self.assertNoHeader('Transfer-Encoding')
+
+                # Make another request on the same connection, which should
+                # error.
+                self.assertRaises(NotConnected, self.getPage, '/')
+
+    def test_HTTP10_KeepAlive(self):
+        self.PROTOCOL = 'HTTP/1.0'
+        if self.scheme == 'https':
+            self.HTTP_CONN = HTTPSConnection
+        else:
+            self.HTTP_CONN = HTTPConnection
+
+        # Test a normal HTTP/1.0 request.
+        self.getPage('/page2')
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        # Apache, for example, may emit a Connection header even for HTTP/1.0
+        # self.assertNoHeader("Connection")
+
+        # Test a keep-alive HTTP/1.0 request.
+        self.persistent = True
+
+        self.getPage('/page3', headers=[('Connection', 'Keep-Alive')])
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        self.assertHeader('Connection', 'Keep-Alive')
+
+        # Remove the keep-alive header again.
+        self.getPage('/page3')
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        # Apache, for example, may emit a Connection header even for HTTP/1.0
+        # self.assertNoHeader("Connection")
+
+
+class PipelineTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_HTTP11_Timeout(self):
+        # If we timeout without sending any data,
+        # the server will close the conn with a 408.
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        # Connect but send nothing.
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.auto_open = False
+        conn.connect()
+
+        # Wait for our socket timeout
+        time.sleep(timeout * 2)
+
+        # The request should have returned 408 already.
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 408)
+        conn.close()
+
+        # Connect but send half the headers only.
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.auto_open = False
+        conn.connect()
+        conn.send(b'GET /hello HTTP/1.1')
+        conn.send(('Host: %s' % self.HOST).encode('ascii'))
+
+        # Wait for our socket timeout
+        time.sleep(timeout * 2)
+
+        # The conn should have already sent 408.
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 408)
+        conn.close()
+
+    def test_HTTP11_Timeout_after_request(self):
+        # If we timeout after at least one request has succeeded,
+        # the server will close the conn without 408.
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        # Make an initial request
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest('GET', '/timeout?t=%s' % timeout, skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 200)
+        self.body = response.read()
+        self.assertBody(str(timeout))
+
+        # Make a second request on the same socket
+        conn._output(b'GET /hello HTTP/1.1')
+        conn._output(ntob('Host: %s' % self.HOST, 'ascii'))
+        conn._send_output()
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 200)
+        self.body = response.read()
+        self.assertBody('Hello, world!')
+
+        # Wait for our socket timeout
+        time.sleep(timeout * 2)
+
+        # Make another request on the same socket, which should error
+        conn._output(b'GET /hello HTTP/1.1')
+        conn._output(ntob('Host: %s' % self.HOST, 'ascii'))
+        conn._send_output()
+        response = conn.response_class(conn.sock, method='GET')
+        try:
+            response.begin()
+        except Exception:
+            if not isinstance(sys.exc_info()[1],
+                              (socket.error, BadStatusLine)):
+                self.fail("Writing to timed out socket didn't fail"
+                          ' as it should have: %s' % sys.exc_info()[1])
+        else:
+            if response.status != 408:
+                self.fail("Writing to timed out socket didn't fail"
+                          ' as it should have: %s' %
+                          response.read())
+
+        conn.close()
+
+        # Make another request on a new socket, which should work
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest('GET', '/', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 200)
+        self.body = response.read()
+        self.assertBody(pov)
+
+        # Make another request on the same socket,
+        # but timeout on the headers
+        conn.send(b'GET /hello HTTP/1.1')
+        # Wait for our socket timeout
+        time.sleep(timeout * 2)
+        response = conn.response_class(conn.sock, method='GET')
+        try:
+            response.begin()
+        except Exception:
+            if not isinstance(sys.exc_info()[1],
+                              (socket.error, BadStatusLine)):
+                self.fail("Writing to timed out socket didn't fail"
+                          ' as it should have: %s' % sys.exc_info()[1])
+        else:
+            self.fail("Writing to timed out socket didn't fail"
+                      ' as it should have: %s' %
+                      response.read())
+
+        conn.close()
+
+        # Retry the request on a new connection, which should work
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest('GET', '/', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.assertEqual(response.status, 200)
+        self.body = response.read()
+        self.assertBody(pov)
+        conn.close()
+
+    def test_HTTP11_pipelining(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        # Test pipelining. httplib doesn't support this directly.
+        self.persistent = True
+        conn = self.HTTP_CONN
+
+        # Put request 1
+        conn.putrequest('GET', '/hello', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+
+        for trial in range(5):
+            # Put next request
+            conn._output(b'GET /hello HTTP/1.1')
+            conn._output(ntob('Host: %s' % self.HOST, 'ascii'))
+            conn._send_output()
+
+            # Retrieve previous response
+            response = conn.response_class(conn.sock, method='GET')
+            # there is a bug in python3 regarding the buffering of
+            # ``conn.sock``. Until that bug get's fixed we will
+            # monkey patch the ``response`` instance.
+            # https://bugs.python.org/issue23377
+            if six.PY3:
+                response.fp = conn.sock.makefile('rb', 0)
+            response.begin()
+            body = response.read(13)
+            self.assertEqual(response.status, 200)
+            self.assertEqual(body, b'Hello, world!')
+
+        # Retrieve final response
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        body = response.read()
+        self.assertEqual(response.status, 200)
+        self.assertEqual(body, b'Hello, world!')
+
+        conn.close()
+
+    def test_100_Continue(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        self.persistent = True
+        conn = self.HTTP_CONN
+
+        # Try a page without an Expect request header first.
+        # Note that httplib's response.begin automatically ignores
+        # 100 Continue responses, so we must manually check for it.
+        try:
+            conn.putrequest('POST', '/upload', skip_host=True)
+            conn.putheader('Host', self.HOST)
+            conn.putheader('Content-Type', 'text/plain')
+            conn.putheader('Content-Length', '4')
+            conn.endheaders()
+            conn.send(ntob("d'oh"))
+            response = conn.response_class(conn.sock, method='POST')
+            version, status, reason = response._read_status()
+            self.assertNotEqual(status, 100)
+        finally:
+            conn.close()
+
+        # Now try a page with an Expect header...
+        try:
+            conn.connect()
+            conn.putrequest('POST', '/upload', skip_host=True)
+            conn.putheader('Host', self.HOST)
+            conn.putheader('Content-Type', 'text/plain')
+            conn.putheader('Content-Length', '17')
+            conn.putheader('Expect', '100-continue')
+            conn.endheaders()
+            response = conn.response_class(conn.sock, method='POST')
+
+            # ...assert and then skip the 100 response
+            version, status, reason = response._read_status()
+            self.assertEqual(status, 100)
+            while True:
+                line = response.fp.readline().strip()
+                if line:
+                    self.fail(
+                        '100 Continue should not output any headers. Got %r' %
+                        line)
+                else:
+                    break
+
+            # ...send the body
+            body = b'I am a small file'
+            conn.send(body)
+
+            # ...get the final response
+            response.begin()
+            self.status, self.headers, self.body = webtest.shb(response)
+            self.assertStatus(200)
+            self.assertBody("thanks for '%s'" % body)
+        finally:
+            conn.close()
+
+
+class ConnectionTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_readall_or_close(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        if self.scheme == 'https':
+            self.HTTP_CONN = HTTPSConnection
+        else:
+            self.HTTP_CONN = HTTPConnection
+
+        # Test a max of 0 (the default) and then reset to what it was above.
+        old_max = cherrypy.server.max_request_body_size
+        for new_max in (0, old_max):
+            cherrypy.server.max_request_body_size = new_max
+
+            self.persistent = True
+            conn = self.HTTP_CONN
+
+            # Get a POST page with an error
+            conn.putrequest('POST', '/err_before_read', skip_host=True)
+            conn.putheader('Host', self.HOST)
+            conn.putheader('Content-Type', 'text/plain')
+            conn.putheader('Content-Length', '1000')
+            conn.putheader('Expect', '100-continue')
+            conn.endheaders()
+            response = conn.response_class(conn.sock, method='POST')
+
+            # ...assert and then skip the 100 response
+            version, status, reason = response._read_status()
+            self.assertEqual(status, 100)
+            while True:
+                skip = response.fp.readline().strip()
+                if not skip:
+                    break
+
+            # ...send the body
+            conn.send(ntob('x' * 1000))
+
+            # ...get the final response
+            response.begin()
+            self.status, self.headers, self.body = webtest.shb(response)
+            self.assertStatus(500)
+
+            # Now try a working page with an Expect header...
+            conn._output(b'POST /upload HTTP/1.1')
+            conn._output(ntob('Host: %s' % self.HOST, 'ascii'))
+            conn._output(b'Content-Type: text/plain')
+            conn._output(b'Content-Length: 17')
+            conn._output(b'Expect: 100-continue')
+            conn._send_output()
+            response = conn.response_class(conn.sock, method='POST')
+
+            # ...assert and then skip the 100 response
+            version, status, reason = response._read_status()
+            self.assertEqual(status, 100)
+            while True:
+                skip = response.fp.readline().strip()
+                if not skip:
+                    break
+
+            # ...send the body
+            body = b'I am a small file'
+            conn.send(body)
+
+            # ...get the final response
+            response.begin()
+            self.status, self.headers, self.body = webtest.shb(response)
+            self.assertStatus(200)
+            self.assertBody("thanks for '%s'" % body)
+            conn.close()
+
+    def test_No_Message_Body(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        # Set our HTTP_CONN to an instance so it persists between requests.
+        self.persistent = True
+
+        # Make the first request and assert there's no "Connection: close".
+        self.getPage('/')
+        self.assertStatus('200 OK')
+        self.assertBody(pov)
+        self.assertNoHeader('Connection')
+
+        # Make a 204 request on the same connection.
+        self.getPage('/custom/204')
+        self.assertStatus(204)
+        self.assertNoHeader('Content-Length')
+        self.assertBody('')
+        self.assertNoHeader('Connection')
+
+        # Make a 304 request on the same connection.
+        self.getPage('/custom/304')
+        self.assertStatus(304)
+        self.assertNoHeader('Content-Length')
+        self.assertBody('')
+        self.assertNoHeader('Connection')
+
+    def test_Chunked_Encoding(self):
+        if cherrypy.server.protocol_version != 'HTTP/1.1':
+            return self.skip()
+
+        if (hasattr(self, 'harness') and
+                'modpython' in self.harness.__class__.__name__.lower()):
+            # mod_python forbids chunked encoding
+            return self.skip()
+
+        self.PROTOCOL = 'HTTP/1.1'
+
+        # Set our HTTP_CONN to an instance so it persists between requests.
+        self.persistent = True
+        conn = self.HTTP_CONN
+
+        # Try a normal chunked request (with extensions)
+        body = ntob('8;key=value\r\nxx\r\nxxxx\r\n5\r\nyyyyy\r\n0\r\n'
+                    'Content-Type: application/json\r\n'
+                    '\r\n')
+        conn.putrequest('POST', '/upload', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.putheader('Transfer-Encoding', 'chunked')
+        conn.putheader('Trailer', 'Content-Type')
+        # Note that this is somewhat malformed:
+        # we shouldn't be sending Content-Length.
+        # RFC 2616 says the server should ignore it.
+        conn.putheader('Content-Length', '3')
+        conn.endheaders()
+        conn.send(body)
+        response = conn.getresponse()
+        self.status, self.headers, self.body = webtest.shb(response)
+        self.assertStatus('200 OK')
+        self.assertBody("thanks for '%s'" % b'xx\r\nxxxxyyyyy')
+
+        # Try a chunked request that exceeds server.max_request_body_size.
+        # Note that the delimiters and trailer are included.
+        body = ntob('3e3\r\n' + ('x' * 995) + '\r\n0\r\n\r\n')
+        conn.putrequest('POST', '/upload', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.putheader('Transfer-Encoding', 'chunked')
+        conn.putheader('Content-Type', 'text/plain')
+        # Chunked requests don't need a content-length
+        # #        conn.putheader("Content-Length", len(body))
+        conn.endheaders()
+        conn.send(body)
+        response = conn.getresponse()
+        self.status, self.headers, self.body = webtest.shb(response)
+        self.assertStatus(413)
+        conn.close()
+
+    def test_Content_Length_in(self):
+        # Try a non-chunked request where Content-Length exceeds
+        # server.max_request_body_size. Assert error before body send.
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest('POST', '/upload', skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.putheader('Content-Type', 'text/plain')
+        conn.putheader('Content-Length', '9999')
+        conn.endheaders()
+        response = conn.getresponse()
+        self.status, self.headers, self.body = webtest.shb(response)
+        self.assertStatus(413)
+        self.assertBody('The entity sent with the request exceeds '
+                        'the maximum allowed bytes.')
+        conn.close()
+
+    def test_Content_Length_out_preheaders(self):
+        # Try a non-chunked response where Content-Length is less than
+        # the actual bytes in the response body.
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest('GET', '/custom_cl?body=I+have+too+many+bytes&cl=5',
+                        skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+        response = conn.getresponse()
+        self.status, self.headers, self.body = webtest.shb(response)
+        self.assertStatus(500)
+        self.assertBody(
+            'The requested resource returned more bytes than the '
+            'declared Content-Length.')
+        conn.close()
+
+    def test_Content_Length_out_postheaders(self):
+        # Try a non-chunked response where Content-Length is less than
+        # the actual bytes in the response body.
+        self.persistent = True
+        conn = self.HTTP_CONN
+        conn.putrequest(
+            'GET', '/custom_cl?body=I+too&body=+have+too+many&cl=5',
+            skip_host=True)
+        conn.putheader('Host', self.HOST)
+        conn.endheaders()
+        response = conn.getresponse()
+        self.status, self.headers, self.body = webtest.shb(response)
+        self.assertStatus(200)
+        self.assertBody('I too')
+        conn.close()
+
+    def test_598(self):
+        tmpl = '{scheme}://{host}:{port}/one_megabyte_of_a/'
+        url = tmpl.format(
+            scheme=self.scheme,
+            host=self.HOST,
+            port=self.PORT,
+        )
+        remote_data_conn = urllib.request.urlopen(url)
+        buf = remote_data_conn.read(512)
+        time.sleep(timeout * 0.6)
+        remaining = (1024 * 1024) - 512
+        while remaining:
+            data = remote_data_conn.read(remaining)
+            if not data:
+                break
+            else:
+                buf += data
+            remaining -= len(data)
+
+        self.assertEqual(len(buf), 1024 * 1024)
+        self.assertEqual(buf, ntob('a' * 1024 * 1024))
+        self.assertEqual(remaining, 0)
+        remote_data_conn.close()
+
+
+def setup_upload_server():
+
+    class Root:
+        @cherrypy.expose
+        def upload(self):
+            if not cherrypy.request.method == 'POST':
+                raise AssertionError("'POST' != request.method %r" %
+                                     cherrypy.request.method)
+            return "thanks for '%s'" % tonative(cherrypy.request.body.read())
+
+    cherrypy.tree.mount(Root())
+    cherrypy.config.update({
+        'server.max_request_body_size': 1001,
+        'server.socket_timeout': 10,
+        'server.accepted_queue_size': 5,
+        'server.accepted_queue_timeout': 0.1,
+    })
+
+
+reset_names = 'ECONNRESET', 'WSAECONNRESET'
+socket_reset_errors = [
+    getattr(errno, name)
+    for name in reset_names
+    if hasattr(errno, name)
+]
+'reset error numbers available on this platform'
+
+socket_reset_errors += [
+    # Python 3.5 raises an http.client.RemoteDisconnected
+    # with this message
+    'Remote end closed connection without response',
+]
+
+
+class LimitedRequestQueueTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_upload_server)
+
+    @pytest.mark.xfail(reason='#1535')
+    def test_queue_full(self):
+        conns = []
+        overflow_conn = None
+
+        try:
+            # Make 15 initial requests and leave them open, which should use
+            # all of wsgiserver's WorkerThreads and fill its Queue.
+            for i in range(15):
+                conn = self.HTTP_CONN(self.HOST, self.PORT)
+                conn.putrequest('POST', '/upload', skip_host=True)
+                conn.putheader('Host', self.HOST)
+                conn.putheader('Content-Type', 'text/plain')
+                conn.putheader('Content-Length', '4')
+                conn.endheaders()
+                conns.append(conn)
+
+            # Now try a 16th conn, which should be closed by the
+            # server immediately.
+            overflow_conn = self.HTTP_CONN(self.HOST, self.PORT)
+            # Manually connect since httplib won't let us set a timeout
+            for res in socket.getaddrinfo(self.HOST, self.PORT, 0,
+                                          socket.SOCK_STREAM):
+                af, socktype, proto, canonname, sa = res
+                overflow_conn.sock = socket.socket(af, socktype, proto)
+                overflow_conn.sock.settimeout(5)
+                overflow_conn.sock.connect(sa)
+                break
+
+            overflow_conn.putrequest('GET', '/', skip_host=True)
+            overflow_conn.putheader('Host', self.HOST)
+            overflow_conn.endheaders()
+            response = overflow_conn.response_class(
+                overflow_conn.sock,
+                method='GET',
+            )
+            try:
+                response.begin()
+            except socket.error as exc:
+                if exc.args[0] in socket_reset_errors:
+                    pass  # Expected.
+                else:
+                    tmpl = (
+                        'Overflow conn did not get RST. '
+                        'Got {exc.args!r} instead'
+                    )
+                    raise AssertionError(tmpl.format(**locals()))
+            except BadStatusLine:
+                # This is a special case in OS X. Linux and Windows will
+                # RST correctly.
+                assert sys.platform == 'darwin'
+            else:
+                raise AssertionError('Overflow conn did not get RST ')
+        finally:
+            for conn in conns:
+                conn.send(b'done')
+                response = conn.response_class(conn.sock, method='POST')
+                response.begin()
+                self.body = response.read()
+                self.assertBody("thanks for 'done'")
+                self.assertEqual(response.status, 200)
+                conn.close()
+            if overflow_conn:
+                overflow_conn.close()
+
+
+class BadRequestTests(helper.CPWebCase):
+    setup_server = staticmethod(setup_server)
+
+    def test_No_CRLF(self):
+        self.persistent = True
+
+        conn = self.HTTP_CONN
+        conn.send(b'GET /hello HTTP/1.1\n\n')
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.body = response.read()
+        self.assertBody('HTTP requires CRLF terminators')
+        conn.close()
+
+        conn.connect()
+        conn.send(b'GET /hello HTTP/1.1\r\n\n')
+        response = conn.response_class(conn.sock, method='GET')
+        response.begin()
+        self.body = response.read()
+        self.assertBody('HTTP requires CRLF terminators')
+        conn.close()
diff --git a/lib/cherrypy/test/test_core.py b/lib/cherrypy/test/test_core.py
new file mode 100644
index 00000000..9834c1f3
--- /dev/null
+++ b/lib/cherrypy/test/test_core.py
@@ -0,0 +1,823 @@
+# coding: utf-8
+
+"""Basic tests for the CherryPy core: request handling."""
+
+import os
+import sys
+import types
+
+import six
+
+import cherrypy
+from cherrypy._cpcompat import ntou
+from cherrypy import _cptools, tools
+from cherrypy.lib import httputil, static
+
+from cherrypy.test._test_decorators import ExposeExamples
+from cherrypy.test import helper
+
+
+localDir = os.path.dirname(__file__)
+favicon_path = os.path.join(os.getcwd(), localDir, '../favicon.ico')
+
+#                             Client-side code                             #
+
+
+class CoreRequestHandlingTest(helper.CPWebCase):
+
+    @staticmethod
+    def setup_server():
+        class Root:
+
+            @cherrypy.expose
+            def index(self):
+                return 'hello'
+
+            favicon_ico = tools.staticfile.handler(filename=favicon_path)
+
+            @cherrypy.expose
+            def defct(self, newct):
+                newct = 'text/%s' % newct
+                cherrypy.config.update({'tools.response_headers.on': True,
+                                        'tools.response_headers.headers':
+                                        [('Content-Type', newct)]})
+
+            @cherrypy.expose
+            def baseurl(self, path_info, relative=None):
+                return cherrypy.url(path_info, relative=bool(relative))
+
+        root = Root()
+        root.expose_dec = ExposeExamples()
+
+        class TestType(type):
+
+            """Metaclass which automatically exposes all functions in each
+            subclass, and adds an instance of the subclass as an attribute
+            of root.
+            """
+            def __init__(cls, name, bases, dct):
+                type.__init__(cls, name, bases, dct)
+                for value in six.itervalues(dct):
+                    if isinstance(value, types.FunctionType):
+                        value.exposed = True
+                setattr(root, name.lower(), cls())
+        Test = TestType('Test', (object, ), {})
+
+        @cherrypy.config(**{'tools.trailing_slash.on': False})
+        class URL(Test):
+
+            def index(self, path_info, relative=None):
+                if relative != 'server':
+                    relative = bool(relative)
+                return cherrypy.url(path_info, relative=relative)
+
+            def leaf(self, path_info, relative=None):
+                if relative != 'server':
+                    relative = bool(relative)
+                return cherrypy.url(path_info, relative=relative)
+
+            def qs(self, qs):
+                return cherrypy.url(qs=qs)
+
+        def log_status():
+            Status.statuses.append(cherrypy.response.status)
+        cherrypy.tools.log_status = cherrypy.Tool(
+            'on_end_resource', log_status)
+
+        class Status(Test):
+
+            def index(self):
+                return 'normal'
+
+            def blank(self):
+                cherrypy.response.status = ''
+
+            # According to RFC 2616, new status codes are OK as long as they
+            # are between 100 and 599.
+
+            # Here is an illegal code...
+            def illegal(self):
+                cherrypy.response.status = 781
+                return 'oops'
+
+            # ...and here is an unknown but legal code.
+            def unknown(self):
+                cherrypy.response.status = '431 My custom error'
+                return 'funky'
+
+            # Non-numeric code
+            def bad(self):
+                cherrypy.response.status = 'error'
+                return 'bad news'
+
+            statuses = []
+
+            @cherrypy.config(**{'tools.log_status.on': True})
+            def on_end_resource_stage(self):
+                return repr(self.statuses)
+
+        class Redirect(Test):
+
+            @cherrypy.config(**{
+                'tools.err_redirect.on': True,
+                'tools.err_redirect.url': '/errpage',
+                'tools.err_redirect.internal': False,
+            })
+            class Error:
+                @cherrypy.expose
+                def index(self):
+                    raise NameError('redirect_test')
+
+            error = Error()
+
+            def index(self):
+                return 'child'
+
+            def custom(self, url, code):
+                raise cherrypy.HTTPRedirect(url, code)
+
+            @cherrypy.config(**{'tools.trailing_slash.extra': True})
+            def by_code(self, code):
+                raise cherrypy.HTTPRedirect('somewhere%20else', code)
+
+            def nomodify(self):
+                raise cherrypy.HTTPRedirect('', 304)
+
+            def proxy(self):
+                raise cherrypy.HTTPRedirect('proxy', 305)
+
+            def stringify(self):
+                return str(cherrypy.HTTPRedirect('/'))
+
+            def fragment(self, frag):
+                raise cherrypy.HTTPRedirect('/some/url#%s' % frag)
+
+            def url_with_quote(self):
+                raise cherrypy.HTTPRedirect("/some\"url/that'we/want")
+
+            def url_with_xss(self):
+                raise cherrypy.HTTPRedirect(
+                    "/someurl/that'we/want")
+
+            def url_with_unicode(self):
+                raise cherrypy.HTTPRedirect(ntou('тест', 'utf-8'))
+
+        def login_redir():
+            if not getattr(cherrypy.request, 'login', None):
+                raise cherrypy.InternalRedirect('/internalredirect/login')
+        tools.login_redir = _cptools.Tool('before_handler', login_redir)
+
+        def redir_custom():
+            raise cherrypy.InternalRedirect('/internalredirect/custom_err')
+
+        class InternalRedirect(Test):
+
+            def index(self):
+                raise cherrypy.InternalRedirect('/')
+
+            @cherrypy.expose
+            @cherrypy.config(**{'hooks.before_error_response': redir_custom})
+            def choke(self):
+                return 3 / 0
+
+            def relative(self, a, b):
+                raise cherrypy.InternalRedirect('cousin?t=6')
+
+            def cousin(self, t):
+                assert cherrypy.request.prev.closed
+                return cherrypy.request.prev.query_string
+
+            def petshop(self, user_id):
+                if user_id == 'parrot':
+                    # Trade it for a slug when redirecting
+                    raise cherrypy.InternalRedirect(
+                        '/image/getImagesByUser?user_id=slug')
+                elif user_id == 'terrier':
+                    # Trade it for a fish when redirecting
+                    raise cherrypy.InternalRedirect(
+                        '/image/getImagesByUser?user_id=fish')
+                else:
+                    # This should pass the user_id through to getImagesByUser
+                    raise cherrypy.InternalRedirect(
+                        '/image/getImagesByUser?user_id=%s' % str(user_id))
+
+            # We support Python 2.3, but the @-deco syntax would look like
+            # this:
+            # @tools.login_redir()
+            def secure(self):
+                return 'Welcome!'
+            secure = tools.login_redir()(secure)
+            # Since calling the tool returns the same function you pass in,
+            # you could skip binding the return value, and just write:
+            # tools.login_redir()(secure)
+
+            def login(self):
+                return 'Please log in'
+
+            def custom_err(self):
+                return 'Something went horribly wrong.'
+
+            @cherrypy.config(**{'hooks.before_request_body': redir_custom})
+            def early_ir(self, arg):
+                return 'whatever'
+
+        class Image(Test):
+
+            def getImagesByUser(self, user_id):
+                return '0 images for %s' % user_id
+
+        class Flatten(Test):
+
+            def as_string(self):
+                return 'content'
+
+            def as_list(self):
+                return ['con', 'tent']
+
+            def as_yield(self):
+                yield b'content'
+
+            @cherrypy.config(**{'tools.flatten.on': True})
+            def as_dblyield(self):
+                yield self.as_yield()
+
+            def as_refyield(self):
+                for chunk in self.as_yield():
+                    yield chunk
+
+        class Ranges(Test):
+
+            def get_ranges(self, bytes):
+                return repr(httputil.get_ranges('bytes=%s' % bytes, 8))
+
+            def slice_file(self):
+                path = os.path.join(os.getcwd(), os.path.dirname(__file__))
+                return static.serve_file(
+                    os.path.join(path, 'static/index.html'))
+
+        class Cookies(Test):
+
+            def single(self, name):
+                cookie = cherrypy.request.cookie[name]
+                # Python2's SimpleCookie.__setitem__ won't take unicode keys.
+                cherrypy.response.cookie[str(name)] = cookie.value
+
+            def multiple(self, names):
+                list(map(self.single, names))
+
+        def append_headers(header_list, debug=False):
+            if debug:
+                cherrypy.log(
+                    'Extending response headers with %s' % repr(header_list),
+                    'TOOLS.APPEND_HEADERS')
+            cherrypy.serving.response.header_list.extend(header_list)
+        cherrypy.tools.append_headers = cherrypy.Tool(
+            'on_end_resource', append_headers)
+
+        class MultiHeader(Test):
+
+            def header_list(self):
+                pass
+            header_list = cherrypy.tools.append_headers(header_list=[
+                (b'WWW-Authenticate', b'Negotiate'),
+                (b'WWW-Authenticate', b'Basic realm="foo"'),
+            ])(header_list)
+
+            def commas(self):
+                cherrypy.response.headers[
+                    'WWW-Authenticate'] = 'Negotiate,Basic realm="foo"'
+
+        cherrypy.tree.mount(root)
+
+    def testStatus(self):
+        self.getPage('/status/')
+        self.assertBody('normal')
+        self.assertStatus(200)
+
+        self.getPage('/status/blank')
+        self.assertBody('')
+        self.assertStatus(200)
+
+        self.getPage('/status/illegal')
+        self.assertStatus(500)
+        msg = 'Illegal response status from server (781 is out of range).'
+        self.assertErrorPage(500, msg)
+
+        if not getattr(cherrypy.server, 'using_apache', False):
+            self.getPage('/status/unknown')
+            self.assertBody('funky')
+            self.assertStatus(431)
+
+        self.getPage('/status/bad')
+        self.assertStatus(500)
+        msg = "Illegal response status from server ('error' is non-numeric)."
+        self.assertErrorPage(500, msg)
+
+    def test_on_end_resource_status(self):
+        self.getPage('/status/on_end_resource_stage')
+        self.assertBody('[]')
+        self.getPage('/status/on_end_resource_stage')
+        self.assertBody(repr(['200 OK']))
+
+    def testSlashes(self):
+        # Test that requests for index methods without a trailing slash
+        # get redirected to the same URI path with a trailing slash.
+        # Make sure GET params are preserved.
+        self.getPage('/redirect?id=3')
+        self.assertStatus(301)
+        self.assertMatchesBody(
+            ''
+            '%s/redirect/[?]id=3' % (self.base(), self.base())
+        )
+
+        if self.prefix():
+            # Corner case: the "trailing slash" redirect could be tricky if
+            # we're using a virtual root and the URI is "/vroot" (no slash).
+            self.getPage('')
+            self.assertStatus(301)
+            self.assertMatchesBody("%s/" %
+                                   (self.base(), self.base()))
+
+        # Test that requests for NON-index methods WITH a trailing slash
+        # get redirected to the same URI path WITHOUT a trailing slash.
+        # Make sure GET params are preserved.
+        self.getPage('/redirect/by_code/?code=307')
+        self.assertStatus(301)
+        self.assertMatchesBody(
+            ""
+            '%s/redirect/by_code[?]code=307'
+            % (self.base(), self.base())
+        )
+
+        # If the trailing_slash tool is off, CP should just continue
+        # as if the slashes were correct. But it needs some help
+        # inside cherrypy.url to form correct output.
+        self.getPage('/url?path_info=page1')
+        self.assertBody('%s/url/page1' % self.base())
+        self.getPage('/url/leaf/?path_info=page1')
+        self.assertBody('%s/url/page1' % self.base())
+
+    def testRedirect(self):
+        self.getPage('/redirect/')
+        self.assertBody('child')
+        self.assertStatus(200)
+
+        self.getPage('/redirect/by_code?code=300')
+        self.assertMatchesBody(
+            r"\2somewhere%20else")
+        self.assertStatus(300)
+
+        self.getPage('/redirect/by_code?code=301')
+        self.assertMatchesBody(
+            r"\2somewhere%20else")
+        self.assertStatus(301)
+
+        self.getPage('/redirect/by_code?code=302')
+        self.assertMatchesBody(
+            r"\2somewhere%20else")
+        self.assertStatus(302)
+
+        self.getPage('/redirect/by_code?code=303')
+        self.assertMatchesBody(
+            r"\2somewhere%20else")
+        self.assertStatus(303)
+
+        self.getPage('/redirect/by_code?code=307')
+        self.assertMatchesBody(
+            r"\2somewhere%20else")
+        self.assertStatus(307)
+
+        self.getPage('/redirect/nomodify')
+        self.assertBody('')
+        self.assertStatus(304)
+
+        self.getPage('/redirect/proxy')
+        self.assertBody('')
+        self.assertStatus(305)
+
+        # HTTPRedirect on error
+        self.getPage('/redirect/error/')
+        self.assertStatus(('302 Found', '303 See Other'))
+        self.assertInBody('/errpage')
+
+        # Make sure str(HTTPRedirect()) works.
+        self.getPage('/redirect/stringify', protocol='HTTP/1.0')
+        self.assertStatus(200)
+        self.assertBody("(['%s/'], 302)" % self.base())
+        if cherrypy.server.protocol_version == 'HTTP/1.1':
+            self.getPage('/redirect/stringify', protocol='HTTP/1.1')
+            self.assertStatus(200)
+            self.assertBody("(['%s/'], 303)" % self.base())
+
+        # check that #fragments are handled properly
+        # http://skrb.org/ietf/http_errata.html#location-fragments
+        frag = 'foo'
+        self.getPage('/redirect/fragment/%s' % frag)
+        self.assertMatchesBody(
+            r"\2\/some\/url\#%s" % (
+                frag, frag))
+        loc = self.assertHeader('Location')
+        assert loc.endswith('#%s' % frag)
+        self.assertStatus(('302 Found', '303 See Other'))
+
+        # check injection protection
+        # See https://github.com/cherrypy/cherrypy/issues/1003
+        self.getPage(
+            '/redirect/custom?'
+            'code=303&url=/foobar/%0d%0aSet-Cookie:%20somecookie=someval')
+        self.assertStatus(303)
+        loc = self.assertHeader('Location')
+        assert 'Set-Cookie' in loc
+        self.assertNoHeader('Set-Cookie')
+
+        def assertValidXHTML():
+            from xml.etree import ElementTree
+            try:
+                ElementTree.fromstring(
+                    '%s' % self.body,
+                )
+            except ElementTree.ParseError:
+                self._handlewebError(
+                    'automatically generated redirect did not '
+                    'generate well-formed html',
+                )
+
+        # check redirects to URLs generated valid HTML - we check this
+        # by seeing if it appears as valid XHTML.
+        self.getPage('/redirect/by_code?code=303')
+        self.assertStatus(303)
+        assertValidXHTML()
+
+        # do the same with a url containing quote characters.
+        self.getPage('/redirect/url_with_quote')
+        self.assertStatus(303)
+        assertValidXHTML()
+
+    def test_redirect_with_xss(self):
+        """A redirect to a URL with HTML injected should result
+        in page contents escaped."""
+        self.getPage('/redirect/url_with_xss')
+        self.assertStatus(303)
+        assert b'