mirror of
https://github.com/clinton-hall/nzbToMedia.git
synced 2025-08-20 05:13:16 -07:00
update request lib. #620
This commit is contained in:
clinton-hall
parent
17b7fcd43a
commit
41e9ad2fa3
43 changed files with 2442 additions and 1469 deletions
|
|
@ -13,13 +13,15 @@ from collections import Mapping
|
|||
from datetime import datetime
|
||||
|
||||
from .auth import _basic_auth_str
|
||||
from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str
|
||||
from .compat import cookielib, OrderedDict, urljoin, urlparse
|
||||
from .cookies import (
|
||||
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
|
||||
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
|
||||
from .hooks import default_hooks, dispatch_hook
|
||||
from .utils import to_key_val_list, default_headers, to_native_string
|
||||
from .exceptions import TooManyRedirects, InvalidSchema
|
||||
from .exceptions import (
|
||||
TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
|
||||
from .packages.urllib3._collections import RecentlyUsedContainer
|
||||
from .structures import CaseInsensitiveDict
|
||||
|
||||
from .adapters import HTTPAdapter
|
||||
|
|
@ -34,6 +36,8 @@ from .status_codes import codes
|
|||
# formerly defined here, reexposed here for backward compatibility
|
||||
from .models import REDIRECT_STATI
|
||||
|
||||
REDIRECT_CACHE_SIZE = 1000
|
||||
|
||||
|
||||
def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
|
||||
"""
|
||||
|
|
@ -90,11 +94,21 @@ class SessionRedirectMixin(object):
|
|||
"""Receives a Response. Returns a generator of Responses."""
|
||||
|
||||
i = 0
|
||||
hist = [] # keep track of history
|
||||
|
||||
while resp.is_redirect:
|
||||
prepared_request = req.copy()
|
||||
|
||||
resp.content # Consume socket so it can be released
|
||||
if i > 0:
|
||||
# Update history and keep track of redirects.
|
||||
hist.append(resp)
|
||||
new_hist = list(hist)
|
||||
resp.history = new_hist
|
||||
|
||||
try:
|
||||
resp.content # Consume socket so it can be released
|
||||
except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
|
||||
resp.raw.read(decode_content=False)
|
||||
|
||||
if i >= self.max_redirects:
|
||||
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects)
|
||||
|
|
@ -114,17 +128,20 @@ class SessionRedirectMixin(object):
|
|||
parsed = urlparse(url)
|
||||
url = parsed.geturl()
|
||||
|
||||
# Facilitate non-RFC2616-compliant 'location' headers
|
||||
# Facilitate relative 'location' headers, as allowed by RFC 7231.
|
||||
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
||||
# Compliant with RFC3986, we percent encode the url.
|
||||
if not urlparse(url).netloc:
|
||||
if not parsed.netloc:
|
||||
url = urljoin(resp.url, requote_uri(url))
|
||||
else:
|
||||
url = requote_uri(url)
|
||||
|
||||
prepared_request.url = to_native_string(url)
|
||||
# Cache the url, unless it redirects to itself.
|
||||
if resp.is_permanent_redirect and req.url != prepared_request.url:
|
||||
self.redirect_cache[req.url] = prepared_request.url
|
||||
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
|
||||
# http://tools.ietf.org/html/rfc7231#section-6.4.4
|
||||
if (resp.status_code == codes.see_other and
|
||||
method != 'HEAD'):
|
||||
method = 'GET'
|
||||
|
|
@ -142,7 +159,7 @@ class SessionRedirectMixin(object):
|
|||
prepared_request.method = method
|
||||
|
||||
# https://github.com/kennethreitz/requests/issues/1084
|
||||
if resp.status_code not in (codes.temporary, codes.resume):
|
||||
if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect):
|
||||
if 'Content-Length' in prepared_request.headers:
|
||||
del prepared_request.headers['Content-Length']
|
||||
|
||||
|
|
@ -218,11 +235,11 @@ class SessionRedirectMixin(object):
|
|||
"""
|
||||
headers = prepared_request.headers
|
||||
url = prepared_request.url
|
||||
new_proxies = {}
|
||||
scheme = urlparse(url).scheme
|
||||
new_proxies = proxies.copy() if proxies is not None else {}
|
||||
|
||||
if not should_bypass_proxies(url):
|
||||
if self.trust_env and not should_bypass_proxies(url):
|
||||
environ_proxies = get_environ_proxies(url)
|
||||
scheme = urlparse(url).scheme
|
||||
|
||||
proxy = environ_proxies.get(scheme)
|
||||
|
||||
|
|
@ -257,9 +274,10 @@ class Session(SessionRedirectMixin):
|
|||
"""
|
||||
|
||||
__attrs__ = [
|
||||
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
|
||||
'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
|
||||
'trust_env', 'max_redirects']
|
||||
'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify',
|
||||
'cert', 'prefetch', 'adapters', 'stream', 'trust_env',
|
||||
'max_redirects',
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
|
||||
|
|
@ -312,6 +330,9 @@ class Session(SessionRedirectMixin):
|
|||
self.mount('https://', HTTPAdapter())
|
||||
self.mount('http://', HTTPAdapter())
|
||||
|
||||
# Only store 1000 redirects to prevent using infinite memory
|
||||
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
|
|
@ -349,6 +370,7 @@ class Session(SessionRedirectMixin):
|
|||
url=request.url,
|
||||
files=request.files,
|
||||
data=request.data,
|
||||
json=request.json,
|
||||
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
|
||||
params=merge_setting(request.params, self.params),
|
||||
auth=merge_setting(auth, self.auth),
|
||||
|
|
@ -370,7 +392,8 @@ class Session(SessionRedirectMixin):
|
|||
hooks=None,
|
||||
stream=None,
|
||||
verify=None,
|
||||
cert=None):
|
||||
cert=None,
|
||||
json=None):
|
||||
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
|
||||
Returns :class:`Response <Response>` object.
|
||||
|
||||
|
|
@ -380,17 +403,22 @@ class Session(SessionRedirectMixin):
|
|||
string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the
|
||||
:class:`Request`.
|
||||
:param json: (optional) json to send in the body of the
|
||||
:class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the
|
||||
:class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the
|
||||
:class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects
|
||||
:param files: (optional) Dictionary of ``'filename': file-like-objects``
|
||||
for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple or callable to enable
|
||||
Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the
|
||||
request in seconds.
|
||||
:param allow_redirects: (optional) Boolean. Set to True by default.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a (`connect timeout, read
|
||||
timeout <user/advanced.html#timeouts>`_) tuple.
|
||||
:type timeout: float or tuple
|
||||
:param allow_redirects: (optional) Set to True by default.
|
||||
:type allow_redirects: bool
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of
|
||||
the proxy.
|
||||
:param stream: (optional) whether to immediately download the response
|
||||
|
|
@ -401,7 +429,7 @@ class Session(SessionRedirectMixin):
|
|||
If Tuple, ('cert', 'key') pair.
|
||||
"""
|
||||
|
||||
method = builtin_str(method)
|
||||
method = to_native_string(method)
|
||||
|
||||
# Create the Request.
|
||||
req = Request(
|
||||
|
|
@ -410,6 +438,7 @@ class Session(SessionRedirectMixin):
|
|||
headers = headers,
|
||||
files = files,
|
||||
data = data or {},
|
||||
json = json,
|
||||
params = params or {},
|
||||
auth = auth,
|
||||
cookies = cookies,
|
||||
|
|
@ -419,36 +448,16 @@ class Session(SessionRedirectMixin):
|
|||
|
||||
proxies = proxies or {}
|
||||
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
env_proxies = get_environ_proxies(url) or {}
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Look for configuration.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('REQUESTS_CA_BUNDLE')
|
||||
|
||||
# Curl compatibility.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('CURL_CA_BUNDLE')
|
||||
|
||||
# Merge all the kwargs.
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
settings = self.merge_environment_settings(
|
||||
prep.url, proxies, stream, verify, cert
|
||||
)
|
||||
|
||||
# Send the request.
|
||||
send_kwargs = {
|
||||
'stream': stream,
|
||||
'timeout': timeout,
|
||||
'verify': verify,
|
||||
'cert': cert,
|
||||
'proxies': proxies,
|
||||
'allow_redirects': allow_redirects,
|
||||
}
|
||||
send_kwargs.update(settings)
|
||||
resp = self.send(prep, **send_kwargs)
|
||||
|
||||
return resp
|
||||
|
|
@ -483,15 +492,16 @@ class Session(SessionRedirectMixin):
|
|||
kwargs.setdefault('allow_redirects', False)
|
||||
return self.request('HEAD', url, **kwargs)
|
||||
|
||||
def post(self, url, data=None, **kwargs):
|
||||
def post(self, url, data=None, json=None, **kwargs):
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('POST', url, data=data, **kwargs)
|
||||
return self.request('POST', url, data=data, json=json, **kwargs)
|
||||
|
||||
def put(self, url, data=None, **kwargs):
|
||||
"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
|
@ -536,6 +546,14 @@ class Session(SessionRedirectMixin):
|
|||
if not isinstance(request, PreparedRequest):
|
||||
raise ValueError('You can only send PreparedRequests.')
|
||||
|
||||
checked_urls = set()
|
||||
while request.url in self.redirect_cache:
|
||||
checked_urls.add(request.url)
|
||||
new_url = self.redirect_cache.get(request.url)
|
||||
if new_url in checked_urls:
|
||||
break
|
||||
request.url = new_url
|
||||
|
||||
# Set up variables needed for resolve_redirects and dispatching of hooks
|
||||
allow_redirects = kwargs.pop('allow_redirects', True)
|
||||
stream = kwargs.get('stream')
|
||||
|
|
@ -588,8 +606,35 @@ class Session(SessionRedirectMixin):
|
|||
r = history.pop()
|
||||
r.history = history
|
||||
|
||||
if not stream:
|
||||
r.content
|
||||
|
||||
return r
|
||||
|
||||
def merge_environment_settings(self, url, proxies, stream, verify, cert):
|
||||
"""Check the environment and merge it with some settings."""
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
env_proxies = get_environ_proxies(url) or {}
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Look for requests environment configuration and be compatible
|
||||
# with cURL.
|
||||
if verify is True or verify is None:
|
||||
verify = (os.environ.get('REQUESTS_CA_BUNDLE') or
|
||||
os.environ.get('CURL_CA_BUNDLE'))
|
||||
|
||||
# Merge all the kwargs.
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
|
||||
return {'verify': verify, 'proxies': proxies, 'stream': stream,
|
||||
'cert': cert}
|
||||
|
||||
def get_adapter(self, url):
|
||||
"""Returns the appropriate connnection adapter for the given URL."""
|
||||
for (prefix, adapter) in self.adapters.items():
|
||||
|
|
@ -617,12 +662,19 @@ class Session(SessionRedirectMixin):
|
|||
self.adapters[key] = self.adapters.pop(key)
|
||||
|
||||
def __getstate__(self):
|
||||
return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
|
||||
state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
|
||||
state['redirect_cache'] = dict(self.redirect_cache)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
redirect_cache = state.pop('redirect_cache', {})
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE)
|
||||
for redirect, to in redirect_cache.items():
|
||||
self.redirect_cache[redirect] = to
|
||||
|
||||
|
||||
def session():
|
||||
"""Returns a :class:`Session` for context-management."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue