mirror of
https://github.com/Tautulli/Tautulli.git
synced 2025-07-11 07:46:07 -07:00
Clean up http_handler using requests
* Restore ThreadPool for multiple requests * Use a requests.Session() * Update requests.exceptions
This commit is contained in:
parent
6d0831ceaa
commit
e705bedc91
1 changed files with 62 additions and 37 deletions
|
@ -19,14 +19,12 @@ from __future__ import unicode_literals
|
||||||
from future.builtins import object
|
from future.builtins import object
|
||||||
from future.builtins import str
|
from future.builtins import str
|
||||||
|
|
||||||
from functools import partial
|
|
||||||
from multiprocessing.dummy import Pool as ThreadPool
|
from multiprocessing.dummy import Pool as ThreadPool
|
||||||
from future.moves.urllib.parse import urljoin
|
from future.moves.urllib.parse import urljoin
|
||||||
|
|
||||||
import os
|
|
||||||
import certifi
|
import certifi
|
||||||
import urllib3
|
|
||||||
import requests
|
import requests
|
||||||
|
import urllib3
|
||||||
|
|
||||||
import plexpy
|
import plexpy
|
||||||
if plexpy.PYTHON2:
|
if plexpy.PYTHON2:
|
||||||
|
@ -43,6 +41,7 @@ class HTTPHandler(object):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, urls, headers=None, token=None, timeout=10, ssl_verify=True, silent=False):
|
def __init__(self, urls, headers=None, token=None, timeout=10, ssl_verify=True, silent=False):
|
||||||
|
self._valid_request_types = {'GET', 'POST', 'PUT', 'DELETE'}
|
||||||
self._silent = silent
|
self._silent = silent
|
||||||
|
|
||||||
if isinstance(urls, str):
|
if isinstance(urls, str):
|
||||||
|
@ -53,7 +52,8 @@ class HTTPHandler(object):
|
||||||
if headers:
|
if headers:
|
||||||
self.headers = headers
|
self.headers = headers
|
||||||
else:
|
else:
|
||||||
self.headers = {'X-Plex-Product': plexpy.common.PRODUCT,
|
self.headers = {
|
||||||
|
'X-Plex-Product': plexpy.common.PRODUCT,
|
||||||
'X-Plex-Version': plexpy.common.RELEASE,
|
'X-Plex-Version': plexpy.common.RELEASE,
|
||||||
'X-Plex-Client-Identifier': plexpy.CONFIG.PMS_UUID,
|
'X-Plex-Client-Identifier': plexpy.CONFIG.PMS_UUID,
|
||||||
'X-Plex-Platform': plexpy.common.PLATFORM,
|
'X-Plex-Platform': plexpy.common.PLATFORM,
|
||||||
|
@ -67,11 +67,19 @@ class HTTPHandler(object):
|
||||||
if self.token:
|
if self.token:
|
||||||
self.headers['X-Plex-Token'] = self.token
|
self.headers['X-Plex-Token'] = self.token
|
||||||
|
|
||||||
|
self._session = requests.Session()
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
|
|
||||||
self.ssl_verify = certifi.where() if ssl_verify else False
|
self.ssl_verify = certifi.where() if ssl_verify else False
|
||||||
|
if not self.ssl_verify:
|
||||||
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||||
|
|
||||||
self.valid_request_types = ('GET', 'POST', 'PUT', 'DELETE')
|
self.uri = None
|
||||||
|
self.data = None
|
||||||
|
self.request_type = 'GET'
|
||||||
|
self.output_format = 'raw'
|
||||||
|
self.return_type = False
|
||||||
|
self.callback = None
|
||||||
|
self.request_kwargs = {}
|
||||||
|
|
||||||
def make_request(self,
|
def make_request(self,
|
||||||
uri=None,
|
uri=None,
|
||||||
|
@ -99,7 +107,7 @@ class HTTPHandler(object):
|
||||||
self.timeout = timeout or self.timeout
|
self.timeout = timeout or self.timeout
|
||||||
self.request_kwargs = request_kwargs
|
self.request_kwargs = request_kwargs
|
||||||
|
|
||||||
if self.request_type not in self.valid_request_types:
|
if self.request_type not in self._valid_request_types:
|
||||||
logger.debug("HTTP request made but unsupported request type given.")
|
logger.debug("HTTP request made but unsupported request type given.")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -118,39 +126,60 @@ class HTTPHandler(object):
|
||||||
return responses[0]
|
return responses[0]
|
||||||
|
|
||||||
else:
|
else:
|
||||||
logger.debug("HTTP request made but no enpoint given.")
|
logger.debug("HTTP request made but no uri endpoint provided.")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _http_requests_pool(self, urls):
|
def _http_requests_pool(self, urls, workers=10, chunk=None):
|
||||||
"""Generator function to request multiple urls"""
|
"""Generator function to request urls in chunks"""
|
||||||
|
# From cpython
|
||||||
|
if chunk is None:
|
||||||
|
chunk, extra = divmod(len(urls), workers * 4)
|
||||||
|
if extra:
|
||||||
|
chunk += 1
|
||||||
|
if len(urls) == 0:
|
||||||
|
chunk = 0
|
||||||
|
|
||||||
if len(urls) == 1:
|
if len(urls) == 1:
|
||||||
yield self._http_requests_single(urls[0])
|
yield self._http_requests_single(urls[0])
|
||||||
else:
|
else:
|
||||||
for url in urls:
|
pool = ThreadPool(workers)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
yield self._http_requests_single(url)
|
for work in pool.imap_unordered(self._http_requests_single, urls, chunk):
|
||||||
|
yield work
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if not self._silent:
|
if not self._silent:
|
||||||
logger.error("Failed to yield request: %s" % e)
|
logger.error("Failed to yield request: %s" % e)
|
||||||
|
finally:
|
||||||
|
pool.close()
|
||||||
|
pool.join()
|
||||||
|
|
||||||
def _http_requests_single(self, url):
|
def _http_requests_single(self, url):
|
||||||
"""Request the data from the url"""
|
"""Request the data from the url"""
|
||||||
|
error_msg = "Failed to access uri endpoint %s. " % self.uri
|
||||||
try:
|
try:
|
||||||
r = requests.request(self.request_type, url, data=self.data, headers=self.headers,
|
r = self._session.request(self.request_type, url, headers=self.headers, data=self.data,
|
||||||
timeout=self.timeout, verify=self.ssl_verify)
|
timeout=self.timeout, verify=self.ssl_verify, **self.request_kwargs)
|
||||||
|
r.raise_for_status()
|
||||||
except IOError as e:
|
except requests.exceptions.Timeout as e:
|
||||||
if not self._silent:
|
if not self._silent:
|
||||||
logger.warn("Failed to access uri endpoint %s with error %s" % (self.uri, e))
|
logger.error(error_msg + "Request timed out: %s", e)
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except requests.exceptions.SSLError as e:
|
||||||
if not self._silent:
|
if not self._silent:
|
||||||
logger.warn("Failed to access uri endpoint %s. Is your server maybe accepting SSL connections only? %s" % (self.uri, e))
|
logger.error(error_msg + "Is your server maybe accepting SSL connections only? %s", e)
|
||||||
return None
|
return None
|
||||||
except:
|
except requests.exceptions.HTTPError as e:
|
||||||
if not self._silent:
|
if not self._silent:
|
||||||
logger.warn("Failed to access uri endpoint %s with Uncaught exception." % self.uri)
|
logger.error(error_msg + "Status code %s", e)
|
||||||
|
return None
|
||||||
|
except requests.exceptions.ConnectionError as e:
|
||||||
|
if not self._silent:
|
||||||
|
logger.error(error_msg + "Connection error: %s", e)
|
||||||
|
return None
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
if not self._silent:
|
||||||
|
logger.error(error_msg + "Uncaught exception: %s", e)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
response_status = r.status_code
|
response_status = r.status_code
|
||||||
|
@ -159,10 +188,6 @@ class HTTPHandler(object):
|
||||||
|
|
||||||
if response_status in (200, 201):
|
if response_status in (200, 201):
|
||||||
return self._http_format_output(response_content, response_headers)
|
return self._http_format_output(response_content, response_headers)
|
||||||
else:
|
|
||||||
if not self._silent:
|
|
||||||
logger.warn("Failed to access uri endpoint %s. Status code %r" % (self.uri, response_status))
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _http_format_output(self, response_content, response_headers):
|
def _http_format_output(self, response_content, response_headers):
|
||||||
"""Formats the request response to the desired type"""
|
"""Formats the request response to the desired type"""
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue