From 80df2b0fadd3e3eff6e32ba03c40d96a662a02fb Mon Sep 17 00:00:00 2001 From: JonnyWong16 Date: Sun, 25 Mar 2018 13:47:49 -0700 Subject: [PATCH] Add Imgur rate limiting --- data/interfaces/default/settings.html | 4 +- lib/ratelimit/__init__.py | 59 +++++++++++++++++++++++++++ lib/ratelimit/version.py | 9 ++++ plexpy/helpers.py | 39 ++++++++++++++++-- 4 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 lib/ratelimit/__init__.py create mode 100644 lib/ratelimit/version.py diff --git a/data/interfaces/default/settings.html b/data/interfaces/default/settings.html index 12ea4b63..e6a345ea 100644 --- a/data/interfaces/default/settings.html +++ b/data/interfaces/default/settings.html @@ -977,7 +977,7 @@

Enter your Imgur API client ID in order to upload posters. You can register a new application here. -

> +

@@ -2543,7 +2543,7 @@ $(document).ready(function() { }); function newsletterUploadEnabled() { - if ($('#notify_upload_posters').is(':checked') || $('#newsletter_self_hosted').is(':checked')) { + if ($('#notify_upload_posters').val() === '1' || $('#newsletter_self_hosted').is(':checked')) { $('#newsletter_upload_warning').hide(); } else { $('#newsletter_upload_warning').show(); diff --git a/lib/ratelimit/__init__.py b/lib/ratelimit/__init__.py new file mode 100644 index 00000000..115a65dd --- /dev/null +++ b/lib/ratelimit/__init__.py @@ -0,0 +1,59 @@ +from math import floor + +import time +import sys +import threading +import functools + + +def clamp(value): + ''' + Clamp integer between 1 and max + + There must be at least 1 method invocation + made over the time period. Make sure the + value passed is at least 1 and is not a + fraction of an invocation. + + :param float value: The number of method invocations. + :return: Clamped number of invocations. + :rtype: int + ''' + return max(1, min(sys.maxsize, floor(value))) + + +class RateLimitDecorator: + def __init__(self, period=1, every=1.0): + self.frequency = abs(every) / float(clamp(period)) + self.last_called = 0.0 + self.lock = threading.RLock() + + def __call__(self, func): + ''' + Extend the behaviour of the following + function, forwarding method invocations + if the time window hes elapsed. + + :param function func: The function to decorate. + :return: Decorated function. + :rtype: function + ''' + @functools.wraps(func) + def wrapper(*args, **kwargs): + '''Decorator wrapper function''' + with self.lock: + elapsed = time.time() - self.last_called + left_to_wait = self.frequency - elapsed + if left_to_wait > 0: + time.sleep(left_to_wait) + self.last_called = time.time() + return func(*args, **kwargs) + return wrapper + + +rate_limited = RateLimitDecorator + + +__all__ = [ + 'rate_limited' +] diff --git a/lib/ratelimit/version.py b/lib/ratelimit/version.py new file mode 100644 index 00000000..68a84cc9 --- /dev/null +++ b/lib/ratelimit/version.py @@ -0,0 +1,9 @@ +class Version(object): + '''Version of the package''' + + def __setattr__(self, *args): + raise TypeError('cannot modify immutable instance') + __delattr__ = __setattr__ + + def __init__(self, num): + super(Version, self).__setattr__('number', num) diff --git a/plexpy/helpers.py b/plexpy/helpers.py index 917b139c..6b1b9af7 100644 --- a/plexpy/helpers.py +++ b/plexpy/helpers.py @@ -28,6 +28,7 @@ import math import maxminddb from operator import itemgetter import os +from ratelimit import rate_limited import re import socket import sys @@ -76,6 +77,7 @@ def addtoapi(*dargs, **dkwargs): return rd + def multikeysort(items, columns): comparers = [((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] @@ -161,6 +163,7 @@ def convert_milliseconds(ms): return minutes + def convert_milliseconds_to_minutes(ms): if str(ms).isdigit(): @@ -171,6 +174,7 @@ def convert_milliseconds_to_minutes(ms): return 0 + def convert_seconds(s): gmtime = time.gmtime(s) @@ -181,6 +185,7 @@ def convert_seconds(s): return minutes + def convert_seconds_to_minutes(s): if str(s).isdigit(): @@ -201,6 +206,7 @@ def now(): now = datetime.datetime.now() return now.strftime("%Y-%m-%d %H:%M:%S") + def human_duration(s, sig='dhms'): hd = '' @@ -233,6 +239,7 @@ def human_duration(s, sig='dhms'): return hd + def get_age(date): try: @@ -385,6 +392,7 @@ def split_string(mystring, splitvar=','): mylist.append(each_word.strip()) return mylist + def create_https_certificates(ssl_cert, ssl_key): """ Create a self-signed HTTPS certificate and store in it in @@ -424,12 +432,14 @@ def cast_to_int(s): except (ValueError, TypeError): return 0 + def cast_to_float(s): try: return float(s) except (ValueError, TypeError): return 0 + def convert_xml_to_json(xml): o = xmltodict.parse(xml) return json.dumps(o) @@ -452,12 +462,14 @@ def get_percent(value1, value2): return math.trunc(percent) + def hex_to_int(hex): try: return int(hex, 16) except (ValueError, TypeError): return 0 + def parse_xml(unparsed=None): if unparsed: try: @@ -473,10 +485,11 @@ def parse_xml(unparsed=None): logger.warn("XML parse request made but no data received.") return [] -""" -Validate xml keys to make sure they exist and return their attribute value, return blank value is none found -""" + def get_xml_attr(xml_key, attribute, return_bool=False, default_return=''): + """ + Validate xml keys to make sure they exist and return their attribute value, return blank value is none found + """ if xml_key.getAttribute(attribute): if return_bool: return True @@ -488,6 +501,7 @@ def get_xml_attr(xml_key, attribute, return_bool=False, default_return=''): else: return default_return + def process_json_kwargs(json_kwargs): params = {} if json_kwargs: @@ -495,18 +509,21 @@ def process_json_kwargs(json_kwargs): return params + def sanitize(string): if string: return unicode(string).replace('<','<').replace('>','>') else: return '' + def is_public_ip(host): ip = is_valid_ip(get_ip(host)) if ip and ip.iptype() == 'PUBLIC': return True return False + def get_ip(host): ip_address = '' if is_valid_ip(host): @@ -519,6 +536,7 @@ def get_ip(host): logger.error(u"IP Checker :: Bad IP or hostname provided.") return ip_address + def is_valid_ip(address): try: return IP(address) @@ -527,6 +545,7 @@ def is_valid_ip(address): except ValueError: return False + def install_geoip_db(): maxmind_url = 'http://geolite.maxmind.com/download/geoip/database/' geolite2_gz = 'GeoLite2-City.mmdb.gz' @@ -587,6 +606,7 @@ def install_geoip_db(): return True + def uninstall_geoip_db(): logger.debug(u"Tautulli Helpers :: Uninstalling the GeoLite2 database...") try: @@ -600,6 +620,7 @@ def uninstall_geoip_db(): logger.debug(u"Tautulli Helpers :: GeoLite2 database uninstalled successfully.") return True + def geoip_lookup(ip_address): if not plexpy.CONFIG.GEOIP_DB: return 'GeoLite2 database not installed. Please install from the ' \ @@ -638,6 +659,7 @@ def geoip_lookup(ip_address): return geo_info + def whois_lookup(ip_address): nets = [] @@ -674,6 +696,7 @@ def whois_lookup(ip_address): return whois_info + # Taken from SickRage def anon_url(*url): """ @@ -682,6 +705,7 @@ def anon_url(*url): return '' if None in url else '%s%s' % (plexpy.CONFIG.ANON_REDIRECT, ''.join(str(s) for s in url)) +@rate_limited(450, 3600) def upload_to_imgur(img_data, img_title='', rating_key='', fallback=''): """ Uploads an image to Imgur """ client_id = plexpy.CONFIG.IMGUR_CLIENT_ID @@ -769,6 +793,7 @@ def cache_image(url, image=None): return imagefile, imagetype + def build_datatables_json(kwargs, dt_columns, default_sort_col=None): """ Builds datatables json data @@ -793,6 +818,7 @@ def build_datatables_json(kwargs, dt_columns, default_sort_col=None): } return json.dumps(json_data) + def humanFileSize(bytes, si=False): if str(bytes).isdigit(): bytes = int(bytes) @@ -816,6 +842,7 @@ def humanFileSize(bytes, si=False): return "{0:.1f} {1}".format(bytes, units[u]) + def parse_condition_logic_string(s, num_cond=0): """ Parse a logic string into a nested list Based on http://stackoverflow.com/a/23185606 @@ -900,6 +927,7 @@ def parse_condition_logic_string(s, num_cond=0): return stack.pop() + def nested_list_to_string(l): for i, x in enumerate(l): if isinstance(x, list): @@ -907,6 +935,7 @@ def nested_list_to_string(l): s = '(' + ' '.join(l) + ')' return s + def eval_logic_groups_to_bool(logic_groups, eval_conds): first_cond = logic_groups[0] @@ -928,6 +957,7 @@ def eval_logic_groups_to_bool(logic_groups, eval_conds): return result + def get_plexpy_url(hostname=None): if plexpy.CONFIG.ENABLE_HTTPS: scheme = 'https' @@ -961,6 +991,7 @@ def get_plexpy_url(hostname=None): return scheme + '://' + hostname + port + root + def momentjs_to_arrow(format, duration=False): invalid_formats = ['Mo', 'DDDo', 'do'] if duration: @@ -974,4 +1005,4 @@ def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n - return izip_longest(fillvalue=fillvalue, *args) \ No newline at end of file + return izip_longest(fillvalue=fillvalue, *args)