diff --git a/lib/cloudinary/__init__.py b/lib/cloudinary/__init__.py index 56e7b5b7..5e315049 100644 --- a/lib/cloudinary/__init__.py +++ b/lib/cloudinary/__init__.py @@ -38,7 +38,7 @@ CL_BLANK = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAA URI_SCHEME = "cloudinary" API_VERSION = "v1_1" -VERSION = "1.28.1" +VERSION = "1.29.0" USER_AGENT = "CloudinaryPython/{} (Python {})".format(VERSION, python_version()) """ :const: USER_AGENT """ diff --git a/lib/cloudinary/api_client/tcp_keep_alive_manager.py b/lib/cloudinary/api_client/tcp_keep_alive_manager.py new file mode 100644 index 00000000..b2c7b75f --- /dev/null +++ b/lib/cloudinary/api_client/tcp_keep_alive_manager.py @@ -0,0 +1,119 @@ +import socket +import sys + +from urllib3 import HTTPSConnectionPool, HTTPConnectionPool, PoolManager, ProxyManager + +# Inspired by: +# https://github.com/finbourne/lusid-sdk-python/blob/b813882e4f1777ea78670a03a7596486639e6f40/sdk/lusid/tcp/tcp_keep_alive_probes.py + +# The content to send on Mac OS in the TCP Keep Alive probe +TCP_KEEPALIVE = 0x10 +# The maximum time to keep the connection idle before sending probes +TCP_KEEP_IDLE = 60 +# The interval between probes +TCP_KEEPALIVE_INTERVAL = 60 +# The maximum number of failed probes before terminating the connection +TCP_KEEP_CNT = 3 + + +class TCPKeepAliveValidationMethods: + """ + This class contains a single method whose sole purpose is to set up TCP Keep Alive probes on the socket for a + connection. This is necessary for long-running requests which will be silently terminated by the AWS Network Load + Balancer which kills a connection if it is idle for more than 350 seconds. + """ + + @staticmethod + def adjust_connection_socket(conn, protocol="https"): + """ + Adjusts the socket settings so that the client sends a TCP keep alive probe over the connection. This is only + applied where possible, if the ability to set the socket options is not available, for example using Anaconda, + then the settings will be left as is. + :param conn: The connection to update the socket settings for + :param str protocol: The protocol of the connection + :return: None + """ + + if protocol == "http": + # It isn't clear how to set this up over HTTP, it seems to differ from HTTPs + return + + # TCP Keep Alive Probes for different platforms + platform = sys.platform + # TCP Keep Alive Probes for Linux + if (platform == 'linux' and hasattr(conn.sock, "setsockopt") and hasattr(socket, "SO_KEEPALIVE") and + hasattr(socket, "TCP_KEEPIDLE") and hasattr(socket, "TCP_KEEPINTVL") and hasattr(socket, + "TCP_KEEPCNT")): + conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + conn.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, TCP_KEEP_IDLE) + conn.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, TCP_KEEPALIVE_INTERVAL) + conn.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, TCP_KEEP_CNT) + + # TCP Keep Alive Probes for Windows OS + elif platform == 'win32' and hasattr(socket, "SIO_KEEPALIVE_VALS") and getattr(conn.sock, "ioctl", + None) is not None: + conn.sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, TCP_KEEP_IDLE * 1000, TCP_KEEPALIVE_INTERVAL * 1000)) + + # TCP Keep Alive Probes for Mac OS + elif platform == 'darwin' and getattr(conn.sock, "setsockopt", None) is not None: + conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + conn.sock.setsockopt(socket.IPPROTO_TCP, TCP_KEEPALIVE, TCP_KEEPALIVE_INTERVAL) + + +class TCPKeepAliveHTTPSConnectionPool(HTTPSConnectionPool): + """ + This class overrides the _validate_conn method in the HTTPSConnectionPool class. This is the entry point to use + for modifying the socket as it is called after the socket is created and before the request is made. + """ + + def _validate_conn(self, conn): + """ + Called right before a request is made, after the socket is created. + """ + # Call the method on the base class + super(TCPKeepAliveHTTPSConnectionPool, self)._validate_conn(conn) + + # Set up TCP Keep Alive probes, this is the only line added to this function + TCPKeepAliveValidationMethods.adjust_connection_socket(conn, "https") + + +class TCPKeepAliveHTTPConnectionPool(HTTPConnectionPool): + """ + This class overrides the _validate_conn method in the HTTPSConnectionPool class. This is the entry point to use + for modifying the socket as it is called after the socket is created and before the request is made. + In the base class this method is passed completely. + """ + + def _validate_conn(self, conn): + """ + Called right before a request is made, after the socket is created. + """ + # Call the method on the base class + super(TCPKeepAliveHTTPConnectionPool, self)._validate_conn(conn) + + # Set up TCP Keep Alive probes, this is the only line added to this function + TCPKeepAliveValidationMethods.adjust_connection_socket(conn, "http") + + +class TCPKeepAlivePoolManager(PoolManager): + """ + This Pool Manager has only had the pool_classes_by_scheme variable changed. This now points at the TCPKeepAlive + connection pools rather than the default connection pools. + """ + + def __init__(self, num_pools=10, headers=None, **connection_pool_kw): + super(TCPKeepAlivePoolManager, self).__init__(num_pools=num_pools, headers=headers, **connection_pool_kw) + self.pool_classes_by_scheme = {"http": TCPKeepAliveHTTPConnectionPool, "https": TCPKeepAliveHTTPSConnectionPool} + + +class TCPKeepAliveProxyManager(ProxyManager): + """ + This Proxy Manager has only had the pool_classes_by_scheme variable changed. This now points at the TCPKeepAlive + connection pools rather than the default connection pools. + """ + + def __init__(self, proxy_url, num_pools=10, headers=None, proxy_headers=None, **connection_pool_kw): + super(TCPKeepAliveProxyManager, self).__init__(proxy_url=proxy_url, num_pools=num_pools, headers=headers, + proxy_headers=proxy_headers, + **connection_pool_kw) + self.pool_classes_by_scheme = {"http": TCPKeepAliveHTTPConnectionPool, "https": TCPKeepAliveHTTPSConnectionPool} diff --git a/lib/cloudinary/uploader.py b/lib/cloudinary/uploader.py index 3b1c63b3..3b118142 100644 --- a/lib/cloudinary/uploader.py +++ b/lib/cloudinary/uploader.py @@ -277,18 +277,48 @@ def explode(public_id, **options): return call_api("explode", params, **options) -# options may include 'exclusive' (boolean) which causes clearing this tag from all other resources def add_tag(tag, public_ids=None, **options): + """ + Adds a single tag or a list of tags or a comma-separated tags to the assets. + + :param tag: The tag or tags to assign. Can specify multiple tags in a single string, + separated by commas - "t1,t2,t3" or list of tags - ["t1","t2","t3"]. + :param public_ids: A list of public IDs (up to 1000). + :param options: Configuration options may include 'exclusive' (boolean) which causes + clearing this tag from all other assets. + + :return: Dictionary with a list of public IDs that were updated. + """ exclusive = options.pop("exclusive", None) command = "set_exclusive" if exclusive else "add" return call_tags_api(tag, command, public_ids, **options) def remove_tag(tag, public_ids=None, **options): + """ + Removes a single tag or a list of tags or a comma-separated tags from the assets. + + :param tag: The tag or tags to assign. Can specify multiple tags in a single string, + separated by commas - "t1,t2,t3" or list of tags - ["t1","t2","t3"]. + :param public_ids: A list of public IDs (up to 1000). + :param options: Additional options. + + :return: Dictionary with a list of public IDs that were updated. + """ return call_tags_api(tag, "remove", public_ids, **options) def replace_tag(tag, public_ids=None, **options): + """ + Replaces all existing tags with a single tag or a list of tags or a comma-separated tags of the assets. + + :param tag: The tag or tags to assign. Can specify multiple tags in a single string, + separated by commas - "t1,t2,t3" or list of tags - ["t1","t2","t3"]. + :param public_ids: A list of public IDs (up to 1000). + :param options: Additional options. + + :return: Dictionary with a list of public IDs that were updated. + """ return call_tags_api(tag, "replace", public_ids, **options) diff --git a/lib/cloudinary/utils.py b/lib/cloudinary/utils.py index f936b845..1fade15b 100644 --- a/lib/cloudinary/utils.py +++ b/lib/cloudinary/utils.py @@ -17,11 +17,12 @@ from fractions import Fraction from numbers import Number import six.moves.urllib.parse -from six import iteritems, string_types +from six import iteritems from urllib3 import ProxyManager, PoolManager import cloudinary from cloudinary import auth_token +from cloudinary.api_client.tcp_keep_alive_manager import TCPKeepAlivePoolManager, TCPKeepAliveProxyManager from cloudinary.compat import PY3, to_bytes, to_bytearray, to_string, string_types, urlparse VAR_NAME_RE = r'(\$\([a-zA-Z]\w+\))' @@ -1509,7 +1510,7 @@ def verify_notification_signature(body, timestamp, signature, valid_for=7200, al def get_http_connector(conf, options): """ - Used to create http connector, depends on api_proxy configuration parameter + Used to create http connector, depends on api_proxy and disable_tcp_keep_alive configuration parameters. :param conf: configuration object :param options: additional options @@ -1517,10 +1518,16 @@ def get_http_connector(conf, options): :return: ProxyManager if api_proxy is set, otherwise PoolManager object """ if conf.api_proxy: - return ProxyManager(conf.api_proxy, **options) - else: + if conf.disable_tcp_keep_alive: + return ProxyManager(conf.api_proxy, **options) + + return TCPKeepAliveProxyManager(conf.api_proxy, **options) + + if conf.disable_tcp_keep_alive: return PoolManager(**options) + return TCPKeepAlivePoolManager(**options) + def encode_list(obj): if isinstance(obj, list): diff --git a/requirements.txt b/requirements.txt index 5ec7d931..ed3dc9c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ bleach==4.1.0 certifi==2021.10.8 cheroot==8.6.0 cherrypy==18.6.1 -cloudinary==1.28.1 +cloudinary==1.29.0 distro==1.6.0 dnspython==2.2.0 facebook-sdk==3.1.0