github/nzbToMedia
1
0
Fork 0
mirror of https://github.com/clinton-hall/nzbToMedia.git synced 2025-08-21 05:43:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updated stevedore to 2.0.1

Browse source
This commit is contained in:
Labrys of Knossos 2022-11-29 01:47:46 -05:00
commit fb6011f88d
52 changed files with 581 additions and 1960 deletions
Download patch file Download diff file Expand all files Collapse all files

273
libs/common/dogpile/cache/backends/redis.py vendored
View file

@ -8,23 +8,20 @@ Provides backends for talking to `Redis <http://redis.io>`_.
from __future__ import absolute_import
import typing
import warnings
from ..api import BytesBackend
from ..api import CacheBackend
from ..api import NO_VALUE
from ...util.compat import pickle
from ...util.compat import u
if typing.TYPE_CHECKING:
import redis
else:
# delayed import
redis = None # noqa F811
redis = None
__all__ = ("RedisBackend", "RedisSentinelBackend")
__all__ = ("RedisBackend",)
class RedisBackend(BytesBackend):
r"""A `Redis <http://redis.io/>`_ backend, using the
class RedisBackend(CacheBackend):
"""A `Redis <http://redis.io/>`_ backend, using the
`redis-py <http://pypi.python.org/pypi/redis/>`_ backend.
Example configuration::
@ -49,10 +46,14 @@ class RedisBackend(BytesBackend):
:param url: string. If provided, will override separate host/port/db
params. The format is that accepted by ``StrictRedis.from_url()``.
.. versionadded:: 0.4.1
:param host: string, default is ``localhost``.
:param password: string, default is no password.
.. versionadded:: 0.4.1
:param port: integer, default is ``6379``.
:param db: integer, default is ``0``.
@ -70,31 +71,33 @@ class RedisBackend(BytesBackend):
Redis should expire it. This argument is only valid when
``distributed_lock`` is ``True``.
.. versionadded:: 0.5.0
:param socket_timeout: float, seconds for socket timeout.
Default is None (no timeout).
.. versionadded:: 0.5.4
:param lock_sleep: integer, number of seconds to sleep when failed to
acquire a lock. This argument is only valid when
``distributed_lock`` is ``True``.
.. versionadded:: 0.5.0
:param connection_pool: ``redis.ConnectionPool`` object. If provided,
this object supersedes other connection arguments passed to the
``redis.StrictRedis`` instance, including url and/or host as well as
socket_timeout, and will be passed to ``redis.StrictRedis`` as the
source of connectivity.
.. versionadded:: 0.5.4
:param thread_local_lock: bool, whether a thread-local Redis lock object
should be used. This is the default, but is not compatible with
asynchronous runners, as they run in a different thread than the one
used to create the lock.
:param connection_kwargs: dict, additional keyword arguments are passed
along to the
``StrictRedis.from_url()`` method or ``StrictRedis()`` constructor
directly, including parameters like ``ssl``, ``ssl_certfile``,
``charset``, etc.
.. versionadded:: 1.1.6 Added ``connection_kwargs`` parameter.
.. versionadded:: 0.9.1
"""
@ -106,12 +109,12 @@ class RedisBackend(BytesBackend):
self.password = arguments.pop("password", None)
self.port = arguments.pop("port", 6379)
self.db = arguments.pop("db", 0)
self.distributed_lock = arguments.pop("distributed_lock", False)
self.distributed_lock = arguments.get("distributed_lock", False)
self.socket_timeout = arguments.pop("socket_timeout", None)
self.lock_timeout = arguments.pop("lock_timeout", None)
self.lock_sleep = arguments.pop("lock_sleep", 0.1)
self.thread_local_lock = arguments.pop("thread_local_lock", True)
self.connection_kwargs = arguments.pop("connection_kwargs", {})
self.lock_timeout = arguments.get("lock_timeout", None)
self.lock_sleep = arguments.get("lock_sleep", 0.1)
self.thread_local_lock = arguments.get("thread_local_lock", True)
if self.distributed_lock and self.thread_local_lock:
warnings.warn(
@ -120,8 +123,8 @@ class RedisBackend(BytesBackend):
)
self.redis_expiration_time = arguments.pop("redis_expiration_time", 0)
self.connection_pool = arguments.pop("connection_pool", None)
self._create_client()
self.connection_pool = arguments.get("connection_pool", None)
self.client = self._create_client()
def _imports(self):
# defer imports until backend is used
@ -133,207 +136,73 @@ class RedisBackend(BytesBackend):
# the connection pool already has all other connection
# options present within, so here we disregard socket_timeout
# and others.
self.writer_client = redis.StrictRedis(
connection_pool=self.connection_pool
)
self.reader_client = self.writer_client
else:
args = {}
args.update(self.connection_kwargs)
if self.socket_timeout:
args["socket_timeout"] = self.socket_timeout
return redis.StrictRedis(connection_pool=self.connection_pool)
if self.url is not None:
args.update(url=self.url)
self.writer_client = redis.StrictRedis.from_url(**args)
self.reader_client = self.writer_client
else:
args.update(
host=self.host,
password=self.password,
port=self.port,
db=self.db,
)
self.writer_client = redis.StrictRedis(**args)
self.reader_client = self.writer_client
args = {}
if self.socket_timeout:
args["socket_timeout"] = self.socket_timeout
if self.url is not None:
args.update(url=self.url)
return redis.StrictRedis.from_url(**args)
else:
args.update(
host=self.host,
password=self.password,
port=self.port,
db=self.db,
)
return redis.StrictRedis(**args)
def get_mutex(self, key):
if self.distributed_lock:
return _RedisLockWrapper(
self.writer_client.lock(
"_lock{0}".format(key),
timeout=self.lock_timeout,
sleep=self.lock_sleep,
thread_local=self.thread_local_lock,
)
return self.client.lock(
u("_lock{0}").format(key),
timeout=self.lock_timeout,
sleep=self.lock_sleep,
thread_local=self.thread_local_lock,
)
else:
return None
def get_serialized(self, key):
value = self.reader_client.get(key)
def get(self, key):
value = self.client.get(key)
if value is None:
return NO_VALUE
return value
return pickle.loads(value)
def get_serialized_multi(self, keys):
def get_multi(self, keys):
if not keys:
return []
values = self.reader_client.mget(keys)
return [v if v is not None else NO_VALUE for v in values]
values = self.client.mget(keys)
return [pickle.loads(v) if v is not None else NO_VALUE for v in values]
def set_serialized(self, key, value):
def set(self, key, value):
if self.redis_expiration_time:
self.writer_client.setex(key, self.redis_expiration_time, value)
self.client.setex(
key,
self.redis_expiration_time,
pickle.dumps(value, pickle.HIGHEST_PROTOCOL),
)
else:
self.writer_client.set(key, value)
self.client.set(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
def set_multi(self, mapping):
mapping = dict(
(k, pickle.dumps(v, pickle.HIGHEST_PROTOCOL))
for k, v in mapping.items()
)
def set_serialized_multi(self, mapping):
if not self.redis_expiration_time:
self.writer_client.mset(mapping)
self.client.mset(mapping)
else:
pipe = self.writer_client.pipeline()
pipe = self.client.pipeline()
for key, value in mapping.items():
pipe.setex(key, self.redis_expiration_time, value)
pipe.execute()
def delete(self, key):
self.writer_client.delete(key)
self.client.delete(key)
def delete_multi(self, keys):
self.writer_client.delete(*keys)
class _RedisLockWrapper:
__slots__ = ("mutex", "__weakref__")
def __init__(self, mutex: typing.Any):
self.mutex = mutex
def acquire(self, wait: bool = True) -> typing.Any:
return self.mutex.acquire(blocking=wait)
def release(self) -> typing.Any:
return self.mutex.release()
def locked(self) -> bool:
return self.mutex.locked() # type: ignore
class RedisSentinelBackend(RedisBackend):
"""A `Redis <http://redis.io/>`_ backend, using the
`redis-py <http://pypi.python.org/pypi/redis/>`_ backend.
It will use the Sentinel of a Redis cluster.
.. versionadded:: 1.0.0
Example configuration::
from dogpile.cache import make_region
region = make_region().configure(
'dogpile.cache.redis_sentinel',
arguments = {
'sentinels': [
['redis_sentinel_1', 26379],
['redis_sentinel_2', 26379]
],
'db': 0,
'redis_expiration_time': 60*60*2, # 2 hours
'distributed_lock': True,
'thread_local_lock': False
}
)
Arguments accepted in the arguments dictionary:
:param db: integer, default is ``0``.
:param redis_expiration_time: integer, number of seconds after setting
a value that Redis should expire it. This should be larger than dogpile's
cache expiration. By default no expiration is set.
:param distributed_lock: boolean, when True, will use a
redis-lock as the dogpile lock. Use this when multiple processes will be
talking to the same redis instance. When False, dogpile will
coordinate on a regular threading mutex, Default is True.
:param lock_timeout: integer, number of seconds after acquiring a lock that
Redis should expire it. This argument is only valid when
``distributed_lock`` is ``True``.
:param socket_timeout: float, seconds for socket timeout.
Default is None (no timeout).
:param sentinels: is a list of sentinel nodes. Each node is represented by
a pair (hostname, port).
Default is None (not in sentinel mode).
:param service_name: str, the service name.
Default is 'mymaster'.
:param sentinel_kwargs: is a dictionary of connection arguments used when
connecting to sentinel instances. Any argument that can be passed to
a normal Redis connection can be specified here.
Default is {}.
:param connection_kwargs: dict, additional keyword arguments are passed
along to the
``StrictRedis.from_url()`` method or ``StrictRedis()`` constructor
directly, including parameters like ``ssl``, ``ssl_certfile``,
``charset``, etc.
:param lock_sleep: integer, number of seconds to sleep when failed to
acquire a lock. This argument is only valid when
``distributed_lock`` is ``True``.
:param thread_local_lock: bool, whether a thread-local Redis lock object
should be used. This is the default, but is not compatible with
asynchronous runners, as they run in a different thread than the one
used to create the lock.
"""
def __init__(self, arguments):
arguments = arguments.copy()
self.sentinels = arguments.pop("sentinels", None)
self.service_name = arguments.pop("service_name", "mymaster")
self.sentinel_kwargs = arguments.pop("sentinel_kwargs", {})
super().__init__(
arguments={
"distributed_lock": True,
"thread_local_lock": False,
**arguments,
}
)
def _imports(self):
# defer imports until backend is used
global redis
import redis.sentinel # noqa
def _create_client(self):
sentinel_kwargs = {}
sentinel_kwargs.update(self.sentinel_kwargs)
sentinel_kwargs.setdefault("password", self.password)
connection_kwargs = {}
connection_kwargs.update(self.connection_kwargs)
connection_kwargs.setdefault("password", self.password)
if self.db is not None:
connection_kwargs.setdefault("db", self.db)
sentinel_kwargs.setdefault("db", self.db)
if self.socket_timeout is not None:
connection_kwargs.setdefault("socket_timeout", self.socket_timeout)
sentinel = redis.sentinel.Sentinel(
self.sentinels,
sentinel_kwargs=sentinel_kwargs,
**connection_kwargs,
)
self.writer_client = sentinel.master_for(self.service_name)
self.reader_client = sentinel.slave_for(self.service_name)
self.client.delete(*keys)