Initial Commit

lib/apscheduler/jobstores/base.py

@@ -0,0 +1,127 @@
+from abc import ABCMeta, abstractmethod
+import logging
+
+import six
+
+
+class JobLookupError(KeyError):
+    """Raised when the job store cannot find a job for update or removal."""
+
+    def __init__(self, job_id):
+        super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id)
+
+
+class ConflictingIdError(KeyError):
+    """Raised when the uniqueness of job IDs is being violated."""
+
+    def __init__(self, job_id):
+        super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id)
+
+
+class TransientJobError(ValueError):
+    """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected."""
+
+    def __init__(self, job_id):
+        super(TransientJobError, self).__init__(
+            six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be '
+                  'determined.') % job_id)
+
+
+class BaseJobStore(six.with_metaclass(ABCMeta)):
+    """Abstract base class that defines the interface that every job store must implement."""
+
+    _scheduler = None
+    _alias = None
+    _logger = logging.getLogger('apscheduler.jobstores')
+
+    def start(self, scheduler, alias):
+        """
+        Called by the scheduler when the scheduler is being started or when the job store is being added to an already
+        running scheduler.
+
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store
+        :param str|unicode alias: alias of this job store as it was assigned to the scheduler
+        """
+
+        self._scheduler = scheduler
+        self._alias = alias
+        self._logger = logging.getLogger('apscheduler.jobstores.%s' % alias)
+
+    def shutdown(self):
+        """Frees any resources still bound to this job store."""
+
+    @abstractmethod
+    def lookup_job(self, job_id):
+        """
+        Returns a specific job, or ``None`` if it isn't found..
+
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to
+        point to the scheduler and itself, respectively.
+
+        :param str|unicode job_id: identifier of the job
+        :rtype: Job
+        """
+
+    @abstractmethod
+    def get_due_jobs(self, now):
+        """
+        Returns the list of jobs that have ``next_run_time`` earlier or equal to ``now``.
+        The returned jobs must be sorted by next run time (ascending).
+
+        :param datetime.datetime now: the current (timezone aware) datetime
+        :rtype: list[Job]
+        """
+
+    @abstractmethod
+    def get_next_run_time(self):
+        """
+        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs.
+
+        :rtype: datetime.datetime
+        """
+
+    @abstractmethod
+    def get_all_jobs(self):
+        """
+        Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending).
+        Paused jobs (next_run_time is None) should be sorted last.
+
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to
+        point to the scheduler and itself, respectively.
+
+        :rtype: list[Job]
+        """
+
+    @abstractmethod
+    def add_job(self, job):
+        """
+        Adds the given job to this store.
+
+        :param Job job: the job to add
+        :raises ConflictingIdError: if there is another job in this store with the same ID
+        """
+
+    @abstractmethod
+    def update_job(self, job):
+        """
+        Replaces the job in the store with the given newer version.
+
+        :param Job job: the job to update
+        :raises JobLookupError: if the job does not exist
+        """
+
+    @abstractmethod
+    def remove_job(self, job_id):
+        """
+        Removes the given job from this store.
+
+        :param str|unicode job_id: identifier of the job
+        :raises JobLookupError: if the job does not exist
+        """
+
+    @abstractmethod
+    def remove_all_jobs(self):
+        """Removes all jobs from this store."""
+
+    def __repr__(self):
+        return '<%s>' % self.__class__.__name__

lib/apscheduler/jobstores/memory.py

@@ -0,0 +1,107 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import datetime_to_utc_timestamp
+
+
+class MemoryJobStore(BaseJobStore):
+    """
+    Stores jobs in an array in RAM. Provides no persistence support.
+
+    Plugin alias: ``memory``
+    """
+
+    def __init__(self):
+        super(MemoryJobStore, self).__init__()
+        self._jobs = []  # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        self._jobs_index = {}  # id -> (job, timestamp) lookup table
+
+    def lookup_job(self, job_id):
+        return self._jobs_index.get(job_id, (None, None))[0]
+
+    def get_due_jobs(self, now):
+        now_timestamp = datetime_to_utc_timestamp(now)
+        pending = []
+        for job, timestamp in self._jobs:
+            if timestamp is None or timestamp > now_timestamp:
+                break
+            pending.append(job)
+
+        return pending
+
+    def get_next_run_time(self):
+        return self._jobs[0][0].next_run_time if self._jobs else None
+
+    def get_all_jobs(self):
+        return [j[0] for j in self._jobs]
+
+    def add_job(self, job):
+        if job.id in self._jobs_index:
+            raise ConflictingIdError(job.id)
+
+        timestamp = datetime_to_utc_timestamp(job.next_run_time)
+        index = self._get_job_index(timestamp, job.id)
+        self._jobs.insert(index, (job, timestamp))
+        self._jobs_index[job.id] = (job, timestamp)
+
+    def update_job(self, job):
+        old_job, old_timestamp = self._jobs_index.get(job.id, (None, None))
+        if old_job is None:
+            raise JobLookupError(job.id)
+
+        # If the next run time has not changed, simply replace the job in its present index.
+        # Otherwise, reinsert the job to the list to preserve the ordering.
+        old_index = self._get_job_index(old_timestamp, old_job.id)
+        new_timestamp = datetime_to_utc_timestamp(job.next_run_time)
+        if old_timestamp == new_timestamp:
+            self._jobs[old_index] = (job, new_timestamp)
+        else:
+            del self._jobs[old_index]
+            new_index = self._get_job_index(new_timestamp, job.id)
+            self._jobs.insert(new_index, (job, new_timestamp))
+
+        self._jobs_index[old_job.id] = (job, new_timestamp)
+
+    def remove_job(self, job_id):
+        job, timestamp = self._jobs_index.get(job_id, (None, None))
+        if job is None:
+            raise JobLookupError(job_id)
+
+        index = self._get_job_index(timestamp, job_id)
+        del self._jobs[index]
+        del self._jobs_index[job.id]
+
+    def remove_all_jobs(self):
+        self._jobs = []
+        self._jobs_index = {}
+
+    def shutdown(self):
+        self.remove_all_jobs()
+
+    def _get_job_index(self, timestamp, job_id):
+        """
+        Returns the index of the given job, or if it's not found, the index where the job should be inserted based on
+        the given timestamp.
+
+        :type timestamp: int
+        :type job_id: str
+        """
+
+        lo, hi = 0, len(self._jobs)
+        timestamp = float('inf') if timestamp is None else timestamp
+        while lo < hi:
+            mid = (lo + hi) // 2
+            mid_job, mid_timestamp = self._jobs[mid]
+            mid_timestamp = float('inf') if mid_timestamp is None else mid_timestamp
+            if mid_timestamp > timestamp:
+                hi = mid
+            elif mid_timestamp < timestamp:
+                lo = mid + 1
+            elif mid_job.id > job_id:
+                hi = mid
+            elif mid_job.id < job_id:
+                lo = mid + 1
+            else:
+                return mid
+
+        return lo

lib/apscheduler/jobstores/mongodb.py

@@ -0,0 +1,124 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from bson.binary import Binary
+    from pymongo.errors import DuplicateKeyError
+    from pymongo import MongoClient, ASCENDING
+except ImportError:  # pragma: nocover
+    raise ImportError('MongoDBJobStore requires PyMongo installed')
+
+
+class MongoDBJobStore(BaseJobStore):
+    """
+    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient
+    <http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_.
+
+    Plugin alias: ``mongodb``
+
+    :param str database: database to store jobs in
+    :param str collection: collection to store jobs in
+    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection
+                   arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    """
+
+    def __init__(self, database='apscheduler', collection='jobs', client=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(MongoDBJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+
+        if not database:
+            raise ValueError('The "database" parameter must not be empty')
+        if not collection:
+            raise ValueError('The "collection" parameter must not be empty')
+
+        if client:
+            self.connection = maybe_ref(client)
+        else:
+            connect_args.setdefault('w', 1)
+            self.connection = MongoClient(**connect_args)
+
+        self.collection = self.connection[database][collection]
+        self.collection.ensure_index('next_run_time', sparse=True)
+
+    def lookup_job(self, job_id):
+        document = self.collection.find_one(job_id, ['job_state'])
+        return self._reconstitute_job(document['job_state']) if document else None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        return self._get_jobs({'next_run_time': {'$lte': timestamp}})
+
+    def get_next_run_time(self):
+        document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'],
+                                            sort=[('next_run_time', ASCENDING)])
+        return utc_timestamp_to_datetime(document['next_run_time']) if document else None
+
+    def get_all_jobs(self):
+        return self._get_jobs({})
+
+    def add_job(self, job):
+        try:
+            self.collection.insert({
+                '_id': job.id,
+                'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+                'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            })
+        except DuplicateKeyError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        result = self.collection.update({'_id': job.id}, {'$set': changes})
+        if result and result['n'] == 0:
+            raise JobLookupError(id)
+
+    def remove_job(self, job_id):
+        result = self.collection.remove(job_id)
+        if result and result['n'] == 0:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        self.collection.remove()
+
+    def shutdown(self):
+        self.connection.disconnect()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, conditions):
+        jobs = []
+        failed_job_ids = []
+        for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]):
+            try:
+                jobs.append(self._reconstitute_job(document['job_state']))
+            except:
+                self._logger.exception('Unable to restore job "%s" -- removing it', document['_id'])
+                failed_job_ids.append(document['_id'])
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            self.collection.remove({'_id': {'$in': failed_job_ids}})
+
+        return jobs
+
+    def __repr__(self):
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.connection)

lib/apscheduler/jobstores/redis.py

@@ -0,0 +1,138 @@
+from __future__ import absolute_import
+
+import six
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from redis import StrictRedis
+except ImportError:  # pragma: nocover
+    raise ImportError('RedisJobStore requires redis installed')
+
+
+class RedisJobStore(BaseJobStore):
+    """
+    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis.
+
+    Plugin alias: ``redis``
+
+    :param int db: the database number to store jobs in
+    :param str jobs_key: key to store jobs in
+    :param str run_times_key: key to store the jobs' run times in
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    """
+
+    def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(RedisJobStore, self).__init__()
+
+        if db is None:
+            raise ValueError('The "db" parameter must not be empty')
+        if not jobs_key:
+            raise ValueError('The "jobs_key" parameter must not be empty')
+        if not run_times_key:
+            raise ValueError('The "run_times_key" parameter must not be empty')
+
+        self.pickle_protocol = pickle_protocol
+        self.jobs_key = jobs_key
+        self.run_times_key = run_times_key
+        self.redis = StrictRedis(db=int(db), **connect_args)
+
+    def lookup_job(self, job_id):
+        job_state = self.redis.hget(self.jobs_key, job_id)
+        return self._reconstitute_job(job_state) if job_state else None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        job_ids = self.redis.zrangebyscore(self.run_times_key, 0, timestamp)
+        if job_ids:
+            job_states = self.redis.hmget(self.jobs_key, *job_ids)
+            return self._reconstitute_jobs(six.moves.zip(job_ids, job_states))
+        return []
+
+    def get_next_run_time(self):
+        next_run_time = self.redis.zrange(self.run_times_key, 0, 0, withscores=True)
+        if next_run_time:
+            return utc_timestamp_to_datetime(next_run_time[0][1])
+
+    def get_all_jobs(self):
+        job_states = self.redis.hgetall(self.jobs_key)
+        jobs = self._reconstitute_jobs(six.iteritems(job_states))
+        return sorted(jobs, key=lambda job: job.next_run_time)
+
+    def add_job(self, job):
+        if self.redis.hexists(self.jobs_key, job.id):
+            raise ConflictingIdError(job.id)
+
+        with self.redis.pipeline() as pipe:
+            pipe.multi()
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+            pipe.execute()
+
+    def update_job(self, job):
+        if not self.redis.hexists(self.jobs_key, job.id):
+            raise JobLookupError(job.id)
+
+        with self.redis.pipeline() as pipe:
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            if job.next_run_time:
+                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+            else:
+                pipe.zrem(self.run_times_key, job.id)
+            pipe.execute()
+
+    def remove_job(self, job_id):
+        if not self.redis.hexists(self.jobs_key, job_id):
+            raise JobLookupError(job_id)
+
+        with self.redis.pipeline() as pipe:
+            pipe.hdel(self.jobs_key, job_id)
+            pipe.zrem(self.run_times_key, job_id)
+            pipe.execute()
+
+    def remove_all_jobs(self):
+        with self.redis.pipeline() as pipe:
+            pipe.delete(self.jobs_key)
+            pipe.delete(self.run_times_key)
+            pipe.execute()
+
+    def shutdown(self):
+        self.redis.connection_pool.disconnect()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _reconstitute_jobs(self, job_states):
+        jobs = []
+        failed_job_ids = []
+        for job_id, job_state in job_states:
+            try:
+                jobs.append(self._reconstitute_job(job_state))
+            except:
+                self._logger.exception('Unable to restore job "%s" -- removing it', job_id)
+                failed_job_ids.append(job_id)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            with self.redis.pipeline() as pipe:
+                pipe.hdel(self.jobs_key, *failed_job_ids)
+                pipe.zrem(self.run_times_key, *failed_job_ids)
+                pipe.execute()
+
+        return jobs
+
+    def __repr__(self):
+        return '<%s>' % self.__class__.__name__

lib/apscheduler/jobstores/sqlalchemy.py

@@ -0,0 +1,137 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select
+    from sqlalchemy.exc import IntegrityError
+except ImportError:  # pragma: nocover
+    raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed')
+
+
+class SQLAlchemyJobStore(BaseJobStore):
+    """
+    Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database.
+
+    Plugin alias: ``sqlalchemy``
+
+    :param str url: connection string (see `SQLAlchemy documentation
+                    <http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#database-urls>`_
+                    on this)
+    :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url``
+    :param str tablename: name of the table to store jobs in
+    :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    """
+
+    def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL):
+        super(SQLAlchemyJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+        metadata = maybe_ref(metadata) or MetaData()
+
+        if engine:
+            self.engine = maybe_ref(engine)
+        elif url:
+            self.engine = create_engine(url)
+        else:
+            raise ValueError('Need either "engine" or "url" defined')
+
+        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float
+        self.jobs_t = Table(
+            tablename, metadata,
+            Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True),
+            Column('next_run_time', Float(25), index=True),
+            Column('job_state', LargeBinary, nullable=False)
+        )
+
+        self.jobs_t.create(self.engine, True)
+
+    def lookup_job(self, job_id):
+        selectable = select([self.jobs_t.c.job_state]).where(self.jobs_t.c.id == job_id)
+        job_state = self.engine.execute(selectable).scalar()
+        return self._reconstitute_job(job_state) if job_state else None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp)
+
+    def get_next_run_time(self):
+        selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\
+            order_by(self.jobs_t.c.next_run_time).limit(1)
+        next_run_time = self.engine.execute(selectable).scalar()
+        return utc_timestamp_to_datetime(next_run_time)
+
+    def get_all_jobs(self):
+        return self._get_jobs()
+
+    def add_job(self, job):
+        insert = self.jobs_t.insert().values(**{
+            'id': job.id,
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
+        })
+        try:
+            self.engine.execute(insert)
+        except IntegrityError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        update = self.jobs_t.update().values(**{
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
+        }).where(self.jobs_t.c.id == job.id)
+        result = self.engine.execute(update)
+        if result.rowcount == 0:
+            raise JobLookupError(id)
+
+    def remove_job(self, job_id):
+        delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)
+        result = self.engine.execute(delete)
+        if result.rowcount == 0:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        delete = self.jobs_t.delete()
+        self.engine.execute(delete)
+
+    def shutdown(self):
+        self.engine.dispose()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job_state['jobstore'] = self
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, *conditions):
+        jobs = []
+        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time)
+        selectable = selectable.where(*conditions) if conditions else selectable
+        failed_job_ids = set()
+        for row in self.engine.execute(selectable):
+            try:
+                jobs.append(self._reconstitute_job(row.job_state))
+            except:
+                self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
+                failed_job_ids.add(row.id)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
+            self.engine.execute(delete)
+
+        return jobs
+
+    def __repr__(self):
+        return '<%s (url=%s)>' % (self.__class__.__name__, self.engine.url)