Add zc.lockfile-2.0

lib/zc/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

lib/zc/lockfile/README.txt

@@ -0,0 +1,70 @@
+Lock file support
+=================
+
+The ZODB lock_file module provides support for creating file system
+locks.  These are locks that are implemented with lock files and
+OS-provided locking facilities.  To create a lock, instantiate a
+LockFile object with a file name:
+
+    >>> import zc.lockfile
+    >>> lock = zc.lockfile.LockFile('lock')
+
+If we try to lock the same name, we'll get a lock error:
+
+    >>> import zope.testing.loggingsupport
+    >>> handler = zope.testing.loggingsupport.InstalledHandler('zc.lockfile')
+    >>> try:
+    ...     zc.lockfile.LockFile('lock')
+    ... except zc.lockfile.LockError:
+    ...     print("Can't lock file")
+    Can't lock file
+
+.. We don't log failure to acquire.
+
+    >>> for record in handler.records: # doctest: +ELLIPSIS
+    ...     print(record.levelname+' '+record.getMessage())
+
+To release the lock, use it's close method:
+
+    >>> lock.close()
+
+The lock file is not removed.  It is left behind:
+
+    >>> import os
+    >>> os.path.exists('lock')
+    True
+
+Of course, now that we've released the lock, we can create it again:
+
+    >>> lock = zc.lockfile.LockFile('lock')
+    >>> lock.close()
+
+.. Cleanup
+
+    >>> import os
+    >>> os.remove('lock')
+
+Hostname in lock file
+=====================
+
+In a container environment (e.g. Docker), the PID is typically always
+identical even if multiple containers are running under the same operating
+system instance.
+
+Clearly, inspecting lock files doesn't then help much in debugging. To identify
+the container which created the lock file, we need information about the
+container in the lock file. Since Docker uses the container identifier or name
+as the hostname, this information can be stored in the lock file in addition to
+or instead of the PID.
+
+Use the ``content_template`` keyword argument to ``LockFile`` to specify a
+custom lock file content format:
+
+    >>> lock = zc.lockfile.LockFile('lock', content_template='{pid};{hostname}')
+    >>> lock.close()
+
+If you now inspected the lock file, you would see e.g.:
+
+    $ cat lock
+     123;myhostname
+

lib/zc/lockfile/__init__.py

@@ -0,0 +1,125 @@
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import os
+import errno
+import logging
+logger = logging.getLogger("zc.lockfile")
+
+__metaclass__ = type
+
+class LockError(Exception):
+    """Couldn't get a lock
+    """
+
+try:
+    import fcntl
+except ImportError:
+    try:
+        import msvcrt
+    except ImportError:
+        def _lock_file(file):
+            raise TypeError('No file-locking support on this platform')
+        def _unlock_file(file):
+            raise TypeError('No file-locking support on this platform')
+
+    else:
+        # Windows
+        def _lock_file(file):
+            # Lock just the first byte
+            try:
+                msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
+            except IOError:
+                raise LockError("Couldn't lock %r" % file.name)
+
+        def _unlock_file(file):
+            try:
+                file.seek(0)
+                msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
+            except IOError:
+                raise LockError("Couldn't unlock %r" % file.name)
+
+else:
+    # Unix
+    _flags = fcntl.LOCK_EX | fcntl.LOCK_NB
+
+    def _lock_file(file):
+        try:
+            fcntl.flock(file.fileno(), _flags)
+        except IOError:
+            raise LockError("Couldn't lock %r" % file.name)
+
+    def _unlock_file(file):
+        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
+
+class LazyHostName:
+    """Avoid importing socket and calling gethostname() unnecessarily"""
+    def __str__(self):
+        import socket
+        return socket.gethostname()
+
+
+class SimpleLockFile:
+
+    _fp = None
+
+    def __init__(self, path):
+        self._path = path
+        try:
+            # Try to open for writing without truncation:
+            fp = open(path, 'r+')
+        except IOError:
+            # If the file doesn't exist, we'll get an IO error, try a+
+            # Note that there may be a race here. Multiple processes
+            # could fail on the r+ open and open the file a+, but only
+            # one will get the the lock and write a pid.
+            fp = open(path, 'a+')
+
+        try:
+            _lock_file(fp)
+            self._fp = fp
+        except:
+            fp.close()
+            raise
+
+        # Lock acquired
+        self._on_lock()
+        fp.flush()
+
+    def close(self):
+        if self._fp is not None:
+            _unlock_file(self._fp)
+            self._fp.close()
+            self._fp = None
+
+    def _on_lock(self):
+        """
+        Allow subclasses to supply behavior to occur following
+        lock acquisition.
+        """
+
+
+class LockFile(SimpleLockFile):
+
+    def __init__(self, path, content_template='{pid}'):
+        self._content_template = content_template
+        super(LockFile, self).__init__(path)
+
+    def _on_lock(self):
+        content = self._content_template.format(
+            pid=os.getpid(),
+            hostname=LazyHostName(),
+        )
+        self._fp.write(" %s\n" % content)
+        self._fp.truncate()

lib/zc/lockfile/tests.py

@@ -0,0 +1,201 @@
+##############################################################################
+#
+# Copyright (c) 2004 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import os, re, sys, unittest, doctest
+import zc.lockfile, time, threading
+from zope.testing import renormalizing, setupstack
+import tempfile
+try:
+    from unittest.mock import Mock, patch
+except ImportError:
+    from mock import Mock, patch
+
+checker = renormalizing.RENormalizing([
+    # Python 3 adds module path to error class name.
+    (re.compile("zc\.lockfile\.LockError:"),
+     r"LockError:"),
+    ])
+
+def inc():
+    while 1:
+        try:
+            lock = zc.lockfile.LockFile('f.lock')
+        except zc.lockfile.LockError:
+            continue
+        else:
+            break
+    f = open('f', 'r+b')
+    v = int(f.readline().strip())
+    time.sleep(0.01)
+    v += 1
+    f.seek(0)
+    f.write(('%d\n' % v).encode('ASCII'))
+    f.close()
+    lock.close()
+
+def many_threads_read_and_write():
+    r"""
+    >>> with open('f', 'w+b') as file:
+    ...     _ = file.write(b'0\n')
+    >>> with open('f.lock', 'w+b') as file:
+    ...     _ = file.write(b'0\n')
+
+    >>> n = 50
+    >>> threads = [threading.Thread(target=inc) for i in range(n)]
+    >>> _ = [thread.start() for thread in threads]
+    >>> _ = [thread.join() for thread in threads]
+    >>> with open('f', 'rb') as file:
+    ...     saved = int(file.read().strip())
+    >>> saved == n
+    True
+
+    >>> os.remove('f')
+
+    We should only have one pid in the lock file:
+
+    >>> f = open('f.lock')
+    >>> len(f.read().strip().split())
+    1
+    >>> f.close()
+
+    >>> os.remove('f.lock')
+
+    """
+
+def pid_in_lockfile():
+    r"""
+    >>> import os, zc.lockfile
+    >>> pid = os.getpid()
+    >>> lock = zc.lockfile.LockFile("f.lock")
+    >>> f = open("f.lock")
+    >>> _ = f.seek(1)
+    >>> f.read().strip() == str(pid)
+    True
+    >>> f.close()
+
+    Make sure that locking twice does not overwrite the old pid:
+
+    >>> lock = zc.lockfile.LockFile("f.lock")
+    Traceback (most recent call last):
+      ...
+    LockError: Couldn't lock 'f.lock'
+
+    >>> f = open("f.lock")
+    >>> _ = f.seek(1)
+    >>> f.read().strip() == str(pid)
+    True
+    >>> f.close()
+
+    >>> lock.close()
+    """
+
+
+def hostname_in_lockfile():
+    r"""
+    hostname is correctly written into the lock file when it's included in the
+    lock file content template
+
+    >>> import zc.lockfile
+    >>> with patch('socket.gethostname', Mock(return_value='myhostname')):
+    ...     lock = zc.lockfile.LockFile("f.lock", content_template='{hostname}')
+    >>> f = open("f.lock")
+    >>> _ = f.seek(1)
+    >>> f.read().rstrip()
+    'myhostname'
+    >>> f.close()
+
+    Make sure that locking twice does not overwrite the old hostname:
+
+    >>> lock = zc.lockfile.LockFile("f.lock", content_template='{hostname}')
+    Traceback (most recent call last):
+      ...
+    LockError: Couldn't lock 'f.lock'
+
+    >>> f = open("f.lock")
+    >>> _ = f.seek(1)
+    >>> f.read().rstrip()
+    'myhostname'
+    >>> f.close()
+
+    >>> lock.close()
+    """
+
+
+class TestLogger(object):
+    def __init__(self):
+        self.log_entries = []
+
+    def exception(self, msg, *args):
+        self.log_entries.append((msg,) + args)
+
+
+class LockFileLogEntryTestCase(unittest.TestCase):
+    """Tests for logging in case of lock failure"""
+    def setUp(self):
+        self.here = os.getcwd()
+        self.tmp = tempfile.mkdtemp(prefix='zc.lockfile-test-')
+        os.chdir(self.tmp)
+
+    def tearDown(self):
+        os.chdir(self.here)
+        setupstack.rmtree(self.tmp)
+
+    def test_log_formatting(self):
+        # PID and hostname are parsed and logged from lock file on failure
+        with patch('os.getpid', Mock(return_value=123)):
+            with patch('socket.gethostname', Mock(return_value='myhostname')):
+                lock = zc.lockfile.LockFile('f.lock',
+                                            content_template='{pid}/{hostname}')
+                with open('f.lock') as f:
+                    self.assertEqual(' 123/myhostname\n', f.read())
+
+                lock.close()
+
+    def test_unlock_and_lock_while_multiprocessing_process_running(self):
+        import multiprocessing
+
+        lock = zc.lockfile.LockFile('l')
+        q = multiprocessing.Queue()
+        p = multiprocessing.Process(target=q.get)
+        p.daemon = True
+        p.start()
+
+        # release and re-acquire should work (obviously)
+        lock.close()
+        lock = zc.lockfile.LockFile('l')
+        self.assertTrue(p.is_alive())
+
+        q.put(0)
+        lock.close()
+        p.join()
+
+    def test_simple_lock(self):
+        assert isinstance(zc.lockfile.SimpleLockFile, type)
+        lock = zc.lockfile.SimpleLockFile('s')
+        with self.assertRaises(zc.lockfile.LockError):
+            zc.lockfile.SimpleLockFile('s')
+        lock.close()
+        zc.lockfile.SimpleLockFile('s').close()
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(doctest.DocFileSuite(
+        'README.txt', checker=checker,
+        setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown))
+    suite.addTest(doctest.DocTestSuite(
+        setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown,
+        checker=checker))
+    # Add unittest test cases from this module
+    suite.addTest(unittest.defaultTestLoader.loadTestsFromName(__name__))
+    return suite