diff --git a/lib/zc/__init__.py b/lib/zc/__init__.py new file mode 100644 index 00000000..de40ea7c --- /dev/null +++ b/lib/zc/__init__.py @@ -0,0 +1 @@ +__import__('pkg_resources').declare_namespace(__name__) diff --git a/lib/zc/lockfile/README.txt b/lib/zc/lockfile/README.txt new file mode 100644 index 00000000..89ef33e9 --- /dev/null +++ b/lib/zc/lockfile/README.txt @@ -0,0 +1,70 @@ +Lock file support +================= + +The ZODB lock_file module provides support for creating file system +locks. These are locks that are implemented with lock files and +OS-provided locking facilities. To create a lock, instantiate a +LockFile object with a file name: + + >>> import zc.lockfile + >>> lock = zc.lockfile.LockFile('lock') + +If we try to lock the same name, we'll get a lock error: + + >>> import zope.testing.loggingsupport + >>> handler = zope.testing.loggingsupport.InstalledHandler('zc.lockfile') + >>> try: + ... zc.lockfile.LockFile('lock') + ... except zc.lockfile.LockError: + ... print("Can't lock file") + Can't lock file + +.. We don't log failure to acquire. + + >>> for record in handler.records: # doctest: +ELLIPSIS + ... print(record.levelname+' '+record.getMessage()) + +To release the lock, use it's close method: + + >>> lock.close() + +The lock file is not removed. It is left behind: + + >>> import os + >>> os.path.exists('lock') + True + +Of course, now that we've released the lock, we can create it again: + + >>> lock = zc.lockfile.LockFile('lock') + >>> lock.close() + +.. Cleanup + + >>> import os + >>> os.remove('lock') + +Hostname in lock file +===================== + +In a container environment (e.g. Docker), the PID is typically always +identical even if multiple containers are running under the same operating +system instance. + +Clearly, inspecting lock files doesn't then help much in debugging. To identify +the container which created the lock file, we need information about the +container in the lock file. Since Docker uses the container identifier or name +as the hostname, this information can be stored in the lock file in addition to +or instead of the PID. + +Use the ``content_template`` keyword argument to ``LockFile`` to specify a +custom lock file content format: + + >>> lock = zc.lockfile.LockFile('lock', content_template='{pid};{hostname}') + >>> lock.close() + +If you now inspected the lock file, you would see e.g.: + + $ cat lock + 123;myhostname + diff --git a/lib/zc/lockfile/__init__.py b/lib/zc/lockfile/__init__.py new file mode 100644 index 00000000..b541fa2d --- /dev/null +++ b/lib/zc/lockfile/__init__.py @@ -0,0 +1,125 @@ +############################################################################## +# +# Copyright (c) 2001, 2002 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## + +import os +import errno +import logging +logger = logging.getLogger("zc.lockfile") + +__metaclass__ = type + +class LockError(Exception): + """Couldn't get a lock + """ + +try: + import fcntl +except ImportError: + try: + import msvcrt + except ImportError: + def _lock_file(file): + raise TypeError('No file-locking support on this platform') + def _unlock_file(file): + raise TypeError('No file-locking support on this platform') + + else: + # Windows + def _lock_file(file): + # Lock just the first byte + try: + msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1) + except IOError: + raise LockError("Couldn't lock %r" % file.name) + + def _unlock_file(file): + try: + file.seek(0) + msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1) + except IOError: + raise LockError("Couldn't unlock %r" % file.name) + +else: + # Unix + _flags = fcntl.LOCK_EX | fcntl.LOCK_NB + + def _lock_file(file): + try: + fcntl.flock(file.fileno(), _flags) + except IOError: + raise LockError("Couldn't lock %r" % file.name) + + def _unlock_file(file): + fcntl.flock(file.fileno(), fcntl.LOCK_UN) + +class LazyHostName: + """Avoid importing socket and calling gethostname() unnecessarily""" + def __str__(self): + import socket + return socket.gethostname() + + +class SimpleLockFile: + + _fp = None + + def __init__(self, path): + self._path = path + try: + # Try to open for writing without truncation: + fp = open(path, 'r+') + except IOError: + # If the file doesn't exist, we'll get an IO error, try a+ + # Note that there may be a race here. Multiple processes + # could fail on the r+ open and open the file a+, but only + # one will get the the lock and write a pid. + fp = open(path, 'a+') + + try: + _lock_file(fp) + self._fp = fp + except: + fp.close() + raise + + # Lock acquired + self._on_lock() + fp.flush() + + def close(self): + if self._fp is not None: + _unlock_file(self._fp) + self._fp.close() + self._fp = None + + def _on_lock(self): + """ + Allow subclasses to supply behavior to occur following + lock acquisition. + """ + + +class LockFile(SimpleLockFile): + + def __init__(self, path, content_template='{pid}'): + self._content_template = content_template + super(LockFile, self).__init__(path) + + def _on_lock(self): + content = self._content_template.format( + pid=os.getpid(), + hostname=LazyHostName(), + ) + self._fp.write(" %s\n" % content) + self._fp.truncate() diff --git a/lib/zc/lockfile/tests.py b/lib/zc/lockfile/tests.py new file mode 100644 index 00000000..4c890539 --- /dev/null +++ b/lib/zc/lockfile/tests.py @@ -0,0 +1,201 @@ +############################################################################## +# +# Copyright (c) 2004 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +import os, re, sys, unittest, doctest +import zc.lockfile, time, threading +from zope.testing import renormalizing, setupstack +import tempfile +try: + from unittest.mock import Mock, patch +except ImportError: + from mock import Mock, patch + +checker = renormalizing.RENormalizing([ + # Python 3 adds module path to error class name. + (re.compile("zc\.lockfile\.LockError:"), + r"LockError:"), + ]) + +def inc(): + while 1: + try: + lock = zc.lockfile.LockFile('f.lock') + except zc.lockfile.LockError: + continue + else: + break + f = open('f', 'r+b') + v = int(f.readline().strip()) + time.sleep(0.01) + v += 1 + f.seek(0) + f.write(('%d\n' % v).encode('ASCII')) + f.close() + lock.close() + +def many_threads_read_and_write(): + r""" + >>> with open('f', 'w+b') as file: + ... _ = file.write(b'0\n') + >>> with open('f.lock', 'w+b') as file: + ... _ = file.write(b'0\n') + + >>> n = 50 + >>> threads = [threading.Thread(target=inc) for i in range(n)] + >>> _ = [thread.start() for thread in threads] + >>> _ = [thread.join() for thread in threads] + >>> with open('f', 'rb') as file: + ... saved = int(file.read().strip()) + >>> saved == n + True + + >>> os.remove('f') + + We should only have one pid in the lock file: + + >>> f = open('f.lock') + >>> len(f.read().strip().split()) + 1 + >>> f.close() + + >>> os.remove('f.lock') + + """ + +def pid_in_lockfile(): + r""" + >>> import os, zc.lockfile + >>> pid = os.getpid() + >>> lock = zc.lockfile.LockFile("f.lock") + >>> f = open("f.lock") + >>> _ = f.seek(1) + >>> f.read().strip() == str(pid) + True + >>> f.close() + + Make sure that locking twice does not overwrite the old pid: + + >>> lock = zc.lockfile.LockFile("f.lock") + Traceback (most recent call last): + ... + LockError: Couldn't lock 'f.lock' + + >>> f = open("f.lock") + >>> _ = f.seek(1) + >>> f.read().strip() == str(pid) + True + >>> f.close() + + >>> lock.close() + """ + + +def hostname_in_lockfile(): + r""" + hostname is correctly written into the lock file when it's included in the + lock file content template + + >>> import zc.lockfile + >>> with patch('socket.gethostname', Mock(return_value='myhostname')): + ... lock = zc.lockfile.LockFile("f.lock", content_template='{hostname}') + >>> f = open("f.lock") + >>> _ = f.seek(1) + >>> f.read().rstrip() + 'myhostname' + >>> f.close() + + Make sure that locking twice does not overwrite the old hostname: + + >>> lock = zc.lockfile.LockFile("f.lock", content_template='{hostname}') + Traceback (most recent call last): + ... + LockError: Couldn't lock 'f.lock' + + >>> f = open("f.lock") + >>> _ = f.seek(1) + >>> f.read().rstrip() + 'myhostname' + >>> f.close() + + >>> lock.close() + """ + + +class TestLogger(object): + def __init__(self): + self.log_entries = [] + + def exception(self, msg, *args): + self.log_entries.append((msg,) + args) + + +class LockFileLogEntryTestCase(unittest.TestCase): + """Tests for logging in case of lock failure""" + def setUp(self): + self.here = os.getcwd() + self.tmp = tempfile.mkdtemp(prefix='zc.lockfile-test-') + os.chdir(self.tmp) + + def tearDown(self): + os.chdir(self.here) + setupstack.rmtree(self.tmp) + + def test_log_formatting(self): + # PID and hostname are parsed and logged from lock file on failure + with patch('os.getpid', Mock(return_value=123)): + with patch('socket.gethostname', Mock(return_value='myhostname')): + lock = zc.lockfile.LockFile('f.lock', + content_template='{pid}/{hostname}') + with open('f.lock') as f: + self.assertEqual(' 123/myhostname\n', f.read()) + + lock.close() + + def test_unlock_and_lock_while_multiprocessing_process_running(self): + import multiprocessing + + lock = zc.lockfile.LockFile('l') + q = multiprocessing.Queue() + p = multiprocessing.Process(target=q.get) + p.daemon = True + p.start() + + # release and re-acquire should work (obviously) + lock.close() + lock = zc.lockfile.LockFile('l') + self.assertTrue(p.is_alive()) + + q.put(0) + lock.close() + p.join() + + def test_simple_lock(self): + assert isinstance(zc.lockfile.SimpleLockFile, type) + lock = zc.lockfile.SimpleLockFile('s') + with self.assertRaises(zc.lockfile.LockError): + zc.lockfile.SimpleLockFile('s') + lock.close() + zc.lockfile.SimpleLockFile('s').close() + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(doctest.DocFileSuite( + 'README.txt', checker=checker, + setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown)) + suite.addTest(doctest.DocTestSuite( + setUp=setupstack.setUpDirectory, tearDown=setupstack.tearDown, + checker=checker)) + # Add unittest test cases from this module + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(__name__)) + return suite