diff options
author | Hervé Beraud <herveberaud.pro@gmail.com> | 2019-03-02 11:01:17 -0500 |
---|---|---|
committer | Mike Bayer <mike_mp@zzzcomputing.com> | 2019-03-04 11:33:57 -0500 |
commit | f88e9d311f5562b0cbf793dd4e8496dde520611e (patch) | |
tree | 9178f02393f4d58d8faabcc3966c429416506026 /tests | |
parent | 7c5b64583db1ba169c1cf643819b9ac4d2e7d2e5 (diff) | |
download | dogpile-cache-f88e9d311f5562b0cbf793dd4e8496dde520611e.tar.gz |
Format code for PEP8 compliance.
These changes format code by using:
```
$ black -l79
$ zimports
```
and some manual changes to format the code as PEP8 compliance
Closes: #146
Pull-request: https://github.com/sqlalchemy/dogpile.cache/pull/146
Pull-request-sha: 6920c1cbe5351a80d157104507dea19bdf7482bf
Change-Id: I6532b61d3163695e89c97e744b6a2061cd83a97e
Diffstat (limited to 'tests')
-rw-r--r-- | tests/cache/__init__.py | 12 | ||||
-rw-r--r-- | tests/cache/_fixtures.py | 139 | ||||
-rw-r--r-- | tests/cache/plugins/test_mako_cache.py | 47 | ||||
-rw-r--r-- | tests/cache/test_dbm_backend.py | 44 | ||||
-rw-r--r-- | tests/cache/test_decorator.py | 291 | ||||
-rw-r--r-- | tests/cache/test_mako.py | 3 | ||||
-rw-r--r-- | tests/cache/test_memcached_backend.py | 134 | ||||
-rw-r--r-- | tests/cache/test_null_backend.py | 28 | ||||
-rw-r--r-- | tests/cache/test_redis_backend.py | 98 | ||||
-rw-r--r-- | tests/cache/test_region.py | 415 | ||||
-rw-r--r-- | tests/conftest.py | 11 | ||||
-rw-r--r-- | tests/test_backgrounding.py | 10 | ||||
-rw-r--r-- | tests/test_lock.py | 139 | ||||
-rw-r--r-- | tests/test_utils.py | 18 | ||||
-rw-r--r-- | tests/util/test_nameregistry.py | 19 |
15 files changed, 741 insertions, 667 deletions
diff --git a/tests/cache/__init__.py b/tests/cache/__init__.py index 63514a4..ae0f846 100644 --- a/tests/cache/__init__.py +++ b/tests/cache/__init__.py @@ -1,9 +1,12 @@ +from functools import wraps import re +import time + import pytest -from functools import wraps + from dogpile.util import compat -from dogpile.util.compat import configparser, io # noqa -import time +from dogpile.util.compat import configparser # noqa +from dogpile.util.compat import io # noqa def eq_(a, b, msg=None): @@ -34,7 +37,7 @@ def winsleep(): # sufficient for windows time.time() # to change if compat.win32: - time.sleep(.001) + time.sleep(0.001) def requires_py3k(fn): @@ -43,4 +46,5 @@ def requires_py3k(fn): if compat.py2k: pytest.skip("Python 3 required") return fn(*arg, **kw) + return wrap diff --git a/tests/cache/_fixtures.py b/tests/cache/_fixtures.py index fd8248a..6a4ebca 100644 --- a/tests/cache/_fixtures.py +++ b/tests/cache/_fixtures.py @@ -1,23 +1,28 @@ -from dogpile.cache.api import CacheBackend, NO_VALUE -from dogpile.cache import register_backend, CacheRegion -from dogpile.cache.region import _backend_loader -from . import eq_, assert_raises_message +import collections import itertools +import random +from threading import Lock +from threading import Thread import time -import pytest -from threading import Thread, Lock from unittest import TestCase -import random -import collections +import pytest + +from dogpile.cache import CacheRegion +from dogpile.cache import register_backend +from dogpile.cache.api import CacheBackend +from dogpile.cache.api import NO_VALUE +from dogpile.cache.region import _backend_loader +from . import assert_raises_message +from . import eq_ -class _GenericBackendFixture(object): +class _GenericBackendFixture(object): @classmethod def setup_class(cls): backend_cls = _backend_loader.load(cls.backend) try: - arguments = cls.config_args.get('arguments', {}) + arguments = cls.config_args.get("arguments", {}) backend = backend_cls(arguments) except ImportError: pytest.skip("Backend %s not installed" % cls.backend) @@ -54,6 +59,7 @@ class _GenericBackendFixture(object): key = existing_key_mangler(key) self._keys.add(key) return key + self._region_inst = reg = CacheRegion(**_region_args) existing_key_mangler = self._region_inst.key_mangler @@ -66,13 +72,12 @@ class _GenericBackendFixture(object): def _backend(self): backend_cls = _backend_loader.load(self.backend) _config_args = self.config_args.copy() - arguments = _config_args.get('arguments', {}) + arguments = _config_args.get("arguments", {}) self._backend_inst = backend_cls(arguments) return self._backend_inst class _GenericBackendTest(_GenericBackendFixture, TestCase): - def test_backend_get_nothing(self): backend = self._backend() eq_(backend.get("some_key"), NO_VALUE) @@ -99,11 +104,11 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def test_region_set_multiple_values(self): reg = self._region() - values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'} + values = {"key1": "value1", "key2": "value2", "key3": "value3"} reg.set_multi(values) - eq_(values['key1'], reg.get('key1')) - eq_(values['key2'], reg.get('key2')) - eq_(values['key3'], reg.get('key3')) + eq_(values["key1"], reg.get("key1")) + eq_(values["key2"], reg.get("key2")) + eq_(values["key3"], reg.get("key3")) def test_region_get_zero_multiple_values(self): reg = self._region() @@ -121,35 +126,34 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def test_region_get_or_create_multi_w_should_cache_none(self): reg = self._region() values = reg.get_or_create_multi( - ['key1', 'key2', 'key3'], lambda *k: [None, None, None], - should_cache_fn=lambda v: v is not None) + ["key1", "key2", "key3"], + lambda *k: [None, None, None], + should_cache_fn=lambda v: v is not None, + ) eq_(values, [None, None, None]) def test_region_get_multiple_values(self): reg = self._region() - key1 = 'value1' - key2 = 'value2' - key3 = 'value3' - reg.set('key1', key1) - reg.set('key2', key2) - reg.set('key3', key3) - values = reg.get_multi(['key1', 'key2', 'key3']) - eq_( - [key1, key2, key3], values - ) + key1 = "value1" + key2 = "value2" + key3 = "value3" + reg.set("key1", key1) + reg.set("key2", key2) + reg.set("key3", key3) + values = reg.get_multi(["key1", "key2", "key3"]) + eq_([key1, key2, key3], values) def test_region_get_nothing_multiple(self): reg = self._region() - reg.delete_multi(['key1', 'key2', 'key3', 'key4', 'key5']) - values = {'key1': 'value1', 'key3': 'value3', 'key5': 'value5'} + reg.delete_multi(["key1", "key2", "key3", "key4", "key5"]) + values = {"key1": "value1", "key3": "value3", "key5": "value5"} reg.set_multi(values) reg_values = reg.get_multi( - ['key1', 'key2', 'key3', 'key4', 'key5', 'key6']) + ["key1", "key2", "key3", "key4", "key5", "key6"] + ) eq_( reg_values, - ["value1", NO_VALUE, "value3", NO_VALUE, - "value5", NO_VALUE - ] + ["value1", NO_VALUE, "value3", NO_VALUE, "value5", NO_VALUE], ) def test_region_get_empty_multiple(self): @@ -159,13 +163,13 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def test_region_delete_multiple(self): reg = self._region() - values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'} + values = {"key1": "value1", "key2": "value2", "key3": "value3"} reg.set_multi(values) - reg.delete_multi(['key2', 'key10']) - eq_(values['key1'], reg.get('key1')) - eq_(NO_VALUE, reg.get('key2')) - eq_(values['key3'], reg.get('key3')) - eq_(NO_VALUE, reg.get('key10')) + reg.delete_multi(["key2", "key10"]) + eq_(values["key1"], reg.get("key1")) + eq_(NO_VALUE, reg.get("key2")) + eq_(values["key3"], reg.get("key3")) + eq_(NO_VALUE, reg.get("key10")) def test_region_set_get_nothing(self): reg = self._region() @@ -177,20 +181,21 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def creator(): return "some value" + eq_(reg.get_or_create("some key", creator), "some value") def test_threaded_dogpile(self): # run a basic dogpile concurrency test. # note the concurrency of dogpile itself # is intensively tested as part of dogpile. - reg = self._region(config_args={"expiration_time": .25}) + reg = self._region(config_args={"expiration_time": 0.25}) lock = Lock() canary = [] def creator(): ack = lock.acquire(False) canary.append(ack) - time.sleep(.25) + time.sleep(0.25) if ack: lock.release() return "some value" @@ -198,7 +203,7 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def f(): for x in range(5): reg.get_or_create("some key", creator) - time.sleep(.5) + time.sleep(0.5) threads = [Thread(target=f) for i in range(10)] for t in threads: @@ -212,7 +217,7 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): assert False in canary def test_threaded_get_multi(self): - reg = self._region(config_args={"expiration_time": .25}) + reg = self._region(config_args={"expiration_time": 0.25}) locks = dict((str(i), Lock()) for i in range(11)) canary = collections.defaultdict(list) @@ -230,7 +235,7 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): for acq, key in zip(ack, keys): canary[key].append(acq) - time.sleep(.5) + time.sleep(0.5) for acq, key in zip(ack, keys): if acq: @@ -240,10 +245,14 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): def f(): for x in range(5): reg.get_or_create_multi( - [str(random.randint(1, 10)) - for i in range(random.randint(1, 5))], - creator) - time.sleep(.5) + [ + str(random.randint(1, 10)) + for i in range(random.randint(1, 5)) + ], + creator, + ) + time.sleep(0.5) + f() return threads = [Thread(target=f) for i in range(5)] @@ -264,13 +273,14 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): eq_(reg.get("some key"), NO_VALUE) def test_region_expire(self): - reg = self._region(config_args={"expiration_time": .25}) + reg = self._region(config_args={"expiration_time": 0.25}) counter = itertools.count(1) def creator(): return "some value %d" % next(counter) + eq_(reg.get_or_create("some key", creator), "some value 1") - time.sleep(.4) + time.sleep(0.4) eq_(reg.get("some key", ignore_expiration=True), "some value 1") eq_(reg.get_or_create("some key", creator), "some value 2") eq_(reg.get("some key"), "some value 2") @@ -307,14 +317,11 @@ class _GenericBackendTest(_GenericBackendFixture, TestCase): raise Exception("boom") assert_raises_message( - Exception, - "boom", - reg.get_or_create, "some_key", boom + Exception, "boom", reg.get_or_create, "some_key", boom ) class _GenericMutexTest(_GenericBackendFixture, TestCase): - def test_mutex(self): backend = self._backend() mutex = backend.get_mutex("foo") @@ -342,11 +349,11 @@ class _GenericMutexTest(_GenericBackendFixture, TestCase): for y in range(5): ack = lock.acquire(False) canary.append(ack) - time.sleep(.002) + time.sleep(0.002) if ack: lock.release() mutex.release() - time.sleep(.02) + time.sleep(0.02) threads = [Thread(target=f) for i in range(5)] for t in threads: @@ -381,18 +388,11 @@ class _GenericMutexTest(_GenericBackendFixture, TestCase): def create_bar(): return "bar" - eq_( - reg.get_or_create("foo", create_foo), - "foobar" - ) - eq_( - reg.get_or_create("foo", create_foo), - "foobar" - ) + eq_(reg.get_or_create("foo", create_foo), "foobar") + eq_(reg.get_or_create("foo", create_foo), "foobar") class MockMutex(object): - def __init__(self, key): self.key = key @@ -404,7 +404,6 @@ class MockMutex(object): class MockBackend(CacheBackend): - def __init__(self, arguments): self.arguments = arguments self._cache = {} @@ -419,9 +418,7 @@ class MockBackend(CacheBackend): return NO_VALUE def get_multi(self, keys): - return [ - self.get(key) for key in keys - ] + return [self.get(key) for key in keys] def set(self, key, value): self._cache[key] = value @@ -436,4 +433,6 @@ class MockBackend(CacheBackend): def delete_multi(self, keys): for key in keys: self.delete(key) + + register_backend("mock", __name__, "MockBackend") diff --git a/tests/cache/plugins/test_mako_cache.py b/tests/cache/plugins/test_mako_cache.py index 325412e..7bd5080 100644 --- a/tests/cache/plugins/test_mako_cache.py +++ b/tests/cache/plugins/test_mako_cache.py @@ -1,54 +1,53 @@ -from .. import eq_ from unittest import TestCase + +from mako.cache import register_plugin +from mako.template import Template +import mock import pytest +from .. import eq_ + try: import mako # noqa except ImportError: raise pytest.skip("this test suite requires mako templates") -from mako.template import Template -from mako.cache import register_plugin -import mock - register_plugin( - "dogpile.cache", "dogpile.cache.plugins.mako_cache", "MakoPlugin") + "dogpile.cache", "dogpile.cache.plugins.mako_cache", "MakoPlugin" +) class TestMakoPlugin(TestCase): - def _mock_fixture(self): reg = mock.MagicMock() reg.get_or_create.return_value = "hello world" - my_regions = { - "myregion": reg - } - return { - 'cache_impl': 'dogpile.cache', - 'cache_args': {'regions': my_regions} - }, reg + my_regions = {"myregion": reg} + return ( + { + "cache_impl": "dogpile.cache", + "cache_args": {"regions": my_regions}, + }, + reg, + ) def test_basic(self): kw, reg = self._mock_fixture() - t = Template( - '<%page cached="True" cache_region="myregion"/>hi', - **kw - ) + t = Template('<%page cached="True" cache_region="myregion"/>hi', **kw) t.render() eq_(reg.get_or_create.call_count, 1) def test_timeout(self): kw, reg = self._mock_fixture() - t = Template(""" + t = Template( + """ <%def name="mydef()" cached="True" cache_region="myregion" cache_timeout="20"> some content </%def> ${mydef()} - """, **kw) - t.render() - eq_( - reg.get_or_create.call_args[1], - {"expiration_time": 20} + """, + **kw ) + t.render() + eq_(reg.get_or_create.call_args[1], {"expiration_time": 20}) diff --git a/tests/cache/test_dbm_backend.py b/tests/cache/test_dbm_backend.py index 7c65ba1..c7781dc 100644 --- a/tests/cache/test_dbm_backend.py +++ b/tests/cache/test_dbm_backend.py @@ -1,19 +1,21 @@ -from ._fixtures import _GenericBackendTest, _GenericMutexTest -from . import assert_raises_message import os import sys -from dogpile.util.readwrite_lock import ReadWriteMutex + from dogpile.cache.backends.file import AbstractFileLock +from dogpile.util.readwrite_lock import ReadWriteMutex +from . import assert_raises_message +from ._fixtures import _GenericBackendTest +from ._fixtures import _GenericMutexTest try: import fcntl # noqa + has_fcntl = True except ImportError: has_fcntl = False class MutexLock(AbstractFileLock): - def __init__(self, filename): self.mutex = ReadWriteMutex() @@ -31,27 +33,22 @@ class MutexLock(AbstractFileLock): def release_write_lock(self): return self.mutex.release_write_lock() + test_fname = "test_%s.db" % sys.hexversion if has_fcntl: + class DBMBackendTest(_GenericBackendTest): backend = "dogpile.cache.dbm" - config_args = { - "arguments": { - "filename": test_fname - } - } + config_args = {"arguments": {"filename": test_fname}} class DBMBackendConditionTest(_GenericBackendTest): backend = "dogpile.cache.dbm" config_args = { - "arguments": { - "filename": test_fname, - "lock_factory": MutexLock - } + "arguments": {"filename": test_fname, "lock_factory": MutexLock} } @@ -74,9 +71,7 @@ class _DBMMutexTest(_GenericMutexTest): backend = self._backend() m1 = backend.get_mutex("foo") assert_raises_message( - AssertionError, - "this thread didn't do the acquire", - m1.release + AssertionError, "this thread didn't do the acquire", m1.release ) def test_release_assertion_key(self): @@ -87,28 +82,21 @@ class _DBMMutexTest(_GenericMutexTest): m1.acquire() try: assert_raises_message( - AssertionError, - "No acquire held for key 'bar'", - m2.release + AssertionError, "No acquire held for key 'bar'", m2.release ) finally: m1.release() + if has_fcntl: + class DBMMutexFileTest(_DBMMutexTest): - config_args = { - "arguments": { - "filename": test_fname, - } - } + config_args = {"arguments": {"filename": test_fname}} class DBMMutexConditionTest(_DBMMutexTest): config_args = { - "arguments": { - "filename": test_fname, - "lock_factory": MutexLock - } + "arguments": {"filename": test_fname, "lock_factory": MutexLock} } diff --git a/tests/cache/test_decorator.py b/tests/cache/test_decorator.py index d82b1e8..76a67f7 100644 --- a/tests/cache/test_decorator.py +++ b/tests/cache/test_decorator.py @@ -1,47 +1,56 @@ #! coding: utf-8 -from ._fixtures import _GenericBackendFixture -from . import eq_, requires_py3k, winsleep -from unittest import TestCase +import itertools import time +from unittest import TestCase + from dogpile.cache import util -from dogpile.util import compat -import itertools from dogpile.cache.api import NO_VALUE +from dogpile.util import compat +from . import eq_ +from . import requires_py3k +from . import winsleep +from ._fixtures import _GenericBackendFixture class DecoratorTest(_GenericBackendFixture, TestCase): backend = "dogpile.cache.memory" - def _fixture(self, namespace=None, expiration_time=None, - key_generator=None): - reg = self._region(config_args={"expiration_time": .25}) + def _fixture( + self, namespace=None, expiration_time=None, key_generator=None + ): + reg = self._region(config_args={"expiration_time": 0.25}) counter = itertools.count(1) @reg.cache_on_arguments( namespace=namespace, expiration_time=expiration_time, - function_key_generator=key_generator) + function_key_generator=key_generator, + ) def go(a, b): val = next(counter) return val, a, b + return go - def _multi_fixture(self, namespace=None, expiration_time=None, - key_generator=None): + def _multi_fixture( + self, namespace=None, expiration_time=None, key_generator=None + ): - reg = self._region(config_args={"expiration_time": .25}) + reg = self._region(config_args={"expiration_time": 0.25}) counter = itertools.count(1) @reg.cache_multi_on_arguments( namespace=namespace, expiration_time=expiration_time, - function_multi_key_generator=key_generator) + function_multi_key_generator=key_generator, + ) def go(*args): val = next(counter) return ["%d %s" % (val, arg) for arg in args] + return go def test_decorator(self): @@ -49,7 +58,7 @@ class DecoratorTest(_GenericBackendFixture, TestCase): eq_(go(1, 2), (1, 1, 2)) eq_(go(3, 4), (2, 3, 4)) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (3, 1, 2)) def test_decorator_namespace(self): @@ -59,27 +68,27 @@ class DecoratorTest(_GenericBackendFixture, TestCase): eq_(go(1, 2), (1, 1, 2)) eq_(go(3, 4), (2, 3, 4)) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (3, 1, 2)) def test_decorator_custom_expire(self): - go = self._fixture(expiration_time=.5) + go = self._fixture(expiration_time=0.5) eq_(go(1, 2), (1, 1, 2)) eq_(go(3, 4), (2, 3, 4)) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (3, 1, 2)) def test_decorator_expire_callable(self): - go = self._fixture(expiration_time=lambda: .5) + go = self._fixture(expiration_time=lambda: 0.5) eq_(go(1, 2), (1, 1, 2)) eq_(go(3, 4), (2, 3, 4)) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 2), (3, 1, 2)) def test_decorator_expire_callable_zero(self): @@ -119,18 +128,18 @@ class DecoratorTest(_GenericBackendFixture, TestCase): def test_explicit_get_multi(self): go = self._multi_fixture(expiration_time=1) - eq_(go(1, 2), ['1 1', '1 2']) - eq_(go.get(1, 2), ['1 1', '1 2']) - eq_(go.get(3, 1), [NO_VALUE, '1 1']) - eq_(go(3, 1), ['2 3', '1 1']) - eq_(go.get(3, 1), ['2 3', '1 1']) + eq_(go(1, 2), ["1 1", "1 2"]) + eq_(go.get(1, 2), ["1 1", "1 2"]) + eq_(go.get(3, 1), [NO_VALUE, "1 1"]) + eq_(go(3, 1), ["2 3", "1 1"]) + eq_(go.get(3, 1), ["2 3", "1 1"]) def test_explicit_set_multi(self): go = self._multi_fixture(expiration_time=1) - eq_(go(1, 2), ['1 1', '1 2']) - eq_(go(1, 2), ['1 1', '1 2']) - go.set({1: '1 5', 2: '1 6'}) - eq_(go(1, 2), ['1 5', '1 6']) + eq_(go(1, 2), ["1 1", "1 2"]) + eq_(go(1, 2), ["1 1", "1 2"]) + go.set({1: "1 5", 2: "1 6"}) + eq_(go(1, 2), ["1 5", "1 6"]) def test_explicit_refresh(self): go = self._fixture(expiration_time=1) @@ -143,11 +152,11 @@ class DecoratorTest(_GenericBackendFixture, TestCase): def test_explicit_refresh_multi(self): go = self._multi_fixture(expiration_time=1) - eq_(go(1, 2), ['1 1', '1 2']) - eq_(go(1, 2), ['1 1', '1 2']) - eq_(go.refresh(1, 2), ['2 1', '2 2']) - eq_(go(1, 2), ['2 1', '2 2']) - eq_(go(1, 2), ['2 1', '2 2']) + eq_(go(1, 2), ["1 1", "1 2"]) + eq_(go(1, 2), ["1 1", "1 2"]) + eq_(go.refresh(1, 2), ["2 1", "2 2"]) + eq_(go(1, 2), ["2 1", "2 2"]) + eq_(go(1, 2), ["2 1", "2 2"]) def test_decorator_key_generator(self): def my_key_generator(namespace, fn, **kw): @@ -155,13 +164,14 @@ class DecoratorTest(_GenericBackendFixture, TestCase): def generate_key_with_first_argument(*args): return fname + "_" + str(args[0]) + return generate_key_with_first_argument go = self._fixture(key_generator=my_key_generator) eq_(go(1, 2), (1, 1, 2)) eq_(go(3, 4), (2, 3, 4)) eq_(go(1, 3), (1, 1, 2)) - time.sleep(.3) + time.sleep(0.3) eq_(go(1, 3), (3, 1, 3)) def test_decorator_key_generator_multi(self): @@ -169,25 +179,26 @@ class DecoratorTest(_GenericBackendFixture, TestCase): fname = fn.__name__ def generate_key_with_reversed_order(*args): - return [fname + '_' + str(a) for a in args][::-1] + return [fname + "_" + str(a) for a in args][::-1] + return generate_key_with_reversed_order go = self._multi_fixture(key_generator=my_key_generator) - eq_(go(1, 2), ['1 1', '1 2']) - eq_(go.get(1, 2), ['1 1', '1 2']) - eq_(go.get(3, 1), ['1 2', NO_VALUE]) - eq_(go(3, 1), ['1 2', '2 1']) - eq_(go.get(3, 1), ['1 2', '2 1']) + eq_(go(1, 2), ["1 1", "1 2"]) + eq_(go.get(1, 2), ["1 1", "1 2"]) + eq_(go.get(3, 1), ["1 2", NO_VALUE]) + eq_(go(3, 1), ["1 2", "2 1"]) + eq_(go.get(3, 1), ["1 2", "2 1"]) class KeyGenerationTest(TestCase): - def _keygen_decorator(self, namespace=None, **kw): canary = [] def decorate(fn): canary.append(util.function_key_generator(namespace, fn, **kw)) return fn + return decorate, canary def _multi_keygen_decorator(self, namespace=None, **kw): @@ -195,8 +206,10 @@ class KeyGenerationTest(TestCase): def decorate(fn): canary.append( - util.function_multi_key_generator(namespace, fn, **kw)) + util.function_multi_key_generator(namespace, fn, **kw) + ) return fn + return decorate, canary def _kwarg_keygen_decorator(self, namespace=None, **kw): @@ -204,8 +217,10 @@ class KeyGenerationTest(TestCase): def decorate(fn): canary.append( - util.kwarg_function_key_generator(namespace, fn, **kw)) + util.kwarg_function_key_generator(namespace, fn, **kw) + ) return fn + return decorate, canary def test_default_keygen_kwargs_raises_value_error(self): @@ -224,6 +239,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] result_key = "tests.cache.test_decorator:one|1 2" @@ -239,6 +255,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b=None): pass + gen = canary[0] result_key = "tests.cache.test_decorator:one|1 2" @@ -255,6 +272,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a=True, b=None): pass + gen = canary[0] result_key = "tests.cache.test_decorator:one|1 2" @@ -265,8 +283,8 @@ class KeyGenerationTest(TestCase): eq_(gen(b=2, a=1), result_key) eq_(gen(a=1), "tests.cache.test_decorator:one|1 None") eq_(gen(1), "tests.cache.test_decorator:one|1 None") - eq_(gen(), "tests.cache.test_decorator:one|True None") - eq_(gen(b=2), "tests.cache.test_decorator:one|True 2") + eq_(gen(), "tests.cache.test_decorator:one|True None") + eq_(gen(b=2), "tests.cache.test_decorator:one|True 2") def test_keygen_fn(self): decorate, canary = self._keygen_decorator() @@ -274,6 +292,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] eq_(gen(1, 2), "tests.cache.test_decorator:one|1 2") @@ -285,12 +304,16 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] - eq_(gen(1, 2), [ - "tests.cache.test_decorator:one|1", - "tests.cache.test_decorator:one|2" - ]) + eq_( + gen(1, 2), + [ + "tests.cache.test_decorator:one|1", + "tests.cache.test_decorator:one|2", + ], + ) def test_keygen_fn_namespace(self): decorate, canary = self._keygen_decorator("mynamespace") @@ -298,6 +321,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] eq_(gen(1, 2), "tests.cache.test_decorator:one|mynamespace|1 2") @@ -309,6 +333,7 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] eq_(gen(1, 2), "tests.cache.test_decorator:one|mynamespace|1 2") @@ -320,9 +345,10 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass + gen = canary[0] - assert isinstance(gen('foo'), str) + assert isinstance(gen("foo"), str) def test_kwarg_kwgen_key_isnt_unicode_bydefault(self): decorate, canary = self._kwarg_keygen_decorator("mynamespace") @@ -330,88 +356,109 @@ class KeyGenerationTest(TestCase): @decorate def one(a, b): pass - gen = canary[0] - assert isinstance(gen('foo'), str) + gen = canary[0] + assert isinstance(gen("foo"), str) def test_unicode_key(self): - decorate, canary = self._keygen_decorator("mynamespace", - to_str=compat.text_type) + decorate, canary = self._keygen_decorator( + "mynamespace", to_str=compat.text_type + ) @decorate def one(a, b): pass + gen = canary[0] - eq_(gen(compat.u('méil'), compat.u('drôle')), - compat.ue("tests.cache.test_decorator:" - "one|mynamespace|m\xe9il dr\xf4le")) + eq_( + gen(compat.u("méil"), compat.u("drôle")), + compat.ue( + "tests.cache.test_decorator:" + "one|mynamespace|m\xe9il dr\xf4le" + ), + ) def test_unicode_key_kwarg_generator(self): decorate, canary = self._kwarg_keygen_decorator( - "mynamespace", - to_str=compat.text_type) + "mynamespace", to_str=compat.text_type + ) @decorate def one(a, b): pass + gen = canary[0] - eq_(gen(compat.u('méil'), compat.u('drôle')), - compat.ue("tests.cache.test_decorator:" - "one|mynamespace|m\xe9il dr\xf4le")) + eq_( + gen(compat.u("méil"), compat.u("drôle")), + compat.ue( + "tests.cache.test_decorator:" + "one|mynamespace|m\xe9il dr\xf4le" + ), + ) def test_unicode_key_multi(self): decorate, canary = self._multi_keygen_decorator( - "mynamespace", - to_str=compat.text_type) + "mynamespace", to_str=compat.text_type + ) @decorate def one(a, b): pass + gen = canary[0] - eq_(gen(compat.u('méil'), compat.u('drôle')), + eq_( + gen(compat.u("méil"), compat.u("drôle")), [ compat.ue( - 'tests.cache.test_decorator:one|mynamespace|m\xe9il'), + "tests.cache.test_decorator:one|mynamespace|m\xe9il" + ), compat.ue( - 'tests.cache.test_decorator:one|mynamespace|dr\xf4le') - ]) + "tests.cache.test_decorator:one|mynamespace|dr\xf4le" + ), + ], + ) @requires_py3k def test_unicode_key_by_default(self): - decorate, canary = self._keygen_decorator("mynamespace", - to_str=compat.text_type) + decorate, canary = self._keygen_decorator( + "mynamespace", to_str=compat.text_type + ) @decorate def one(a, b): pass + gen = canary[0] - assert isinstance(gen('méil'), str) + assert isinstance(gen("méil"), str) - eq_(gen('méil', 'drôle'), - "tests.cache.test_decorator:" - "one|mynamespace|m\xe9il dr\xf4le") + eq_( + gen("méil", "drôle"), + "tests.cache.test_decorator:" "one|mynamespace|m\xe9il dr\xf4le", + ) @requires_py3k def test_unicode_key_by_default_kwarg_generator(self): decorate, canary = self._kwarg_keygen_decorator( - "mynamespace", - to_str=compat.text_type) + "mynamespace", to_str=compat.text_type + ) @decorate def one(a, b): pass + gen = canary[0] - assert isinstance(gen('méil'), str) + assert isinstance(gen("méil"), str) - eq_(gen('méil', 'drôle'), - "tests.cache.test_decorator:" - "one|mynamespace|m\xe9il dr\xf4le") + eq_( + gen("méil", "drôle"), + "tests.cache.test_decorator:" "one|mynamespace|m\xe9il dr\xf4le", + ) class CacheDecoratorTest(_GenericBackendFixture, TestCase): @@ -452,14 +499,12 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): # if these two classes get the same namespace, # you get a reentrant deadlock. class Foo(object): - @classmethod @reg.cache_on_arguments(namespace="foo") def generate(cls, x, y): return next(counter) + x + y class Bar(object): - @classmethod @reg.cache_on_arguments(namespace="bar") def generate(cls, x, y): @@ -476,14 +521,14 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): def generate(*args): return ["%d %d" % (arg, next(counter)) for arg in args] - eq_(generate(2, 8, 10), ['2 2', '8 3', '10 1']) - eq_(generate(2, 9, 10), ['2 2', '9 4', '10 1']) + eq_(generate(2, 8, 10), ["2 2", "8 3", "10 1"]) + eq_(generate(2, 9, 10), ["2 2", "9 4", "10 1"]) generate.invalidate(2) - eq_(generate(2, 7, 10), ['2 5', '7 6', '10 1']) + eq_(generate(2, 7, 10), ["2 5", "7 6", "10 1"]) generate.set({7: 18, 10: 15}) - eq_(generate(2, 7, 10), ['2 5', 18, 15]) + eq_(generate(2, 7, 10), ["2 5", 18, 15]) def test_multi_asdict(self): reg = self._region() @@ -496,20 +541,17 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): [(arg, "%d %d" % (arg, next(counter))) for arg in args] ) - eq_(generate(2, 8, 10), {2: '2 2', 8: '8 3', 10: '10 1'}) - eq_(generate(2, 9, 10), {2: '2 2', 9: '9 4', 10: '10 1'}) + eq_(generate(2, 8, 10), {2: "2 2", 8: "8 3", 10: "10 1"}) + eq_(generate(2, 9, 10), {2: "2 2", 9: "9 4", 10: "10 1"}) generate.invalidate(2) - eq_(generate(2, 7, 10), {2: '2 5', 7: '7 6', 10: '10 1'}) + eq_(generate(2, 7, 10), {2: "2 5", 7: "7 6", 10: "10 1"}) generate.set({7: 18, 10: 15}) - eq_(generate(2, 7, 10), {2: '2 5', 7: 18, 10: 15}) + eq_(generate(2, 7, 10), {2: "2 5", 7: 18, 10: 15}) - eq_( - generate.refresh(2, 7), - {2: '2 7', 7: '7 8'} - ) - eq_(generate(2, 7, 10), {2: '2 7', 10: 15, 7: '7 8'}) + eq_(generate.refresh(2, 7), {2: "2 7", 7: "7 8"}) + eq_(generate(2, 7, 10), {2: "2 7", 10: 15, 7: "7 8"}) def test_multi_asdict_keys_missing(self): reg = self._region() @@ -519,20 +561,23 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): @reg.cache_multi_on_arguments(asdict=True) def generate(*args): return dict( - [(arg, "%d %d" % (arg, next(counter))) - for arg in args if arg != 10] + [ + (arg, "%d %d" % (arg, next(counter))) + for arg in args + if arg != 10 + ] ) - eq_(generate(2, 8, 10), {2: '2 1', 8: '8 2'}) - eq_(generate(2, 9, 10), {2: '2 1', 9: '9 3'}) + eq_(generate(2, 8, 10), {2: "2 1", 8: "8 2"}) + eq_(generate(2, 9, 10), {2: "2 1", 9: "9 3"}) assert reg.get(10) is NO_VALUE generate.invalidate(2) - eq_(generate(2, 7, 10), {2: '2 4', 7: '7 5'}) + eq_(generate(2, 7, 10), {2: "2 4", 7: "7 5"}) generate.set({7: 18, 10: 15}) - eq_(generate(2, 7, 10), {2: '2 4', 7: 18, 10: 15}) + eq_(generate(2, 7, 10), {2: "2 4", 7: 18, 10: 15}) def test_multi_asdict_keys_missing_existing_cache_fn(self): reg = self._region() @@ -540,26 +585,29 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): counter = itertools.count(1) @reg.cache_multi_on_arguments( - asdict=True, - should_cache_fn=lambda v: not v.startswith('8 ')) + asdict=True, should_cache_fn=lambda v: not v.startswith("8 ") + ) def generate(*args): return dict( - [(arg, "%d %d" % (arg, next(counter))) - for arg in args if arg != 10] + [ + (arg, "%d %d" % (arg, next(counter))) + for arg in args + if arg != 10 + ] ) - eq_(generate(2, 8, 10), {2: '2 1', 8: '8 2'}) - eq_(generate(2, 8, 10), {2: '2 1', 8: '8 3'}) - eq_(generate(2, 8, 10), {2: '2 1', 8: '8 4'}) - eq_(generate(2, 9, 10), {2: '2 1', 9: '9 5'}) + eq_(generate(2, 8, 10), {2: "2 1", 8: "8 2"}) + eq_(generate(2, 8, 10), {2: "2 1", 8: "8 3"}) + eq_(generate(2, 8, 10), {2: "2 1", 8: "8 4"}) + eq_(generate(2, 9, 10), {2: "2 1", 9: "9 5"}) assert reg.get(10) is NO_VALUE generate.invalidate(2) - eq_(generate(2, 7, 10), {2: '2 6', 7: '7 7'}) + eq_(generate(2, 7, 10), {2: "2 6", 7: "7 7"}) generate.set({7: 18, 10: 15}) - eq_(generate(2, 7, 10), {2: '2 6', 7: 18, 10: 15}) + eq_(generate(2, 7, 10), {2: "2 6", 7: 18, 10: 15}) def test_multi_namespace(self): reg = self._region() @@ -570,22 +618,23 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): def generate(*args): return ["%d %d" % (arg, next(counter)) for arg in args] - eq_(generate(2, 8, 10), ['2 2', '8 3', '10 1']) - eq_(generate(2, 9, 10), ['2 2', '9 4', '10 1']) + eq_(generate(2, 8, 10), ["2 2", "8 3", "10 1"]) + eq_(generate(2, 9, 10), ["2 2", "9 4", "10 1"]) eq_( sorted(list(reg.backend._cache)), [ - 'tests.cache.test_decorator:generate|foo|10', - 'tests.cache.test_decorator:generate|foo|2', - 'tests.cache.test_decorator:generate|foo|8', - 'tests.cache.test_decorator:generate|foo|9'] + "tests.cache.test_decorator:generate|foo|10", + "tests.cache.test_decorator:generate|foo|2", + "tests.cache.test_decorator:generate|foo|8", + "tests.cache.test_decorator:generate|foo|9", + ], ) generate.invalidate(2) - eq_(generate(2, 7, 10), ['2 5', '7 6', '10 1']) + eq_(generate(2, 7, 10), ["2 5", "7 6", "10 1"]) generate.set({7: 18, 10: 15}) - eq_(generate(2, 7, 10), ['2 5', 18, 15]) + eq_(generate(2, 7, 10), ["2 5", 18, 15]) def test_cache_preserve_sig(self): reg = self._region() diff --git a/tests/cache/test_mako.py b/tests/cache/test_mako.py index fedf675..e816e72 100644 --- a/tests/cache/test_mako.py +++ b/tests/cache/test_mako.py @@ -12,5 +12,6 @@ class MakoTest(TestCase): # if the entrypoint isn't there, just pass, as the tests can be run
# without any setuptools install
for impl in pkg_resources.iter_entry_points(
- "mako.cache", "dogpile.cache"):
+ "mako.cache", "dogpile.cache"
+ ):
impl.load()
diff --git a/tests/cache/test_memcached_backend.py b/tests/cache/test_memcached_backend.py index 3793d27..f673ac6 100644 --- a/tests/cache/test_memcached_backend.py +++ b/tests/cache/test_memcached_backend.py @@ -1,53 +1,52 @@ -from ._fixtures import _GenericBackendTest, _GenericMutexTest -from . import eq_ -from unittest import TestCase +import os from threading import Thread import time -import pytest -from dogpile.util import compat -import os +from unittest import TestCase import weakref +import pytest -MEMCACHED_PORT = os.getenv('DOGPILE_MEMCACHED_PORT', '11211') +from dogpile.cache.backends.memcached import GenericMemcachedBackend +from dogpile.cache.backends.memcached import MemcachedBackend +from dogpile.cache.backends.memcached import PylibmcBackend +from . import eq_ +from ._fixtures import _GenericBackendTest +from ._fixtures import _GenericMutexTest + + +MEMCACHED_PORT = os.getenv("DOGPILE_MEMCACHED_PORT", "11211") MEMCACHED_URL = "127.0.0.1:%s" % MEMCACHED_PORT -expect_memcached_running = bool(os.getenv('DOGPILE_MEMCACHED_PORT')) +expect_memcached_running = bool(os.getenv("DOGPILE_MEMCACHED_PORT")) LOCK_TIMEOUT = 1 class _TestMemcachedConn(object): - @classmethod def _check_backend_available(cls, backend): try: client = backend._create_client() client.set("x", "y") assert client.get("x") == "y" - except: + except Exception: if not expect_memcached_running: pytest.skip( "memcached is not running or " - "otherwise not functioning correctly") + "otherwise not functioning correctly" + ) else: raise class _NonDistributedMemcachedTest(_TestMemcachedConn, _GenericBackendTest): - region_args = { - "key_mangler": lambda x: x.replace(" ", "_") - } - config_args = { - "arguments": { - "url": MEMCACHED_URL - } - } + region_args = {"key_mangler": lambda x: x.replace(" ", "_")} + config_args = {"arguments": {"url": MEMCACHED_URL}} -class _DistributedMemcachedWithTimeoutTest(_TestMemcachedConn, _GenericBackendTest): - region_args = { - "key_mangler": lambda x: x.replace(" ", "_") - } +class _DistributedMemcachedWithTimeoutTest( + _TestMemcachedConn, _GenericBackendTest +): + region_args = {"key_mangler": lambda x: x.replace(" ", "_")} config_args = { "arguments": { "url": MEMCACHED_URL, @@ -58,27 +57,21 @@ class _DistributedMemcachedWithTimeoutTest(_TestMemcachedConn, _GenericBackendTe class _DistributedMemcachedTest(_TestMemcachedConn, _GenericBackendTest): - region_args = { - "key_mangler": lambda x: x.replace(" ", "_") - } + region_args = {"key_mangler": lambda x: x.replace(" ", "_")} config_args = { - "arguments": { - "url": MEMCACHED_URL, - "distributed_lock": True, - } + "arguments": {"url": MEMCACHED_URL, "distributed_lock": True} } class _DistributedMemcachedMutexTest(_TestMemcachedConn, _GenericMutexTest): config_args = { - "arguments": { - "url": MEMCACHED_URL, - "distributed_lock": True - } + "arguments": {"url": MEMCACHED_URL, "distributed_lock": True} } -class _DistributedMemcachedMutexWithTimeoutTest(_TestMemcachedConn, _GenericMutexTest): +class _DistributedMemcachedMutexWithTimeoutTest( + _TestMemcachedConn, _GenericMutexTest +): config_args = { "arguments": { "url": MEMCACHED_URL, @@ -119,22 +112,24 @@ class BMemcachedTest(BMemcachedSkips, _NonDistributedMemcachedTest): class BMemcachedDistributedWithTimeoutTest( - BMemcachedSkips, _DistributedMemcachedWithTimeoutTest): + BMemcachedSkips, _DistributedMemcachedWithTimeoutTest +): backend = "dogpile.cache.bmemcached" -class BMemcachedDistributedTest( - BMemcachedSkips, _DistributedMemcachedTest): +class BMemcachedDistributedTest(BMemcachedSkips, _DistributedMemcachedTest): backend = "dogpile.cache.bmemcached" class BMemcachedDistributedMutexTest( - BMemcachedSkips, _DistributedMemcachedMutexTest): + BMemcachedSkips, _DistributedMemcachedMutexTest +): backend = "dogpile.cache.bmemcached" class BMemcachedDistributedMutexWithTimeoutTest( - BMemcachedSkips, _DistributedMemcachedMutexWithTimeoutTest): + BMemcachedSkips, _DistributedMemcachedMutexWithTimeoutTest +): backend = "dogpile.cache.bmemcached" @@ -150,13 +145,7 @@ class MemcachedDistributedMutexTest(_DistributedMemcachedMutexTest): backend = "dogpile.cache.memcached" -from dogpile.cache.backends.memcached import GenericMemcachedBackend -from dogpile.cache.backends.memcached import PylibmcBackend -from dogpile.cache.backends.memcached import MemcachedBackend - - class MockGenericMemcachedBackend(GenericMemcachedBackend): - def _imports(self): pass @@ -165,7 +154,6 @@ class MockGenericMemcachedBackend(GenericMemcachedBackend): class MockMemcacheBackend(MemcachedBackend): - def _imports(self): pass @@ -174,15 +162,13 @@ class MockMemcacheBackend(MemcachedBackend): class MockPylibmcBackend(PylibmcBackend): - def _imports(self): pass def _create_client(self): - return MockClient(self.url, - binary=self.binary, - behaviors=self.behaviors - ) + return MockClient( + self.url, binary=self.binary, behaviors=self.behaviors + ) class MockClient(object): @@ -215,61 +201,64 @@ class MockClient(object): class PylibmcArgsTest(TestCase): - def test_binary_flag(self): - backend = MockPylibmcBackend(arguments={'url': 'foo', 'binary': True}) + backend = MockPylibmcBackend(arguments={"url": "foo", "binary": True}) eq_(backend._create_client().kw["binary"], True) def test_url_list(self): - backend = MockPylibmcBackend(arguments={'url': ["a", "b", "c"]}) + backend = MockPylibmcBackend(arguments={"url": ["a", "b", "c"]}) eq_(backend._create_client().arg[0], ["a", "b", "c"]) def test_url_scalar(self): - backend = MockPylibmcBackend(arguments={'url': "foo"}) + backend = MockPylibmcBackend(arguments={"url": "foo"}) eq_(backend._create_client().arg[0], ["foo"]) def test_behaviors(self): - backend = MockPylibmcBackend(arguments={'url': "foo", - "behaviors": {"q": "p"}}) + backend = MockPylibmcBackend( + arguments={"url": "foo", "behaviors": {"q": "p"}} + ) eq_(backend._create_client().kw["behaviors"], {"q": "p"}) def test_set_time(self): - backend = MockPylibmcBackend(arguments={'url': "foo", - "memcached_expire_time": 20}) + backend = MockPylibmcBackend( + arguments={"url": "foo", "memcached_expire_time": 20} + ) backend.set("foo", "bar") eq_(backend._clients.memcached.canary, [{"time": 20}]) def test_set_min_compress_len(self): - backend = MockPylibmcBackend(arguments={'url': "foo", - "min_compress_len": 20}) + backend = MockPylibmcBackend( + arguments={"url": "foo", "min_compress_len": 20} + ) backend.set("foo", "bar") eq_(backend._clients.memcached.canary, [{"min_compress_len": 20}]) def test_no_set_args(self): - backend = MockPylibmcBackend(arguments={'url': "foo"}) + backend = MockPylibmcBackend(arguments={"url": "foo"}) backend.set("foo", "bar") eq_(backend._clients.memcached.canary, [{}]) class MemcachedArgstest(TestCase): - def test_set_time(self): - backend = MockMemcacheBackend(arguments={'url': "foo", - "memcached_expire_time": 20}) + backend = MockMemcacheBackend( + arguments={"url": "foo", "memcached_expire_time": 20} + ) backend.set("foo", "bar") eq_(backend._clients.memcached.canary, [{"time": 20}]) def test_set_min_compress_len(self): - backend = MockMemcacheBackend(arguments={'url': "foo", - "min_compress_len": 20}) + backend = MockMemcacheBackend( + arguments={"url": "foo", "min_compress_len": 20} + ) backend.set("foo", "bar") eq_(backend._clients.memcached.canary, [{"min_compress_len": 20}]) class LocalThreadTest(TestCase): - def setUp(self): import gc + gc.collect() eq_(MockClient.number_of_clients(), 0) @@ -283,7 +272,7 @@ class LocalThreadTest(TestCase): self._test_client_cleanup(10) def _test_client_cleanup(self, count): - backend = MockGenericMemcachedBackend(arguments={'url': 'foo'}) + backend = MockGenericMemcachedBackend(arguments={"url": "foo"}) canary = [] flag = [False] @@ -292,9 +281,9 @@ class LocalThreadTest(TestCase): backend._clients.memcached canary.append(MockClient.number_of_clients()) while not flag[0]: - time.sleep(.02) + time.sleep(0.02) - threads = [Thread(target=f, args=(count - i, )) for i in range(count)] + threads = [Thread(target=f, args=(count - i,)) for i in range(count)] for t in threads: t.start() flag[0] = True @@ -303,5 +292,6 @@ class LocalThreadTest(TestCase): eq_(canary, [i + 1 for i in range(count)]) import gc + gc.collect() eq_(MockClient.number_of_clients(), 0) diff --git a/tests/cache/test_null_backend.py b/tests/cache/test_null_backend.py index 5899a9f..1166708 100644 --- a/tests/cache/test_null_backend.py +++ b/tests/cache/test_null_backend.py @@ -1,8 +1,9 @@ +import itertools from unittest import TestCase -from ._fixtures import _GenericBackendFixture + from dogpile.cache.api import NO_VALUE from . import eq_ -import itertools +from ._fixtures import _GenericBackendFixture class NullBackendTest(_GenericBackendFixture, TestCase): @@ -26,32 +27,17 @@ class NullBackendTest(_GenericBackendFixture, TestCase): def test_get_multi(self): reg = self._region() - eq_( - reg.get_multi(['a', 'b', 'c']), - [NO_VALUE, NO_VALUE, NO_VALUE] - ) + eq_(reg.get_multi(["a", "b", "c"]), [NO_VALUE, NO_VALUE, NO_VALUE]) def test_set_multi(self): reg = self._region() - reg.set_multi( - { - "a": 1, - "b": 2, - "c": 3 - } - ) - eq_( - reg.get_multi(['a', 'b', 'c']), - [NO_VALUE, NO_VALUE, NO_VALUE] - ) + reg.set_multi({"a": 1, "b": 2, "c": 3}) + eq_(reg.get_multi(["a", "b", "c"]), [NO_VALUE, NO_VALUE, NO_VALUE]) def test_delete_multi(self): reg = self._region() reg.delete_multi(["a", "b", "c"]) - eq_( - reg.get_multi(['a', 'b', 'c']), - [NO_VALUE, NO_VALUE, NO_VALUE] - ) + eq_(reg.get_multi(["a", "b", "c"]), [NO_VALUE, NO_VALUE, NO_VALUE]) def test_decorator(self): reg = self._region() diff --git a/tests/cache/test_redis_backend.py b/tests/cache/test_redis_backend.py index fbf90dd..8baa3e8 100644 --- a/tests/cache/test_redis_backend.py +++ b/tests/cache/test_redis_backend.py @@ -1,17 +1,20 @@ -from dogpile.cache.region import _backend_loader -from ._fixtures import _GenericBackendTest, _GenericMutexTest +import os from unittest import TestCase -from mock import patch, Mock + +from mock import Mock +from mock import patch import pytest -import os -REDIS_HOST = '127.0.0.1' -REDIS_PORT = int(os.getenv('DOGPILE_REDIS_PORT', '6379')) -expect_redis_running = os.getenv('DOGPILE_REDIS_PORT') is not None +from dogpile.cache.region import _backend_loader +from ._fixtures import _GenericBackendTest +from ._fixtures import _GenericMutexTest +REDIS_HOST = "127.0.0.1" +REDIS_PORT = int(os.getenv("DOGPILE_REDIS_PORT", "6379")) +expect_redis_running = os.getenv("DOGPILE_REDIS_PORT") is not None -class _TestRedisConn(object): +class _TestRedisConn(object): @classmethod def _check_backend_available(cls, backend): try: @@ -20,42 +23,43 @@ class _TestRedisConn(object): # on py3k it appears to return b"y" assert client.get("x").decode("ascii") == "y" client.delete("x") - except: + except Exception: if not expect_redis_running: pytest.skip( "redis is not running or " - "otherwise not functioning correctly") + "otherwise not functioning correctly" + ) else: raise class RedisTest(_TestRedisConn, _GenericBackendTest): - backend = 'dogpile.cache.redis' + backend = "dogpile.cache.redis" config_args = { "arguments": { - 'host': REDIS_HOST, - 'port': REDIS_PORT, - 'db': 0, - "foo": "barf" + "host": REDIS_HOST, + "port": REDIS_PORT, + "db": 0, + "foo": "barf", } } class RedisDistributedMutexTest(_TestRedisConn, _GenericMutexTest): - backend = 'dogpile.cache.redis' + backend = "dogpile.cache.redis" config_args = { "arguments": { - 'host': REDIS_HOST, - 'port': REDIS_PORT, - 'db': 0, - 'distributed_lock': True, + "host": REDIS_HOST, + "port": REDIS_PORT, + "db": 0, + "distributed_lock": True, } } -@patch('redis.StrictRedis', autospec=True) +@patch("redis.StrictRedis", autospec=True) class RedisConnectionTest(TestCase): - backend = 'dogpile.cache.redis' + backend = "dogpile.cache.redis" @classmethod def setup_class(cls): @@ -75,53 +79,49 @@ class RedisConnectionTest(TestCase): def test_connect_with_defaults(self, MockStrictRedis): # The defaults, used if keys are missing from the arguments dict. arguments = { - 'host': 'localhost', - 'password': None, - 'port': 6379, - 'db': 0, + "host": "localhost", + "password": None, + "port": 6379, + "db": 0, } self._test_helper(MockStrictRedis, arguments, {}) def test_connect_with_basics(self, MockStrictRedis): arguments = { - 'host': '127.0.0.1', - 'password': None, - 'port': 6379, - 'db': 0, + "host": "127.0.0.1", + "password": None, + "port": 6379, + "db": 0, } self._test_helper(MockStrictRedis, arguments) def test_connect_with_password(self, MockStrictRedis): arguments = { - 'host': '127.0.0.1', - 'password': 'some password', - 'port': 6379, - 'db': 0, + "host": "127.0.0.1", + "password": "some password", + "port": 6379, + "db": 0, } self._test_helper(MockStrictRedis, arguments) def test_connect_with_socket_timeout(self, MockStrictRedis): arguments = { - 'host': '127.0.0.1', - 'port': 6379, - 'socket_timeout': 0.5, - 'password': None, - 'db': 0, + "host": "127.0.0.1", + "port": 6379, + "socket_timeout": 0.5, + "password": None, + "db": 0, } self._test_helper(MockStrictRedis, arguments) def test_connect_with_connection_pool(self, MockStrictRedis): pool = Mock() - arguments = { - 'connection_pool': pool, - 'socket_timeout': 0.5 - } - expected_args = {'connection_pool': pool} - self._test_helper(MockStrictRedis, expected_args, - connection_args=arguments) + arguments = {"connection_pool": pool, "socket_timeout": 0.5} + expected_args = {"connection_pool": pool} + self._test_helper( + MockStrictRedis, expected_args, connection_args=arguments + ) def test_connect_with_url(self, MockStrictRedis): - arguments = { - 'url': 'redis://redis:password@127.0.0.1:6379/0' - } + arguments = {"url": "redis://redis:password@127.0.0.1:6379/0"} self._test_helper(MockStrictRedis.from_url, arguments) diff --git a/tests/cache/test_region.py b/tests/cache/test_region.py index 3fa6d4b..b49076b 100644 --- a/tests/cache/test_region.py +++ b/tests/cache/test_region.py @@ -1,18 +1,26 @@ +from collections import defaultdict +import datetime +import itertools +import time from unittest import TestCase -from dogpile.cache.api import NO_VALUE, CachedValue -from dogpile.cache.api import CacheBackend + +import mock + +from dogpile.cache import CacheRegion from dogpile.cache import exception -from dogpile.cache import make_region, CacheRegion +from dogpile.cache import make_region +from dogpile.cache.api import CacheBackend +from dogpile.cache.api import CachedValue +from dogpile.cache.api import NO_VALUE from dogpile.cache.proxy import ProxyBackend from dogpile.cache.region import _backend_loader from dogpile.cache.region import RegionInvalidationStrategy from dogpile.util import compat -from . import eq_, is_, assert_raises_message, io, configparser -import time -import datetime -import itertools -import mock -from collections import defaultdict +from . import assert_raises_message +from . import configparser +from . import eq_ +from . import io +from . import is_ from ._fixtures import MockBackend @@ -26,54 +34,55 @@ class APITest(TestCase): class RegionTest(TestCase): - def _region(self, init_args={}, config_args={}, backend="mock"): reg = CacheRegion(**init_args) reg.configure(backend, **config_args) return reg def test_set_name(self): - my_region = make_region(name='my-name') - eq_(my_region.name, 'my-name') + my_region = make_region(name="my-name") + eq_(my_region.name, "my-name") def test_instance_from_dict(self): my_conf = { - 'cache.example.backend': 'mock', - 'cache.example.expiration_time': 600, - 'cache.example.arguments.url': '127.0.0.1' + "cache.example.backend": "mock", + "cache.example.expiration_time": 600, + "cache.example.arguments.url": "127.0.0.1", } my_region = make_region() - my_region.configure_from_config(my_conf, 'cache.example.') + my_region.configure_from_config(my_conf, "cache.example.") eq_(my_region.expiration_time, 600) assert isinstance(my_region.backend, MockBackend) is True - eq_(my_region.backend.arguments, {'url': '127.0.0.1'}) + eq_(my_region.backend.arguments, {"url": "127.0.0.1"}) def test_instance_from_config_string(self): - my_conf = \ - '[xyz]\n'\ - 'cache.example.backend=mock\n'\ - 'cache.example.expiration_time=600\n'\ - 'cache.example.arguments.url=127.0.0.1\n'\ - 'cache.example.arguments.dogpile_lockfile=false\n'\ - 'cache.example.arguments.xyz=None\n' + my_conf = ( + "[xyz]\n" + "cache.example.backend=mock\n" + "cache.example.expiration_time=600\n" + "cache.example.arguments.url=127.0.0.1\n" + "cache.example.arguments.dogpile_lockfile=false\n" + "cache.example.arguments.xyz=None\n" + ) my_region = make_region() config = configparser.ConfigParser() compat.read_config_file(config, io.StringIO(my_conf)) my_region.configure_from_config( - dict(config.items('xyz')), 'cache.example.') + dict(config.items("xyz")), "cache.example." + ) eq_(my_region.expiration_time, 600) assert isinstance(my_region.backend, MockBackend) is True - eq_(my_region.backend.arguments, { - 'url': '127.0.0.1', - 'dogpile_lockfile': False, 'xyz': None}) + eq_( + my_region.backend.arguments, + {"url": "127.0.0.1", "dogpile_lockfile": False, "xyz": None}, + ) def test_datetime_expiration_time(self): my_region = make_region() my_region.configure( - backend='mock', - expiration_time=datetime.timedelta(days=1, hours=8) + backend="mock", expiration_time=datetime.timedelta(days=1, hours=8) ) eq_(my_region.expiration_time, 32 * 60 * 60) @@ -83,7 +92,9 @@ class RegionTest(TestCase): assert_raises_message( exception.ValidationError, "expiration_time is not a number or timedelta.", - my_region.configure, 'mock', 'one hour' + my_region.configure, + "mock", + "one hour", ) def test_key_mangler_argument(self): @@ -106,7 +117,8 @@ class RegionTest(TestCase): eq_(reg.get("some key"), "some value") eq_( reg.get_or_create("some key", lambda: "some new value"), - "some value") + "some value", + ) reg.delete("some key") eq_(reg.get("some key"), NO_VALUE) @@ -116,7 +128,8 @@ class RegionTest(TestCase): assert_raises_message( exception.RegionAlreadyConfigured, "This region is already configured", - reg.configure, "mock" + reg.configure, + "mock", ) eq_(reg.is_configured, True) @@ -151,7 +164,9 @@ class RegionTest(TestCase): assert_raises_message( exception.RegionNotConfigured, "No backend is configured on this region.", - getattr, reg, "backend" + getattr, + reg, + "backend", ) eq_(reg.is_configured, False) @@ -160,7 +175,8 @@ class RegionTest(TestCase): assert_raises_message( exception.PluginNotFound, "Couldn't find cache plugin to load: unknown", - reg.configure, 'unknown' + reg.configure, + "unknown", ) eq_(reg.is_configured, False) @@ -181,6 +197,7 @@ class RegionTest(TestCase): def creator(): return "some value" + eq_(reg.get_or_create("some key", creator), "some value") def test_multi_creator(self): @@ -188,8 +205,11 @@ class RegionTest(TestCase): def creator(*keys): return ["some value %s" % key for key in keys] - eq_(reg.get_or_create_multi(["k3", "k2", "k5"], creator), - ['some value k3', 'some value k2', 'some value k5']) + + eq_( + reg.get_or_create_multi(["k3", "k2", "k5"], creator), + ["some value k3", "some value k2", "some value k5"], + ) def test_remove(self): reg = self._region() @@ -204,12 +224,15 @@ class RegionTest(TestCase): def creator(): return "some value %d" % next(counter) + eq_(reg.get_or_create("some key", creator), "some value 1") time.sleep(2) is_(reg.get("some key"), NO_VALUE) eq_(reg.get("some key", ignore_expiration=True), "some value 1") - eq_(reg.get_or_create("some key", creator, expiration_time=-1), - "some value 1") + eq_( + reg.get_or_create("some key", creator, expiration_time=-1), + "some value 1", + ) eq_(reg.get_or_create("some key", creator), "some value 2") eq_(reg.get("some key"), "some value 2") @@ -219,33 +242,40 @@ class RegionTest(TestCase): def creator(*keys): return ["some value %s %d" % (key, next(counter)) for key in keys] - eq_(reg.get_or_create_multi(["k3", "k2", "k5"], creator), - ['some value k3 2', 'some value k2 1', 'some value k5 3']) + + eq_( + reg.get_or_create_multi(["k3", "k2", "k5"], creator), + ["some value k3 2", "some value k2 1", "some value k5 3"], + ) time.sleep(2) is_(reg.get("k2"), NO_VALUE) eq_(reg.get("k2", ignore_expiration=True), "some value k2 1") - eq_(reg.get_or_create_multi(["k3", "k2"], creator, expiration_time=-1), - ['some value k3 2', 'some value k2 1']) - eq_(reg.get_or_create_multi(["k3", "k2"], creator), - ['some value k3 5', 'some value k2 4']) + eq_( + reg.get_or_create_multi(["k3", "k2"], creator, expiration_time=-1), + ["some value k3 2", "some value k2 1"], + ) + eq_( + reg.get_or_create_multi(["k3", "k2"], creator), + ["some value k3 5", "some value k2 4"], + ) eq_(reg.get("k2"), "some value k2 4") def test_expire_on_get(self): - reg = self._region(config_args={"expiration_time": .5}) + reg = self._region(config_args={"expiration_time": 0.5}) reg.set("some key", "some value") eq_(reg.get("some key"), "some value") time.sleep(1) is_(reg.get("some key"), NO_VALUE) def test_ignore_expire_on_get(self): - reg = self._region(config_args={"expiration_time": .5}) + reg = self._region(config_args={"expiration_time": 0.5}) reg.set("some key", "some value") eq_(reg.get("some key"), "some value") time.sleep(1) eq_(reg.get("some key", ignore_expiration=True), "some value") def test_override_expire_on_get(self): - reg = self._region(config_args={"expiration_time": .5}) + reg = self._region(config_args={"expiration_time": 0.5}) reg.set("some key", "some value") eq_(reg.get("some key"), "some value") time.sleep(1) @@ -258,18 +288,23 @@ class RegionTest(TestCase): def creator(): return "some value %d" % next(counter) - eq_(reg.get_or_create("some key", creator, expiration_time=1), - "some value 1") + + eq_( + reg.get_or_create("some key", creator, expiration_time=1), + "some value 1", + ) time.sleep(2) eq_(reg.get("some key"), "some value 1") - eq_(reg.get_or_create("some key", creator, expiration_time=1), - "some value 2") + eq_( + reg.get_or_create("some key", creator, expiration_time=1), + "some value 2", + ) eq_(reg.get("some key"), "some value 2") def test_hard_invalidate_get(self): reg = self._region() reg.set("some key", "some value") - time.sleep(.1) + time.sleep(0.1) reg.invalidate() is_(reg.get("some key"), NO_VALUE) @@ -279,23 +314,19 @@ class RegionTest(TestCase): def creator(): return "some value %d" % next(counter) - eq_(reg.get_or_create("some key", creator), - "some value 1") - time.sleep(.1) + eq_(reg.get_or_create("some key", creator), "some value 1") + + time.sleep(0.1) reg.invalidate() - eq_(reg.get_or_create("some key", creator), - "some value 2") + eq_(reg.get_or_create("some key", creator), "some value 2") - eq_(reg.get_or_create("some key", creator), - "some value 2") + eq_(reg.get_or_create("some key", creator), "some value 2") reg.invalidate() - eq_(reg.get_or_create("some key", creator), - "some value 3") + eq_(reg.get_or_create("some key", creator), "some value 3") - eq_(reg.get_or_create("some key", creator), - "some value 3") + eq_(reg.get_or_create("some key", creator), "some value 3") def test_hard_invalidate_get_or_create_multi(self): reg = self._region() @@ -304,28 +335,38 @@ class RegionTest(TestCase): def creator(*keys): return ["some value %s %d" % (k, next(counter)) for k in keys] - eq_(reg.get_or_create_multi(["k1", "k2"], creator), - ["some value k1 1", "some value k2 2"]) + eq_( + reg.get_or_create_multi(["k1", "k2"], creator), + ["some value k1 1", "some value k2 2"], + ) - time.sleep(.1) + time.sleep(0.1) reg.invalidate() - eq_(reg.get_or_create_multi(["k1", "k2"], creator), - ["some value k1 3", "some value k2 4"]) + eq_( + reg.get_or_create_multi(["k1", "k2"], creator), + ["some value k1 3", "some value k2 4"], + ) - eq_(reg.get_or_create_multi(["k1", "k2"], creator), - ["some value k1 3", "some value k2 4"]) + eq_( + reg.get_or_create_multi(["k1", "k2"], creator), + ["some value k1 3", "some value k2 4"], + ) reg.invalidate() - eq_(reg.get_or_create_multi(["k1", "k2"], creator), - ["some value k1 5", "some value k2 6"]) + eq_( + reg.get_or_create_multi(["k1", "k2"], creator), + ["some value k1 5", "some value k2 6"], + ) - eq_(reg.get_or_create_multi(["k1", "k2"], creator), - ["some value k1 5", "some value k2 6"]) + eq_( + reg.get_or_create_multi(["k1", "k2"], creator), + ["some value k1 5", "some value k2 6"], + ) def test_soft_invalidate_get(self): reg = self._region(config_args={"expiration_time": 1}) reg.set("some key", "some value") - time.sleep(.1) + time.sleep(0.1) reg.invalidate(hard=False) is_(reg.get("some key"), NO_VALUE) @@ -335,13 +376,12 @@ class RegionTest(TestCase): def creator(): return "some value %d" % next(counter) - eq_(reg.get_or_create("some key", creator), - "some value 1") - time.sleep(.1) + eq_(reg.get_or_create("some key", creator), "some value 1") + + time.sleep(0.1) reg.invalidate(hard=False) - eq_(reg.get_or_create("some key", creator), - "some value 2") + eq_(reg.get_or_create("some key", creator), "some value 2") def test_soft_invalidate_get_or_create_multi(self): reg = self._region(config_args={"expiration_time": 5}) @@ -350,13 +390,12 @@ class RegionTest(TestCase): def creator(*keys): v = values.pop(0) return [v for k in keys] - ret = reg.get_or_create_multi( - [1, 2], creator) + + ret = reg.get_or_create_multi([1, 2], creator) eq_(ret, [1, 1]) - time.sleep(.1) + time.sleep(0.1) reg.invalidate(hard=False) - ret = reg.get_or_create_multi( - [1, 2], creator) + ret = reg.get_or_create_multi([1, 2], creator) eq_(ret, [2, 2]) def test_soft_invalidate_requires_expire_time_get(self): @@ -365,7 +404,9 @@ class RegionTest(TestCase): assert_raises_message( exception.DogpileCacheException, "Non-None expiration time required for soft invalidation", - reg.get_or_create, "some key", lambda: "x" + reg.get_or_create, + "some key", + lambda: "x", ) def test_soft_invalidate_requires_expire_time_get_multi(self): @@ -374,7 +415,9 @@ class RegionTest(TestCase): assert_raises_message( exception.DogpileCacheException, "Non-None expiration time required for soft invalidation", - reg.get_or_create_multi, ["k1", "k2"], lambda k: "x" + reg.get_or_create_multi, + ["k1", "k2"], + lambda k: "x", ) def test_should_cache_fn(self): @@ -383,25 +426,26 @@ class RegionTest(TestCase): def creator(): return values.pop(0) - should_cache_fn = lambda val: val in (1, 3) + + should_cache_fn = lambda val: val in (1, 3) # noqa ret = reg.get_or_create( - "some key", creator, - should_cache_fn=should_cache_fn) + "some key", creator, should_cache_fn=should_cache_fn + ) eq_(ret, 1) - eq_(reg.backend._cache['some key'][0], 1) - time.sleep(.1) + eq_(reg.backend._cache["some key"][0], 1) + time.sleep(0.1) reg.invalidate() ret = reg.get_or_create( - "some key", creator, - should_cache_fn=should_cache_fn) + "some key", creator, should_cache_fn=should_cache_fn + ) eq_(ret, 2) - eq_(reg.backend._cache['some key'][0], 1) + eq_(reg.backend._cache["some key"][0], 1) reg.invalidate() ret = reg.get_or_create( - "some key", creator, - should_cache_fn=should_cache_fn) + "some key", creator, should_cache_fn=should_cache_fn + ) eq_(ret, 3) - eq_(reg.backend._cache['some key'][0], 3) + eq_(reg.backend._cache["some key"][0], 3) def test_should_cache_fn_multi(self): reg = self._region() @@ -410,70 +454,67 @@ class RegionTest(TestCase): def creator(*keys): v = values.pop(0) return [v for k in keys] - should_cache_fn = lambda val: val in (1, 3) + + should_cache_fn = lambda val: val in (1, 3) # noqa ret = reg.get_or_create_multi( - [1, 2], creator, - should_cache_fn=should_cache_fn) + [1, 2], creator, should_cache_fn=should_cache_fn + ) eq_(ret, [1, 1]) eq_(reg.backend._cache[1][0], 1) - time.sleep(.1) + time.sleep(0.1) reg.invalidate() ret = reg.get_or_create_multi( - [1, 2], creator, - should_cache_fn=should_cache_fn) + [1, 2], creator, should_cache_fn=should_cache_fn + ) eq_(ret, [2, 2]) eq_(reg.backend._cache[1][0], 1) - time.sleep(.1) + time.sleep(0.1) reg.invalidate() ret = reg.get_or_create_multi( - [1, 2], creator, - should_cache_fn=should_cache_fn) + [1, 2], creator, should_cache_fn=should_cache_fn + ) eq_(ret, [3, 3]) eq_(reg.backend._cache[1][0], 3) def test_should_set_multiple_values(self): reg = self._region() - values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'} + values = {"key1": "value1", "key2": "value2", "key3": "value3"} reg.set_multi(values) - eq_(values['key1'], reg.get('key1')) - eq_(values['key2'], reg.get('key2')) - eq_(values['key3'], reg.get('key3')) + eq_(values["key1"], reg.get("key1")) + eq_(values["key2"], reg.get("key2")) + eq_(values["key3"], reg.get("key3")) def test_should_get_multiple_values(self): reg = self._region() - values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'} + values = {"key1": "value1", "key2": "value2", "key3": "value3"} reg.set_multi(values) - reg_values = reg.get_multi(['key1', 'key2', 'key3']) - eq_( - reg_values, - ["value1", "value2", "value3"] - ) + reg_values = reg.get_multi(["key1", "key2", "key3"]) + eq_(reg_values, ["value1", "value2", "value3"]) def test_should_delete_multiple_values(self): reg = self._region() - values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'} + values = {"key1": "value1", "key2": "value2", "key3": "value3"} reg.set_multi(values) - reg.delete_multi(['key2', 'key1000']) - eq_(values['key1'], reg.get('key1')) - eq_(NO_VALUE, reg.get('key2')) - eq_(values['key3'], reg.get('key3')) + reg.delete_multi(["key2", "key1000"]) + eq_(values["key1"], reg.get("key1")) + eq_(NO_VALUE, reg.get("key2")) + eq_(values["key3"], reg.get("key3")) class ProxyRegionTest(RegionTest): - ''' This is exactly the same as the region test above, but it goes through + """ This is exactly the same as the region test above, but it goes through a dummy proxy. The purpose of this is to make sure the tests still run - successfully even when there is a proxy ''' + successfully even when there is a proxy """ class MockProxy(ProxyBackend): - @property def _cache(self): return self.proxied._cache def _region(self, init_args={}, config_args={}, backend="mock"): reg = CacheRegion(**init_args) - config_args['wrap'] = [ProxyRegionTest.MockProxy] + config_args["wrap"] = [ProxyRegionTest.MockProxy] reg.configure(backend, **config_args) return reg @@ -489,7 +530,6 @@ class CustomInvalidationStrategyTest(RegionTest): """ class CustomInvalidationStrategy(RegionInvalidationStrategy): - def __init__(self): self._soft_invalidated = None self._hard_invalidated = None @@ -503,24 +543,27 @@ class CustomInvalidationStrategyTest(RegionTest): self._hard_invalidated = None def is_invalidated(self, timestamp): - return ((self._soft_invalidated and - timestamp < self._soft_invalidated) or - (self._hard_invalidated and - timestamp < self._hard_invalidated)) + return ( + self._soft_invalidated and timestamp < self._soft_invalidated + ) or ( + self._hard_invalidated and timestamp < self._hard_invalidated + ) def was_hard_invalidated(self): return bool(self._hard_invalidated) def is_hard_invalidated(self, timestamp): - return (self._hard_invalidated and - timestamp < self._hard_invalidated) + return ( + self._hard_invalidated and timestamp < self._hard_invalidated + ) def was_soft_invalidated(self): return bool(self._soft_invalidated) def is_soft_invalidated(self, timestamp): - return (self._soft_invalidated and - timestamp < self._soft_invalidated) + return ( + self._soft_invalidated and timestamp < self._soft_invalidated + ) def _region(self, init_args={}, config_args={}, backend="mock"): reg = CacheRegion(**init_args) @@ -536,7 +579,6 @@ class TestProxyValue(object): class AsyncCreatorTest(TestCase): def _fixture(self): - def async_creation_runner(cache, somekey, creator, mutex): try: value = creator() @@ -549,7 +591,7 @@ class AsyncCreatorTest(TestCase): def test_get_or_create(self): acr = self._fixture() reg = CacheRegion(async_creation_runner=acr) - reg.configure("mock", expiration_time=.2) + reg.configure("mock", expiration_time=0.2) def some_value(): return "some value" @@ -558,18 +600,17 @@ class AsyncCreatorTest(TestCase): return "some new value" eq_(reg.get_or_create("some key", some_value), "some value") - time.sleep(.5) - eq_( - reg.get_or_create("some key", some_new_value), - "some value") + time.sleep(0.5) + eq_(reg.get_or_create("some key", some_new_value), "some value") + eq_(reg.get_or_create("some key", some_new_value), "some new value") eq_( - reg.get_or_create("some key", some_new_value), - "some new value") - eq_( - acr.mock_calls, [ - mock.call(reg, "some key", - some_new_value, reg._mutex("some key")) - ]) + acr.mock_calls, + [ + mock.call( + reg, "some key", some_new_value, reg._mutex("some key") + ) + ], + ) def test_fn_decorator(self): acr = self._fixture() @@ -596,23 +637,28 @@ class AsyncCreatorTest(TestCase): eq_(go(1, 2), 3) - eq_(canary.mock_calls, [ - mock.call(1, 2), mock.call(3, 4), mock.call(1, 2)]) + eq_( + canary.mock_calls, + [mock.call(1, 2), mock.call(3, 4), mock.call(1, 2)], + ) eq_( - acr.mock_calls, [ - mock.call(reg, "tests.cache.test_region:go|1 2", - mock.ANY, - reg._mutex("tests.cache.test_region:go|1 2")) - ]) + acr.mock_calls, + [ + mock.call( + reg, + "tests.cache.test_region:go|1 2", + mock.ANY, + reg._mutex("tests.cache.test_region:go|1 2"), + ) + ], + ) def test_fn_decorator_with_kw(self): acr = self._fixture() reg = CacheRegion(async_creation_runner=acr) reg.configure("mock", expiration_time=5) - canary = mock.Mock() - @reg.cache_on_arguments() def go(x, **kw): return x @@ -628,8 +674,8 @@ class AsyncCreatorTest(TestCase): result = go2(x=test_value) self.assertTrue(isinstance(result, TestProxyValue)) -class ProxyBackendTest(TestCase): +class ProxyBackendTest(TestCase): class GetCounterProxy(ProxyBackend): counter = 0 @@ -646,11 +692,12 @@ class ProxyBackendTest(TestCase): class UsedKeysProxy(ProxyBackend): - ''' Keep a counter of hose often we set a particular key''' + """ Keep a counter of hose often we set a particular key""" def __init__(self, *args, **kwargs): super(ProxyBackendTest.UsedKeysProxy, self).__init__( - *args, **kwargs) + *args, **kwargs + ) self._key_count = defaultdict(lambda: 0) def setcount(self, key): @@ -662,12 +709,13 @@ class ProxyBackendTest(TestCase): class NeverSetProxy(ProxyBackend): - ''' A totally contrived example of a Proxy that we pass arguments to. - Never set a key that matches never_set ''' + """ A totally contrived example of a Proxy that we pass arguments to. + Never set a key that matches never_set """ def __init__(self, never_set, *args, **kwargs): super(ProxyBackendTest.NeverSetProxy, self).__init__( - *args, **kwargs) + *args, **kwargs + ) self.never_set = never_set self._key_count = defaultdict(lambda: 0) @@ -691,8 +739,9 @@ class ProxyBackendTest(TestCase): return reg def test_cachedvalue_passed(self): - reg = self._region(config_args={"wrap": [ - ProxyBackendTest.CanModifyCachedValueProxy]}) + reg = self._region( + config_args={"wrap": [ProxyBackendTest.CanModifyCachedValueProxy]} + ) reg.set("some key", "some value") eq_(reg.get("some key"), "some value") @@ -702,9 +751,14 @@ class ProxyBackendTest(TestCase): # to the backend properly. Test that methods not overridden # continue to work - reg = self._region(config_args={"wrap": [ - ProxyBackendTest.GetCounterProxy, - ProxyBackendTest.SetCounterProxy]}) + reg = self._region( + config_args={ + "wrap": [ + ProxyBackendTest.GetCounterProxy, + ProxyBackendTest.SetCounterProxy, + ] + } + ) ProxyBackendTest.GetCounterProxy.counter = 0 ProxyBackendTest.SetCounterProxy.counter = 0 @@ -738,21 +792,21 @@ class ProxyBackendTest(TestCase): reg_abc = self._region(config_args={"wrap": [proxy_abc]}) for i in range(10): reg_num.set(i, True) - reg_abc.set(chr(ord('a') + i), True) + reg_abc.set(chr(ord("a") + i), True) for i in range(5): reg_num.set(i, True) - reg_abc.set(chr(ord('a') + i), True) + reg_abc.set(chr(ord("a") + i), True) # make sure proxy_num has the right counts per key eq_(proxy_num.setcount(1), 2) eq_(proxy_num.setcount(9), 1) - eq_(proxy_num.setcount('a'), 0) + eq_(proxy_num.setcount("a"), 0) # make sure proxy_abc has the right counts per key - eq_(proxy_abc.setcount('a'), 2) - eq_(proxy_abc.setcount('g'), 1) - eq_(proxy_abc.setcount('9'), 0) + eq_(proxy_abc.setcount("a"), 2) + eq_(proxy_abc.setcount("g"), 1) + eq_(proxy_abc.setcount("9"), 0) def test_argument_proxies(self): # Test that we can pass an argument to Proxy on creation @@ -768,9 +822,14 @@ class ProxyBackendTest(TestCase): def test_actual_backend_proxied(self): # ensure that `reg.actual_backend` is the actual backend # also ensure that `reg.backend` is a proxied backend - reg = self._region(config_args={"wrap": [ - ProxyBackendTest.GetCounterProxy, - ProxyBackendTest.SetCounterProxy]}) + reg = self._region( + config_args={ + "wrap": [ + ProxyBackendTest.GetCounterProxy, + ProxyBackendTest.SetCounterProxy, + ] + } + ) assert isinstance(reg.backend, ProxyBackend) assert isinstance(reg.actual_backend, CacheBackend) diff --git a/tests/conftest.py b/tests/conftest.py index e5e36d3..10afafb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ -from _pytest.unittest import UnitTestCase -import sys import logging - import logging.config +import sys + +from _pytest.unittest import UnitTestCase + logging.config.fileConfig("log_tests.ini") @@ -12,14 +13,14 @@ def is_unittest(obj): Lifted from older versions of py.test, as this seems to be removed. """ - unittest = sys.modules.get('unittest') + unittest = sys.modules.get("unittest") if unittest is None: return # nobody can have derived unittest.TestCase try: return issubclass(obj, unittest.TestCase) except KeyboardInterrupt: raise - except: + except Exception: return False diff --git a/tests/test_backgrounding.py b/tests/test_backgrounding.py index 0c71005..e471991 100644 --- a/tests/test_backgrounding.py +++ b/tests/test_backgrounding.py @@ -1,5 +1,6 @@ -import unittest import threading +import unittest + import dogpile @@ -12,14 +13,13 @@ class TestAsyncRunner(unittest.TestCase): mutex.release() mutex = threading.Lock() - create = lambda: ("value", 1) - get = lambda: ("value", 1) + create = lambda: ("value", 1) # noqa + get = lambda: ("value", 1) # noqa expiretime = 1 assert not self.called - with dogpile.Lock(mutex, create, get, expiretime, runner) as l: + with dogpile.Lock(mutex, create, get, expiretime, runner) as _: assert self.called assert self.called - diff --git a/tests/test_lock.py b/tests/test_lock.py index 6bf277b..dd7f058 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -1,12 +1,16 @@ -from unittest import TestCase -import time -import threading -from dogpile import Lock, NeedRegenerationException -from dogpile.util import ReadWriteMutex import contextlib -import math import logging +import math +import threading +import time +from unittest import TestCase + import mock + +from dogpile import Lock +from dogpile import NeedRegenerationException +from dogpile.util import ReadWriteMutex + log = logging.getLogger(__name__) @@ -16,14 +20,10 @@ class ConcurrencyTest(TestCase): _assertion_lock = threading.Lock() def test_quick(self): - self._test_multi( - 10, 2, .5, 50, .05, .1, - ) + self._test_multi(10, 2, 0.5, 50, 0.05, 0.1) def test_slow(self): - self._test_multi( - 10, 5, 2, 50, .1, .1, - ) + self._test_multi(10, 5, 2, 50, 0.1, 0.1) # TODO: this is a port from the legacy test_dogpile test. # sequence and calculations need to be revised. @@ -34,32 +34,21 @@ class ConcurrencyTest(TestCase): # ) def test_return_while_in_progress(self): - self._test_multi( - 10, 5, 2, 50, 1, .1 - ) + self._test_multi(10, 5, 2, 50, 1, 0.1) def test_get_value_plus_created_long_create(self): - self._test_multi( - 10, 2, 2.5, 50, .05, .1, - ) + self._test_multi(10, 2, 2.5, 50, 0.05, 0.1) def test_get_value_plus_created_registry_unsafe_cache(self): self._test_multi( - 10, 1, .6, 100, .05, .1, - cache_expire_time='unsafe' + 10, 1, 0.6, 100, 0.05, 0.1, cache_expire_time="unsafe" ) def test_get_value_plus_created_registry_safe_cache_quick(self): - self._test_multi( - 10, 2, .5, 50, .05, .1, - cache_expire_time='safe' - ) + self._test_multi(10, 2, 0.5, 50, 0.05, 0.1, cache_expire_time="safe") def test_get_value_plus_created_registry_safe_cache_slow(self): - self._test_multi( - 10, 5, 2, 50, .1, .1, - cache_expire_time='safe' - ) + self._test_multi(10, 5, 2, 50, 0.1, 0.1, cache_expire_time="safe") def _assert_synchronized(self): acq = self._assertion_lock.acquire(False) @@ -69,10 +58,11 @@ class ConcurrencyTest(TestCase): def go(): try: yield {} - except: + except Exception: raise finally: self._assertion_lock.release() + return go() def _assert_log(self, cond, msg, *args): @@ -83,14 +73,15 @@ class ConcurrencyTest(TestCase): assert False, msg % args def _test_multi( - self, num_threads, + self, + num_threads, expiretime, creation_time, num_usages, usage_time, delay_time, cache_expire_time=None, - slow_write_time=None + slow_write_time=None, ): mutex = threading.Lock() @@ -99,10 +90,10 @@ class ConcurrencyTest(TestCase): unsafe_cache = False if cache_expire_time: - if cache_expire_time == 'unsafe': + if cache_expire_time == "unsafe": unsafe_cache = True - cache_expire_time = expiretime * .8 - elif cache_expire_time == 'safe': + cache_expire_time = expiretime * 0.8 + elif cache_expire_time == "safe": cache_expire_time = (expiretime + creation_time) * 1.1 else: assert False, cache_expire_time @@ -116,8 +107,11 @@ class ConcurrencyTest(TestCase): effective_creation_time = creation_time max_stale = ( - effective_expiretime + effective_creation_time + - usage_time + delay_time) * 1.1 + effective_expiretime + + effective_creation_time + + usage_time + + delay_time + ) * 1.1 the_resource = [] slow_waiters = [0] @@ -126,7 +120,8 @@ class ConcurrencyTest(TestCase): def create_resource(): with self._assert_synchronized(): log.debug( - "creating resource, will take %f sec" % creation_time) + "creating resource, will take %f sec" % creation_time + ) time.sleep(creation_time) if slow_write_time: @@ -178,15 +173,15 @@ class ConcurrencyTest(TestCase): for i in range(num_usages): now = time.time() with Lock( - mutex, create_resource, - get_value, expiretime) as value: + mutex, create_resource, get_value, expiretime + ) as value: waited = time.time() - now - if waited > .01: + if waited > 0.01: slow_waiters[0] += 1 check_value(value, waited) time.sleep(usage_time) time.sleep(delay_time) - except: + except Exception: log.error("thread failed", exc_info=True) failures[0] += 1 @@ -199,10 +194,10 @@ class ConcurrencyTest(TestCase): self._assert_log( time_since_create < max_stale, - "Time since create %.4f max stale time %s, " - "total waited %s", - time_since_create, max_stale, - slow_waiters[0] + "Time since create %.4f max stale time %s, " "total waited %s", + time_since_create, + max_stale, + slow_waiters[0], ) started_at = time.time() @@ -219,7 +214,8 @@ class ConcurrencyTest(TestCase): expected_run_time = (num_usages * (usage_time + delay_time)) * 1.1 expected_generations = math.ceil( - expected_run_time / effective_expiretime) + expected_run_time / effective_expiretime + ) if unsafe_cache: expected_slow_waiters = expected_generations * num_threads @@ -235,7 +231,8 @@ class ConcurrencyTest(TestCase): # and a fudged version of the periodic waiting time anticipated # for a single thread... expected_run_time += ( - expected_slow_waiters * effective_creation_time) / num_threads + expected_slow_waiters * effective_creation_time + ) / num_threads expected_run_time *= 1.1 log.info("Test Summary") @@ -243,40 +240,48 @@ class ConcurrencyTest(TestCase): "num threads: %s; expiretime: %s; creation_time: %s; " "num_usages: %s; " "usage_time: %s; delay_time: %s", - num_threads, expiretime, creation_time, num_usages, - usage_time, delay_time + num_threads, + expiretime, + creation_time, + num_usages, + usage_time, + delay_time, ) log.info( "cache expire time: %s; unsafe cache: %s", - cache_expire_time, unsafe_cache) + cache_expire_time, + unsafe_cache, + ) log.info( "Estimated run time %.2f actual run time %.2f", - expected_run_time, actual_run_time) + expected_run_time, + actual_run_time, + ) log.info( "Effective expiretime (min(cache_exp_time, exptime)) %s", - effective_expiretime) + effective_expiretime, + ) log.info( "Expected slow waits %s, Total slow waits %s", - expected_slow_waiters, slow_waiters[0]) + expected_slow_waiters, + slow_waiters[0], + ) log.info( - "Total generations %s Max generations expected %s" % ( - len(the_resource), expected_generations - ) + "Total generations %s Max generations expected %s" + % (len(the_resource), expected_generations) ) assert not failures[0], "%s failures occurred" % failures[0] assert actual_run_time <= expected_run_time - assert slow_waiters[0] <= expected_slow_waiters, \ - "Number of slow waiters %s exceeds expected slow waiters %s" % ( - slow_waiters[0], - expected_slow_waiters + assert slow_waiters[0] <= expected_slow_waiters, ( + "Number of slow waiters %s exceeds expected slow waiters %s" + % (slow_waiters[0], expected_slow_waiters) + ) + assert len(the_resource) <= expected_generations, ( + "Number of resource generations %d exceeded " + "expected %d" % (len(the_resource), expected_generations) ) - assert len(the_resource) <= expected_generations,\ - "Number of resource generations %d exceeded "\ - "expected %d" % ( - len(the_resource), - expected_generations) class RaceConditionTests(TestCase): @@ -306,6 +311,4 @@ class RaceConditionTests(TestCase): ) as entered_value: self.assertEqual("the value", entered_value) - self.assertEqual( - value_and_created_fn.call_count, 1 - ) + self.assertEqual(value_and_created_fn.call_count, 1) diff --git a/tests/test_utils.py b/tests/test_utils.py index a69a1d3..64442c7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,19 +9,19 @@ class UtilsTest(TestCase): """ def test_coerce_string_conf(self): - settings = {'expiration_time': '-1'} + settings = {"expiration_time": "-1"} coerced = util.coerce_string_conf(settings) - self.assertEqual(coerced['expiration_time'], -1) + self.assertEqual(coerced["expiration_time"], -1) - settings = {'expiration_time': '+1'} + settings = {"expiration_time": "+1"} coerced = util.coerce_string_conf(settings) - self.assertEqual(coerced['expiration_time'], 1) - self.assertEqual(type(coerced['expiration_time']), int) + self.assertEqual(coerced["expiration_time"], 1) + self.assertEqual(type(coerced["expiration_time"]), int) - settings = {'arguments.lock_sleep': '0.1'} + settings = {"arguments.lock_sleep": "0.1"} coerced = util.coerce_string_conf(settings) - self.assertEqual(coerced['arguments.lock_sleep'], 0.1) + self.assertEqual(coerced["arguments.lock_sleep"], 0.1) - settings = {'arguments.lock_sleep': '-3.14e-10'} + settings = {"arguments.lock_sleep": "-3.14e-10"} coerced = util.coerce_string_conf(settings) - self.assertEqual(coerced['arguments.lock_sleep'], -3.14e-10) + self.assertEqual(coerced["arguments.lock_sleep"], -3.14e-10) diff --git a/tests/util/test_nameregistry.py b/tests/util/test_nameregistry.py index 2170c36..8369813 100644 --- a/tests/util/test_nameregistry.py +++ b/tests/util/test_nameregistry.py @@ -1,15 +1,15 @@ -from unittest import TestCase -import time +import logging +import random import threading +import time +from unittest import TestCase + from dogpile.util import NameRegistry -import random -import logging log = logging.getLogger(__name__) class NameRegistryTest(TestCase): - def test_name_registry(self): success = [True] num_operations = [0] @@ -20,11 +20,7 @@ class NameRegistryTest(TestCase): registry = NameRegistry(create) - baton = { - "beans":False, - "means":False, - "please":False - } + baton = {"beans": False, "means": False, "please": False} def do_something(name): for iteration in range(20): @@ -38,7 +34,7 @@ class NameRegistryTest(TestCase): break baton[name] = True try: - time.sleep(random.random() * .01) + time.sleep(random.random() * 0.01) finally: num_operations[0] += 1 baton[name] = False @@ -56,4 +52,3 @@ class NameRegistryTest(TestCase): t.join() assert success[0] - |