diff options
| -rw-r--r-- | README.md | 9 | ||||
| -rw-r--r-- | requests_cache/backends/mongodb.py | 108 | ||||
| -rw-r--r-- | requests_cache/backends/redis.py | 34 |
3 files changed, 109 insertions, 42 deletions
@@ -38,8 +38,8 @@ Complete project documentation can be found at [requests-cache.readthedocs.io](h eagerly cache everything for long-term storage, use [URL patterns](https://requests-cache.readthedocs.io/en/stable/user_guide/expiration.html#expiration-with-url-patterns) for selective caching, or any combination of strategies -* ✔️ **Compatibility:** Can be combined with other popular - [libraries based on requests](https://requests-cache.readthedocs.io/en/stable/user_guide/compatibility.html) +* ✔️ **Compatibility:** Can be combined with other + [popular libraries based on requests](https://requests-cache.readthedocs.io/en/stable/user_guide/compatibility.html) ## Quickstart First, install with pip: @@ -116,9 +116,6 @@ To find out more about what you can do with requests-cache, see: * [User Guide](https://requests-cache.readthedocs.io/en/stable/user_guide.html) * [API Reference](https://requests-cache.readthedocs.io/en/stable/reference.html) +* [Examples](https://requests-cache.readthedocs.io/en/stable/examples.html) * [Project Info](https://requests-cache.readthedocs.io/en/stable/project_info.html) -* A working example at Real Python: - [Caching External API Requests](https://realpython.com/blog/python/caching-external-api-requests) -* More examples in the - [examples/](https://github.com/reclosedev/requests-cache/tree/master/examples) folder <!-- END-RTD-IGNORE --> diff --git a/requests_cache/backends/mongodb.py b/requests_cache/backends/mongodb.py index 8ba68da..0fb83f2 100644 --- a/requests_cache/backends/mongodb.py +++ b/requests_cache/backends/mongodb.py @@ -14,14 +14,54 @@ already have a MongoDB instance you're using for other purposes, or if you find Expiration ^^^^^^^^^^ -MongoDB natively supports TTL, and can automatically remove expired responses from the cache. -Note that this is `not guaranteed to happen immediately -<https://www.mongodb.com/docs/v4.0/core/index-ttl/#timing-of-the-delete-operation>`_. This is the -recommended way to expire responses, and you can leave the session ``expire_after`` as the default -(never expire). Example: +MongoDB `natively supports TTL <https://www.mongodb.com/docs/v4.0/core/index-ttl>`_, and can +automatically remove expired responses from the cache. - >>> backend = MongoCache(ttl=3600) - >>> session = CachedSession('http_cache', backend=backend) +**Notes:** + +* TTL is set for a whole collection, and cannot be set on a per-document basis. +* It will persist until explicitly removed or overwritten, or if the collection is deleted. +* Expired items are + `not guaranteed to be removed immediately <https://www.mongodb.com/docs/v4.0/core/index-ttl/#timing-of-the-delete-operation>`_. + Typically it happens within 60 seconds. +* If you want, you can rely entirely on MongoDB TTL instead of requests-cache + :ref:`expiration settings <expiration>`. +* Or you can set both values, to be certain that you don't get an expired response before MongoDB + removes it. +* If you intend to reuse expired responses, e.g. with :ref:`conditional-requests` or ``stale_if_error``, + you can set TTL to a larger value than your session ``expire_after``, or disable it altogether. + +**Examples:** + +Create a TTL index: + +>>> backend = MongoCache() +>>> backend.set_ttl(3600) + +Overwrite it with a new value: + +>>> backend = MongoCache() +>>> backend.set_ttl(timedelta(days=1), overwrite=True) + +Remove the TTL index: + +>>> backend = MongoCache() +>>> backend.set_ttl(None, overwrite=True) + +Use both MongoDB TTL and requests-cache expiration: + +>>> ttl = timedelta(days=1) +>>> backend = MongoCache() +>>> backend.set_ttl(ttl) +>>> session = CachedSession(backend=backend, expire_after=ttl) + +**Recommended:** Set MongoDB TTL to a longer value than your :py:class:`.CachedSession` expiration. +This allows expired responses to be eventually cleaned up, but still be reused for conditional +requests for some period of time: + + >>> backend = MongoCache() + >>> backend.set_ttl(timedelta(days=7)) + >>> session = CachedSession(backend=backend, expire_after=timedelta(days=1)) Connection Options ^^^^^^^^^^^^^^^^^^ @@ -37,15 +77,20 @@ API Reference :classes-only: :nosignatures: """ -from typing import Iterable +from datetime import timedelta +from typing import Iterable, Union from pymongo import MongoClient +from pymongo.errors import OperationFailure from .._utils import get_valid_kwargs +from ..expiration import NEVER_EXPIRE, get_expiration_seconds from ..serializers import dict_serializer from . import BaseCache, BaseStorage +# TODO: TTL tests +# TODO: Example of viewing responses with MongoDB VSCode plugin or other GUI class MongoCache(BaseCache): """MongoDB cache backend @@ -55,29 +100,32 @@ class MongoCache(BaseCache): kwargs: Additional keyword arguments for :py:class:`pymongo.mongo_client.MongoClient` """ - def __init__( - self, - db_name: str = 'http_cache', - connection: MongoClient = None, - ttl: int = None, - **kwargs, - ): + def __init__(self, db_name: str = 'http_cache', connection: MongoClient = None, **kwargs): super().__init__(**kwargs) - self.responses = MongoPickleDict( + self.responses: MongoDict = MongoPickleDict( db_name, collection_name='responses', connection=connection, - ttl=ttl, **kwargs, ) - self.redirects = MongoDict( + self.redirects: MongoDict = MongoDict( db_name, collection_name='redirects', connection=self.responses.connection, - ttl=ttl, **kwargs, ) + def set_ttl(self, ttl: Union[int, timedelta], overwrite: bool = False): + """Set MongoDB TTL for all collections. Notes: + + * This will have no effect if TTL is already set + * To overwrite an existing TTL index, use ``overwrite=True`` + * Use ``ttl=None, overwrite=True`` to remove the TTL index + * This may take some time to complete, depending on the size of your cache + """ + self.responses.set_ttl(ttl, overwrite=overwrite) + self.redirects.set_ttl(ttl, overwrite=overwrite) + class MongoDict(BaseStorage): """A dictionary-like interface for a MongoDB collection @@ -94,24 +142,22 @@ class MongoDict(BaseStorage): db_name: str, collection_name: str = 'http_cache', connection: MongoClient = None, - ttl: int = None, **kwargs, ): super().__init__(**kwargs) connection_kwargs = get_valid_kwargs(MongoClient, kwargs) self.connection = connection or MongoClient(**connection_kwargs) self.collection = self.connection[db_name][collection_name] - # Index will not be recreated if it already exists - # TODO: If TTL changes, drop and recreate index? Or just document that you need to manually - # call update_ttl()? - # TODO: Accept timedelta TTL - if ttl: - self.collection.create_index('created_at', expireAfterSeconds=ttl) - - def update_ttl(self, ttl: int = None): - self.collection.drop_index('created_at') - if ttl: - self.collection.create_index('created_at', expireAfterSeconds=ttl) + + def set_ttl(self, ttl: Union[int, timedelta], overwrite: bool = False): + if overwrite: + try: + self.collection.drop_index('ttl_idx') + except OperationFailure: + pass + ttl = get_expiration_seconds(ttl) + if ttl and ttl != NEVER_EXPIRE: + self.collection.create_index('created_at', name='ttl_idx', expireAfterSeconds=ttl) def __getitem__(self, key): result = self.collection.find_one({'_id': key}) diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py index fe05cf4..a2430e2 100644 --- a/requests_cache/backends/redis.py +++ b/requests_cache/backends/redis.py @@ -16,6 +16,16 @@ optimized for performance, with a minor risk of data loss, and is usually the be for a cache. If you need different behavior, the frequency and type of persistence can be customized or disabled entirely. See `Redis Persistence <https://redis.io/topics/persistence>`_ for details. +Expiration +^^^^^^^^^^ +Redis natively supports TTL on a per-key basis, and can automatically remove expired responses from +the cache. This will be set by by default, according to normal :ref:`expiration settings <expiration>`. + +If you intend to reuse expired responses, e.g. with :ref:`conditional-requests` or ``stale_if_error``, +you can disable this behavior with the ``ttl`` argument: + + >>> backend = RedisCache(ttl=False) + Connection Options ^^^^^^^^^^^^^^^^^^ The Redis backend accepts any keyword arguments for :py:class:`redis.client.Redis`. These can be @@ -28,7 +38,7 @@ Or you can pass an existing ``Redis`` object: >>> from redis import Redis >>> connection = Redis(host='192.168.1.63', port=6379) - >>> backend=RedisCache(connection=connection)) + >>> backend = RedisCache(connection=connection)) >>> session = CachedSession('http_cache', backend=backend) Redislite @@ -36,6 +46,7 @@ Redislite If you can't easily set up your own Redis server, another option is `redislite <https://github.com/yahoo/redislite>`_. It contains its own lightweight, embedded Redis database, and can be used as a drop-in replacement for redis-py. Usage example: + >>> from redislite import Redis >>> from requests_cache import CachedSession, RedisCache >>> @@ -60,18 +71,23 @@ from . import BaseCache, BaseStorage logger = getLogger(__name__) +# TODO: TTL tests +# TODO: Option to set a different (typically longer) TTL than expire_after, like MongoCache class RedisCache(BaseCache): """Redis cache backend Args: namespace: Redis namespace connection: Redis connection instance to use instead of creating a new one + ttl: Use Redis TTL to automatically remove expired items kwargs: Additional keyword arguments for :py:class:`redis.client.Redis` """ - def __init__(self, namespace='http_cache', connection: Redis = None, **kwargs): + def __init__( + self, namespace='http_cache', connection: Redis = None, ttl: bool = True, **kwargs + ): super().__init__(**kwargs) - self.responses = RedisDict(namespace, connection=connection, **kwargs) + self.responses = RedisDict(namespace, connection=connection, ttl=ttl, **kwargs) self.redirects = RedisHashDict( namespace, 'redirects', connection=self.responses.connection, **kwargs ) @@ -85,12 +101,20 @@ class RedisDict(BaseStorage): * Supports TTL """ - def __init__(self, namespace: str, collection_name: str = None, connection=None, **kwargs): + def __init__( + self, + namespace: str, + collection_name: str = None, + connection=None, + ttl: bool = True, + **kwargs, + ): super().__init__(**kwargs) connection_kwargs = get_valid_kwargs(Redis, kwargs) self.connection = connection or StrictRedis(**connection_kwargs) self.namespace = namespace + self.ttl = ttl def _bkey(self, key: str) -> bytes: """Get a full hash key as bytes""" @@ -110,7 +134,7 @@ class RedisDict(BaseStorage): def __setitem__(self, key, item): """Save an item to the cache, optionally with TTL""" - if getattr(item, 'ttl', None): + if self.ttl and getattr(item, 'ttl', None): self.connection.setex(self._bkey(key), item.ttl, self.serializer.dumps(item)) else: self.connection.set(self._bkey(key), self.serializer.dumps(item)) |