summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.md9
-rw-r--r--requests_cache/backends/mongodb.py108
-rw-r--r--requests_cache/backends/redis.py34
3 files changed, 109 insertions, 42 deletions
diff --git a/README.md b/README.md
index f46aa7a..53c16e2 100644
--- a/README.md
+++ b/README.md
@@ -38,8 +38,8 @@ Complete project documentation can be found at [requests-cache.readthedocs.io](h
eagerly cache everything for long-term storage, use
[URL patterns](https://requests-cache.readthedocs.io/en/stable/user_guide/expiration.html#expiration-with-url-patterns)
for selective caching, or any combination of strategies
-* ✔️ **Compatibility:** Can be combined with other popular
- [libraries based on requests](https://requests-cache.readthedocs.io/en/stable/user_guide/compatibility.html)
+* ✔️ **Compatibility:** Can be combined with other
+ [popular libraries based on requests](https://requests-cache.readthedocs.io/en/stable/user_guide/compatibility.html)
## Quickstart
First, install with pip:
@@ -116,9 +116,6 @@ To find out more about what you can do with requests-cache, see:
* [User Guide](https://requests-cache.readthedocs.io/en/stable/user_guide.html)
* [API Reference](https://requests-cache.readthedocs.io/en/stable/reference.html)
+* [Examples](https://requests-cache.readthedocs.io/en/stable/examples.html)
* [Project Info](https://requests-cache.readthedocs.io/en/stable/project_info.html)
-* A working example at Real Python:
- [Caching External API Requests](https://realpython.com/blog/python/caching-external-api-requests)
-* More examples in the
- [examples/](https://github.com/reclosedev/requests-cache/tree/master/examples) folder
<!-- END-RTD-IGNORE -->
diff --git a/requests_cache/backends/mongodb.py b/requests_cache/backends/mongodb.py
index 8ba68da..0fb83f2 100644
--- a/requests_cache/backends/mongodb.py
+++ b/requests_cache/backends/mongodb.py
@@ -14,14 +14,54 @@ already have a MongoDB instance you're using for other purposes, or if you find
Expiration
^^^^^^^^^^
-MongoDB natively supports TTL, and can automatically remove expired responses from the cache.
-Note that this is `not guaranteed to happen immediately
-<https://www.mongodb.com/docs/v4.0/core/index-ttl/#timing-of-the-delete-operation>`_. This is the
-recommended way to expire responses, and you can leave the session ``expire_after`` as the default
-(never expire). Example:
+MongoDB `natively supports TTL <https://www.mongodb.com/docs/v4.0/core/index-ttl>`_, and can
+automatically remove expired responses from the cache.
- >>> backend = MongoCache(ttl=3600)
- >>> session = CachedSession('http_cache', backend=backend)
+**Notes:**
+
+* TTL is set for a whole collection, and cannot be set on a per-document basis.
+* It will persist until explicitly removed or overwritten, or if the collection is deleted.
+* Expired items are
+ `not guaranteed to be removed immediately <https://www.mongodb.com/docs/v4.0/core/index-ttl/#timing-of-the-delete-operation>`_.
+ Typically it happens within 60 seconds.
+* If you want, you can rely entirely on MongoDB TTL instead of requests-cache
+ :ref:`expiration settings <expiration>`.
+* Or you can set both values, to be certain that you don't get an expired response before MongoDB
+ removes it.
+* If you intend to reuse expired responses, e.g. with :ref:`conditional-requests` or ``stale_if_error``,
+ you can set TTL to a larger value than your session ``expire_after``, or disable it altogether.
+
+**Examples:**
+
+Create a TTL index:
+
+>>> backend = MongoCache()
+>>> backend.set_ttl(3600)
+
+Overwrite it with a new value:
+
+>>> backend = MongoCache()
+>>> backend.set_ttl(timedelta(days=1), overwrite=True)
+
+Remove the TTL index:
+
+>>> backend = MongoCache()
+>>> backend.set_ttl(None, overwrite=True)
+
+Use both MongoDB TTL and requests-cache expiration:
+
+>>> ttl = timedelta(days=1)
+>>> backend = MongoCache()
+>>> backend.set_ttl(ttl)
+>>> session = CachedSession(backend=backend, expire_after=ttl)
+
+**Recommended:** Set MongoDB TTL to a longer value than your :py:class:`.CachedSession` expiration.
+This allows expired responses to be eventually cleaned up, but still be reused for conditional
+requests for some period of time:
+
+ >>> backend = MongoCache()
+ >>> backend.set_ttl(timedelta(days=7))
+ >>> session = CachedSession(backend=backend, expire_after=timedelta(days=1))
Connection Options
^^^^^^^^^^^^^^^^^^
@@ -37,15 +77,20 @@ API Reference
:classes-only:
:nosignatures:
"""
-from typing import Iterable
+from datetime import timedelta
+from typing import Iterable, Union
from pymongo import MongoClient
+from pymongo.errors import OperationFailure
from .._utils import get_valid_kwargs
+from ..expiration import NEVER_EXPIRE, get_expiration_seconds
from ..serializers import dict_serializer
from . import BaseCache, BaseStorage
+# TODO: TTL tests
+# TODO: Example of viewing responses with MongoDB VSCode plugin or other GUI
class MongoCache(BaseCache):
"""MongoDB cache backend
@@ -55,29 +100,32 @@ class MongoCache(BaseCache):
kwargs: Additional keyword arguments for :py:class:`pymongo.mongo_client.MongoClient`
"""
- def __init__(
- self,
- db_name: str = 'http_cache',
- connection: MongoClient = None,
- ttl: int = None,
- **kwargs,
- ):
+ def __init__(self, db_name: str = 'http_cache', connection: MongoClient = None, **kwargs):
super().__init__(**kwargs)
- self.responses = MongoPickleDict(
+ self.responses: MongoDict = MongoPickleDict(
db_name,
collection_name='responses',
connection=connection,
- ttl=ttl,
**kwargs,
)
- self.redirects = MongoDict(
+ self.redirects: MongoDict = MongoDict(
db_name,
collection_name='redirects',
connection=self.responses.connection,
- ttl=ttl,
**kwargs,
)
+ def set_ttl(self, ttl: Union[int, timedelta], overwrite: bool = False):
+ """Set MongoDB TTL for all collections. Notes:
+
+ * This will have no effect if TTL is already set
+ * To overwrite an existing TTL index, use ``overwrite=True``
+ * Use ``ttl=None, overwrite=True`` to remove the TTL index
+ * This may take some time to complete, depending on the size of your cache
+ """
+ self.responses.set_ttl(ttl, overwrite=overwrite)
+ self.redirects.set_ttl(ttl, overwrite=overwrite)
+
class MongoDict(BaseStorage):
"""A dictionary-like interface for a MongoDB collection
@@ -94,24 +142,22 @@ class MongoDict(BaseStorage):
db_name: str,
collection_name: str = 'http_cache',
connection: MongoClient = None,
- ttl: int = None,
**kwargs,
):
super().__init__(**kwargs)
connection_kwargs = get_valid_kwargs(MongoClient, kwargs)
self.connection = connection or MongoClient(**connection_kwargs)
self.collection = self.connection[db_name][collection_name]
- # Index will not be recreated if it already exists
- # TODO: If TTL changes, drop and recreate index? Or just document that you need to manually
- # call update_ttl()?
- # TODO: Accept timedelta TTL
- if ttl:
- self.collection.create_index('created_at', expireAfterSeconds=ttl)
-
- def update_ttl(self, ttl: int = None):
- self.collection.drop_index('created_at')
- if ttl:
- self.collection.create_index('created_at', expireAfterSeconds=ttl)
+
+ def set_ttl(self, ttl: Union[int, timedelta], overwrite: bool = False):
+ if overwrite:
+ try:
+ self.collection.drop_index('ttl_idx')
+ except OperationFailure:
+ pass
+ ttl = get_expiration_seconds(ttl)
+ if ttl and ttl != NEVER_EXPIRE:
+ self.collection.create_index('created_at', name='ttl_idx', expireAfterSeconds=ttl)
def __getitem__(self, key):
result = self.collection.find_one({'_id': key})
diff --git a/requests_cache/backends/redis.py b/requests_cache/backends/redis.py
index fe05cf4..a2430e2 100644
--- a/requests_cache/backends/redis.py
+++ b/requests_cache/backends/redis.py
@@ -16,6 +16,16 @@ optimized for performance, with a minor risk of data loss, and is usually the be
for a cache. If you need different behavior, the frequency and type of persistence can be customized
or disabled entirely. See `Redis Persistence <https://redis.io/topics/persistence>`_ for details.
+Expiration
+^^^^^^^^^^
+Redis natively supports TTL on a per-key basis, and can automatically remove expired responses from
+the cache. This will be set by by default, according to normal :ref:`expiration settings <expiration>`.
+
+If you intend to reuse expired responses, e.g. with :ref:`conditional-requests` or ``stale_if_error``,
+you can disable this behavior with the ``ttl`` argument:
+
+ >>> backend = RedisCache(ttl=False)
+
Connection Options
^^^^^^^^^^^^^^^^^^
The Redis backend accepts any keyword arguments for :py:class:`redis.client.Redis`. These can be
@@ -28,7 +38,7 @@ Or you can pass an existing ``Redis`` object:
>>> from redis import Redis
>>> connection = Redis(host='192.168.1.63', port=6379)
- >>> backend=RedisCache(connection=connection))
+ >>> backend = RedisCache(connection=connection))
>>> session = CachedSession('http_cache', backend=backend)
Redislite
@@ -36,6 +46,7 @@ Redislite
If you can't easily set up your own Redis server, another option is
`redislite <https://github.com/yahoo/redislite>`_. It contains its own lightweight, embedded Redis
database, and can be used as a drop-in replacement for redis-py. Usage example:
+
>>> from redislite import Redis
>>> from requests_cache import CachedSession, RedisCache
>>>
@@ -60,18 +71,23 @@ from . import BaseCache, BaseStorage
logger = getLogger(__name__)
+# TODO: TTL tests
+# TODO: Option to set a different (typically longer) TTL than expire_after, like MongoCache
class RedisCache(BaseCache):
"""Redis cache backend
Args:
namespace: Redis namespace
connection: Redis connection instance to use instead of creating a new one
+ ttl: Use Redis TTL to automatically remove expired items
kwargs: Additional keyword arguments for :py:class:`redis.client.Redis`
"""
- def __init__(self, namespace='http_cache', connection: Redis = None, **kwargs):
+ def __init__(
+ self, namespace='http_cache', connection: Redis = None, ttl: bool = True, **kwargs
+ ):
super().__init__(**kwargs)
- self.responses = RedisDict(namespace, connection=connection, **kwargs)
+ self.responses = RedisDict(namespace, connection=connection, ttl=ttl, **kwargs)
self.redirects = RedisHashDict(
namespace, 'redirects', connection=self.responses.connection, **kwargs
)
@@ -85,12 +101,20 @@ class RedisDict(BaseStorage):
* Supports TTL
"""
- def __init__(self, namespace: str, collection_name: str = None, connection=None, **kwargs):
+ def __init__(
+ self,
+ namespace: str,
+ collection_name: str = None,
+ connection=None,
+ ttl: bool = True,
+ **kwargs,
+ ):
super().__init__(**kwargs)
connection_kwargs = get_valid_kwargs(Redis, kwargs)
self.connection = connection or StrictRedis(**connection_kwargs)
self.namespace = namespace
+ self.ttl = ttl
def _bkey(self, key: str) -> bytes:
"""Get a full hash key as bytes"""
@@ -110,7 +134,7 @@ class RedisDict(BaseStorage):
def __setitem__(self, key, item):
"""Save an item to the cache, optionally with TTL"""
- if getattr(item, 'ttl', None):
+ if self.ttl and getattr(item, 'ttl', None):
self.connection.setex(self._bkey(key), item.ttl, self.serializer.dumps(item))
else:
self.connection.set(self._bkey(key), self.serializer.dumps(item))