diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | docs/api.rst | 17 | ||||
| -rw-r--r-- | requests_cache/__init__.py | 6 | ||||
| -rw-r--r-- | requests_cache/core.py | 387 | ||||
| -rw-r--r-- | requests_cache/patcher.py | 131 | ||||
| -rw-r--r-- | requests_cache/session.py | 263 | ||||
| -rw-r--r-- | tests/conftest.py | 2 | ||||
| -rw-r--r-- | tests/unit/test_cache.py | 40 | ||||
| -rw-r--r-- | tests/unit/test_patcher.py (renamed from tests/unit/test_monkey_patch.py) | 1 |
9 files changed, 441 insertions, 410 deletions
@@ -40,10 +40,10 @@ pip install requests-cache ## General Usage There are two main ways of using `requests-cache`: -* **Sessions:** Use [requests_cache.CachedSession](https://requests-cache.readthedocs.io/en/latest/api.html#requests_cache.core.CachedSession) +* **Sessions:** Use [requests_cache.CachedSession](https://requests-cache.readthedocs.io/en/latest/api.html#requests_cache.session.CachedSession) in place of [requests.Session](https://requests.readthedocs.io/en/master/user/advanced/#session-objects) (recommended) * **Patching:** Globally patch `requests` using - [requests_cache.install_cache](https://requests-cache.readthedocs.io/en/latest/api.html#requests_cache.core.install_cache) + [requests_cache.install_cache](https://requests-cache.readthedocs.io/en/latest/api.html#requests_cache.patcher.install_cache) ### Sessions `CachedSession` wraps `requests.Session` with caching features, and otherwise behaves the same as a diff --git a/docs/api.rst b/docs/api.rst index 52b90a3..eb9157d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -2,24 +2,25 @@ API === This section covers all the public interfaces of ``requests-cache`` -Public API ----------- +Sessions +-------- .. Explicitly show inherited method docs on CachedSession instead of CachedMixin -.. autoclass:: requests_cache.core.CachedSession +.. autoclass:: requests_cache.session.CachedSession :members: send, request, cache_disabled, remove_expired_responses :show-inheritance: -.. autoclass:: requests_cache.core.CacheMixin +.. autoclass:: requests_cache.session.CacheMixin -.. automodule:: requests_cache.core +Patching +-------- +.. automodule:: requests_cache.patcher :members: - :exclude-members: CachedSession, CacheMixin +Responses +--------- .. automodule:: requests_cache.response :members: ----------------------------------------------- - Cache Backends -------------- .. automodule:: requests_cache.backends.base diff --git a/requests_cache/__init__.py b/requests_cache/__init__.py index 1b1789a..d72d155 100644 --- a/requests_cache/__init__.py +++ b/requests_cache/__init__.py @@ -4,10 +4,8 @@ __version__ = '0.6.0' try: from .response import AnyResponse, CachedHTTPResponse, CachedResponse, ExpirationTime - from .core import ( - ALL_METHODS, - CachedSession, - CacheMixin, + from .session import ALL_METHODS, CachedSession, CacheMixin + from .patcher import ( clear, disabled, enabled, diff --git a/requests_cache/core.py b/requests_cache/core.py index 393c4f4..7d3999e 100644 --- a/requests_cache/core.py +++ b/requests_cache/core.py @@ -1,383 +1,8 @@ -"""Core functions for configuring cache and monkey patching ``requests``""" -from contextlib import contextmanager -from fnmatch import fnmatch -from logging import getLogger -from typing import Any, Callable, Dict, Iterable, Optional, Type +"""Placeholder module for backwards-compatibility""" +import warnings -import requests -from requests import PreparedRequest -from requests import Session as OriginalSession -from requests.hooks import dispatch_hook +from .patcher import * # noqa: F401, F403 +from .session import * # noqa: F401, F403 -from .backends import BACKEND_KWARGS, BackendSpecifier, BaseCache, init_backend -from .cache_keys import normalize_dict -from .response import AnyResponse, ExpirationTime, set_response_defaults - -ALL_METHODS = ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'DELETE'] -logger = getLogger(__name__) - - -class CacheMixin: - """Mixin class that extends :py:class:`requests.Session` with caching features. - See :py:class:`.CachedSession` for usage information. - """ - - def __init__( - self, - cache_name: str = 'http-cache', - backend: BackendSpecifier = None, - expire_after: ExpirationTime = -1, - urls_expire_after: Dict[str, ExpirationTime] = None, - allowable_codes: Iterable[int] = (200,), - allowable_methods: Iterable['str'] = ('GET', 'HEAD'), - filter_fn: Callable = None, - old_data_on_error: bool = False, - **kwargs, - ): - self.cache = init_backend(backend, cache_name, **kwargs) - self.allowable_codes = allowable_codes - self.allowable_methods = allowable_methods - self.expire_after = expire_after - self.urls_expire_after = urls_expire_after - self.filter_fn = filter_fn or (lambda r: True) - self.old_data_on_error = old_data_on_error - - self._cache_name = cache_name - self._request_expire_after: ExpirationTime = None - self._disabled = False - - # Remove any requests-cache-specific kwargs before passing along to superclass - session_kwargs = {k: v for k, v in kwargs.items() if k not in BACKEND_KWARGS} - super().__init__(**session_kwargs) - - def request( - self, - method: str, - url: str, - params: Dict = None, - data: Any = None, - json: Dict = None, - expire_after: ExpirationTime = None, - **kwargs, - ) -> AnyResponse: - """This method prepares and sends a request while automatically performing any necessary - caching operations. This will be called by any other method-specific ``requests`` functions - (get, post, etc.). This does not include prepared requests, which will still be cached via - ``send()``. - - See :py:meth:`requests.Session.request` for parameters. Additional parameters: - - Args: - expire_after: Expiration time to set only for this request; see details below. - Overrides ``CachedSession.expire_after``. Accepts all the same values as - ``CachedSession.expire_after`` except for ``None``; use ``-1`` to disable expiration - on a per-request basis. - - Returns: - Either a new or cached response - - **Order of operations:** A request will pass through the following methods: - - 1. :py:func:`requests.get`/:py:meth:`requests.Session.get` or other method-specific functions (optional) - 2. :py:meth:`.CachedSession.request` - 3. :py:meth:`requests.Session.request` - 4. :py:meth:`.CachedSession.send` - 5. :py:meth:`.BaseCache.get_response` - 6. :py:meth:`requests.Session.send` (if not cached) - """ - with self.request_expire_after(expire_after): - response = super().request( - method, - url, - params=normalize_dict(params), - data=normalize_dict(data), - json=normalize_dict(json), - **kwargs, - ) - if self._disabled: - return response - - # If the request has been filtered out, delete previously cached response if it exists - cache_key = self.cache.create_key(response.request, **kwargs) - if not response.from_cache and not self.filter_fn(response): - logger.info(f'Deleting filtered response for URL: {response.url}') - self.cache.delete(cache_key) - return response - - # Cache redirect history - for r in response.history: - self.cache.save_redirect(r.request, cache_key) - return response - - def send(self, request: PreparedRequest, **kwargs) -> AnyResponse: - """Send a prepared request, with caching.""" - # If we shouldn't cache the response, just send the request - if not self._is_cacheable(request): - logger.info(f'Request for URL {request.url} is not cacheable') - response = super().send(request, **kwargs) - return set_response_defaults(response) - - # Attempt to fetch the cached response - cache_key = self.cache.create_key(request, **kwargs) - response = self.cache.get_response(cache_key) - - # Attempt to fetch and cache a new response, if needed - if response is None: - return self._send_and_cache(request, cache_key, **kwargs) - if response.is_expired: - return self._handle_expired_response(request, response, cache_key, **kwargs) - - # Dispatch hook here, because we've removed it before pickling - return dispatch_hook('response', request.hooks, response, **kwargs) - - def _is_cacheable(self, request: PreparedRequest) -> bool: - criteria = [ - not self._disabled, - str(request.method) in self.allowable_methods, - self.filter_fn(request), - ] - return all(criteria) - - def _handle_expired_response(self, request, response, cache_key, **kwargs) -> AnyResponse: - """Determine what to do with an expired response, depending on old_data_on_error setting""" - # Attempt to send the request and cache the new response - logger.info('Expired response; attempting to re-send request') - try: - return self._send_and_cache(request, cache_key, **kwargs) - # Return the expired/invalid response on error, if specified; otherwise reraise - except Exception as e: - logger.exception(e) - if self.old_data_on_error: - logger.warning('Request failed; using stale cache data') - return response - self.cache.delete(cache_key) - raise - - def _send_and_cache(self, request, cache_key, **kwargs): - logger.info(f'Sending request and caching response for URL: {request.url}') - response = super().send(request, **kwargs) - if response.status_code in self.allowable_codes: - self.cache.save_response(cache_key, response, self.get_expiration(request.url)) - return set_response_defaults(response) - - @contextmanager - def cache_disabled(self): - """ - Context manager for temporary disabling the cache - :: - - >>> s = CachedSession() - >>> with s.cache_disabled(): - ... s.get('http://httpbin.org/ip') - """ - if self._disabled: - yield - else: - self._disabled = True - try: - yield - finally: - self._disabled = False - - def get_expiration(self, url: str = None) -> ExpirationTime: - """Get the appropriate expiration, in order of precedence: - 1. Per-request expiration - 2. Per-URL expiration - 3. Per-session expiration - """ - return self._request_expire_after or self.url_expire_after(url) or self.expire_after - - @contextmanager - def request_expire_after(self, expire_after: ExpirationTime = None): - """Temporarily override ``expire_after`` for an individual request""" - self._request_expire_after = expire_after - yield - self._request_expire_after = None - - def url_expire_after(self, url: str) -> ExpirationTime: - """Get the expiration time for a URL, if a matching pattern is defined""" - for pattern, expire_after in (self.urls_expire_after or {}).items(): - if url_match(url, pattern): - return expire_after - return None - - def remove_expired_responses(self, expire_after: ExpirationTime = None): - """Remove expired responses from the cache, optionally with revalidation - - Args: - expire_after: A new expiration time used to revalidate the cache - """ - self.cache.remove_expired_responses(expire_after) - - def __repr__(self): - return ( - f"<CachedSession({self.cache.__class__.__name__}('{self._cache_name}', ...), " - f"expire_after={self.expire_after}, allowable_methods={self.allowable_methods})>" - ) - - -class CachedSession(CacheMixin, OriginalSession): - """Class that extends :py:class:`requests.Session` with caching features. - - See individual :ref:`backend classes <cache-backends>` for additional backend-specific arguments. - Also see :ref:`advanced-usage` for more details and examples on how the following arguments - affect cache behavior. - - Args: - cache_name: Cache prefix or namespace, depending on backend - backend: Cache backend name, class, or instance; name may be one of - ``['sqlite', 'mongodb', 'gridfs', 'redis', 'dynamodb', 'memory']``. - expire_after: Time after which cached items will expire - urls_expire_after: Expiration times to apply for different URL patterns - allowable_codes: Only cache responses with one of these codes - allowable_methods: Cache only responses for one of these HTTP methods - include_get_headers: Make request headers part of the cache key - ignored_parameters: List of request parameters to be excluded from the cache key - filter_fn: function that takes a :py:class:`aiohttp.ClientResponse` object and - returns a boolean indicating whether or not that response should be cached. Will be - applied to both new and previously cached responses. - old_data_on_error: Return expired cached responses if new request fails - secret_key: Optional secret key used to sign cache items for added security - - """ - - -def url_match(url: str, pattern: str) -> bool: - """Determine if a URL matches a pattern. - - Args: - url: URL to test. Its base URL (without protocol) will be used. - pattern: Glob pattern to match against. A recursive wildcard will be added if not present - - Example: - >>> url_match('https://httpbin.org/delay/1', 'httpbin.org/delay') - True - >>> url_match('https://httpbin.org/stream/1', 'httpbin.org/*/1') - True - >>> url_match('https://httpbin.org/stream/2', 'httpbin.org/*/1') - False - """ - if not url: - return False - url = url.split('://')[-1] - pattern = pattern.split('://')[-1].rstrip('*') + '**' - return fnmatch(url, pattern) - - -def install_cache( - cache_name: str = 'cache', - backend: str = None, - expire_after: ExpirationTime = None, - urls_expire_after: Dict[str, ExpirationTime] = None, - allowable_codes: Iterable[int] = (200,), - allowable_methods: Iterable['str'] = ('GET', 'HEAD'), - filter_fn: Callable = None, - old_data_on_error: bool = False, - session_factory: Type[OriginalSession] = CachedSession, - **kwargs, -): - """ - Installs cache for all ``requests`` functions by monkey-patching ``Session`` - - Parameters are the same as in :py:class:`CachedSession`. Additional parameters: - - Args: - session_factory: Session class to use. It must inherit from either :py:class:`CachedSession` - or :py:class:`CacheMixin` - """ - - class _ConfiguredCachedSession(session_factory): - def __init__(self): - super().__init__( - cache_name=cache_name, - backend=backend, - expire_after=expire_after, - urls_expire_after=urls_expire_after, - allowable_codes=allowable_codes, - allowable_methods=allowable_methods, - filter_fn=filter_fn, - old_data_on_error=old_data_on_error, - **kwargs, - ) - - _patch_session_factory(_ConfiguredCachedSession) - - -def uninstall_cache(): - """Restores ``requests.Session`` and disables cache""" - _patch_session_factory(OriginalSession) - - -@contextmanager -def disabled(): - """ - Context manager for temporary disabling globally installed cache - - .. warning:: not thread-safe - - :: - - >>> with requests_cache.disabled(): - ... requests.get('http://httpbin.org/ip') - ... requests.get('http://httpbin.org/get') - - """ - previous = requests.Session - uninstall_cache() - try: - yield - finally: - _patch_session_factory(previous) - - -@contextmanager -def enabled(*args, **kwargs): - """ - Context manager for temporary installing global cache. - - Accepts same arguments as :func:`install_cache` - - .. warning:: not thread-safe - - :: - - >>> with requests_cache.enabled('cache_db'): - ... requests.get('http://httpbin.org/get') - - """ - install_cache(*args, **kwargs) - try: - yield - finally: - uninstall_cache() - - -def get_cache() -> Optional[BaseCache]: - """Returns internal cache object from globally installed ``CachedSession``""" - return requests.Session().cache if is_installed() else None - - -def is_installed(): - """Indicate whether or not requests-cache is installed""" - return isinstance(requests.Session(), CachedSession) - - -def clear(): - """Clears globally installed cache""" - if get_cache(): - get_cache().clear() - - -def remove_expired_responses(expire_after: ExpirationTime = None): - """Remove expired responses from the cache, optionally with revalidation - - Args: - expire_after: A new expiration time used to revalidate the cache - """ - if is_installed(): - return requests.Session().remove_expired_responses(expire_after) - - -def _patch_session_factory(session_factory: Type[OriginalSession] = CachedSession): - logger.info(f'Patching requests.Session with class: {type(session_factory).__name__}') - requests.Session = requests.sessions.Session = session_factory # noqa +msg = 'The module `requests_cache.core` is deprecated; please import from `requests_cache`.' +warnings.warn(DeprecationWarning(msg)) diff --git a/requests_cache/patcher.py b/requests_cache/patcher.py new file mode 100644 index 0000000..d49d49c --- /dev/null +++ b/requests_cache/patcher.py @@ -0,0 +1,131 @@ +"""Functions for monkey-patching ``requests``""" +from contextlib import contextmanager +from logging import getLogger +from typing import Callable, Dict, Iterable, Optional, Type + +import requests + +from .backends import BackendSpecifier, BaseCache +from .response import ExpirationTime +from .session import CachedSession, OriginalSession + +logger = getLogger(__name__) + + +def install_cache( + cache_name: str = 'http-cache', + backend: BackendSpecifier = None, + expire_after: ExpirationTime = -1, + urls_expire_after: Dict[str, ExpirationTime] = None, + allowable_codes: Iterable[int] = (200,), + allowable_methods: Iterable['str'] = ('GET', 'HEAD'), + filter_fn: Callable = None, + old_data_on_error: bool = False, + session_factory: Type[OriginalSession] = CachedSession, + **kwargs, +): + """ + Installs cache for all ``requests`` functions by monkey-patching ``Session`` + + Parameters are the same as in :py:class:`.CachedSession`. Additional parameters: + + Args: + session_factory: Session class to use. It must inherit from either + :py:class:`.CachedSession` or :py:class:`.CacheMixin` + """ + + class _ConfiguredCachedSession(session_factory): + def __init__(self): + super().__init__( + cache_name=cache_name, + backend=backend, + expire_after=expire_after, + urls_expire_after=urls_expire_after, + allowable_codes=allowable_codes, + allowable_methods=allowable_methods, + filter_fn=filter_fn, + old_data_on_error=old_data_on_error, + **kwargs, + ) + + _patch_session_factory(_ConfiguredCachedSession) + + +def uninstall_cache(): + """Restores ``requests.Session`` and disables cache""" + _patch_session_factory(OriginalSession) + + +@contextmanager +def disabled(): + """ + Context manager for temporary disabling globally installed cache + + .. warning:: not thread-safe + + :: + + >>> with requests_cache.disabled(): + ... requests.get('http://httpbin.org/ip') + ... requests.get('http://httpbin.org/get') + + """ + previous = requests.Session + uninstall_cache() + try: + yield + finally: + _patch_session_factory(previous) + + +@contextmanager +def enabled(*args, **kwargs): + """ + Context manager for temporary installing global cache. + + Accepts same arguments as :func:`install_cache` + + .. warning:: not thread-safe + + :: + + >>> with requests_cache.enabled('cache_db'): + ... requests.get('http://httpbin.org/get') + + """ + install_cache(*args, **kwargs) + try: + yield + finally: + uninstall_cache() + + +def get_cache() -> Optional[BaseCache]: + """Returns internal cache object from globally installed ``CachedSession``""" + return requests.Session().cache if is_installed() else None + + +def is_installed(): + """Indicate whether or not requests-cache is installed""" + return isinstance(requests.Session(), CachedSession) + + +def clear(): + """Clears globally installed cache""" + if get_cache(): + get_cache().clear() + + +def remove_expired_responses(expire_after: ExpirationTime = None): + """Remove expired responses from the cache, optionally with revalidation + + Args: + expire_after: A new expiration time used to revalidate the cache + """ + if is_installed(): + return requests.Session().remove_expired_responses(expire_after) + + +def _patch_session_factory(session_factory: Type[OriginalSession] = CachedSession): + logger.info(f'Patching requests.Session with class: {type(session_factory).__name__}') + requests.Session = requests.sessions.Session = session_factory # noqa diff --git a/requests_cache/session.py b/requests_cache/session.py new file mode 100644 index 0000000..15ef8f7 --- /dev/null +++ b/requests_cache/session.py @@ -0,0 +1,263 @@ +"""Main classes to add caching features to ``requests.Session``""" +from contextlib import contextmanager +from fnmatch import fnmatch +from logging import getLogger +from typing import Any, Callable, Dict, Iterable + +from requests import PreparedRequest +from requests import Session as OriginalSession +from requests.hooks import dispatch_hook + +from .backends import BACKEND_KWARGS, BackendSpecifier, init_backend +from .cache_keys import normalize_dict +from .response import AnyResponse, ExpirationTime, set_response_defaults + +ALL_METHODS = ['GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'DELETE'] +logger = getLogger(__name__) + + +class CacheMixin: + """Mixin class that extends :py:class:`requests.Session` with caching features. + See :py:class:`.CachedSession` for usage information. + """ + + def __init__( + self, + cache_name: str = 'http-cache', + backend: BackendSpecifier = None, + expire_after: ExpirationTime = -1, + urls_expire_after: Dict[str, ExpirationTime] = None, + allowable_codes: Iterable[int] = (200,), + allowable_methods: Iterable['str'] = ('GET', 'HEAD'), + filter_fn: Callable = None, + old_data_on_error: bool = False, + **kwargs, + ): + self.cache = init_backend(backend, cache_name, **kwargs) + self.allowable_codes = allowable_codes + self.allowable_methods = allowable_methods + self.expire_after = expire_after + self.urls_expire_after = urls_expire_after + self.filter_fn = filter_fn or (lambda r: True) + self.old_data_on_error = old_data_on_error + + self._cache_name = cache_name + self._request_expire_after: ExpirationTime = None + self._disabled = False + + # Remove any requests-cache-specific kwargs before passing along to superclass + session_kwargs = {k: v for k, v in kwargs.items() if k not in BACKEND_KWARGS} + super().__init__(**session_kwargs) + + def request( + self, + method: str, + url: str, + params: Dict = None, + data: Any = None, + json: Dict = None, + expire_after: ExpirationTime = None, + **kwargs, + ) -> AnyResponse: + """This method prepares and sends a request while automatically performing any necessary + caching operations. This will be called by any other method-specific ``requests`` functions + (get, post, etc.). This does not include prepared requests, which will still be cached via + ``send()``. + + See :py:meth:`requests.Session.request` for parameters. Additional parameters: + + Args: + expire_after: Expiration time to set only for this request; see details below. + Overrides ``CachedSession.expire_after``. Accepts all the same values as + ``CachedSession.expire_after`` except for ``None``; use ``-1`` to disable expiration + on a per-request basis. + + Returns: + Either a new or cached response + + **Order of operations:** A request will pass through the following methods: + + 1. :py:func:`requests.get`/:py:meth:`requests.Session.get` or other method-specific functions (optional) + 2. :py:meth:`.CachedSession.request` + 3. :py:meth:`requests.Session.request` + 4. :py:meth:`.CachedSession.send` + 5. :py:meth:`.BaseCache.get_response` + 6. :py:meth:`requests.Session.send` (if not cached) + """ + with self.request_expire_after(expire_after): + response = super().request( + method, + url, + params=normalize_dict(params), + data=normalize_dict(data), + json=normalize_dict(json), + **kwargs, + ) + if self._disabled: + return response + + # If the request has been filtered out, delete previously cached response if it exists + cache_key = self.cache.create_key(response.request, **kwargs) + if not response.from_cache and not self.filter_fn(response): + logger.info(f'Deleting filtered response for URL: {response.url}') + self.cache.delete(cache_key) + return response + + # Cache redirect history + for r in response.history: + self.cache.save_redirect(r.request, cache_key) + return response + + def send(self, request: PreparedRequest, **kwargs) -> AnyResponse: + """Send a prepared request, with caching.""" + # If we shouldn't cache the response, just send the request + if not self._is_cacheable(request): + logger.info(f'Request for URL {request.url} is not cacheable') + response = super().send(request, **kwargs) + return set_response_defaults(response) + + # Attempt to fetch the cached response + cache_key = self.cache.create_key(request, **kwargs) + response = self.cache.get_response(cache_key) + + # Attempt to fetch and cache a new response, if needed + if response is None: + return self._send_and_cache(request, cache_key, **kwargs) + if response.is_expired: + return self._handle_expired_response(request, response, cache_key, **kwargs) + + # Dispatch hook here, because we've removed it before pickling + return dispatch_hook('response', request.hooks, response, **kwargs) + + def _is_cacheable(self, request: PreparedRequest) -> bool: + criteria = [ + not self._disabled, + str(request.method) in self.allowable_methods, + self.filter_fn(request), + ] + return all(criteria) + + def _handle_expired_response(self, request, response, cache_key, **kwargs) -> AnyResponse: + """Determine what to do with an expired response, depending on old_data_on_error setting""" + # Attempt to send the request and cache the new response + logger.info('Expired response; attempting to re-send request') + try: + return self._send_and_cache(request, cache_key, **kwargs) + # Return the expired/invalid response on error, if specified; otherwise reraise + except Exception as e: + logger.exception(e) + if self.old_data_on_error: + logger.warning('Request failed; using stale cache data') + return response + self.cache.delete(cache_key) + raise + + def _send_and_cache(self, request, cache_key, **kwargs): + logger.info(f'Sending request and caching response for URL: {request.url}') + response = super().send(request, **kwargs) + if response.status_code in self.allowable_codes: + self.cache.save_response(cache_key, response, self.get_expiration(request.url)) + return set_response_defaults(response) + + @contextmanager + def cache_disabled(self): + """ + Context manager for temporary disabling the cache + :: + + >>> s = CachedSession() + >>> with s.cache_disabled(): + ... s.get('http://httpbin.org/ip') + """ + if self._disabled: + yield + else: + self._disabled = True + try: + yield + finally: + self._disabled = False + + def get_expiration(self, url: str = None) -> ExpirationTime: + """Get the appropriate expiration, in order of precedence: + 1. Per-request expiration + 2. Per-URL expiration + 3. Per-session expiration + """ + return self._request_expire_after or self.url_expire_after(url) or self.expire_after + + @contextmanager + def request_expire_after(self, expire_after: ExpirationTime = None): + """Temporarily override ``expire_after`` for an individual request""" + self._request_expire_after = expire_after + yield + self._request_expire_after = None + + def url_expire_after(self, url: str) -> ExpirationTime: + """Get the expiration time for a URL, if a matching pattern is defined""" + for pattern, expire_after in (self.urls_expire_after or {}).items(): + if url_match(url, pattern): + return expire_after + return None + + def remove_expired_responses(self, expire_after: ExpirationTime = None): + """Remove expired responses from the cache, optionally with revalidation + + Args: + expire_after: A new expiration time used to revalidate the cache + """ + self.cache.remove_expired_responses(expire_after) + + def __repr__(self): + return ( + f"<CachedSession({self.cache.__class__.__name__}('{self._cache_name}', ...), " + f"expire_after={self.expire_after}, allowable_methods={self.allowable_methods})>" + ) + + +class CachedSession(CacheMixin, OriginalSession): + """Class that extends :py:class:`requests.Session` with caching features. + + See individual :ref:`backend classes <cache-backends>` for additional backend-specific arguments. + Also see :ref:`advanced-usage` for more details and examples on how the following arguments + affect cache behavior. + + Args: + cache_name: Cache prefix or namespace, depending on backend + backend: Cache backend name, class, or instance; name may be one of + ``['sqlite', 'mongodb', 'gridfs', 'redis', 'dynamodb', 'memory']``. + expire_after: Time after which cached items will expire + urls_expire_after: Expiration times to apply for different URL patterns + allowable_codes: Only cache responses with one of these codes + allowable_methods: Cache only responses for one of these HTTP methods + include_get_headers: Make request headers part of the cache key + ignored_parameters: List of request parameters to be excluded from the cache key + filter_fn: function that takes a :py:class:`aiohttp.ClientResponse` object and + returns a boolean indicating whether or not that response should be cached. Will be + applied to both new and previously cached responses. + old_data_on_error: Return expired cached responses if new request fails + secret_key: Optional secret key used to sign cache items for added security + + """ + + +def url_match(url: str, pattern: str) -> bool: + """Determine if a URL matches a pattern. + + Args: + url: URL to test. Its base URL (without protocol) will be used. + pattern: Glob pattern to match against. A recursive wildcard will be added if not present + + Example: + >>> url_match('https://httpbin.org/delay/1', 'httpbin.org/delay') + True + >>> url_match('https://httpbin.org/stream/1', 'httpbin.org/*/1') + True + >>> url_match('https://httpbin.org/stream/2', 'httpbin.org/*/1') + False + """ + if not url: + return False + url = url.split('://')[-1] + pattern = pattern.split('://')[-1].rstrip('*') + '**' + return fnmatch(url, pattern) diff --git a/tests/conftest.py b/tests/conftest.py index 466dd41..999f456 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from requests_mock import ANY as ANY_METHOD from requests_mock import Adapter import requests_cache -from requests_cache.core import ALL_METHODS, CachedSession +from requests_cache.session import ALL_METHODS, CachedSession MOCKED_URL = 'http+mock://requests-cache.com/text' MOCKED_URL_HTTPS = 'https+mock://requests-cache.com/text' diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 28f038e..84004b2 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -1,5 +1,4 @@ """CachedSession + BaseCache tests that use mocked responses only""" -# flake8: noqa: F841 # TODO: This could be split up into some smaller test modules import json import pickle @@ -55,6 +54,11 @@ def test_init_backend_class(): assert isinstance(session.cache, MyCache) +def test_import_compat(): + """Just make sure that we can still import from requests_cache.core""" + from requests_cache.core import CachedSession, install_cache # noqa: F401 + + @pytest.mark.parametrize('method', ALL_METHODS) @pytest.mark.parametrize('field', ['params', 'data', 'json']) def test_all_methods(field, method, mock_session): @@ -104,10 +108,10 @@ def test_verify(mock_session): def test_response_history(mock_session): - r1 = mock_session.get(MOCKED_URL_REDIRECT) - r2 = mock_session.get(MOCKED_URL_REDIRECT_TARGET) + mock_session.get(MOCKED_URL_REDIRECT) + r = mock_session.get(MOCKED_URL_REDIRECT_TARGET) - assert r2.from_cache is True + assert r.from_cache is True assert len(mock_session.cache.redirects) == 1 @@ -161,10 +165,18 @@ def test_hooks(mock_session): return r for i in range(5): - r = mock_session.get(MOCKED_URL, hooks={hook: hook_func}) + mock_session.get(MOCKED_URL, hooks={hook: hook_func}) assert state[hook] == 5 +@pytest.mark.parametrize('method', ['POST', 'PUT']) +def test_raw_data(method, mock_session): + """POST and PUT requests with different data (raw) should be cached under different keys""" + assert mock_session.request(method, MOCKED_URL, data='raw data').from_cache is False + assert mock_session.request(method, MOCKED_URL, data='raw data').from_cache is True + assert mock_session.request(method, MOCKED_URL, data='new raw data').from_cache is False + + @pytest.mark.parametrize('mapping_class', [dict, UserDict, CaseInsensitiveDict]) @pytest.mark.parametrize('field', ['params', 'data', 'json']) def test_normalize_params(field, mapping_class, mock_session): @@ -322,14 +334,6 @@ def test_old_data_on_error(mock_session): assert response.from_cache is True and response.is_expired is True -@pytest.mark.parametrize('method', ['POST', 'PUT']) -def test_raw_data(method, mock_session): - """POST and PUT requests with different data (raw) should be cached under different keys""" - assert mock_session.request(method, MOCKED_URL, data='raw data').from_cache is False - assert mock_session.request(method, MOCKED_URL, data='raw data').from_cache is True - assert mock_session.request(method, MOCKED_URL, data='new raw data').from_cache is False - - def test_cache_disabled(mock_session): mock_session.get(MOCKED_URL) with mock_session.cache_disabled(): @@ -338,6 +342,16 @@ def test_cache_disabled(mock_session): assert mock_session.get(MOCKED_URL).from_cache is True +def test_cache_disabled__nested(mock_session): + mock_session.get(MOCKED_URL) + with mock_session.cache_disabled(): + mock_session.get(MOCKED_URL) + with mock_session.cache_disabled(): + for i in range(2): + assert mock_session.get(MOCKED_URL).from_cache is False + assert mock_session.get(MOCKED_URL).from_cache is True + + @pytest.mark.parametrize( 'url, expected_expire_after', [ diff --git a/tests/unit/test_monkey_patch.py b/tests/unit/test_patcher.py index 54c5c33..4a3e417 100644 --- a/tests/unit/test_monkey_patch.py +++ b/tests/unit/test_patcher.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python from unittest.mock import patch import requests |