diff options
author | Jordan Cook <jordan.cook.git@proton.me> | 2022-09-29 12:10:54 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-29 12:10:54 -0500 |
commit | 3fa37177263baa7e93c62947b92d6079c53b7c8e (patch) | |
tree | 5a7683e4cb01359a151742223c4d3a4301d75f50 | |
parent | 09de36a70e966925ed7a86652a3fbea704cf293b (diff) | |
parent | 5204d487fea1d0b89e9eecdbc4a38621d2d4dbe4 (diff) | |
download | requests-cache-3fa37177263baa7e93c62947b92d6079c53b7c8e.tar.gz |
Merge pull request #694 from requests-cache/match-ignored-params
Match whether ignored_parameters are present in a request
-rw-r--r-- | .github/workflows/build.yml | 3 | ||||
-rw-r--r-- | .github/workflows/deploy.yml | 4 | ||||
-rw-r--r-- | HISTORY.md | 8 | ||||
-rw-r--r-- | poetry.lock | 159 | ||||
-rw-r--r-- | requests_cache/cache_keys.py | 21 | ||||
-rw-r--r-- | tests/integration/base_cache_test.py | 24 | ||||
-rw-r--r-- | tests/integration/test_upgrade.py (renamed from tests/integration/test_compat.py) | 2 | ||||
-rw-r--r-- | tests/sample_data/sample.db.0.9.3 | bin | 0 -> 118784 bytes | |||
-rw-r--r-- | tests/sample_data/sample.db.0.9.4 | bin | 0 -> 118784 bytes | |||
-rw-r--r-- | tests/sample_data/sample.db.0.9.5 | bin | 0 -> 118784 bytes | |||
-rw-r--r-- | tests/sample_data/sample.db.0.9.6 | bin | 0 -> 118784 bytes | |||
-rw-r--r-- | tests/sample_data/sample.db.1.0.0-alpha | bin | 0 -> 122880 bytes | |||
-rw-r--r-- | tests/unit/test_cache_keys.py | 2 | ||||
-rw-r--r-- | tests/unit/test_session.py | 33 |
14 files changed, 134 insertions, 122 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 14d9404..e2f5e9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10', 3.11.0-beta.4] + python-version: [3.7, 3.8, 3.9, '3.10', 3.11.0-rc.2] fail-fast: false services: nginx: @@ -33,7 +33,6 @@ jobs: python-version: ${{ matrix.python-version }} - uses: snok/install-poetry@v1.3 with: - version: 1.2.0b1 virtualenvs-in-project: true # Start integration test databases diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ea290a2..8c3d14b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -49,7 +49,6 @@ jobs: python-version: ${{ matrix.python-version }} - uses: snok/install-poetry@v1.3 with: - version: 1.2.0b1 virtualenvs-in-project: true # Start integration test databases @@ -122,9 +121,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ env.LATEST_PY_VERSION }} - - uses: snok/install-poetry@v1.3 + - uses: snok/install-psoetry@v1.3 with: - version: 1.2.0b1 virtualenvs-in-project: true - name: Set pre-release version @@ -54,8 +54,12 @@ **Request matching & filtering:** * Add serializer name to cache keys to avoid errors due to switching serializers * Always skip both cache read and write for requests excluded by `allowable_methods` (previously only skipped write) -* Ignore and redact common authentication headers and request parameters by default. This provides some default recommended values for `ignored_parameters`, to avoid accidentally storing common credentials (e.g., OAuth tokens) in the cache. This will have no effect if you are already setting `ignored_parameters`. -* Support distinct matching for requests that differ by duplicate request params (e.g, `a=1` vs `?a=1&a=2`) +* Ignore and redact common authentication headers and request parameters by default. This provides + some default recommended values for `ignored_parameters`, to avoid accidentally storing common + credentials in the cache. This will have no effect if `ignored_parameters` is already set. +* Support distinct matching for requests that differ only by a parameter in `ignored_parameters` + (e.g., for a request sent both with and without authentication) +* Support distinct matching for requests that differ only by duplicate request params (e.g, `a=1` vs `?a=1&a=2`) **Cache convenience methods:** * Add `expired` and `invalid` arguments to `BaseCache.delete()` (to replace `remove_expired_responses()`) diff --git a/poetry.lock b/poetry.lock index 7c79818..df69ae1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,14 +32,6 @@ python-versions = ">=3.6" typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} [[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" @@ -81,14 +73,14 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.24.59" +version = "1.24.82" description = "The AWS SDK for Python" category = "dev" optional = true python-versions = ">= 3.7" [package.dependencies] -botocore = ">=1.27.59,<1.28.0" +botocore = ">=1.27.82,<1.28.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -97,7 +89,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.59" +version = "1.27.82" description = "Low-level, data-driven core of boto 3." category = "dev" optional = true @@ -138,7 +130,7 @@ typing_extensions = {version = "*", markers = "python_version >= \"3.7\" and pyt [[package]] name = "certifi" -version = "2022.6.15" +version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false @@ -173,7 +165,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorlog" -version = "6.6.0" +version = "6.7.0" description = "Add colours to the output of Python's logging module." category = "dev" optional = false @@ -194,7 +186,7 @@ optional = false python-versions = "*" [package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] +test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] [[package]] name = "coverage" @@ -226,7 +218,7 @@ dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "im [[package]] name = "distlib" -version = "0.3.5" +version = "0.3.6" description = "Distribution utilities" category = "dev" optional = false @@ -234,15 +226,15 @@ python-versions = "*" [[package]] name = "docutils" -version = "0.18.1" +version = "0.19" description = "Docutils -- Python Documentation Utilities" category = "dev" optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" [[package]] name = "exceptiongroup" -version = "1.0.0rc8" +version = "1.0.0rc9" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -271,12 +263,12 @@ optional = false python-versions = ">=3.7" [package.extras] -testing = ["pytest-timeout (>=2.1)", "pytest-cov (>=3)", "pytest (>=7.1.2)", "coverage (>=6.4.2)", "covdefaults (>=2.2)"] -docs = ["sphinx-autodoc-typehints (>=1.19.1)", "sphinx (>=5.1.1)", "furo (>=2022.6.21)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "furo" -version = "2022.6.21" +version = "2022.9.15" description = "A clean customisable Sphinx documentation theme." category = "dev" optional = true @@ -284,13 +276,13 @@ python-versions = ">=3.7" [package.dependencies] beautifulsoup4 = "*" -pygments = "*" +pygments = ">=2.7" sphinx = ">=4.0,<6.0" sphinx-basic-ng = "*" [[package]] name = "identify" -version = "2.5.3" +version = "2.5.5" description = "File identification library for Python" category = "dev" optional = false @@ -301,7 +293,7 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.3" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "dev" optional = false @@ -412,14 +404,14 @@ mdurl = ">=0.1,<1.0" typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} [package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code_style = ["pre-commit (==2.6)"] -compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] -linkify = ["linkify-it-py (>=1.0,<2.0)"] -plugins = ["mdit-py-plugins"] +testing = ["pytest-regressions", "pytest-cov", "pytest", "coverage"] +rtd = ["sphinx-book-theme", "sphinx-design", "sphinx-copybutton", "sphinx", "pyyaml", "myst-parser", "attrs"] profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx-book-theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] +plugins = ["mdit-py-plugins"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +compare = ["panflute (>=2.1.3,<2.2.0)", "mistune (>=2.0.2,<2.1.0)", "mistletoe (>=0.8.1,<0.9.0)", "markdown (>=3.3.6,<3.4.0)", "commonmark (>=0.9.1,<0.10.0)"] +code_style = ["pre-commit (==2.6)"] +benchmarking = ["pytest-benchmark (>=3.2,<4.0)", "pytest", "psutil"] [[package]] name = "markupsafe" @@ -431,19 +423,19 @@ python-versions = ">=3.7" [[package]] name = "mdit-py-plugins" -version = "0.3.0" +version = "0.3.1" description = "Collection of plugins for markdown-it-py" category = "dev" optional = true -python-versions = "~=3.6" +python-versions = ">=3.7" [package.dependencies] markdown-it-py = ">=1.0.0,<3.0.0" [package.extras] -code_style = ["pre-commit (==2.6)"] -rtd = ["myst-parser (>=0.14.0,<0.15.0)", "sphinx-book-theme (>=0.1.0,<0.2.0)"] -testing = ["coverage", "pytest (>=3.6,<4)", "pytest-cov", "pytest-regressions"] +testing = ["pytest-regressions", "pytest-cov", "pytest", "coverage"] +rtd = ["sphinx-book-theme (>=0.1.0,<0.2.0)", "myst-parser (>=0.16.1,<0.17.0)", "attrs"] +code_style = ["pre-commit"] [[package]] name = "mdurl" @@ -455,26 +447,26 @@ python-versions = ">=3.7" [[package]] name = "myst-parser" -version = "0.18.0" +version = "0.18.1" description = "An extended commonmark compliant parser, with bridges to docutils & sphinx." category = "dev" optional = true python-versions = ">=3.7" [package.dependencies] -docutils = ">=0.15,<0.19" +docutils = ">=0.15,<0.20" jinja2 = "*" markdown-it-py = ">=1.0.0,<3.0.0" -mdit-py-plugins = ">=0.3.0,<0.4.0" +mdit-py-plugins = ">=0.3.1,<0.4.0" pyyaml = "*" sphinx = ">=4,<6" typing-extensions = "*" [package.extras] -testing = ["sphinx-pytest", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "pytest-cov", "pytest (>=6,<7)", "coverage", "beautifulsoup4"] -rtd = ["sphinxext-opengraph (>=0.6.3,<0.7.0)", "sphinxcontrib.mermaid (>=0.7.1,<0.8.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)", "sphinx-design", "sphinx-book-theme", "ipython"] -linkify = ["linkify-it-py (>=1.0,<2.0)"] code_style = ["pre-commit (>=2.12,<3.0)"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +rtd = ["ipython", "sphinx-book-theme", "sphinx-design", "sphinxext-rediraffe (>=0.2.7,<0.3.0)", "sphinxcontrib.mermaid (>=0.7.1,<0.8.0)", "sphinxext-opengraph (>=0.6.3,<0.7.0)"] +testing = ["beautifulsoup4", "coverage", "pytest (>=6,<7)", "pytest-cov", "pytest-regressions", "pytest-param-files (>=0.3.4,<0.4.0)", "sphinx-pytest", "sphinx (<5.2)"] [[package]] name = "nodeenv" @@ -505,7 +497,7 @@ typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} virtualenv = ">=14" [package.extras] -tox_to_nox = ["tox", "jinja2"] +tox_to_nox = ["jinja2", "tox"] [[package]] name = "nox-poetry" @@ -563,8 +555,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "pprintpp" @@ -593,7 +585,7 @@ virtualenv = ">=20.0.8" [[package]] name = "psutil" -version = "5.9.1" +version = "5.9.2" description = "Cross-platform lib for process and system monitoring in Python." category = "dev" optional = false @@ -651,14 +643,13 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" -version = "7.1.2" +version = "7.1.3" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -686,7 +677,7 @@ rich = ">=8.0.0" [[package]] name = "pytest-cov" -version = "3.0.0" +version = "4.0.0" description = "Pytest plugin for measuring coverage." category = "dev" optional = false @@ -807,7 +798,7 @@ use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" -version = "1.9.3" +version = "1.10.0" description = "Mock out responses from the requests package" category = "dev" optional = false @@ -819,7 +810,7 @@ six = "*" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools", "requests-futures"] [[package]] name = "responses" @@ -868,7 +859,7 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "setuptools" -version = "65.3.0" +version = "65.4.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false @@ -905,7 +896,7 @@ python-versions = ">=3.6" [[package]] name = "sphinx" -version = "5.1.1" +version = "5.2.2" description = "Python documentation generator" category = "dev" optional = false @@ -913,16 +904,16 @@ python-versions = ">=3.6" [package.dependencies] alabaster = ">=0.7,<0.8" -babel = ">=1.3" -colorama = {version = ">=0.3.5", markers = "sys_platform == \"win32\""} +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} docutils = ">=0.14,<0.20" -imagesize = "*" -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} -Jinja2 = ">=2.3" -packaging = "*" -Pygments = ">=2.0" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.12" requests = ">=2.5.0" -snowballstemmer = ">=1.1" +snowballstemmer = ">=2.0" sphinxcontrib-applehelp = "*" sphinxcontrib-devhelp = "*" sphinxcontrib-htmlhelp = ">=2.0.0" @@ -932,8 +923,8 @@ sphinxcontrib-serializinghtml = ">=1.1.5" [package.extras] docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "isort", "mypy (>=0.971)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] -test = ["pytest (>=4.6)", "html5lib", "cython", "typed-ast"] +lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] +test = ["pytest (>=4.6)", "html5lib", "typed-ast", "cython"] [[package]] name = "sphinx-autobuild" @@ -953,18 +944,19 @@ test = ["pytest", "pytest-cov"] [[package]] name = "sphinx-autodoc-typehints" -version = "1.19.2" +version = "1.19.4" description = "Type hints (PEP 484) support for the Sphinx autodoc extension" category = "dev" optional = true python-versions = ">=3.7" [package.dependencies] -Sphinx = ">=5.1.1" +sphinx = ">=5.2.1" [package.extras] -type_comments = ["typed-ast (>=1.5.4)"] -testing = ["typing-extensions (>=4.3)", "sphobjinv (>=2.2.2)", "pytest-cov (>=3)", "pytest (>=7.1.2)", "nptyping (>=2.2)", "diff-cover (>=6.5.1)", "coverage (>=6.4.2)", "covdefaults (>=2.2)"] +docs = ["furo (>=2022.9.15)", "sphinx-autodoc-typehints (>=1.19.3)", "sphinx (>=5.2.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.4)", "diff-cover (>=7.0.1)", "nptyping (>=2.3.1)", "pytest-cov (>=3)", "pytest (>=7.1.3)", "sphobjinv (>=2.2.2)", "typing-extensions (>=4.3)"] +type-comment = ["typed-ast (>=1.5.4)"] [[package]] name = "sphinx-automodapi" @@ -1006,8 +998,8 @@ python-versions = ">=3.6" sphinx = ">=1.8" [package.extras] +rtd = ["sphinx-book-theme", "myst-nb", "ipython", "sphinx"] code_style = ["pre-commit (==2.12.1)"] -rtd = ["sphinx", "ipython", "myst-nb", "sphinx-book-theme"] [[package]] name = "sphinx-design" @@ -1065,8 +1057,8 @@ optional = true python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-devhelp" @@ -1077,8 +1069,8 @@ optional = true python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-htmlhelp" @@ -1089,8 +1081,8 @@ optional = true python-versions = ">=3.6" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] -test = ["pytest", "html5lib"] +test = ["html5lib", "pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-jsmath" @@ -1101,7 +1093,7 @@ optional = true python-versions = ">=3.5" [package.extras] -test = ["pytest", "flake8", "mypy"] +test = ["mypy", "flake8", "pytest"] [[package]] name = "sphinxcontrib-qthelp" @@ -1112,8 +1104,8 @@ optional = true python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxcontrib-serializinghtml" @@ -1124,8 +1116,8 @@ optional = true python-versions = ">=3.5" [package.extras] -lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] +lint = ["docutils-stubs", "mypy", "flake8"] [[package]] name = "sphinxext-opengraph" @@ -1140,14 +1132,14 @@ sphinx = ">=2.0" [[package]] name = "tenacity" -version = "8.0.1" +version = "8.1.0" description = "Retry code until it succeeds" category = "dev" optional = false python-versions = ">=3.6" [package.extras] -doc = ["tornado (>=4.5)", "sphinx", "reno"] +doc = ["reno", "sphinx", "tornado (>=4.5)"] [[package]] name = "timeout-decorator" @@ -1175,7 +1167,7 @@ python-versions = ">=3.7" [[package]] name = "tomlkit" -version = "0.11.4" +version = "0.11.5" description = "Style preserving TOML library" category = "dev" optional = false @@ -1206,11 +1198,11 @@ optional = true python-versions = ">=3.6" [package.extras] -test = ["coverage", "pytest", "pytest-cov"] +test = ["pytest-cov", "pytest", "coverage"] [[package]] name = "ujson" -version = "5.4.0" +version = "5.5.0" description = "Ultra fast JSON encoder and decoder for Python" category = "dev" optional = true @@ -1242,7 +1234,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.3" +version = "20.16.5" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1255,8 +1247,8 @@ importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.8\""} platformdirs = ">=2.4,<3" [package.extras] -testing = ["pytest-timeout (>=2.1)", "pytest-randomly (>=3.10.3)", "pytest-mock (>=3.6.1)", "pytest-freezegun (>=0.4.2)", "pytest-env (>=0.6.2)", "pytest (>=7.0.1)", "packaging (>=21.3)", "flaky (>=3.7)", "coverage-enable-subprocess (>=1)", "coverage (>=6.2)"] -docs = ["towncrier (>=21.9)", "sphinx-rtd-theme (>=1)", "sphinx-argparse (>=0.3.1)", "sphinx (>=5.1.1)", "proselint (>=0.13)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "wrapt" @@ -1298,7 +1290,6 @@ content-hash = "ea39161c795ec7ea80f5a3f51caedba1bb995f379c31eb4b994295d2d5052c1f alabaster = [] argcomplete = [] async-timeout = [] -atomicwrites = [] attrs = [] babel = [] beautifulsoup4 = [] diff --git a/requests_cache/cache_keys.py b/requests_cache/cache_keys.py index 1aeed7a..fa60619 100644 --- a/requests_cache/cache_keys.py +++ b/requests_cache/cache_keys.py @@ -40,7 +40,7 @@ __all__ = [ if TYPE_CHECKING: from .models import AnyPreparedRequest, AnyRequest, CachedResponse -# Maximum JSON request body size that will be normalized +# Maximum JSON request body size that will be filtered and normalized MAX_NORM_BODY_SIZE = 10 * 1024 * 1024 KVList = List[Tuple[str, str]] @@ -160,15 +160,16 @@ def normalize_url(url: str, ignored_parameters: ParamList) -> str: def normalize_body(request: AnyPreparedRequest, ignored_parameters: ParamList) -> bytes: """Normalize and filter a request body if possible, depending on Content-Type""" - original_body = request.body or b'' + if not request.body: + return b'' content_type = request.headers.get('Content-Type') # Filter and sort params if possible - filtered_body: Union[str, bytes] = original_body + filtered_body: Union[str, bytes] = request.body if content_type == 'application/json': - filtered_body = normalize_json_body(original_body, ignored_parameters) + filtered_body = normalize_json_body(request.body, ignored_parameters) elif content_type == 'application/x-www-form-urlencoded': - filtered_body = normalize_params(original_body, ignored_parameters) + filtered_body = normalize_params(request.body, ignored_parameters) return encode(filtered_body) @@ -224,14 +225,18 @@ def filter_sort_json(data: Union[List, Mapping], ignored_parameters: ParamList): def filter_sort_dict( data: Mapping[str, str], ignored_parameters: ParamList = None ) -> Dict[str, str]: - return {k: v for k, v in sorted(data.items()) if k not in set(ignored_parameters or [])} + # Note: Any ignored_parameters present will have their values replaced instead of removing the + # parameter, so the cache key will still match whether the parameter was present or not. + ignored_parameters = set(ignored_parameters or []) + return {k: ('REDACTED' if k in ignored_parameters else v) for k, v in sorted(data.items())} def filter_sort_multidict(data: KVList, ignored_parameters: ParamList = None) -> KVList: - return [(k, v) for k, v in sorted(data) if k not in set(ignored_parameters or [])] + ignored_parameters = set(ignored_parameters or []) + return [(k, 'REDACTED' if k in ignored_parameters else v) for k, v in sorted(data)] def filter_sort_list(data: List, ignored_parameters: ParamList = None) -> List: if not ignored_parameters: return sorted(data) - return [k for k in sorted(data) if k not in set(ignored_parameters or [])] + return [k for k in sorted(data) if k not in set(ignored_parameters)] diff --git a/tests/integration/base_cache_test.py b/tests/integration/base_cache_test.py index 5d69ea8..546f6c5 100644 --- a/tests/integration/base_cache_test.py +++ b/tests/integration/base_cache_test.py @@ -313,7 +313,7 @@ class BaseCacheTest: assert response.from_cache is False response = session.request(method, url, headers={"Authorization": "<Secret Key>"}) assert response.from_cache is True - assert response.request.headers.get('Authorization') is None + assert response.request.headers.get('Authorization') == 'REDACTED' @pytest.mark.parametrize('method', HTTPBIN_METHODS) def test_filter_request_query_parameters(self, method): @@ -325,23 +325,25 @@ class BaseCacheTest: assert response.from_cache is True query = urlparse(response.request.url).query query_dict = parse_qs(query) - assert 'api_key' not in query_dict + assert query_dict['api_key'] == ['REDACTED'] @pytest.mark.parametrize('post_type', ['data', 'json']) def test_filter_request_post_data(self, post_type): method = 'POST' url = httpbin(method.lower()) + body = {"api_key": "<Secret Key>"} + headers = {} + if post_type == 'data': + body = json.dumps(body) + headers = {'Content-Type': 'application/json'} session = self.init_session(ignored_parameters=['api_key']) - response = session.request(method, url, **{post_type: {"api_key": "<Secret Key>"}}) - assert response.from_cache is False - response = session.request(method, url, **{post_type: {"api_key": "<Secret Key>"}}) + + response = session.request(method, url, headers=headers, **{post_type: body}) + response = session.request(method, url, headers=headers, **{post_type: body}) assert response.from_cache is True - if post_type == 'data': - body = parse_qs(response.request.body) - assert "api_key" not in body - elif post_type == 'json': - body = json.loads(response.request.body) - assert "api_key" not in body + + parsed_body = json.loads(response.request.body) + assert parsed_body['api_key'] == 'REDACTED' @pytest.mark.parametrize('executor_class', [ThreadPoolExecutor, ProcessPoolExecutor]) @pytest.mark.parametrize('iteration', range(N_ITERATIONS)) diff --git a/tests/integration/test_compat.py b/tests/integration/test_upgrade.py index a49cf39..162024e 100644 --- a/tests/integration/test_compat.py +++ b/tests/integration/test_upgrade.py @@ -6,8 +6,6 @@ from requests_cache import CachedSession from tests.conftest import HTTPBIN_FORMATS, SAMPLE_CACHE_FILES, httpbin -# TODO: Debug why this sometimes fails (mostly just on GitHub Actions) -# @pytest.mark.flaky(reruns=3) @pytest.mark.parametrize('db_path', SAMPLE_CACHE_FILES) def test_version_upgrade(db_path, tempfile_path): """Load SQLite cache files created with older versions of requests-cache. diff --git a/tests/sample_data/sample.db.0.9.3 b/tests/sample_data/sample.db.0.9.3 Binary files differnew file mode 100644 index 0000000..9fd788b --- /dev/null +++ b/tests/sample_data/sample.db.0.9.3 diff --git a/tests/sample_data/sample.db.0.9.4 b/tests/sample_data/sample.db.0.9.4 Binary files differnew file mode 100644 index 0000000..e3611c4 --- /dev/null +++ b/tests/sample_data/sample.db.0.9.4 diff --git a/tests/sample_data/sample.db.0.9.5 b/tests/sample_data/sample.db.0.9.5 Binary files differnew file mode 100644 index 0000000..b8aba8c --- /dev/null +++ b/tests/sample_data/sample.db.0.9.5 diff --git a/tests/sample_data/sample.db.0.9.6 b/tests/sample_data/sample.db.0.9.6 Binary files differnew file mode 100644 index 0000000..9e64314 --- /dev/null +++ b/tests/sample_data/sample.db.0.9.6 diff --git a/tests/sample_data/sample.db.1.0.0-alpha b/tests/sample_data/sample.db.1.0.0-alpha Binary files differnew file mode 100644 index 0000000..b727b22 --- /dev/null +++ b/tests/sample_data/sample.db.1.0.0-alpha diff --git a/tests/unit/test_cache_keys.py b/tests/unit/test_cache_keys.py index cb76f52..40cca7f 100644 --- a/tests/unit/test_cache_keys.py +++ b/tests/unit/test_cache_keys.py @@ -69,7 +69,7 @@ def test_normalize_request__json_body(): headers={'Content-Type': 'application/json'}, ) norm_request = normalize_request(request, ignored_parameters=['param_2']) - assert norm_request.body == b'{"param_1": "value_1"}' + assert norm_request.body == b'{"param_1": "value_1", "param_2": "REDACTED"}' def test_normalize_request__json_body_list(): diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 5151a7d..dda449a 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -143,10 +143,18 @@ def test_all_methods__ignored_parameters__redacted(field, method, mock_session): mock_session.request(method, MOCKED_URL, **{field: params_1}) cached_response = mock_session.request(method, MOCKED_URL, **{field: params_1}) - assert 'ignored' not in cached_response.url - assert 'ignored' not in cached_response.request.url - assert 'ignored' not in cached_response.request.headers - assert 'ignored' not in cached_response.request.body.decode('utf-8') + request_url = cached_response.request.url + headers = cached_response.request.headers + body = cached_response.request.body.decode('utf-8') + + assert 'ignored' not in cached_response.url or 'ignored=REDACTED' in cached_response.url + assert 'ignored' not in request_url or 'ignored=REDACTED' in request_url + assert 'ignored' not in headers or headers['ignored'] == 'REDACTED' + if field == 'data': + assert 'ignored=REDACTED' in body + elif field == 'json': + body = json.loads(body) + assert body['ignored'] == 'REDACTED' # Variations of relevant request arguments @@ -502,12 +510,19 @@ def test_default_ignored_parameters(mock_session): params={'access_token': 'token'}, headers={'Authorization': 'Bearer token'}, ) - response = mock_session.get(MOCKED_URL) - + response = mock_session.get( + MOCKED_URL, + params={'access_token': 'token'}, + headers={'Authorization': 'Bearer token'}, + ) assert response.from_cache is True - assert 'access_token' not in response.url - assert 'access_token' not in response.request.url - assert 'Authorization' not in response.request.headers + + unauthenticated_response = mock_session.get(MOCKED_URL) + assert unauthenticated_response.from_cache is False + + assert 'access_token=REDACTED' in response.url + assert 'access_token=REDACTED' in response.request.url + assert response.request.headers['Authorization'] == 'REDACTED' @patch_normalize_url |