Run multi-threaded tests for all backends, and run with more threads & iterations for merges to master

6 files changed, 67 insertions, 42 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 05fa56a..36e2423 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -69,6 +69,11 @@ jobs:
           pytest --numprocesses=auto tests/unit
           pytest tests/integration
 
+      # Run longer stress tests if this is a release or merge to master
+      - name: Run stress tests
+        if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, '/master')
+        run: STRESS_TEST_MULTIPLIER=5 pytest tests/integration/test_thread_safety.py
+
   # Run code analysis checks
   analyze:
     runs-on: ubuntu-18.04
@@ -100,7 +105,7 @@ jobs:
   # Deploy pre-release builds from dev branch, and stable builds on tags only
   release:
     needs: [test, analyze]
-    if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev'
+    if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, '/dev')
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
diff --git a/runtests.sh b/runtests.sh
index 6d6d31e..f3be637 100755
--- a/runtests.sh
+++ b/runtests.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # Test runner script with useful pytest options
 COVERAGE_ARGS='--cov --cov-report=term --cov-report=html'
+export STRESS_TEST_MULTIPLIER=2
 
 # Run unit tests first (and with multiprocessing) to fail quickly if there are issues
 pytest tests/unit --numprocesses=auto $COVERAGE_ARGS
diff --git a/tests/conftest.py b/tests/conftest.py
index 0b3167a..577cb78 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -29,6 +29,13 @@ MOCKED_URL_REDIRECT = 'http+mock://requests-cache.com/redirect'
 MOCKED_URL_REDIRECT_TARGET = 'http+mock://requests-cache.com/redirect_target'
 MOCK_PROTOCOLS = ['mock://', 'http+mock://', 'https+mock://']
 
+AWS_OPTIONS = {
+    'endpoint_url': 'http://localhost:8000',
+    'region_name': 'us-east-1',
+    'aws_access_key_id': 'placeholder',
+    'aws_secret_access_key': 'placeholder',
+}
+
 # Configure logging to show debug output when tests fail (or with pytest -s)
 basicConfig(level='INFO')
 getLogger('requests_cache').setLevel('DEBUG')
@@ -81,11 +88,7 @@ def mock_session() -> CachedSession:
             allowable_methods=ALL_METHODS,
             suppress_warnings=True,
         )
-        adapter = get_mock_adapter()
-        for protocol in MOCK_PROTOCOLS:
-            session.mount(protocol, adapter)
-        session.mock_adapter = adapter
-        yield session
+        yield mount_mock_adapter(session)
 
 
 @pytest.fixture(scope='function')
@@ -117,6 +120,14 @@ def installed_session() -> CachedSession:
     requests_cache.uninstall_cache()
 
 
+def mount_mock_adapter(session: CachedSession) -> CachedSession:
+    adapter = get_mock_adapter()
+    for protocol in MOCK_PROTOCOLS:
+        session.mount(protocol, adapter)
+    session.mock_adapter = adapter
+    return session
+
+
 def get_mock_adapter() -> Adapter:
     """Get a requests-mock Adapter with some URLs mocked by default"""
     adapter = Adapter()
diff --git a/tests/integration/test_dynamodb.py b/tests/integration/test_dynamodb.py
index 13e40dd..3a4bd23 100644
--- a/tests/integration/test_dynamodb.py
+++ b/tests/integration/test_dynamodb.py
@@ -3,19 +3,12 @@ import unittest
 from unittest.mock import patch
 
 from requests_cache.backends import DynamoDbDict
-from tests.conftest import fail_if_no_connection
+from tests.conftest import AWS_OPTIONS, fail_if_no_connection
 from tests.integration.test_backends import BaseStorageTestCase
 
 # Run this test module last, since the DynamoDB container takes the longest to initialize
 pytestmark = pytest.mark.order(-1)
 
-boto_options = {
-    'endpoint_url': 'http://localhost:8000',
-    'region_name': 'us-east-1',
-    'aws_access_key_id': 'placeholder',
-    'aws_secret_access_key': 'placeholder',
-}
-
 
 @pytest.fixture(scope='module', autouse=True)
 @fail_if_no_connection
@@ -23,13 +16,13 @@ def ensure_connection():
     """Fail all tests in this module if DynamoDB is not running"""
     import boto3
 
-    client = boto3.client('dynamodb', **boto_options)
+    client = boto3.client('dynamodb', **AWS_OPTIONS)
     client.describe_limits()
 
 
 class DynamoDbDictWrapper(DynamoDbDict):
     def __init__(self, namespace, collection_name='dynamodb_dict_data', **options):
-        super().__init__(namespace, collection_name, **options, **boto_options)
+        super().__init__(namespace, collection_name, **options, **AWS_OPTIONS)
 
 
 class DynamoDbTestCase(BaseStorageTestCase, unittest.TestCase):
diff --git a/tests/integration/test_thread_safety.py b/tests/integration/test_thread_safety.py
new file mode 100644
index 0000000..e9f8aca
--- /dev/null
+++ b/tests/integration/test_thread_safety.py
@@ -0,0 +1,41 @@
+import pytest
+from os import getenv
+from threading import Thread
+from time import time
+
+from requests_cache.backends import BACKEND_CLASSES
+from requests_cache.session import CachedSession
+from tests.conftest import AWS_OPTIONS, httpbin
+
+# Allow running longer stress tests with an environment variable
+MULTIPLIER = int(getenv('STRESS_TEST_MULTIPLIER', '1'))
+N_THREADS = 2 * MULTIPLIER
+N_ITERATIONS = 4 * MULTIPLIER
+
+
+@pytest.mark.parametrize('iteration', range(N_ITERATIONS))
+@pytest.mark.parametrize('backend', BACKEND_CLASSES.keys())
+def test_caching_with_threads(backend, iteration):
+    """Run a multi-threaded stress test for each backend"""
+    start = time()
+    session = CachedSession(backend=backend, **AWS_OPTIONS)
+    session.cache.clear()
+    url = httpbin('anything')
+
+    def send_requests():
+        for i in range(N_ITERATIONS):
+            session.get(url, params={f'key_{i}': f'value_{i}'})
+
+    threads = [Thread(target=send_requests) for i in range(N_THREADS)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    elapsed = time() - start
+    average = (elapsed * 1000) / (N_ITERATIONS * N_THREADS)
+    print(f'{backend}: Ran {N_ITERATIONS} iterations with {N_THREADS} threads each in {elapsed} s')
+    print(f'Average time per request: {average} ms')
+
+    for i in range(N_ITERATIONS):
+        assert session.cache.has_url(f'{url}?key_{i}=value_{i}')
diff --git a/tests/unit/test_thread_safety.py b/tests/unit/test_thread_safety.py
deleted file mode 100644
index 55f256c..0000000
--- a/tests/unit/test_thread_safety.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import pytest
-from threading import Thread
-
-from tests.conftest import MOCKED_URL
-
-N_THREADS = 10
-N_ITERATIONS = 20
-
-
-@pytest.mark.parametrize('iteration', range(N_ITERATIONS))
-@pytest.mark.parametrize('backend', ['sqlite', 'mongodb', 'gridfs', 'redis', 'dynamodb'])
-def test_caching_with_threads(backend, iteration, mock_session):
-    """Stress test for multi-threaded caching"""
-
-    def send_requests(url, params):
-        for i in range(10):
-            mock_session.get(url, params=params)
-
-    threads = [Thread(target=send_requests, args=(MOCKED_URL, {'param': i})) for i in range(N_THREADS)]
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join()
-
-    for i in range(N_THREADS):
-        assert mock_session.cache.has_url(f'{MOCKED_URL}?param={i}')