tox integrations with invoke and docker (#1632)

12 files changed, 256 insertions, 95 deletions

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index f08a2c2..2618c33 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -2,12 +2,62 @@ name: CI
 
 on:
   push:
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.rst'
+      - '**/*.md'
   pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.rst'
+      - '**/*.md'
 
 jobs:
-  integration:
+
+  lint:
+    name: Code linters
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: run code linters
+        run: |
+          pip install -r dev_requirements.txt
+          invoke linters
+
+  run-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 6
+      matrix:
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7']
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    name: Python ${{ matrix.python-version }} tests
+    steps:
+      - uses: actions/checkout@v2
+      - name: install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: run tests
+        run: |
+          pip install -r dev_requirements.txt
+          invoke tests
+
+  build_package:
+    name: Validate building and installing the package
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: test
-        run: make test
+      - name: install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: build and install
+        run: |
+          pip install invoke
+          invoke package
diff --git a/.gitignore b/.gitignore
index b59bcdf..05c3846 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ vagrant/.vagrant
 env
 venv
 coverage.xml
+.venv
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index edaa7b6..ba1ddfc 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -29,25 +29,36 @@ Here's how to get started with your code contribution:
 
 1. Create your own fork of redis-py
 2. Do the changes in your fork
-3. If you need a development environment, run ``make dev``
-4. While developing, make sure the tests pass by running ``make test``
+3. Create a virtualenv and install the development dependencies from the dev_requirements.txt file:
+    a. python -m venv .venv
+    b. source .venv/bin/activate
+    c. pip install -r dev_requirements.txt
+3. If you need a development environment, run ``invoke devenv``
+4. While developing, make sure the tests pass by running ``invoke tests``
 5. If you like the change and think the project could use it, send a pull request
 
+To see what else is part of the automation, run ``invoke -l``
+
 The Development Environment
 ---------------------------
 
-Running ``make dev`` will create a Docker-based development environment that starts the following containers:
+Running ``invoke devenv`` installs the development dependencies specified in the dev_requirements.txt. It starts all of the dockers used by this project, and leaves them running. These can be easily cleaned up with ``invoke clean``. NOTE: it is assumed that the user running these tests, can execute docker and its various commands.
 
 * A master Redis node
-* A slave Redis node
+* A Redis replica node
 * Three sentinel Redis nodes
-* A test container
+* A multi-python docker, with your source code mounted in /data
 
-The slave is a replica of the master node, using the `leader-follower replication <https://redis.io/topics/replication>`_ feature.
+The replica node, is a replica of the master node, using the `leader-follower replication <https://redis.io/topics/replication>`_ feature.
 
 The sentinels monitor the master node in a `sentinel high-availability configuration <https://redis.io/topics/sentinel>`_.
 
-Meanwhile, the `test` container hosts the code from your checkout of ``redis-py`` and allows running tests against many Python versions.
+Testing
+-------
+
+Each run of tox starts and stops the various dockers required.  Sometimes things get stuck, an ``invoke clean`` can help.
+
+Continuous Integration uses these same wrappers to run all of these tests against multiple versions of python. Feel free to test your changes against all the python versions supported, as declared by the tox.ini file (eg: tox -e py39). If you have the various python versions on your desktop, you can run *tox* by itself, to test all supported versions. Alternatively, as your source code is mounted in the **lots-of-pythons** docker, you can start exploring from there, with all supported python versions!
 
 Docker Tips
 ^^^^^^^^^^^
@@ -56,17 +67,17 @@ Following are a few tips that can help you work with the Docker-based developmen
 
 To get a bash shell inside of a container:
 
-``$ docker-compose run <service> /bin/bash``
- 
-**Note**: The term "service" refers to the "services" defined in the ``docker-compose.yml`` file: "master", "slave", "sentinel_1", "sentinel_2", "sentinel_3", "test".
+``$ docker run -it <service> /bin/bash``
+
+**Note**: The term "service" refers to the "services" defined in the ``tox.ini`` file at the top of the repo: "master", "replicaof", "sentinel_1", "sentinel_2", "sentinel_3".
 
 Containers run a minimal Debian image that probably lacks tools you want to use. To install packages, first get a bash session (see previous tip) and then run:
 
 ``$ apt update && apt install <package>``
 
-You can see the combined logging output of all containers like this:
+You can see the logging output of a containers like this:
 
-``$ docker-compose logs``
+``$ docker logs -f <service>``
 
 The command `make test` runs all tests in all tested Python environments. To run the tests in a single environment, like Python 3.6, use a command like this:
 
@@ -81,13 +92,11 @@ Our test suite uses ``pytest``. You can run a specific test suite against a spec
 Troubleshooting
 ^^^^^^^^^^^^^^^
 If you get any errors when running ``make dev`` or ``make test``, make sure that you
-are using supported versions of Docker and docker-compose.
+are using supported versions of Docker.
 
-The included Dockerfiles and docker-compose.yml file work with the following
-versions of Docker and docker-compose:
+Please try at least versions of Docker.
 
 * Docker 19.03.12
-* docker-compose 1.26.2
 
 How to Report a Bug
 -------------------
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 2ceb725..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM fkrull/multi-python:latest
-
-RUN apt update \
-    && apt install -y pypy pypy-dev pypy3-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /redis-py
-
-COPY . /redis-py
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 0d09643..0000000
--- a/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-.PHONY: base clean dev test
-
-base:
-		docker build -t redis-py-base docker/base
-
-dev:	base
-		docker-compose up -d --build
-
-test:	dev
-		docker-compose run --rm test /redis-py/docker-entry.sh
-
-clean:
-		docker-compose stop
-		docker-compose rm -f
diff --git a/dev_requirements.txt b/dev_requirements.txt
new file mode 100644
index 0000000..2648127
--- /dev/null
+++ b/dev_requirements.txt
@@ -0,0 +1,5 @@
+flake8>=3.9.2
+pytest==6.2.5
+tox==3.24.4
+tox-docker==3.1.0
+invoke==1.6.0
diff --git a/docker-entry.sh b/docker-entry.sh
deleted file mode 100755
index dc119dc..0000000
--- a/docker-entry.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# This is the entrypoint for "make test". It invokes Tox. If running
-# outside the CI environment, it disables uploading the coverage report to codecov
-
-set -eu
-
-REDIS_MASTER="${REDIS_MASTER_HOST}":"${REDIS_MASTER_PORT}"
-echo "Testing against Redis Server: ${REDIS_MASTER}"
-
-# skip the "codecov" env if not running on Travis
-if [ "${GITHUB_ACTIONS}" = true ] ; then
-    echo "Skipping codecov"
-    export TOX_SKIP_ENV="codecov"
-fi
-
-# use the wait-for-it util to ensure the server is running before invoking Tox
-util/wait-for-it.sh ${REDIS_MASTER} -- tox -- --redis-url=redis://"${REDIS_MASTER}"/9
-
diff --git a/docker/slave/Dockerfile b/docker/replica/Dockerfile
index 7b6bdbe..7b6bdbe 100644
--- a/docker/slave/Dockerfile
+++ b/docker/replica/Dockerfile
diff --git a/docker/replica/redis.conf b/docker/replica/redis.conf
new file mode 100644
index 0000000..4a1dcd7
--- /dev/null
+++ b/docker/replica/redis.conf
@@ -0,0 +1,4 @@
+bind replica 127.0.0.1
+port 6380
+save ""
+replicaof master 6379
diff --git a/docker/slave/redis.conf b/docker/slave/redis.conf
deleted file mode 100644
index 629ac70..0000000
--- a/docker/slave/redis.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-bind slave 127.0.0.1
-port 6380
-save ""
-slaveof master 6379
diff --git a/tasks.py b/tasks.py
new file mode 100644
index 0000000..15e983b
--- /dev/null
+++ b/tasks.py
@@ -0,0 +1,60 @@
+import os
+import shutil
+from invoke import task, run
+
+with open('tox.ini') as fp:
+    lines = fp.read().split("\n")
+    dockers = [line.split("=")[1].strip() for line in lines
+               if line.find("name") != -1]
+
+
+@task
+def devenv(c):
+    """Builds a development environment: downloads, and starts all dockers
+    specified in the tox.ini file.
+    """
+    clean(c)
+    cmd = 'tox -e devenv'
+    for d in dockers:
+        cmd += " --docker-dont-stop={}".format(d)
+    print("Running: {}".format(cmd))
+    run(cmd)
+
+
+@task
+def linters(c):
+    """Run code linters"""
+    run("flake8")
+
+
+@task
+def all_tests(c):
+    """Run all linters, and tests in redis-py. This assumes you have all
+    the python versions specified in the tox.ini file.
+    """
+    linters(c)
+    tests(c)
+
+
+@task
+def tests(c):
+    """Run the redis-py test suite against the current python,
+    with and without hiredis.
+    """
+    run("tox -e plain -e hiredis")
+
+
+@task
+def clean(c):
+    """Stop all dockers, and clean up the built binaries, if generated."""
+    if os.path.isdir("build"):
+        shutil.rmtree("build")
+    if os.path.isdir("dist"):
+        shutil.rmtree("dist")
+    run("docker rm -f {}".format(' '.join(dockers)))
+
+
+@task
+def package(c):
+    """Create the python packages"""
+    run("python setup.py build install")
diff --git a/tox.ini b/tox.ini
index db7492d..bfe937f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,50 +2,128 @@
 addopts = -s
 
 [tox]
-minversion = 2.4
-envlist = {py35,py36,py37,py38,py39,pypy3}-{plain,hiredis}, flake8, covreport, codecov
+minversion = 3.2.0
+requires = tox-docker
+envlist = {py35,py36,py37,py38,py39,pypy3}-{plain,hiredis}, flake8
+
+[docker:master]
+name = master
+image = redis:6.2-bullseye
+ports =
+    6379:6379/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',6379)) else False"
+volumes =
+    bind:rw:{toxinidir}/docker/master/redis.conf:/usr/local/etc/redis/redis.conf
+
+[docker:replica]
+name = replica
+image = redis:6.2-bullseye
+links =
+    master:master
+ports =
+    6380:6380/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',6380)) else False"
+volumes =
+    bind:rw:{toxinidir}/docker/replica/redis.conf:/usr/local/etc/redis/redis.conf
+
+[docker:sentinel_1]
+name = sentinel_1
+image = redis:6.2-bullseye
+links =
+    master:master
+ports =
+    26379:26379/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26379)) else False"
+volumes =
+    bind:rw:{toxinidir}/docker/sentinel_1/sentinel.conf:/usr/local/etc/redis/sentinel.conf
+
+[docker:sentinel_2]
+name = sentinel_2
+image = redis:6.2-bullseye
+links =
+    master:master
+ports =
+    26380:26380/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26380)) else False"
+volumes =
+    bind:rw:{toxinidir}/docker/sentinel_2/sentinel.conf:/usr/local/etc/redis/sentinel.conf
+
+[docker:sentinel_3]
+name = sentinel_3
+image = redis:6.2-bullseye
+links =
+    master:master
+ports =
+    26381:26381/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26381)) else False"
+volumes =
+    bind:rw:{toxinidir}/docker/sentinel_3/sentinel.conf:/usr/local/etc/redis/sentinel.conf
+
+[docker:redismod]
+name = redismod
+image = redislabs/redismod:edge
+ports =
+    16379:16379/tcp
+healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',16379)) else False"
+
+[docker:lots-of-pythons]
+name = lots-of-pythons
+image = redisfab/lots-of-pythons
+volumes =
+    bind:rw:{toxinidir}:/data
 
 [testenv]
-deps =
-    coverage
-    pytest >= 2.7.0
+deps = -r {toxinidir}/dev_requirements.txt
+docker =
+    master
+    replica
+    sentinel_1
+    sentinel_2
+    sentinel_3
+    redismod
 extras =
     hiredis: hiredis
 commands =
-    {envpython} -b -m coverage run -p -m pytest -W always {posargs}
-    {envpython} -b -m coverage combine --append
+    pytest -W always {posargs}
+
+[testenv:devenv]
+skipsdist = true
+skip_install = true
+deps = -r {toxinidir}/dev_requirements.txt
+docker =
+    master
+    replica
+    sentinel_1
+    sentinel_2
+    sentinel_3
+    redismod
+    lots-of-pythons
+commands = echo
 
 [testenv:flake8]
-basepython = python3.6
-deps = flake8
+deps_files = dev_requirements.txt
 commands = flake8
 skipsdist = true
 skip_install = true
 
-[testenv:pypy-plain]
-basepython = pypy
-
-[testenv:pypy-hiredis]
-basepython = pypy
-
 [testenv:pypy3-plain]
 basepython = pypy3
 
 [testenv:pypy3-hiredis]
 basepython = pypy3
 
-[testenv:codecov]
-deps = codecov
-commands = codecov
-passenv =
-  REDIS_*
-  CI
-  CI_*
-  CODECOV_*
-  SHIPPABLE
-  GITHUB_*
-  VCS_*
-
-[testenv:covreport]
-deps = coverage
-commands = coverage report
+#[testenv:codecov]
+#deps = codecov
+#commands = codecov
+#passenv =
+#  REDIS_*
+#  CI
+#  CI_*
+#  CODECOV_*
+#  SHIPPABLE
+#  GITHUB_*
+#  VCS_*
+#
+#[testenv:covreport]
+#deps = coverage
+#commands = coverage report