Use ZooKeeper TLS in tests

This mirrors the configuration in Nodepool for using TLS-enabled
ZooKeeper in tests.  We use the ensure-zookeeper role in order
to get a newer ZooKeeper than is supplied in bionic.

Change-Id: I14413fccbc9a6a7a75b6233d667e2a1d2856d894

16 files changed, 172 insertions, 77 deletions

diff --git a/.gitignore b/.gitignore
index c5b47046e..c2f47953b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,4 @@ zuul/web/static/*
 !.keep
 node_modules
 yarn-error.log
+tools/ca/
diff --git a/.zuul.yaml b/.zuul.yaml
index 711cb69c3..57c69a1a9 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -45,16 +45,70 @@
       zuul_ansible_version: 2.9
 
 - job:
+    name: zuul-tox
+    description: |
+      Zuul unit tests with ZooKeeper running
+    parent: tox
+    nodeset: ubuntu-bionic
+    pre-run: playbooks/zuul-tox/pre.yaml
+    post-run: playbooks/zuul-tox/post-system-logs.yaml
+    vars:
+      tox_environment:
+        ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
+        ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
+        ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
+        ZUUL_TEST_ROOT: /tmp/zuul-test
+        YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
+      test_setup_environment:
+        ZUUL_TEST_ROOT: /tmp/zuul-test
+        YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
+
+- job:
     name: zuul-tox-remote
     parent: tox
+    timeout: 2700  # 45 minutes
+    pre-run: playbooks/zuul-tox/pre.yaml
+    post-run: playbooks/zuul-tox/post-system-logs.yaml
     vars:
       tox_envlist: remote
       tox_environment:
+        ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
+        ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
+        ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
         ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
         ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
         ZUUL_REMOTE_KEEP: "true"
 
 - job:
+    name: zuul-tox-zuul-client
+    parent: zuul-tox
+    description: |
+      Test that Zuul and zuul-client work together.
+    required-projects:
+      - zuul/zuul
+      - zuul/zuul-client
+    vars:
+      zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}"
+      tox_envlist: zuul_client
+    nodeset: ubuntu-bionic
+
+- job:
+    name: zuul-tox-py36
+    parent: zuul-tox
+    timeout: 4800  # 80 minutes
+    vars:
+      tox_envlist: py36
+      python_version: 3.6
+
+- job:
+    name: zuul-tox-py38
+    parent: zuul-tox
+    timeout: 4800  # 80 minutes
+    vars:
+      tox_envlist: py38
+      python_version: 3.8
+
+- job:
     name: zuul-build-dashboard
     parent: build-javascript-deployment
     description: |
@@ -223,22 +277,8 @@
         - tox-linters:
             vars:
               tox_install_bindep: false
-        - tox-py36:
-            nodeset: ubuntu-bionic
-            timeout: 4800  # 80 minutes
-            vars: &zuul_tox_vars
-                test_setup_environment:
-                    ZUUL_TEST_ROOT: /tmp/zuul-test
-                    YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
-                tox_environment:
-                    ZUUL_TEST_ROOT: /tmp/zuul-test
-                    YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
-            post-run: playbooks/common/post-system-logs.yaml
-        - tox-py38:
-            timeout: 4800  # 80 minutes
-            nodeset: ubuntu-bionic
-            vars: *zuul_tox_vars
-            post-run: playbooks/common/post-system-logs.yaml
+        - zuul-tox-py36
+        - zuul-tox-py38
         - zuul-build-dashboard-openstack-whitelabel
         - zuul-build-dashboard-software-factory
         - zuul-build-dashboard-opendev
@@ -253,14 +293,13 @@
               - web/.*
         - zuul-stream-functional-2.8
         - zuul-stream-functional-2.9
-        - zuul-tox-remote:
-            timeout: 2700  # 45 minutes
+        - zuul-tox-remote
         - zuul-quick-start:
             requires: nodepool-container-image
             dependencies: zuul-build-image
         - nodepool-zuul-functional:
             voting: false
-        - zuul-client-zuul-functional
+        - zuul-tox-zuul-client
         - zuul-build-python-release
     gate:
       jobs:
@@ -269,14 +308,8 @@
         - tox-linters:
             vars:
               tox_install_bindep: false
-        - tox-py36:
-            nodeset: ubuntu-bionic
-            timeout: 4800  # 80 minutes
-            vars: *zuul_tox_vars
-        - tox-py38:
-            timeout: 4800  # 80 minutes
-            nodeset: ubuntu-bionic
-            vars: *zuul_tox_vars
+        - zuul-tox-py36
+        - zuul-tox-py38
         - zuul-build-dashboard
         - nodejs-run-lint:
             vars:
@@ -289,12 +322,11 @@
               - web/.*
         - zuul-stream-functional-2.8
         - zuul-stream-functional-2.9
-        - zuul-tox-remote:
-            timeout: 2700  # 45 minutes
+        - zuul-tox-remote
         - zuul-quick-start:
             requires: nodepool-container-image
             dependencies: zuul-upload-image
-        - zuul-client-zuul-functional
+        - zuul-tox-zuul-client
         - zuul-build-python-release
     promote:
       jobs:
diff --git a/bindep.txt b/bindep.txt
index 63bc952c7..d8695dc19 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -8,7 +8,6 @@ postgresql [test]
 libjpeg-dev [test !platform:rpm]
 libjpeg-turbo-devel [test platform:rpm]
 openssl [test]
-zookeeperd [test platform:dpkg]
 musl-dev [compile test platform:apk]
 make [compile test platform:apk]
 linux-headers [compile test platform:apk]
diff --git a/playbooks/common/post-system-logs.yaml b/playbooks/common/post-system-logs.yaml
deleted file mode 100644
index 830899c38..000000000
--- a/playbooks/common/post-system-logs.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
-  tasks:
-
-    - name: Collect zookeeper logs
-      shell: "cp /var/log/zookeeper/zookeeper.log {{ zuul_output_dir }}/logs/zookeeper.log"
diff --git a/playbooks/zuul-tox/post-system-logs.yaml b/playbooks/zuul-tox/post-system-logs.yaml
new file mode 100644
index 000000000..7a9da4821
--- /dev/null
+++ b/playbooks/zuul-tox/post-system-logs.yaml
@@ -0,0 +1,5 @@
+- hosts: all
+  tasks:
+
+    - name: Collect zookeeper logs
+      shell: "cp /opt/zookeeper/logs/* {{ zuul_output_dir }}/logs/"
diff --git a/playbooks/zuul-tox/pre.yaml b/playbooks/zuul-tox/pre.yaml
new file mode 100644
index 000000000..45d213e97
--- /dev/null
+++ b/playbooks/zuul-tox/pre.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: ensure-zookeeper
+      zookeeper_use_tls: true
diff --git a/tests/base.py b/tests/base.py
index 1b0c6791e..351d04d74 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -3352,11 +3352,20 @@ class FakeNodepool(object):
 
     log = logging.getLogger("zuul.test.FakeNodepool")
 
-    def __init__(self, host, port, chroot):
+    def __init__(self, zk_chroot_fixture):
         self.complete_event = threading.Event()
         self.host_keys = None
+
         self.client = kazoo.client.KazooClient(
-            hosts='%s:%s%s' % (host, port, chroot))
+            hosts='%s:%s%s' % (
+                zk_chroot_fixture.zookeeper_host,
+                zk_chroot_fixture.zookeeper_port,
+                zk_chroot_fixture.zookeeper_chroot),
+            use_ssl=True,
+            keyfile=zk_chroot_fixture.zookeeper_key,
+            certfile=zk_chroot_fixture.zookeeper_cert,
+            ca=zk_chroot_fixture.zookeeper_ca,
+        )
         self.client.start()
         self.registerLauncher()
         self._running = True
@@ -3614,10 +3623,25 @@ class ChrootedKazooFixture(fixtures.Fixture):
             host = zk_host
             port = None
 
+        zk_ca = os.environ.get('ZUUL_ZK_CA', None)
+        if not zk_ca:
+            zk_ca = os.path.join(os.path.dirname(__file__),
+                                 '../tools/ca/certs/cacert.pem')
+        self.zookeeper_ca = zk_ca
+        zk_cert = os.environ.get('ZUUL_ZK_CERT', None)
+        if not zk_cert:
+            zk_cert = os.path.join(os.path.dirname(__file__),
+                                   '../tools/ca/certs/client.pem')
+        self.zookeeper_cert = zk_cert
+        zk_key = os.environ.get('ZUUL_ZK_KEY', None)
+        if not zk_key:
+            zk_key = os.path.join(os.path.dirname(__file__),
+                                  '../tools/ca/keys/clientkey.pem')
+        self.zookeeper_key = zk_key
         self.zookeeper_host = host
 
         if not port:
-            self.zookeeper_port = 2181
+            self.zookeeper_port = 2281
         else:
             self.zookeeper_port = int(port)
 
@@ -3636,7 +3660,11 @@ class ChrootedKazooFixture(fixtures.Fixture):
 
         # Ensure the chroot path exists and clean up any pre-existing znodes.
         _tmp_client = kazoo.client.KazooClient(
-            hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10
+            hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10,
+            use_ssl=True,
+            keyfile=self.zookeeper_key,
+            certfile=self.zookeeper_cert,
+            ca=self.zookeeper_ca,
         )
         _tmp_client.start()
 
@@ -3651,7 +3679,12 @@ class ChrootedKazooFixture(fixtures.Fixture):
         '''Remove the chroot path.'''
         # Need a non-chroot'ed client to remove the chroot path
         _tmp_client = kazoo.client.KazooClient(
-            hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
+            hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port),
+            use_ssl=True,
+            keyfile=self.zookeeper_key,
+            certfile=self.zookeeper_cert,
+            ca=self.zookeeper_ca,
+        )
         _tmp_client.start()
         _tmp_client.delete(self.zookeeper_chroot, recursive=True)
         _tmp_client.stop()
@@ -3722,8 +3755,7 @@ class ZuulWebFixture(fixtures.Fixture):
             self.info = zuul.model.WebInfo.fromConfig(config)
         else:
             self.info = info
-        self.zk_client = ZooKeeperClient.fromConfig(config,
-                                                    _require_tls=False)
+        self.zk_client = ZooKeeperClient.fromConfig(config)
         self.zk_client.connect()
         self.test_root = test_root
 
@@ -3995,8 +4027,7 @@ class SchedulerTestApp:
             self.config, self.sched)
         merge_client = RecordingMergeClient(self.config, self.sched)
         nodepool = zuul.nodepool.Nodepool(self.sched)
-        zk_client = ZooKeeperClient.fromConfig(self.config,
-                                               _require_tls=False)
+        zk_client = ZooKeeperClient.fromConfig(self.config)
         zk_client.connect()
 
         self.sched.setExecutor(executor_client)
@@ -4179,10 +4210,7 @@ class ZuulTestCase(BaseTestCase):
         super(ZuulTestCase, self).setUp()
 
         self.setupZK()
-        self.fake_nodepool = FakeNodepool(
-            self.zk_chroot_fixture.zookeeper_host,
-            self.zk_chroot_fixture.zookeeper_port,
-            self.zk_chroot_fixture.zookeeper_chroot)
+        self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture)
 
         if not KEEP_TEMPDIRS:
             tmp_root = self.useFixture(fixtures.TempDir(
@@ -4270,9 +4298,14 @@ class ZuulTestCase(BaseTestCase):
             self.zk_chroot_fixture.zookeeper_chroot)
         self.config.set('zookeeper', 'hosts', zk_hosts)
         self.config.set('zookeeper', 'session_timeout', '30')
-
-        self.zk_client = ZooKeeperClient.fromConfig(self.config,
-                                                    _require_tls=False)
+        self.config.set('zookeeper', 'tls_cert',
+                        self.zk_chroot_fixture.zookeeper_cert)
+        self.config.set('zookeeper', 'tls_key',
+                        self.zk_chroot_fixture.zookeeper_key)
+        self.config.set('zookeeper', 'tls_ca',
+                        self.zk_chroot_fixture.zookeeper_ca)
+
+        self.zk_client = ZooKeeperClient.fromConfig(self.config)
         self.zk_client.connect()
 
         self.rpcclient = zuul.rpcclient.RPCClient(
diff --git a/tests/unit/test_nodepool.py b/tests/unit/test_nodepool.py
index 5a02e5ab9..d34233a0a 100644
--- a/tests/unit/test_nodepool.py
+++ b/tests/unit/test_nodepool.py
@@ -33,12 +33,16 @@ class TestNodepool(BaseTestCase):
         self.statsd = None
         self.zk_chroot_fixture = self.useFixture(
             ChrootedKazooFixture(self.id()))
-        self.zk_config = '%s:%s%s' % (
+        zk_hosts = '%s:%s%s' % (
             self.zk_chroot_fixture.zookeeper_host,
             self.zk_chroot_fixture.zookeeper_port,
             self.zk_chroot_fixture.zookeeper_chroot)
 
-        self.zk_client = ZooKeeperClient(self.zk_config)
+        self.zk_client = ZooKeeperClient(
+            zk_hosts,
+            tls_cert=self.zk_chroot_fixture.zookeeper_cert,
+            tls_key=self.zk_chroot_fixture.zookeeper_key,
+            tls_ca=self.zk_chroot_fixture.zookeeper_ca)
         self.zk_nodepool = ZooKeeperNodepool(self.zk_client)
         self.addCleanup(self.zk_client.disconnect)
         self.zk_client.connect()
@@ -49,10 +53,7 @@ class TestNodepool(BaseTestCase):
         # needs, so we pass 'self' as the scheduler.
         self.nodepool = zuul.nodepool.Nodepool(self)
 
-        self.fake_nodepool = FakeNodepool(
-            self.zk_chroot_fixture.zookeeper_host,
-            self.zk_chroot_fixture.zookeeper_port,
-            self.zk_chroot_fixture.zookeeper_chroot)
+        self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture)
         self.addCleanup(self.fake_nodepool.stop)
 
     def waitForRequests(self):
diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py
index 67d06e13c..7ca2a3b73 100644
--- a/tests/unit/test_streaming.py
+++ b/tests/unit/test_streaming.py
@@ -522,8 +522,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
         logfile = open(ansible_log, 'r')
         self.addCleanup(logfile.close)
 
-        zk_client = ZooKeeperClient.fromConfig(self.config,
-                                               _require_tls=False)
+        zk_client = ZooKeeperClient.fromConfig(self.config)
         zk_client.connect()
         self.addCleanup(zk_client.disconnect)
 
diff --git a/tests/unit/test_zk.py b/tests/unit/test_zk.py
index c6e883dca..ad12b5c1d 100644
--- a/tests/unit/test_zk.py
+++ b/tests/unit/test_zk.py
@@ -30,12 +30,16 @@ class TestZK(BaseTestCase):
 
         self.zk_chroot_fixture = self.useFixture(
             ChrootedKazooFixture(self.id()))
-        self.zk_config = '%s:%s%s' % (
+        zk_hosts = '%s:%s%s' % (
             self.zk_chroot_fixture.zookeeper_host,
             self.zk_chroot_fixture.zookeeper_port,
             self.zk_chroot_fixture.zookeeper_chroot)
 
-        self.zk_client = ZooKeeperClient(self.zk_config)
+        self.zk_client = ZooKeeperClient(
+            zk_hosts,
+            tls_cert=self.zk_chroot_fixture.zookeeper_cert,
+            tls_key=self.zk_chroot_fixture.zookeeper_key,
+            tls_ca=self.zk_chroot_fixture.zookeeper_ca)
         self.zk_nodepool = ZooKeeperNodepool(self.zk_client)
         self.addCleanup(self.zk_client.disconnect)
         self.zk_client.connect()
diff --git a/tools/docker-compose.yaml b/tools/docker-compose.yaml
index d7c3ce0cd..e17750bb2 100644
--- a/tools/docker-compose.yaml
+++ b/tools/docker-compose.yaml
@@ -29,7 +29,10 @@ services:
       - ZOO_AUTOPURGE_PURGEINTERVAL=1
       - ZOO_LOG4J_PROP=WARN
     ports:
-      - "2181:2181"
+      - "2281:2281"
     tmpfs:
       - /data
       - /datalog
+    volumes:
+      - "./ca:/var/certs:z"
+      - "./zoo.cfg:/conf/zoo.cfg:z"
diff --git a/tools/test-setup-docker.sh b/tools/test-setup-docker.sh
index c64dccb41..526f025c7 100755
--- a/tools/test-setup-docker.sh
+++ b/tools/test-setup-docker.sh
@@ -3,6 +3,7 @@
 set -eu
 
 cd $(dirname $0)
+SCRIPT_DIR="$(pwd)"
 
 # Select docker or podman
 if command -v docker > /dev/null; then
@@ -33,6 +34,11 @@ else
   podman-compose down
 fi
 
+CA_DIR=$SCRIPT_DIR/ca
+
+mkdir -p $CA_DIR
+$SCRIPT_DIR/zk-ca.sh $CA_DIR zuul-test-zookeeper
+
 ${COMPOSE} up -d
 
 echo "Waiting for mysql"
diff --git a/tools/test-setup.sh b/tools/test-setup.sh
index cb524f9c5..237763569 100755
--- a/tools/test-setup.sh
+++ b/tools/test-setup.sh
@@ -7,23 +7,15 @@
 # This setup needs to be run as a user that can run sudo.
 TOOLSDIR=$(dirname $0)
 
-# Config Zookeeper to run on tmpfs
-sudo service zookeeper stop
-DATADIR=$(sed -n -e 's/^dataDir=//p' /etc/zookeeper/conf/zoo.cfg)
-sudo mount -t tmpfs -o nodev,nosuid,size=500M none $DATADIR
-echo "autopurge.purgeInterval=1" | sudo tee -a /etc/zookeeper/conf/zoo.cfg
-echo "maxClientCnxns=1000" | sudo tee -a /etc/zookeeper/conf/zoo.cfg
-
 # Prepare a tmpfs for Zuul test root
 if [[ -n "${ZUUL_TEST_ROOT:-}" ]]; then
     sudo mkdir -p "$ZUUL_TEST_ROOT"
     sudo mount -t tmpfs -o noatime,nodev,nosuid,size=64M none "$ZUUL_TEST_ROOT"
 fi
 
-# Be sure mysql and zookeeper are started.
+# Be sure mysql is started.
 sudo service mysql start
 sudo service postgresql start
-sudo service zookeeper start
 
 # The root password for the MySQL database; pass it in via
 # MYSQL_ROOT_PW.
diff --git a/tools/zoo.cfg b/tools/zoo.cfg
new file mode 100644
index 000000000..bf1ac6cf7
--- /dev/null
+++ b/tools/zoo.cfg
@@ -0,0 +1,16 @@
+# zoo.cfg for use in test-setup.sh
+dataDir=/data
+dataLogDir=/datalog
+tickTime=2000
+initLimit=5
+syncLimit=2
+autopurge.snapRetainCount=3
+autopurge.purgeInterval=0
+maxClientCnxns=1000
+standaloneEnabled=true
+admin.enableServer=true
+server.1=nodepool-test-zookeeper:2888:3888
+serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
+secureClientPort=2281
+ssl.keyStore.location=/var/certs/keystores/zuul-test-zookeeper.pem
+ssl.trustStore.location=/var/certs/certs/cacert.pem
diff --git a/tox.ini b/tox.ini
index 517eb01f8..a6f704e82 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,6 +27,10 @@ passenv =
   ZUUL_MYSQL_HOST
   ZUUL_POSTGRES_HOST
   ZUUL_TEST_ROOT
+  ZUUL_ZK_HOST
+  ZUUL_ZK_CA
+  ZUUL_ZK_CERT
+  ZUUL_ZK_KEY
 usedevelop = True
 whitelist_externals = bash
 deps =
@@ -106,6 +110,10 @@ passenv =
   ZUUL_REMOTE_IPV4
   ZUUL_SSH_KEY
   ZUUL_TEST_ROOT
+  ZUUL_ZK_HOST
+  ZUUL_ZK_CA
+  ZUUL_ZK_CERT
+  ZUUL_ZK_KEY
 commands =
   stestr run --test-path ./tests/remote {posargs}
 
diff --git a/zuul/zk/__init__.py b/zuul/zk/__init__.py
index f0388b457..c085d2120 100644
--- a/zuul/zk/__init__.py
+++ b/zuul/zk/__init__.py
@@ -144,17 +144,14 @@ class ZooKeeperClient(object):
             self.client.set_hosts(hosts=hosts)
 
     @classmethod
-    def fromConfig(cls, config: ConfigParser,
-                   _require_tls=True) -> "ZooKeeperClient":
-        # _require_tls is temporary, only used until we move the tests
-        # to use TLS.
+    def fromConfig(cls, config: ConfigParser) -> "ZooKeeperClient":
         hosts = get_default(config, "zookeeper", "hosts")
         if not hosts:
             raise Exception("The zookeeper hosts config value is required")
         tls_key = get_default(config, "zookeeper", "tls_key")
         tls_cert = get_default(config, "zookeeper", "tls_cert")
         tls_ca = get_default(config, "zookeeper", "tls_ca")
-        if _require_tls and not all([tls_key, tls_cert, tls_ca]):
+        if not all([tls_key, tls_cert, tls_ca]):
             raise Exception(
                 "A TLS ZooKeeper connection is required; please supply the "
                 "tls_* zookeeper config values."