*) mod_md: the `MDCertificateAuthority` directive can take more than one URL/name of

     an ACME CA. This gives a failover for renewals when several consecutive attempts
     to get a certificate failed.
     A new directive was added: `MDRetryDelay` sets the delay of retries.
     A new directive was added: `MDRetryFailover` sets the number of errored
     attempts before an alternate CA is selected for certificate renewals.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1900852 13f79535-47bb-0310-9956-ffa450edef68

9 files changed, 144 insertions, 26 deletions

diff --git a/test/modules/md/md_conf.py b/test/modules/md/md_conf.py
index 0b4502a7ac..19d4977f00 100755
--- a/test/modules/md/md_conf.py
+++ b/test/modules/md/md_conf.py
@@ -13,7 +13,9 @@ class MDConf(HttpdConf):
             admin = f"admin@{env.http_tld}"
         if len(admin.strip()):
             self.add_admin(admin)
-
+        self.add([
+            "MDRetryDelay 1s",  # speed up testing a little
+        ])
         if local_ca:
             self.add([
                 f"MDCertificateAuthority {env.acme_url}",
@@ -23,7 +25,7 @@ class MDConf(HttpdConf):
                 ])
         if std_ports:
             self.add(f"MDPortMap 80:{env.http_port} 443:{env.https_port}")
-            if env.ssl_module == "tls":
+            if env.ssl_module == "mod_tls":
                 self.add(f"TLSListen {env.https_port}")
         self.add([
             "<Location /server-status>",
diff --git a/test/modules/md/md_env.py b/test/modules/md/md_env.py
index ca07f96937..e8e36e5b1b 100755
--- a/test/modules/md/md_env.py
+++ b/test/modules/md/md_env.py
@@ -313,7 +313,8 @@ class MDTestEnv(HttpdTestEnv):
         if state >= 0:
             assert md['state'] == state
         if ca:
-            assert md['ca']['url'] == ca
+            assert len(md['ca']['urls']) == 1
+            assert md['ca']['urls'][0] == ca
         if protocol:
             assert md['ca']['proto'] == protocol
         if agreement:
@@ -343,6 +344,7 @@ class MDTestEnv(HttpdTestEnv):
             assert False, f"pkey missing: {pkey_file}: {r.stdout}"
         if not os.path.isfile(cert_file):
             assert False, f"cert missing: {cert_file}: {r.stdout}"
+        return md
 
     def check_md_credentials(self, domain):
         if isinstance(domain, list):
diff --git a/test/modules/md/test_001_store.py b/test/modules/md/test_001_store.py
index c888db9b8e..995d40dc7b 100644
--- a/test/modules/md/test_001_store.py
+++ b/test/modules/md/test_001_store.py
@@ -39,7 +39,7 @@ class TestStore:
                 "domains": [dns],
                 "contacts": [],
                 "ca": {
-                    "url": env.acme_url,
+                    "urls": [env.acme_url],
                     "proto": "ACME"
                 },
                 "state": 0
@@ -55,7 +55,7 @@ class TestStore:
                 "domains": dns,
                 "contacts": [],
                 "ca": {
-                    "url": env.acme_url,
+                    "urls": [env.acme_url],
                     "proto": "ACME"
                 },
                 "state": 0
@@ -76,7 +76,7 @@ class TestStore:
             "domains": dns2,
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": 0
@@ -129,7 +129,7 @@ class TestStore:
                 "domains": domains[i],
                 "contacts": [],
                 "ca": {
-                    "url": env.acme_url,
+                    "urls": [env.acme_url],
                     "proto": "ACME"
                 },
                 "state": 0
@@ -186,10 +186,10 @@ class TestStore:
     def test_md_001_402(self, env: MDTestEnv):
         dns = "test000-402.com"
         args = ["store", "add", dns]
-        assert env.a2md(args).json['output'][0]['ca']['url'] == env.acme_url
+        assert env.a2md(args).json['output'][0]['ca']['urls'][0] == env.acme_url
         nurl = "https://foo.com/"
         args = [env.a2md_bin, "-a", nurl, "-d", env.store_dir, "-j", "store", "update", dns]
-        assert env.run(args).json['output'][0]['ca']['url'] == nurl
+        assert env.run(args).json['output'][0]['ca']['urls'][0] == nurl
 
     # test case: update nonexisting managed domain
     def test_md_001_403(self, env: MDTestEnv):
diff --git a/test/modules/md/test_100_reg_add.py b/test/modules/md/test_100_reg_add.py
index 2b5bd23aa2..1a6d3fe838 100644
--- a/test/modules/md/test_100_reg_add.py
+++ b/test/modules/md/test_100_reg_add.py
@@ -23,7 +23,7 @@ class TestRegAdd:
             "domains": [dns],
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -39,7 +39,7 @@ class TestRegAdd:
             "domains": dns,
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -60,7 +60,7 @@ class TestRegAdd:
             "domains": dns2,
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
diff --git a/test/modules/md/test_110_reg_update.py b/test/modules/md/test_110_reg_update.py
index 3120ced6c4..71b50f8ea3 100644
--- a/test/modules/md/test_110_reg_update.py
+++ b/test/modules/md/test_110_reg_update.py
@@ -37,7 +37,7 @@ class TestRegUpdate:
             "domains": dns,
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -104,7 +104,7 @@ class TestRegUpdate:
             "domains": [self.NAME1, "www.greenbytes2.de", "mail.greenbytes2.de"],
             "contacts": [],
             "ca": {
-                "url": url,
+                "urls": [url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -121,7 +121,7 @@ class TestRegUpdate:
     def test_md_110_102(self, env):
         md = env.a2md(["update", self.NAME1, "ca", env.acme_url, "FOO"]).json['output'][0]
         env.check_json_contains(md['ca'], {
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "FOO"
         })
         assert md['state'] == 1
@@ -137,7 +137,7 @@ class TestRegUpdate:
             "contacts": [],
             "ca": {
                 "account": acc_id,
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -148,7 +148,7 @@ class TestRegUpdate:
         assert env.a2md(["update", self.NAME1, "account", "test.account.id"]).exit_code == 0
         md = env.a2md(["update", self.NAME1, "account"]).json['output'][0]
         env.check_json_contains(md['ca'], {
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "ACME"
         })
         assert md['state'] == 1
@@ -159,7 +159,7 @@ class TestRegUpdate:
         md = env.a2md(["update", self.NAME1, "account", "foo.test.com"]).json['output'][0]
         env.check_json_contains(md['ca'], {
             "account": "foo.test.com",
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "ACME"
         })
         assert md['state'] == 1
@@ -170,7 +170,7 @@ class TestRegUpdate:
                        "test2.account.id"]).json['output'][0]
         env.check_json_contains(md['ca'], {
             "account": "test.account.id",
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "ACME"
         })
         assert md['state'] == 1
@@ -185,7 +185,7 @@ class TestRegUpdate:
             "domains": [self.NAME1, "www.greenbytes2.de", "mail.greenbytes2.de"],
             "contacts": ["mailto:" + mail],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME"
             },
             "state": env.MD_S_INCOMPLETE
@@ -237,7 +237,7 @@ class TestRegUpdate:
             "domains": [self.NAME1, "www.greenbytes2.de", "mail.greenbytes2.de"],
             "contacts": [],
             "ca": {
-                "url": env.acme_url,
+                "urls": [env.acme_url],
                 "proto": "ACME",
                 "agreement": env.acme_tos
             },
@@ -249,7 +249,7 @@ class TestRegUpdate:
         assert env.a2md(["update", self.NAME1, "agreement", env.acme_tos]).exit_code == 0
         md = env.a2md(["update", self.NAME1, "agreement"]).json['output'][0]
         env.check_json_contains(md['ca'], {
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "ACME"
         })
         assert md['state'] == 1
@@ -259,7 +259,7 @@ class TestRegUpdate:
         md = env.a2md(["update", self.NAME1, "agreement",
                        env.acme_tos, "http://invalid.tos/"]).json['output'][0]
         env.check_json_contains(md['ca'], {
-            "url": env.acme_url,
+            "urls": [env.acme_url],
             "proto": "ACME",
             "agreement": env.acme_tos
         })
diff --git a/test/modules/md/test_120_reg_list.py b/test/modules/md/test_120_reg_list.py
index 0c1ce8a664..82e109f723 100644
--- a/test/modules/md/test_120_reg_list.py
+++ b/test/modules/md/test_120_reg_list.py
@@ -39,7 +39,7 @@ class TestRegAdd:
                 "domains": domains[i],
                 "contacts": [],
                 "ca": {
-                    "url": env.acme_url,
+                    "urls": [env.acme_url],
                     "proto": "ACME"
                 },
                 "state": env.MD_S_INCOMPLETE
diff --git a/test/modules/md/test_300_conf_validate.py b/test/modules/md/test_300_conf_validate.py
index e6b0561f82..25b033e393 100644
--- a/test/modules/md/test_300_conf_validate.py
+++ b/test/modules/md/test_300_conf_validate.py
@@ -340,7 +340,7 @@ class TestConf:
         conf.install()
         assert env.apache_restart() == 0, "Server did not accepted CA '{}'".format(ca)
         md = env.get_md_status(domain)
-        assert md['ca']['url'] == url
+        assert md['ca']['urls'][0] == url, f"CA url '{url}' not set in {md}"
 
     # vhost on another address, see #278
     def test_md_300_026(self, env):
@@ -365,3 +365,27 @@ class TestConf:
         conf.install()
         assert env.apache_restart() == 0
 
+    # test case: configure more than 1 CA
+    @pytest.mark.parametrize("cas, should_work", [
+        (["https://acme-v02.api.letsencrypt.org/directory"], True),
+        (["https://acme-v02.api.letsencrypt.org/directory", "buypass"], True),
+        (["x", "buypass"], False),
+        (["letsencrypt", "abc"], False),
+        (["letsencrypt", "buypass"], True),
+    ])
+    def test_md_300_027(self, env, cas, should_work):
+        domain = f"test1.{env.http_tld}"
+        conf = MDConf(env, text=f"""
+            MDCertificateAuthority {' '.join(cas)}
+            MDRenewMode manual
+        """)
+        conf.add_md([domain])
+        conf.install()
+        rv = env.apache_restart()
+        if should_work:
+            assert rv == 0, "Server did not accepted CAs '{}'".format(cas)
+            md = env.get_md_status(domain)
+            assert len(md['ca']['urls']) == len(cas)
+        else:
+            assert rv != 0, "Server should not have accepted CAs '{}'".format(cas)
+
diff --git a/test/modules/md/test_702_auto.py b/test/modules/md/test_702_auto.py
index 57187ad749..8e8f5f155c 100644
--- a/test/modules/md/test_702_auto.py
+++ b/test/modules/md/test_702_auto.py
@@ -1,4 +1,6 @@
 import os
+import time
+
 import pytest
 
 from pyhttpd.conf import HttpdConf
@@ -131,7 +133,8 @@ class TestAutov2:
         assert env.apache_restart() == 0
         env.check_md(domains)
         assert env.await_completion([domain])
-        env.check_md_complete(domain)
+        md = env.check_md_complete(domain)
+        assert md['ca']['url'], f"URL of CA used not set in md: {md}"
         #
         # check: SSL is running OK
         cert_a = env.get_cert(name_a)
diff --git a/test/modules/md/test_790_failover.py b/test/modules/md/test_790_failover.py
new file mode 100644
index 0000000000..a93991233d
--- /dev/null
+++ b/test/modules/md/test_790_failover.py
@@ -0,0 +1,87 @@
+import pytest
+
+from .md_env import MDTestEnv
+from .md_conf import MDConf
+
+
+@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(),
+                    reason="no ACME test server configured")
+class TestFailover:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env, acme):
+        acme.start(config='default')
+        env.check_acme()
+        env.clear_store()
+        conf = MDConf(env)
+        conf.install()
+
+        assert env.apache_restart() == 0
+
+    @pytest.fixture(autouse=True, scope='function')
+    def _method_scope(self, env, request):
+        env.clear_store()
+        self.test_domain = env.get_request_domain(request)
+
+    # set 2 ACME certificata authority, valid + invalid
+    def test_md_790_001(self, env):
+        domain = self.test_domain
+        # generate config with one MD
+        domains = [domain, "www." + domain]
+        conf = MDConf(env)
+        conf.add([
+            "MDRetryDelay 200ms",  # speed up failovers
+        ])
+        conf.start_md(domains)
+        conf.add([
+            f"MDCertificateAuthority {env.acme_url} https://does-not-exist/dir"
+        ])
+        conf.end_md()
+        conf.add_vhost(domains)
+        conf.install()
+        assert env.apache_restart() == 0
+        assert env.await_completion([domain])
+        env.check_md_complete(domain)
+
+    # set 2 ACME certificata authority, invalid + valid
+    def test_md_790_002(self, env):
+        domain = self.test_domain
+        # generate config with one MD
+        domains = [domain, "www." + domain]
+        conf = MDConf(env)
+        conf.add([
+            "MDRetryDelay 100ms",  # speed up failovers
+            "MDRetryFailover 2",
+        ])
+        conf.start_md(domains)
+        conf.add([
+            f"MDCertificateAuthority https://does-not-exist/dir {env.acme_url} "
+        ])
+        conf.end_md()
+        conf.add_vhost(domains)
+        conf.install()
+        assert env.apache_restart() == 0
+        assert env.await_completion([domain])
+        env.check_md_complete(domain)
+
+    # set 3 ACME certificata authority, invalid + invalid + valid
+    def test_md_790_003(self, env):
+        domain = self.test_domain
+        # generate config with one MD
+        domains = [domain, "www." + domain]
+        conf = MDConf(env)
+        conf.add([
+            "MDRetryDelay 100ms",  # speed up failovers
+            "MDRetryFailover 2",
+        ])
+        conf.start_md(domains)
+        conf.add([
+            f"MDCertificateAuthority https://does-not-exist/dir https://does-not-either/ "
+            f"{env.acme_url} "
+        ])
+        conf.end_md()
+        conf.add_vhost(domains)
+        conf.install()
+        assert env.apache_restart() == 0
+        assert env.await_completion([domain])
+        env.check_md_complete(domain)