mod_md v2.4.19 from github sync

  *) mod_md: a new directive `MDStoreLocks` can be used on cluster
     setups with a shared file system for `MDStoreDir` to order
     activation of renewed certificates when several cluster nodes are
     restarted at the same time. Store locks are not enabled by default.

     Restored curl_easy cleanup behaviour from v2.4.14 and refactored
     the use of curl_multi for OCSP requests to work with that.
     Fixes <https://github.com/icing/mod_md/issues/293>.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1903677 13f79535-47bb-0310-9956-ffa450edef68

13 files changed, 347 insertions, 67 deletions

diff --git a/modules/md/md_curl.c b/modules/md/md_curl.c
index 0a399f9a50..217e8579dd 100644
--- a/modules/md/md_curl.c
+++ b/modules/md/md_curl.c
@@ -253,18 +253,17 @@ static apr_status_t internals_setup(md_http_request_t *req)
             rv = APR_EGENERAL;
             goto leave;
         }
+        curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_cb);
+        curl_easy_setopt(curl, CURLOPT_HEADERDATA, NULL);
+        curl_easy_setopt(curl, CURLOPT_READFUNCTION, req_data_cb);
+        curl_easy_setopt(curl, CURLOPT_READDATA, NULL);
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, resp_data_cb);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
     }
     else {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, 0, req->pool, "reusing curl instance from http");
     }
 
-    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_cb);
-    curl_easy_setopt(curl, CURLOPT_HEADERDATA, NULL);
-    curl_easy_setopt(curl, CURLOPT_READFUNCTION, req_data_cb);
-    curl_easy_setopt(curl, CURLOPT_READDATA, NULL);
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, resp_data_cb);
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
-
     internals = apr_pcalloc(req->pool, sizeof(*internals));
     internals->curl = curl;
         
@@ -442,33 +441,41 @@ static void add_to_curlm(md_http_request_t *req, CURLM *curlm)
 {
     md_curl_internals_t *internals = req->internals;
     
-    if (curlm && internals && internals->curlm == NULL) {
-        curl_multi_add_handle(curlm, internals->curl);
+    assert(curlm);
+    assert(internals);
+    if (internals->curlm == NULL) {
         internals->curlm = curlm;
     }
+    assert(internals->curlm == curlm);
+    curl_multi_add_handle(curlm, internals->curl);
 }
 
-static void remove_from_curlm(md_http_request_t *req, CURLM *curlm)
+static void remove_from_curlm_and_destroy(md_http_request_t *req, CURLM *curlm)
 {
     md_curl_internals_t *internals = req->internals;
 
-    if (curlm && internals && internals->curlm == curlm) {
-        curl_multi_remove_handle(curlm, internals->curl);
-        internals->curlm = NULL;
-    }
+    assert(curlm);
+    assert(internals);
+    assert(internals->curlm == curlm);
+    curl_multi_remove_handle(curlm, internals->curl);
+    internals->curlm = NULL;
+    md_http_req_destroy(req);
 }
     
 static apr_status_t md_curl_multi_perform(md_http_t *http, apr_pool_t *p,
                                           md_http_next_req *nextreq, void *baton)
 {
+    md_http_t *sub_http;
     md_http_request_t *req;
     CURLM *curlm = NULL;
     CURLMcode mc;
     struct CURLMsg *curlmsg;
+    apr_array_header_t *http_spares;
     apr_array_header_t *requests;
     int i, running, numfds, slowdown, msgcount;
     apr_status_t rv;
     
+    http_spares = apr_array_make(p, 10, sizeof(md_http_t*));
     requests = apr_array_make(p, 10, sizeof(md_http_request_t*));
     curlm = curl_multi_init();
     if (!curlm) {
@@ -481,35 +488,46 @@ static apr_status_t md_curl_multi_perform(md_http_t *http, apr_pool_t *p,
     while(1) {
         while (1) {
             /* fetch as many requests as nextreq gives us */
-            rv = nextreq(&req, baton, http, requests->nelts);
-            
-            if (APR_SUCCESS == rv) {
-                if (APR_SUCCESS != (rv = internals_setup(req))) {
-                    if (req->cb.on_status) req->cb.on_status(req, rv, req->cb.on_status_data);
-                    md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p, 
+            if (http_spares->nelts > 0) {
+                sub_http = *(md_http_t **)(apr_array_pop(http_spares));
+            }
+            else {
+                rv = md_http_clone(&sub_http, p, http);
+                if (APR_SUCCESS != rv) {
+                    md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p,
                                   "multi_perform[%d reqs]: setup failed", requests->nelts);
+                    goto leave;
                 }
-                else {
-                    APR_ARRAY_PUSH(requests, md_http_request_t*) = req;
-                    add_to_curlm(req, curlm);
-                    md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p, 
-                                  "multi_perform[%d reqs]: added request", requests->nelts);
-                }
-                continue;
             }
-            else if (APR_STATUS_IS_ENOENT(rv)) {
-                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, 0, p, 
+
+            rv = nextreq(&req, baton, sub_http, requests->nelts);
+            if (APR_STATUS_IS_ENOENT(rv)) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, 0, p,
                               "multi_perform[%d reqs]: no more requests", requests->nelts);
                 if (!requests->nelts) {
                     goto leave;
                 }
                 break;
             }
-            else {
-                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p, 
+            else if (APR_SUCCESS != rv) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p,
                               "multi_perform[%d reqs]: nextreq() failed", requests->nelts);
+                APR_ARRAY_PUSH(http_spares, md_http_t*) = sub_http;
                 goto leave;
             }
+
+            if (APR_SUCCESS != (rv = internals_setup(req))) {
+                if (req->cb.on_status) req->cb.on_status(req, rv, req->cb.on_status_data);
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p,
+                              "multi_perform[%d reqs]: setup failed", requests->nelts);
+                APR_ARRAY_PUSH(http_spares, md_http_t*) = sub_http;
+                goto leave;
+            }
+
+            APR_ARRAY_PUSH(requests, md_http_request_t*) = req;
+            add_to_curlm(req, curlm);
+            md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, p,
+                          "multi_perform[%d reqs]: added request", requests->nelts);
         }
     
         mc = curl_multi_perform(curlm, &running);
@@ -544,9 +562,10 @@ static apr_status_t md_curl_multi_perform(md_http_t *http, apr_pool_t *p,
                                   requests->nelts, req->id);
                     update_status(req);
                     fire_status(req, curl_status(curlmsg->data.result));
-                    remove_from_curlm(req, curlm);
                     md_array_remove(requests, req);
-                    md_http_req_destroy(req);
+                    sub_http = req->http;
+                    APR_ARRAY_PUSH(http_spares, md_http_t*) = sub_http;
+                    remove_from_curlm_and_destroy(req, curlm);
                 }
                 else {
                     md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, 
@@ -563,8 +582,9 @@ leave:
     for (i = 0; i < requests->nelts; ++i) {
         req = APR_ARRAY_IDX(requests, i, md_http_request_t*);
         fire_status(req, APR_SUCCESS);
-        remove_from_curlm(req, curlm);
-        md_http_req_destroy(req);
+        sub_http = req->http;
+        APR_ARRAY_PUSH(http_spares, md_http_t*) = sub_http;
+        remove_from_curlm_and_destroy(req, curlm);
     }
     if (curlm) curl_multi_cleanup(curlm);
     return rv;
@@ -585,7 +605,19 @@ static void md_curl_req_cleanup(md_http_request_t *req)
     md_curl_internals_t *internals = req->internals;
     if (internals) {
         if (internals->curl) {
-            curl_easy_cleanup(internals->curl);
+            CURL *curl = md_http_get_impl_data(req->http);
+            if (curl == internals->curl) {
+                /* NOP: we have this curl at the md_http_t already */
+            }
+            else if (!curl) {
+                /* no curl at the md_http_t yet, install this one */
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, 0, req->pool, "register curl instance at http");
+                md_http_set_impl_data(req->http, internals->curl);
+            }
+            else {
+                /* There already is a curl at the md_http_t and it's not this one. */
+                curl_easy_cleanup(internals->curl);
+            }
         }
         if (internals->req_hdrs) curl_slist_free_all(internals->req_hdrs);
         req->internals = NULL;
diff --git a/modules/md/md_http.c b/modules/md/md_http.c
index 74db961301..0d21e7b14c 100644
--- a/modules/md/md_http.c
+++ b/modules/md/md_http.c
@@ -92,6 +92,25 @@ apr_status_t md_http_create(md_http_t **phttp, apr_pool_t *p, const char *user_a
     return APR_SUCCESS;
 }
 
+apr_status_t md_http_clone(md_http_t **phttp,
+                           apr_pool_t *p, md_http_t *source_http)
+{
+    apr_status_t rv;
+
+    rv = md_http_create(phttp, p, source_http->user_agent, source_http->proxy_url);
+    if (APR_SUCCESS == rv) {
+        (*phttp)->resp_limit = source_http->resp_limit;
+        (*phttp)->timeout = source_http->timeout;
+        if (source_http->unix_socket_path) {
+            (*phttp)->unix_socket_path = apr_pstrdup(p, source_http->unix_socket_path);
+        }
+        if (source_http->ca_file) {
+            (*phttp)->ca_file = apr_pstrdup(p, source_http->ca_file);
+        }
+    }
+    return rv;
+}
+
 void md_http_set_impl_data(md_http_t *http, void *data)
 {
     http->impl_data = data;
@@ -183,7 +202,6 @@ static apr_status_t req_set_body_data(md_http_request_t *req, const char *conten
         bbody = apr_brigade_create(req->pool, req->http->bucket_alloc);
         rv = apr_brigade_write(bbody, NULL, NULL, body->data, body->len);
         if (rv != APR_SUCCESS) {
-            md_http_req_destroy(req);
             return rv;
         }
     }
@@ -315,10 +333,16 @@ apr_status_t md_http_POSTd_create(md_http_request_t **preq, md_http_t *http, con
     apr_status_t rv;
     
     rv = req_create(&req, http, "POST", url, headers);
+    if (APR_SUCCESS != rv) goto cleanup;
+    rv = req_set_body_data(req, content_type, body);
+cleanup:
     if (APR_SUCCESS == rv) {
-        rv = req_set_body_data(req, content_type, body);
+        *preq = req;
+    }
+    else {
+        *preq = NULL;
+        if (req) md_http_req_destroy(req);
     }
-    *preq = (APR_SUCCESS == rv)? req : NULL;
     return rv;
 }
 
diff --git a/modules/md/md_http.h b/modules/md/md_http.h
index c210aa9913..2f250f6d76 100644
--- a/modules/md/md_http.h
+++ b/modules/md/md_http.h
@@ -88,6 +88,13 @@ apr_status_t md_http_create(md_http_t **phttp, apr_pool_t *p, const char *user_a
 void md_http_set_response_limit(md_http_t *http, apr_off_t resp_limit);
 
 /**
+ * Clone a http instance, inheriting all settings from source_http.
+ * The cloned instance is not tied in any way to the source.
+ */
+apr_status_t md_http_clone(md_http_t **phttp,
+                           apr_pool_t *p, md_http_t *source_http);
+
+/**
  * Set the timeout for the complete request. This needs to take everything from
  * DNS looksups, to conntects, to transfer of all data into account and should
  * be sufficiently large.
diff --git a/modules/md/md_log.h b/modules/md/md_log.h
index 73885f268e..19e688f7fd 100644
--- a/modules/md/md_log.h
+++ b/modules/md/md_log.h
@@ -38,6 +38,10 @@ typedef enum {
 
 #define MD_LOG_MARK     __FILE__,__LINE__
 
+#ifndef APLOGNO
+#define APLOGNO(n)              "AH" #n ": "
+#endif
+
 const char *md_log_level_name(md_log_level_t level);
 
 int md_log_is_level(apr_pool_t *p, md_log_level_t level);
diff --git a/modules/md/md_reg.c b/modules/md/md_reg.c
index 21374fc1af..8bceb0eb47 100644
--- a/modules/md/md_reg.c
+++ b/modules/md/md_reg.c
@@ -55,6 +55,8 @@ struct md_reg_t {
     void *notify_ctx;
     apr_time_t min_delay;
     int retry_failover;
+    int use_store_locks;
+    apr_time_t lock_wait_timeout;
 };
 
 /**************************************************************************************************/
@@ -83,7 +85,8 @@ static apr_status_t load_props(md_reg_t *reg, apr_pool_t *p)
 
 apr_status_t md_reg_create(md_reg_t **preg, apr_pool_t *p, struct md_store_t *store,
                            const char *proxy_url, const char *ca_file,
-                           apr_time_t min_delay, int retry_failover)
+                           apr_time_t min_delay, int retry_failover,
+                           int use_store_locks, apr_time_t lock_wait_timeout)
 {
     md_reg_t *reg;
     apr_status_t rv;
@@ -100,6 +103,8 @@ apr_status_t md_reg_create(md_reg_t **preg, apr_pool_t *p, struct md_store_t *st
                     apr_pstrdup(p, ca_file) : NULL;
     reg->min_delay = min_delay;
     reg->retry_failover = retry_failover;
+    reg->use_store_locks = use_store_locks;
+    reg->lock_wait_timeout = lock_wait_timeout;
 
     md_timeslice_create(&reg->renew_window, p, MD_TIME_LIFE_NORM, MD_TIME_RENEW_WINDOW_DEF); 
     md_timeslice_create(&reg->warn_window, p, MD_TIME_LIFE_NORM, MD_TIME_WARN_WINDOW_DEF); 
@@ -1235,6 +1240,52 @@ apr_status_t md_reg_load_staging(md_reg_t *reg, const md_t *md, apr_table_t *env
     return md_util_pool_vdo(run_load_staging, reg, p, md, env, result, NULL);
 }
 
+apr_status_t md_reg_load_stagings(md_reg_t *reg, apr_array_header_t *mds,
+                                  apr_table_t *env, apr_pool_t *p)
+{
+    apr_status_t rv = APR_SUCCESS;
+    md_t *md;
+    md_result_t *result;
+    int i;
+
+    for (i = 0; i < mds->nelts; ++i) {
+        md = APR_ARRAY_IDX(mds, i, md_t *);
+        result = md_result_md_make(p, md->name);
+        rv = md_reg_load_staging(reg, md, env, result, p);
+        if (APR_SUCCESS == rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_INFO, rv, p, APLOGNO(10068)
+                          "%s: staged set activated", md->name);
+        }
+        else if (!APR_STATUS_IS_ENOENT(rv)) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, APLOGNO(10069)
+                          "%s: error loading staged set", md->name);
+        }
+    }
+
+    return rv;
+}
+
+apr_status_t md_reg_lock_global(md_reg_t *reg, apr_pool_t *p)
+{
+    apr_status_t rv = APR_SUCCESS;
+
+    if (reg->use_store_locks) {
+        rv = md_store_lock_global(reg->store, p, reg->lock_wait_timeout);
+        if (APR_SUCCESS != rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p,
+                          "unable to acquire global store lock");
+        }
+    }
+    return rv;
+}
+
+void md_reg_unlock_global(md_reg_t *reg, apr_pool_t *p)
+{
+    if (reg->use_store_locks) {
+        md_store_unlock_global(reg->store, p);
+    }
+}
+
 apr_status_t md_reg_freeze_domains(md_reg_t *reg, apr_array_header_t *mds)
 {
     apr_status_t rv = APR_SUCCESS;
diff --git a/modules/md/md_reg.h b/modules/md/md_reg.h
index ccaf10253a..58ee16ac62 100644
--- a/modules/md/md_reg.h
+++ b/modules/md/md_reg.h
@@ -34,10 +34,18 @@ typedef struct md_reg_t md_reg_t;
 
 /**
  * Create the MD registry, using the pool and store.
+ * @param preg on APR_SUCCESS, the create md_reg_t
+ * @param pm memory pool to use for creation
+ * @param store the store to base on
+ * @param proxy_url optional URL of a proxy to use for requests
+ * @param ca_file  optioinal CA trust anchor file to use
+ * @param min_delay minimum delay between renewal attempts for a domain
+ * @param retry_failover numer of failed renewals attempt to fail over to alternate ACME ca
  */
 apr_status_t md_reg_create(md_reg_t **preg, apr_pool_t *pm, md_store_t *store,
                            const char *proxy_url, const char *ca_file,
-                           apr_time_t min_delay, int retry_failover);
+                           apr_time_t min_delay, int retry_failover,
+                           int use_store_locks, apr_time_t lock_wait_timeout);
 
 md_store_t *md_reg_store_get(md_reg_t *reg);
 
@@ -270,9 +278,36 @@ apr_status_t md_reg_renew(md_reg_t *reg, const md_t *md,
 apr_status_t md_reg_load_staging(md_reg_t *reg, const md_t *md, struct apr_table_t *env, 
                                  struct md_result_t *result, apr_pool_t *p);
 
+/**
+ * Check given MDomains for new data in staging areas and, if it exists, load
+ * the new credentials. On encountering errors, leave the credentails as
+ * they are.
+ */
+apr_status_t md_reg_load_stagings(md_reg_t *reg, apr_array_header_t *mds,
+                                  apr_table_t *env, apr_pool_t *p);
+
 void md_reg_set_renew_window_default(md_reg_t *reg, md_timeslice_t *renew_window);
 void md_reg_set_warn_window_default(md_reg_t *reg, md_timeslice_t *warn_window);
 
 struct md_job_t *md_reg_job_make(md_reg_t *reg, const char *mdomain, apr_pool_t *p);
 
+/**
+ * Acquire a cooperative, global lock on registry modifications. Will
+ * do nothing if locking is not configured.
+ *
+ * This will only prevent other children/processes/cluster nodes from
+ * doing the same and does not protect individual store functions from
+ * being called without it.
+ * @param reg the registy
+ * @param p memory pool to use
+ * @param max_wait maximum time to wait in order to acquire
+ * @return APR_SUCCESS when lock was obtained
+ */
+apr_status_t md_reg_lock_global(md_reg_t *reg, apr_pool_t *p);
+
+/**
+ * Realease the global registry lock. Will do nothing if there is no lock.
+ */
+void md_reg_unlock_global(md_reg_t *reg, apr_pool_t *p);
+
 #endif /* mod_md_md_reg_h */
diff --git a/modules/md/md_store.c b/modules/md/md_store.c
index 29f3632d92..59dbd676e9 100644
--- a/modules/md/md_store.c
+++ b/modules/md/md_store.c
@@ -374,3 +374,12 @@ apr_status_t md_store_md_iter(md_store_md_inspect *inspect, void *baton, md_stor
     return md_store_iter(insp_md, &ctx, store, p, group, pattern, MD_FN_MD, MD_SV_JSON);
 }
 
+apr_status_t md_store_lock_global(md_store_t *store, apr_pool_t *p, apr_time_t max_wait)
+{
+    return store->lock_global(store, p, max_wait);
+}
+
+void md_store_unlock_global(md_store_t *store, apr_pool_t *p)
+{
+    store->unlock_global(store, p);
+}
diff --git a/modules/md/md_store.h b/modules/md/md_store.h
index e252c27909..73c840fc57 100644
--- a/modules/md/md_store.h
+++ b/modules/md/md_store.h
@@ -204,7 +204,23 @@ apr_status_t md_store_iter_names(md_store_inspect *inspect, void *baton, md_stor
 apr_time_t md_store_get_modified(md_store_t *store, md_store_group_t group,  
                                  const char *name, const char *aspect, apr_pool_t *p);
 
+/**
+ * Acquire a cooperative, global lock on store modifications.
+
+ * This will only prevent other children/processes/cluster nodes from
+ * doing the same and does not protect individual store functions from
+ * being called without it.
+ * @param store the store
+ * @param p memory pool to use
+ * @param max_wait maximum time to wait in order to acquire
+ * @return APR_SUCCESS when lock was obtained
+ */
+apr_status_t md_store_lock_global(md_store_t *store, apr_pool_t *p, apr_time_t max_wait);
 
+/**
+ * Realease the global store lock. Will do nothing if there is no lock.
+ */
+void md_store_unlock_global(md_store_t *store, apr_pool_t *p);
 
 /**************************************************************************************************/
 /* Storage handling utils */
@@ -303,6 +319,8 @@ typedef apr_time_t md_store_get_modified_cb(md_store_t *store, md_store_group_t
 typedef apr_status_t md_store_remove_nms_cb(md_store_t *store, apr_pool_t *p, 
                                             apr_time_t modified, md_store_group_t group, 
                                             const char *name, const char *aspect);
+typedef apr_status_t md_store_lock_global_cb(md_store_t *store, apr_pool_t *p, apr_time_t max_wait);
+typedef void md_store_unlock_global_cb(md_store_t *store, apr_pool_t *p);
 
 struct md_store_t {
     md_store_save_cb *save;
@@ -317,6 +335,8 @@ struct md_store_t {
     md_store_is_newer_cb *is_newer;
     md_store_get_modified_cb *get_modified;
     md_store_remove_nms_cb *remove_nms;
+    md_store_lock_global_cb *lock_global;
+    md_store_unlock_global_cb *unlock_global;
 };
 
 
diff --git a/modules/md/md_store_fs.c b/modules/md/md_store_fs.c
index 4be7a0e818..79f694ad30 100644
--- a/modules/md/md_store_fs.c
+++ b/modules/md/md_store_fs.c
@@ -39,6 +39,7 @@
 /* file system based implementation of md_store_t */
 
 #define MD_STORE_VERSION        3
+#define MD_FS_LOCK_NAME         "store.lock"
 
 typedef struct {
     apr_fileperms_t dir;
@@ -60,6 +61,8 @@ struct md_store_fs_t {
     
     int port_80;
     int port_443;
+
+    apr_file_t *global_lock;
 };
 
 #define FS_STORE(store)     (md_store_fs_t*)(((char*)store)-offsetof(md_store_fs_t, s))
@@ -101,6 +104,9 @@ static int fs_is_newer(md_store_t *store, md_store_group_t group1, md_store_grou
 static apr_time_t fs_get_modified(md_store_t *store, md_store_group_t group,  
                                   const char *name, const char *aspect, apr_pool_t *p);
 
+static apr_status_t fs_lock_global(md_store_t *store, apr_pool_t *p, apr_time_t max_wait);
+static void fs_unlock_global(md_store_t *store, apr_pool_t *p);
+
 static apr_status_t init_store_file(md_store_fs_t *s_fs, const char *fname, 
                                     apr_pool_t *p, apr_pool_t *ptemp)
 {
@@ -296,7 +302,9 @@ apr_status_t md_store_fs_init(md_store_t **pstore, apr_pool_t *p, const char *pa
     s_fs->s.is_newer = fs_is_newer;
     s_fs->s.get_modified = fs_get_modified;
     s_fs->s.remove_nms = fs_remove_nms;
-    
+    s_fs->s.lock_global = fs_lock_global;
+    s_fs->s.unlock_global = fs_unlock_global;
+
     /* by default, everything is only readable by the current user */ 
     s_fs->def_perms.dir = MD_FPROT_D_UONLY;
     s_fs->def_perms.file = MD_FPROT_F_UONLY;
@@ -1094,3 +1102,67 @@ static apr_status_t fs_rename(md_store_t *store, apr_pool_t *p,
     md_store_fs_t *s_fs = FS_STORE(store);
     return md_util_pool_vdo(pfs_rename, s_fs, p, group, from, to, NULL);
 }
+
+static apr_status_t fs_lock_global(md_store_t *store, apr_pool_t *p, apr_time_t max_wait)
+{
+    md_store_fs_t *s_fs = FS_STORE(store);
+    apr_status_t rv;
+    const char *lpath;
+    apr_time_t end;
+
+    if (s_fs->global_lock) {
+        rv = APR_EEXIST;
+        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, "already locked globally");
+        goto cleanup;
+    }
+
+    rv = md_util_path_merge(&lpath, p, s_fs->base, MD_FS_LOCK_NAME, NULL);
+    if (APR_SUCCESS != rv) goto cleanup;
+    end = apr_time_now() + max_wait;
+
+    md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, p,
+                  "acquire global lock: %s", lpath);
+    while (apr_time_now() < end) {
+        rv = apr_file_open(&s_fs->global_lock, lpath,
+                           (APR_FOPEN_WRITE|APR_FOPEN_CREATE),
+                           MD_FPROT_F_UALL_GREAD, p);
+        if (APR_SUCCESS != rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, rv, p,
+                          "unable to create/open lock file: %s",
+                          lpath);
+            goto next_try;
+        }
+        rv = apr_file_lock(s_fs->global_lock,
+                           APR_FLOCK_EXCLUSIVE|APR_FLOCK_NONBLOCK);
+        if (APR_SUCCESS == rv) {
+            goto cleanup;
+        }
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, rv, p,
+                      "unable to obtain lock on: %s",
+                      lpath);
+
+    next_try:
+        if (s_fs->global_lock) {
+            apr_file_close(s_fs->global_lock);
+            s_fs->global_lock = NULL;
+        }
+        apr_sleep(apr_time_from_msec(100));
+    }
+    rv = APR_EGENERAL;
+    md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, rv, p,
+                  "acquire global lock: %s", lpath);
+
+cleanup:
+    return rv;
+}
+
+static void fs_unlock_global(md_store_t *store, apr_pool_t *p)
+{
+    md_store_fs_t *s_fs = FS_STORE(store);
+
+    (void)p;
+    if (s_fs->global_lock) {
+        apr_file_close(s_fs->global_lock);
+        s_fs->global_lock = NULL;
+    }
+}
diff --git a/modules/md/md_version.h b/modules/md/md_version.h
index 8b09e2ef11..9a8d588263 100644
--- a/modules/md/md_version.h
+++ b/modules/md/md_version.h
@@ -27,7 +27,7 @@
  * @macro
  * Version number of the md module as c string
  */
-#define MOD_MD_VERSION "2.4.17"
+#define MOD_MD_VERSION "2.4.19"
 
 /**
  * @macro
@@ -35,7 +35,7 @@
  * release. This is a 24 bit number with 8 bits for major number, 8 bits
  * for minor and 8 bits for patch. Version 1.2.3 becomes 0x010203.
  */
-#define MOD_MD_VERSION_NUM 0x020410
+#define MOD_MD_VERSION_NUM 0x020413
 
 #define MD_ACME_DEF_URL         "https://acme-v02.api.letsencrypt.org/directory"
 #define MD_TAILSCALE_DEF_URL    "file://localhost/var/run/tailscale/tailscaled.sock"
diff --git a/modules/md/mod_md.c b/modules/md/mod_md.c
index d5237f4699..00ed4baa95 100644
--- a/modules/md/mod_md.c
+++ b/modules/md/mod_md.c
@@ -713,27 +713,6 @@ static apr_status_t merge_mds_with_conf(md_mod_conf_t *mc, apr_pool_t *p,
     return rv;
 }
 
-static void load_staged_data(md_mod_conf_t *mc, server_rec *s, apr_pool_t *p)
-{
-    apr_status_t rv;
-    md_t *md;
-    md_result_t *result;
-    int i;
-
-    for (i = 0; i < mc->mds->nelts; ++i) {
-        md = APR_ARRAY_IDX(mc->mds, i, md_t *);
-        result = md_result_md_make(p, md->name);
-        if (APR_SUCCESS == (rv = md_reg_load_staging(mc->reg, md, mc->env, result, p))) {
-            ap_log_error( APLOG_MARK, APLOG_INFO, rv, s, APLOGNO(10068)
-                         "%s: staged set activated", md->name);
-        }
-        else if (!APR_STATUS_IS_ENOENT(rv)) {
-            ap_log_error( APLOG_MARK, APLOG_ERR, rv, s, APLOGNO(10069)
-                         "%s: error loading staged set", md->name);
-        }
-    }
-}
-
 static apr_status_t check_invalid_duplicates(server_rec *base_server)
 {
     server_rec *s;
@@ -891,7 +870,8 @@ static apr_status_t md_post_config_before_ssl(apr_pool_t *p, apr_pool_t *plog,
     if (APR_SUCCESS != rv) goto leave;
 
     rv = md_reg_create(&mc->reg, p, store, mc->proxy_url, mc->ca_certs,
-                       mc->min_delay, mc->retry_failover);
+                       mc->min_delay, mc->retry_failover,
+                       mc->use_store_locks, mc->lock_wait_timeout);
     if (APR_SUCCESS != rv) {
         ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, APLOGNO(10072) "setup md registry");
         goto leave;
@@ -934,14 +914,24 @@ static apr_status_t md_post_config_before_ssl(apr_pool_t *p, apr_pool_t *plog,
     /*3*/
     if (APR_SUCCESS != (rv = link_mds_to_servers(mc, s, p))) goto leave;
     /*4*/
+    if (APR_SUCCESS != (rv = md_reg_lock_global(mc->reg, ptemp))) {
+        ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, APLOGNO(10398)
+                     "unable to obtain global registry lock, "
+                     "renewed certificates may remain inactive on "
+                     "this httpd instance!");
+        /* FIXME: or should we fail the server start/reload here? */
+        rv = APR_SUCCESS;
+        goto leave;
+    }
     if (APR_SUCCESS != (rv = md_reg_sync_start(mc->reg, mc->mds, ptemp))) {
         ap_log_error(APLOG_MARK, APLOG_ERR, rv, s, APLOGNO(10073)
                      "syncing %d mds to registry", mc->mds->nelts);
         goto leave;
     }
     /*5*/
-    load_staged_data(mc, s, p);
+    md_reg_load_stagings(mc->reg, mc->mds, mc->env, p);
 leave:
+    md_reg_unlock_global(mc->reg, ptemp);
     return rv;
 }
 
diff --git a/modules/md/mod_md_config.c b/modules/md/mod_md_config.c
index f096ad238f..2f19300584 100644
--- a/modules/md/mod_md_config.c
+++ b/modules/md/mod_md_config.c
@@ -86,6 +86,8 @@ static md_mod_conf_t defmc = {
     NULL,                      /* CA cert file to use */
     apr_time_from_sec(5),      /* minimum delay for retries */
     13,                        /* retry_failover after 14 errors, with 5s delay ~ half a day */
+    0,                         /* store locks, disabled by default */
+    apr_time_from_sec(5),      /* max time to wait to obaint a store lock */
 };
 
 static md_timeslice_t def_renew_window = {
@@ -647,6 +649,36 @@ static const char *md_config_set_retry_failover(cmd_parms *cmd, void *dc, const
     return NULL;
 }
 
+static const char *md_config_set_store_locks(cmd_parms *cmd, void *dc, const char *s)
+{
+    md_srv_conf_t *config = md_config_get(cmd->server);
+    const char *err = md_conf_check_location(cmd, MD_LOC_NOT_MD);
+    int use_store_locks;
+    apr_time_t wait_time = 0;
+
+    (void)dc;
+    if (err) {
+        return err;
+    }
+    else if (!apr_strnatcasecmp("off", s)) {
+        use_store_locks = 0;
+    }
+    else if (!apr_strnatcasecmp("on", s)) {
+        use_store_locks = 1;
+    }
+    else {
+        if (md_duration_parse(&wait_time, s, "s") != APR_SUCCESS) {
+            return "neither 'on', 'off' or a duration specified";
+        }
+        use_store_locks = (wait_time != 0);
+    }
+    config->mc->use_store_locks = use_store_locks;
+    if (wait_time) {
+        config->mc->lock_wait_timeout = wait_time;
+    }
+    return NULL;
+}
+
 static const char *md_config_set_require_https(cmd_parms *cmd, void *dc, const char *value)
 {
     md_srv_conf_t *config = md_config_get(cmd->server);
@@ -1215,6 +1247,8 @@ const command_rec md_cmds[] = {
                   "Time length for first retry, doubled on every consecutive error."),
     AP_INIT_TAKE1("MDRetryFailover", md_config_set_retry_failover, NULL, RSRC_CONF,
                   "The number of errors before a failover to another CA is triggered."),
+    AP_INIT_TAKE1("MDStoreLocks", md_config_set_store_locks, NULL, RSRC_CONF,
+                  "Configure locking of store for updates."),
 
     AP_INIT_TAKE1(NULL, NULL, NULL, RSRC_CONF, NULL)
 };
diff --git a/modules/md/mod_md_config.h b/modules/md/mod_md_config.h
index 5d7da4b8d1..b34b92e14c 100644
--- a/modules/md/mod_md_config.h
+++ b/modules/md/mod_md_config.h
@@ -72,6 +72,8 @@ struct md_mod_conf_t {
     const char *ca_certs;              /* root certificates to use for connections */
     apr_time_t min_delay;              /* minimum delay for retries */
     int retry_failover;                /* number of errors to trigger CA failover */
+    int use_store_locks;               /* use locks when updating store */
+    apr_time_t lock_wait_timeout;      /* fail after this time when unable to obtain lock */
 };
 
 typedef struct md_srv_conf_t {