reconnect: Add ability to do a number of retries without backoff.

This is aimed at an upcoming database clustering implementation, where it's
desirable to try all of the cluster members quickly before backing off to
retry them again in sequence.

Signed-off-by: Ben Pfaff <blp@ovn.org>
Acked-by: Russell Bryant <russell@ovn.org>
Signed-off-by: Ben Pfaff <blp@ovn.org>
Acked-by: Justin Pettit <jpettit@ovn.org>

2 files changed, 42 insertions, 17 deletions

diff --git a/lib/reconnect.c b/lib/reconnect.c
index 04cb15b7c..0f21378d7 100644
--- a/lib/reconnect.c
+++ b/lib/reconnect.c
@@ -62,6 +62,7 @@ struct reconnect {
     long long int last_connected;
     long long int last_disconnected;
     unsigned int max_tries;
+    unsigned int backoff_free_tries;
 
     /* These values are simply for statistics reporting, not otherwise used
      * directly by anything internal. */
@@ -206,6 +207,15 @@ reconnect_get_max_tries(struct reconnect *fsm)
     return fsm->max_tries;
 }
 
+/* Sets the number of connection attempts that will be made without backoff to
+ * 'backoff_free_tries'.  Values 0 and 1 both represent a single attempt. */
+void
+reconnect_set_backoff_free_tries(struct reconnect *fsm,
+                                 unsigned int backoff_free_tries)
+{
+    fsm->backoff_free_tries = backoff_free_tries;
+}
+
 /* Configures the backoff parameters for 'fsm'.  'min_backoff' is the minimum
  * number of milliseconds, and 'max_backoff' is the maximum, between connection
  * attempts.  The current backoff is also the duration that 'fsm' is willing to
@@ -346,7 +356,7 @@ reconnect_disconnected(struct reconnect *fsm, long long int now, int error)
                 VLOG(fsm->info, "%s: error listening for connections",
                      fsm->name);
             }
-        } else {
+        } else if (fsm->backoff < fsm->max_backoff) {
             const char *type = fsm->passive ? "listen" : "connection";
             if (error > 0) {
                 VLOG_INFO("%s: %s attempt failed (%s)",
@@ -354,35 +364,47 @@ reconnect_disconnected(struct reconnect *fsm, long long int now, int error)
             } else {
                 VLOG(fsm->info, "%s: %s attempt timed out", fsm->name, type);
             }
+        } else {
+            /* We have reached the maximum backoff, so suppress logging to
+             * avoid wastefully filling the log.  (Previously we logged that we
+             * were suppressing further logging, see below.) */
         }
 
         if (fsm->state & (S_ACTIVE | S_IDLE)) {
             fsm->last_disconnected = now;
         }
+
+        if (!reconnect_may_retry(fsm)) {
+            reconnect_transition__(fsm, now, S_VOID);
+            return;
+        }
+
         /* Back off. */
-        if (fsm->state & (S_ACTIVE | S_IDLE)
-             && (fsm->last_activity - fsm->last_connected >= fsm->backoff
-                 || fsm->passive)) {
+        if (fsm->backoff_free_tries > 1) {
+            fsm->backoff_free_tries--;
+            fsm->backoff = 0;
+        } else if (fsm->state & (S_ACTIVE | S_IDLE)
+                   && (fsm->last_activity - fsm->last_connected >= fsm->backoff
+                       || fsm->passive)) {
             fsm->backoff = fsm->passive ? 0 : fsm->min_backoff;
         } else {
             if (fsm->backoff < fsm->min_backoff) {
                 fsm->backoff = fsm->min_backoff;
-            } else if (fsm->backoff >= fsm->max_backoff / 2) {
-                fsm->backoff = fsm->max_backoff;
-            } else {
+            } else if (fsm->backoff < fsm->max_backoff / 2) {
                 fsm->backoff *= 2;
-            }
-            if (fsm->passive) {
-                VLOG(fsm->info, "%s: waiting %.3g seconds before trying to "
-                          "listen again", fsm->name, fsm->backoff / 1000.0);
+                VLOG(fsm->info, "%s: waiting %.3g seconds before %s",
+                     fsm->name, fsm->backoff / 1000.0,
+                     fsm->passive ? "trying to listen again" : "reconnect");
             } else {
-                VLOG(fsm->info, "%s: waiting %.3g seconds before reconnect",
-                          fsm->name, fsm->backoff / 1000.0);
+                if (fsm->backoff < fsm->max_backoff) {
+                    VLOG_INFO("%s: continuing to %s in the background but "
+                              "suppressing further logging", fsm->name,
+                              fsm->passive ? "try to listen" : "reconnect");
+                }
+                fsm->backoff = fsm->max_backoff;
             }
         }
-
-        reconnect_transition__(fsm, now,
-                               reconnect_may_retry(fsm) ? S_BACKOFF : S_VOID);
+        reconnect_transition__(fsm, now, S_BACKOFF);
     }
 }
 
@@ -397,7 +419,7 @@ reconnect_connecting(struct reconnect *fsm, long long int now)
     if (fsm->state != S_CONNECTING) {
         if (fsm->passive) {
             VLOG(fsm->info, "%s: listening...", fsm->name);
-        } else {
+        } else if (fsm->backoff < fsm->max_backoff) {
             VLOG(fsm->info, "%s: connecting...", fsm->name);
         }
         reconnect_transition__(fsm, now, S_CONNECTING);
diff --git a/lib/reconnect.h b/lib/reconnect.h
index 4446713ce..9f2d469e2 100644
--- a/lib/reconnect.h
+++ b/lib/reconnect.h
@@ -51,6 +51,8 @@ int reconnect_get_probe_interval(const struct reconnect *);
 
 void reconnect_set_max_tries(struct reconnect *, unsigned int max_tries);
 unsigned int reconnect_get_max_tries(struct reconnect *);
+void reconnect_set_backoff_free_tries(struct reconnect *,
+                                      unsigned int backoff_free_tries);
 
 void reconnect_set_backoff(struct reconnect *,
                            int min_backoff, int max_backoff);
@@ -65,6 +67,7 @@ void reconnect_enable(struct reconnect *, long long int now);
 void reconnect_disable(struct reconnect *, long long int now);
 
 void reconnect_force_reconnect(struct reconnect *, long long int now);
+void reconnect_skip_backoff(struct reconnect *);
 
 bool reconnect_is_connected(const struct reconnect *);
 unsigned int reconnect_get_last_connect_elapsed(const struct reconnect *,