Cluster start-up retries to handle slow cman start-up.

 - infinite retry if cpg_initialize returns TRY_AGAIN.
 - infinite retry for cman_is_quorate


git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@744321 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 11 insertions, 7 deletions

diff --git a/qpid/cpp/src/qpid/cluster/Cpg.cpp b/qpid/cpp/src/qpid/cluster/Cpg.cpp
index 453def31cc..c5a1b72003 100644
--- a/qpid/cpp/src/qpid/cluster/Cpg.cpp
+++ b/qpid/cpp/src/qpid/cluster/Cpg.cpp
@@ -18,6 +18,7 @@
 
 #include "Cpg.h"
 #include "qpid/sys/Mutex.h"
+#include "qpid/sys/Time.h"
 #include "qpid/sys/posix/PrivatePosix.h"
 #include "qpid/log/Statement.h"
 
@@ -74,7 +75,13 @@ Cpg::Cpg(Handler& h) : IOHandle(new sys::IOHandlePrivate), handler(h), isShutdow
     ::memset(&callbacks, sizeof(callbacks), 0);
     callbacks.cpg_deliver_fn = &globalDeliver;
     callbacks.cpg_confchg_fn = &globalConfigChange;
-    check(cpg_initialize(&handle, &callbacks), "Cannot initialize CPG");
+    cpg_error_t err = cpg_initialize(&handle, &callbacks);
+    if (err == CPG_ERR_TRY_AGAIN) {
+        QPID_LOG(notice, "Waiting for CPG initialization.");
+        while (CPG_ERR_TRY_AGAIN == (err = cpg_initialize(&handle, &callbacks)))
+            sys::sleep(5);
+    }
+    check(err, "Failed to initialize CPG.");
     check(cpg_context_set(handle, this), "Cannot set CPG context");
     // Note: CPG is currently unix-specific. If CPG is ported to
     // windows then this needs to be refactored into
diff --git a/qpid/cpp/src/qpid/cluster/Quorum_cman.cpp b/qpid/cpp/src/qpid/cluster/Quorum_cman.cpp
index 62c014fcc4..f301f4c877 100644
--- a/qpid/cpp/src/qpid/cluster/Quorum_cman.cpp
+++ b/qpid/cpp/src/qpid/cluster/Quorum_cman.cpp
@@ -35,13 +35,10 @@ void Quorum::init() {
     enable = true;
     cman = cman_init(0);
     if (cman == 0) throw ErrnoException("Can't connect to cman service");
-    // TODO aconway 2008-11-13: configurable max wait.
-    for (int retry = 0;  !cman_is_quorate(cman) && retry < 30; retry++) {
-        QPID_LOG(info, "Waiting for cluster quorum: " << sys::strError(errno));
-        sys::sleep(1);
+    if (!cman_is_quorate(cman)) {
+        QPID_LOG(notice, "Waiting for cluster quorum.");
+        while(!cman_is_quorate(cman)) sys::sleep(5);
     }
-    if (!cman_is_quorate(cman))
-        throw ErrnoException("Timed out waiting for cluster quorum.");
 }
 
 bool Quorum::isQuorate() { return enable ? cman_is_quorate(cman) : true; }