QPID-3121: Cluster management inconsistency when using persistent store.

With the store doing async completions, completion IO callbacks could
be queued differently on different nodes. This led to inconsistent
management changes in a cluster when a connection was modified in an
IO callback.

Fix was to mark IO callback processing as not cluster safe, so
connections don't record management stats during an IO callback.

Test changes:
- enable durable tests in test_management.
- add substitutions to mask known issue of inconsistent "stats changed" messages.
- add transactional client to test_management.
- ignore heartbeat connection close logs in cluster_test_logs.py
- make brokertest.retry more accurate
- fix minor bug in brokertest.log_ready.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk/qpid@1078947 13f79535-47bb-0310-9956-ffa450edef68

2 files changed, 24 insertions, 4 deletions

diff --git a/cpp/src/qpid/sys/ClusterSafe.cpp b/cpp/src/qpid/sys/ClusterSafe.cpp
index b67b04c267..dd37615145 100644
--- a/cpp/src/qpid/sys/ClusterSafe.cpp
+++ b/cpp/src/qpid/sys/ClusterSafe.cpp
@@ -51,6 +51,16 @@ ClusterSafeScope::~ClusterSafeScope() {
     inContext = save;
 }
 
+ClusterUnsafeScope::ClusterUnsafeScope()  {
+    save = inContext;
+    inContext = false;
+}
+
+ClusterUnsafeScope::~ClusterUnsafeScope() {
+    assert(!inContext);
+    inContext = save;
+}
+
 void enableClusterSafe() { inCluster = true; }
 
 }} // namespace qpid::sys
diff --git a/cpp/src/qpid/sys/ClusterSafe.h b/cpp/src/qpid/sys/ClusterSafe.h
index 42e290f4c8..fd3ec32ae6 100644
--- a/cpp/src/qpid/sys/ClusterSafe.h
+++ b/cpp/src/qpid/sys/ClusterSafe.h
@@ -53,10 +53,8 @@ QPID_COMMON_EXTERN void assertClusterSafe();
 QPID_COMMON_EXTERN bool isClusterSafe();
 
 /**
- * Base class for classes that encapsulate state which is replicated
- * to all members of a cluster. Acts as a marker for clustered state
- * and provides functions to assist detecting bugs in cluster
- * behavior.
+ *  Mark a scope as cluster safe. Sets isClusterSafe in constructor and resets
+ *  to previous value in destructor.
  */
 class ClusterSafeScope {
   public:
@@ -67,6 +65,18 @@ class ClusterSafeScope {
 };
 
 /**
+ *  Mark a scope as cluster unsafe. Clears isClusterSafe in constructor and resets
+ *  to previous value in destructor.
+ */
+class ClusterUnsafeScope {
+  public:
+    ClusterUnsafeScope();
+    ~ClusterUnsafeScope();
+  private:
+    bool save;
+};
+
+/**
  * Enable cluster-safe assertions. By default they are no-ops.
  * Called by cluster code.
  */