ZOOKEEPER-4479: C tests: Avoid some jitter which results in flaky tests

With these commits, the various tests in `TestOperations.cc` set the `last_sent` and `last_recv` fields of the "force-connected" ZooKeeper handle to deterministic values (which match the initial value of the time mock).

When that value is not specified, these fields get set to the "real" time at the end of the  `forceConnected` function, which can be sufficiently different from the value held in the mock for some of the tests to fail.

Author: Damien Diederen <ddiederen@apache.org>

Reviewers: Enrico Olivelli <eolivelli@apache.org>

Closes #1827 from ztzg/ZOOKEEPER-4479-flaky-c-test-operations

(cherry picked from commit 640b6dd65fd302ada1ac46c9b8c32f17dadb0d32)
Signed-off-by: Enrico Olivelli <eolivelli@apache.org>

3 files changed, 20 insertions, 15 deletions

diff --git a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc
index ddb9533e8..a36af8417 100644
--- a/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc
+++ b/zookeeper-client/zookeeper-client-c/tests/TestOperations.cc
@@ -130,7 +130,7 @@ public:
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         
         int fd=0;
         int interest=0;
@@ -173,7 +173,7 @@ public:
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         
         int fd=0;
         int interest=0;
@@ -216,7 +216,7 @@ public:
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         
         int fd=0;
         int interest=0;
@@ -267,7 +267,7 @@ public:
         zh=zookeeper_init("localhost:1234",watcher,TIMEOUT*1000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         
         int fd=0;
         int interest=0;
@@ -344,7 +344,7 @@ public:
         zh=zookeeper_init("localhost:1234",watcher,TIMEOUT*1000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
 
         int fd=0;
         int interest=0;
@@ -379,7 +379,7 @@ public:
         zh=zookeeper_init("localhost:1234",watcher,TIMEOUT*1000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         
         int fd=0;
         int interest=0;
@@ -426,7 +426,7 @@ public:
         zh=zookeeper_init("localhost:1234",watcher,TIMEOUT*1000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
         // simulate connected state
-        forceConnected(zh);
+        forceConnected(zh, &now.tv);
         
         // queue up a request; keep it pending (as if the server is busy or has died)
         AsyncGetOperationCompletion res1;
@@ -481,7 +481,7 @@ public:
 
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         zhandle_t* savezh=zh;
 
         // issue a request
@@ -520,7 +520,7 @@ public:
 
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         zhandle_t* savezh=zh;
 
         // will handle completion on request #1 and issue request #2 from it
@@ -594,7 +594,7 @@ public:
 
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         zhandle_t* savezh=zh;
 
         // issue a multi request
@@ -639,7 +639,7 @@ public:
 
         zh=zookeeper_init("localhost:2121",watcher,10000,TEST_CLIENT_ID,0,0);
         CPPUNIT_ASSERT(zh!=0);
-        forceConnected(zh);
+        forceConnected(zh, &timeMock.tv);
         zhandle_t* savezh=zh;
 
         // these shall persist during the test
diff --git a/zookeeper-client/zookeeper-client-c/tests/ZKMocks.cc b/zookeeper-client/zookeeper-client-c/tests/ZKMocks.cc
index 54c0819d9..674945e04 100644
--- a/zookeeper-client/zookeeper-client-c/tests/ZKMocks.cc
+++ b/zookeeper-client/zookeeper-client-c/tests/ZKMocks.cc
@@ -525,7 +525,7 @@ void ZookeeperServer::notifyBufferSent(const std::string& buffer){
     addRecvResponse(e);
 }
 
-void forceConnected(zhandle_t* zh){
+void forceConnected(zhandle_t* zh, const struct timeval *last_recv_send){
     // simulate connected state
     zh->state=ZOO_CONNECTED_STATE;
 
@@ -536,8 +536,13 @@ void forceConnected(zhandle_t* zh){
     zh->addrs.next++;
 
     zh->input_buffer=0;
-    gettimeofday(&zh->last_recv,0);
-    gettimeofday(&zh->last_send,0);
+    if (last_recv_send) {
+        zh->last_recv = *last_recv_send;
+        zh->last_send = *last_recv_send;
+    } else {
+        gettimeofday(&zh->last_recv,0);
+        gettimeofday(&zh->last_send,0);
+    }
 }
 
 void terminateZookeeperThreads(zhandle_t* zh){
diff --git a/zookeeper-client/zookeeper-client-c/tests/ZKMocks.h b/zookeeper-client/zookeeper-client-c/tests/ZKMocks.h
index 48eddabbc..a091f2fe8 100644
--- a/zookeeper-client/zookeeper-client-c/tests/ZKMocks.h
+++ b/zookeeper-client/zookeeper-client-c/tests/ZKMocks.h
@@ -30,7 +30,7 @@
 // sets internal zhandle_t members to certain values to simulate the client 
 // connected state. This function should only be used with the single-threaded
 // Async API tests!
-void forceConnected(zhandle_t* zh); 
+void forceConnected(zhandle_t* zh, const struct timeval *last_recv_send = NULL);
 
 /**
  * Gracefully terminates zookeeper I/O and completion threads.