SERVER-54623 Wait longer for session collection in ShardedClusterFixture

3 files changed, 28 insertions, 14 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index fc1979bdeb1..6a92ce93652 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -72,6 +72,14 @@ class TeardownMode(Enum):
 class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):  # pylint: disable=invalid-metaclass
     """Base class for all fixtures."""
 
+    # Error response codes copied from mongo/base/error_codes.yml.
+    _WRITE_CONCERN_FAILED = 64
+    _NODE_NOT_FOUND = 74
+    _NEW_REPLICA_SET_CONFIGURATION_INCOMPATIBLE = 103
+    _CONFIGURATION_IN_PROGRESS = 109
+    _CURRENT_CONFIG_NOT_COMMITTED_YET = 308
+    _INTERRUPTED_DUE_TO_REPL_STATE_CHANGE = 11602
+
     # We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute
     # is defined for all subclasses of Fixture.
     REGISTERED_NAME = "Fixture"
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 7c8ac94bc6b..6e5d5c07108 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -14,13 +14,6 @@ import buildscripts.resmokelib.testing.fixtures.interface as interface
 class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-instance-attributes
     """Fixture which provides JSTests with a replica set to run against."""
 
-    # Error response codes copied from mongo/base/error_codes.yml.
-    _NODE_NOT_FOUND = 74
-    _NEW_REPLICA_SET_CONFIGURATION_INCOMPATIBLE = 103
-    _CONFIGURATION_IN_PROGRESS = 109
-    _CURRENT_CONFIG_NOT_COMMITTED_YET = 308
-    _INTERRUPTED_DUE_TO_REPL_STATE_CHANGE = 11602
-
     def __init__(  # pylint: disable=too-many-arguments, too-many-locals
             self, logger, job_num, fixturelib, mongod_executable=None, mongod_options=None,
             dbpath_prefix=None, preserve_dbpath=False, num_nodes=2, start_initial_sync_node=False,
@@ -289,7 +282,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
                     raise self.fixturelib.ServerFailure(msg)
                 time.sleep(5)  # Wait a little bit before trying again.
 
-    def await_last_op_committed(self):
+    def await_last_op_committed(self, timeout_secs=None):
         """Wait for the last majority committed op to be visible."""
         primary_client = self.get_primary().mongo_client()
         interface.authenticate(primary_client, self.auth_options)
@@ -308,7 +301,8 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
             return len(up_to_date_nodes) == len(self.nodes)
 
-        self._await_cmd_all_nodes(check_rcmaj_optime, "waiting for last committed optime")
+        self._await_cmd_all_nodes(check_rcmaj_optime, "waiting for last committed optime",
+                                  timeout_secs)
 
     def await_ready(self):
         """Wait for replica set to be ready."""
@@ -506,7 +500,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
         return self._await_cmd_all_nodes(is_primary, "waiting for a primary", timeout_secs)
 
-    def _await_cmd_all_nodes(self, fn, msg, timeout_secs=30):
+    def _await_cmd_all_nodes(self, fn, msg, timeout_secs=None):
         """Run `fn` on all nodes until it returns a truthy value.
 
         Return the node for which makes `fn` become truthy.
@@ -515,6 +509,8 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         the MongoDFixture corresponding to that node.
         """
 
+        if timeout_secs is None:
+            timeout_secs = self.AWAIT_REPL_TIMEOUT_MINS * 60
         start = time.time()
         clients = {}
         while True:
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 738f271176e..5c68f437659 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -109,6 +109,18 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         for shard in self.shards:
             shard.setup()
 
+    def refresh_logical_session_cache(self, target):
+        """Refresh logical session cache with no timeout."""
+        primary = target.get_primary().mongo_client()
+        try:
+            primary.admin.command({"refreshLogicalSessionCacheNow": 1})
+        except pymongo.errors.OperationFailure as err:
+            if err.code != self._WRITE_CONCERN_FAILED:
+                raise err
+            self.logger.info("Ignoring write concern timeout for refreshLogicalSessionCacheNow "
+                             "command and continuing to wait")
+            target.await_last_op_committed(target.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60)
+
     def await_ready(self):
         """Block until the fixture can be used for testing."""
         # Wait for the config server
@@ -167,12 +179,10 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
 
         # Ensure that the sessions collection gets auto-sharded by the config server
         if self.configsvr is not None:
-            primary = self.configsvr.get_primary().mongo_client()
-            primary.admin.command({"refreshLogicalSessionCacheNow": 1})
+            self.refresh_logical_session_cache(self.configsvr)
 
         for shard in self.shards:
-            primary = shard.get_primary().mongo_client()
-            primary.admin.command({"refreshLogicalSessionCacheNow": 1})
+            self.refresh_logical_session_cache(shard)
 
     def _await_mongod_sharding_initialization(self):
         if (self.enable_sharding) and (self.num_rs_nodes_per_shard is not None):