SERVER-20694 Catch IncompatibleCatalogManager errors in cluster find code

8 files changed, 18 insertions, 11 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding.yml b/buildscripts/resmokeconfig/suites/sharding.yml
index cb084fdbf8d..c1a9b396fd1 100644
--- a/buildscripts/resmokeconfig/suites/sharding.yml
+++ b/buildscripts/resmokeconfig/suites/sharding.yml
@@ -4,7 +4,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/replset_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade.js                    # SERVER-20694
     - jstests/sharding/csrs_upgrade_during_migrate.js     # flaky - SERVER-20580
 
 executor:
diff --git a/buildscripts/resmokeconfig/suites/sharding_auth.yml b/buildscripts/resmokeconfig/suites/sharding_auth.yml
index 39f36e53586..f3e1e8df86d 100644
--- a/buildscripts/resmokeconfig/suites/sharding_auth.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_auth.yml
@@ -10,7 +10,6 @@ selector:
     - jstests/sharding/replset_config/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade.js                   # SERVER-20694
     - jstests/sharding/csrs_upgrade_during_migrate.js    # flaky - SERVER-20580
     # Skip any tests that run with auth explicitly.
     - jstests/sharding/*[aA]uth*.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml
index f5a65432ce7..dc9ad86be84 100644
--- a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml
@@ -10,7 +10,6 @@ selector:
     - jstests/sharding/replset_config/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade.js                   # SERVER-20694
     - jstests/sharding/csrs_upgrade_during_migrate.js    # flaky - SERVER-20580
     # Skip any tests that run with auth explicitly.
     - jstests/sharding/*[aA]uth*.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_ese.yml b/buildscripts/resmokeconfig/suites/sharding_ese.yml
index 2c6887ba8bf..af63a4f4f75 100644
--- a/buildscripts/resmokeconfig/suites/sharding_ese.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_ese.yml
@@ -8,7 +8,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/replset_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade.js                     # SERVER-20694
     - jstests/sharding/csrs_upgrade_during_migrate.js      # flaky - SERVER-20580
 
 executor:
diff --git a/buildscripts/resmokeconfig/suites/sharding_legacy.yml b/buildscripts/resmokeconfig/suites/sharding_legacy.yml
index 54021acd2b5..2d59cba5119 100644
--- a/buildscripts/resmokeconfig/suites/sharding_legacy.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_legacy.yml
@@ -4,7 +4,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade.js  # SERVER-20694
     - jstests/sharding/csrs_upgrade_during_migrate.js  # SERVER-20580
 
 executor:
diff --git a/jstests/sharding/csrs_upgrade.js b/jstests/sharding/csrs_upgrade.js
index ad8cce00bce..c3c044c395a 100644
--- a/jstests/sharding/csrs_upgrade.js
+++ b/jstests/sharding/csrs_upgrade.js
@@ -219,11 +219,13 @@ var st;
 
         var i;
         for (i = 0; i < csrsStatus.members.length; ++i) {
-            if (TestData.storageEngine != "wiredTiger" && TestData.storageEngine != "") {
-                // NOTE: "" means default storage engine, which is WiredTiger.
-                if (csrsStatus.members[i].name == csrs[0].name &&
-                    csrsStatus.members[i].stateStr != "REMOVED") {
-
+            if (csrsStatus.members[i].name == csrs[0].name) {
+                var supportsCommitted =
+                    csrs[0].getDB("admin").serverStatus().storageEngine.supportsCommittedReads;
+                var stateIsRemoved = csrsStatus.members[i].stateStr == "REMOVED";
+                // If the storage engine supports committed reads, it shouldn't go into REMOVED
+                // state, but if it does not then it should.
+                if (supportsCommitted ? stateIsRemoved : !stateIsRemoved) {
                     return false;
                 }
             }
diff --git a/src/mongo/s/catalog/forwarding_catalog_manager.cpp b/src/mongo/s/catalog/forwarding_catalog_manager.cpp
index 9ace0d47b0e..d90dc882f24 100644
--- a/src/mongo/s/catalog/forwarding_catalog_manager.cpp
+++ b/src/mongo/s/catalog/forwarding_catalog_manager.cpp
@@ -328,6 +328,7 @@ void ForwardingCatalogManager::_replaceCatalogManager(const TaskExecutor::Callba
     }
     Client::initThreadIfNotAlready();
     auto txn = cc().makeOperationContext();
+    log() << "Swapping sharding Catalog Manager from mirrored (SCCC) to replica set (CSRS) mode";
 
     stdx::lock_guard<RWLock> oplk(_operationLock);
     stdx::lock_guard<stdx::mutex> oblk(_observerMutex);
diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp
index 7cab6adcb28..6b1df73fad1 100644
--- a/src/mongo/s/query/cluster_find.cpp
+++ b/src/mongo/s/query/cluster_find.cpp
@@ -250,7 +250,16 @@ StatusWith<CursorId> runQueryWithoutRetrying(OperationContext* txn,
         // handling for querying config server content with legacy 3-host config servers.
         if (shard->isConfig() && shard->getConnString().type() == ConnectionString::SYNC) {
             invariant(shards.size() == 1U);
-            return runConfigServerQuerySCCC(query, *shard, results);
+            try {
+                return runConfigServerQuerySCCC(query, *shard, results);
+            } catch (const DBException& e) {
+                if (e.getCode() != ErrorCodes::IncompatibleCatalogManager) {
+                    throw;
+                }
+                grid.forwardingCatalogManager()->waitForCatalogManagerChange(txn);
+                // Fall through to normal code path now that the catalog manager mode has been
+                // swapped and the config servers are a normal replica set.
+            }
         }
 
         // Build the find command, and attach shard version if necessary.