SERVER-28151 Sync admin database first during initial sync

(cherry picked from commit 38218c1a0c2a15a05557df36794fe53618ca0db5)

3 files changed, 184 insertions, 7 deletions

diff --git a/src/mongo/db/repl/databases_cloner.cpp b/src/mongo/db/repl/databases_cloner.cpp
index 4a18e1be1b8..c9016c322f4 100644
--- a/src/mongo/db/repl/databases_cloner.cpp
+++ b/src/mongo/db/repl/databases_cloner.cpp
@@ -231,6 +231,44 @@ void DatabasesCloner::setScheduleDbWorkFn_forTest(const CollectionCloner::Schedu
     _scheduleDbWorkFn = work;
 }
 
+StatusWith<std::vector<BSONElement>> DatabasesCloner::parseListDatabasesResponse_forTest(
+    BSONObj dbResponse) {
+    return _parseListDatabasesResponse(dbResponse);
+}
+
+void DatabasesCloner::setAdminAsFirst_forTest(std::vector<BSONElement>& dbsArray) {
+    _setAdminAsFirst(dbsArray);
+}
+
+StatusWith<std::vector<BSONElement>> DatabasesCloner::_parseListDatabasesResponse(
+    BSONObj dbResponse) {
+    if (!dbResponse.hasField("databases")) {
+        return Status(ErrorCodes::BadValue,
+                      "The 'listDatabases' response does not contain a 'databases' field.");
+    }
+    BSONElement response = dbResponse["databases"];
+    try {
+        return response.Array();
+    } catch (const MsgAssertionException& e) {
+        return Status(ErrorCodes::BadValue,
+                      "The 'listDatabases' response is unable to be transformed into an array.");
+    }
+}
+
+void DatabasesCloner::_setAdminAsFirst(std::vector<BSONElement>& dbsArray) {
+    auto adminIter = std::find_if(dbsArray.begin(), dbsArray.end(), [](BSONElement elem) {
+        if (!elem.isABSONObj()) {
+            return false;
+        }
+        auto bsonObj = elem.Obj();
+        std::string databaseName = bsonObj.getStringField("name");
+        return (databaseName == "admin");
+    });
+    if (adminIter != dbsArray.end()) {
+        std::iter_swap(adminIter, dbsArray.begin());
+    }
+}
+
 void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
     Status respStatus = cbd.response.status;
     if (respStatus.isOK()) {
@@ -239,24 +277,44 @@ void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
 
     UniqueLock lk(_mutex);
     if (!respStatus.isOK()) {
-        LOG(1) << "listDatabases failed: " << respStatus;
+        LOG(1) << "'listDatabases' failed: " << respStatus;
         _fail_inlock(&lk, respStatus);
         return;
     }
 
-    const auto respBSON = cbd.response.data;
-    // There should not be any cloners yet
+    // There should not be any cloners yet.
     invariant(_databaseCloners.size() == 0);
-    const auto dbsElem = respBSON["databases"].Obj();
-    BSONForEach(arrayElement, dbsElem) {
+    const auto respBSON = cbd.response.data;
+
+    auto databasesArray = _parseListDatabasesResponse(respBSON);
+    if (!databasesArray.isOK()) {
+        LOG(1) << "'listDatabases' returned a malformed response: "
+               << databasesArray.getStatus().toString();
+        _fail_inlock(&lk, databasesArray.getStatus());
+        return;
+    }
+
+    auto dbsArray = databasesArray.getValue();
+    // Ensure that the 'admin' database is the first element in the array of databases so that it
+    // will be the first to be cloned. This allows users to authenticate against a database while
+    // initial sync is occurring.
+    _setAdminAsFirst(dbsArray);
+
+    for (BSONElement arrayElement : dbsArray) {
         const BSONObj dbBSON = arrayElement.Obj();
 
         // Check to see if we want to exclude this db from the clone.
         if (!_includeDbFn(dbBSON)) {
-            LOG(1) << "excluding db: " << dbBSON;
+            LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON;
             continue;
         }
 
+        if (!dbBSON.hasField("name")) {
+            LOG(1) << "Excluding database due to the 'listDatabases' response not containing a "
+                      "'name' field for this entry: "
+                   << dbBSON;
+        }
+
         const std::string dbName = dbBSON["name"].str();
         std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
 
@@ -321,7 +379,6 @@ void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
         // add cloner to list.
         _databaseCloners.push_back(dbCloner);
     }
-
     if (_databaseCloners.size() == 0) {
         if (_status.isOK()) {
             _succeed_inlock(&lk);
diff --git a/src/mongo/db/repl/databases_cloner.h b/src/mongo/db/repl/databases_cloner.h
index 53cced9ee4f..13f11f1e2a1 100644
--- a/src/mongo/db/repl/databases_cloner.h
+++ b/src/mongo/db/repl/databases_cloner.h
@@ -34,6 +34,7 @@
 
 #include "mongo/base/disallow_copying.h"
 #include "mongo/base/status.h"
+#include "mongo/base/status_with.h"
 #include "mongo/bson/bsonobj.h"
 #include "mongo/client/fetcher.h"
 #include "mongo/db/namespace_string.h"
@@ -104,6 +105,18 @@ public:
      */
     void setScheduleDbWorkFn_forTest(const CollectionCloner::ScheduleDbWorkFn& scheduleDbWorkFn);
 
+    /**
+     * Calls DatabasesCloner::_setAdminAsFirst.
+     * For testing only.
+     */
+    void setAdminAsFirst_forTest(std::vector<BSONElement>& dbsArray);
+
+    /**
+     * Calls DatabasesCloner::_parseListDatabasesResponse.
+     * For testing only.
+     */
+    StatusWith<std::vector<BSONElement>> parseListDatabasesResponse_forTest(BSONObj dbResponse);
+
 private:
     bool _isActive_inlock() const;
 
@@ -135,6 +148,23 @@ private:
 
     void _onListDatabaseFinish(const CommandCallbackArgs& cbd);
 
+    /**
+     * Takes a vector of BSONElements and scans for an element that contains a 'name' field with the
+     * value 'admin'. If found, the element is swapped with the first element in the vector.
+     * Otherwise, return.
+     *
+     * Used to parse the BSONResponse returned by listDatabases.
+     */
+    void _setAdminAsFirst(std::vector<BSONElement>& dbsArray);
+
+    /**
+     * Takes a 'listDatabases' command response and parses the response into a
+     * vector of BSON elements.
+     *
+     * If the input response is malformed, Status ErrorCodes::BadValue will be returned.
+     */
+    StatusWith<std::vector<BSONElement>> _parseListDatabasesResponse(BSONObj dbResponse);
+
     //
     // All member variables are labeled with one of the following codes indicating the
     // synchronization rules for accessing them.
diff --git a/src/mongo/db/repl/databases_cloner_test.cpp b/src/mongo/db/repl/databases_cloner_test.cpp
index 81552201485..e44114e5d58 100644
--- a/src/mongo/db/repl/databases_cloner_test.cpp
+++ b/src/mongo/db/repl/databases_cloner_test.cpp
@@ -314,6 +314,15 @@ protected:
         ASSERT_OK(result);
     };
 
+    std::unique_ptr<DatabasesCloner> makeDummyDatabasesCloner() {
+        return stdx::make_unique<DatabasesCloner>(&getStorage(),
+                                                  &getExecutor(),
+                                                  &getDbWorkThreadPool(),
+                                                  HostAndPort{"local:1234"},
+                                                  [](const BSONObj&) { return true; },
+                                                  [](const Status&) {});
+    }
+
 private:
     executor::ThreadPoolMock::Options makeThreadPoolMockOptions() const override;
 
@@ -450,6 +459,87 @@ TEST_F(DBsClonerTest, StartupReturnsInternalErrorAfterSuccessfulStartup) {
     ASSERT_TRUE(cloner.isActive());
 }
 
+TEST_F(DBsClonerTest, ParseAndSetAdminFirstWhenAdminInListDatabasesResponse) {
+    const Responses responsesWithAdmin = {
+        {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {name:'admin'}]}")},
+        {"listDatabases", fromjson("{ok:1, databases:[{name:'admin'}, {name:'a'}, {name:'b'}]}")},
+    };
+    std::unique_ptr<DatabasesCloner> cloner = makeDummyDatabasesCloner();
+
+    for (auto&& resp : responsesWithAdmin) {
+        auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second);
+        ASSERT_TRUE(parseResponseStatus.isOK());
+        std::vector<BSONElement> dbNamesArray = parseResponseStatus.getValue();
+        cloner->setAdminAsFirst_forTest(dbNamesArray);
+        ASSERT_EQUALS("admin", dbNamesArray[0].Obj().firstElement().str());
+    }
+}
+
+TEST_F(DBsClonerTest, ParseAndAttemptSetAdminFirstWhenAdminNotInListDatabasesResponse) {
+    const Responses responsesWithoutAdmin = {
+        {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {name:'abc'}]}")},
+        {"listDatabases", fromjson("{ok:1, databases:[{name:'foo'}, {name:'a'}, {name:'b'}]}")},
+        {"listDatabases", fromjson("{ok:1, databases:[{name:1}, {name:2}, {name:3}]}")},
+    };
+    std::unique_ptr<DatabasesCloner> cloner = makeDummyDatabasesCloner();
+
+    for (auto&& resp : responsesWithoutAdmin) {
+        auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second);
+        ASSERT_TRUE(parseResponseStatus.isOK());
+        std::vector<BSONElement> dbNamesArray = parseResponseStatus.getValue();
+        std::string expectedResult = dbNamesArray[0].Obj().firstElement().str();
+        cloner->setAdminAsFirst_forTest(dbNamesArray);
+        ASSERT_EQUALS(expectedResult, dbNamesArray[0].Obj().firstElement().str());
+    }
+}
+
+
+TEST_F(DBsClonerTest, ParseListDatabasesResponseWithMalformedResponses) {
+    Status expectedResultForNoDatabasesField{
+        ErrorCodes::BadValue,
+        "The 'listDatabases' command response does not contain a databases field."};
+    Status expectedResultForNoArrayOfDatabases{
+        ErrorCodes::BadValue,
+        "The 'listDatabases' command response is unable to be transformed into an array."};
+
+    const Responses responsesWithoutDatabasesField = {
+        {"listDatabases", fromjson("{ok:1, fake:[{name:'a'}, {name:'aab'}, {name:'foo'}]}")},
+        {"listDatabases", fromjson("{ok:1, fake:[{name:'admin'}, {name:'a'}, {name:'b'}]}")},
+    };
+
+    const Responses responsesWithoutArrayOfDatabases = {
+        {"listDatabases", fromjson("{ok:1, databases:1}")},
+        {"listDatabases", fromjson("{ok:1, databases:'abc'}")},
+    };
+
+    const Responses responsesWithInvalidAdminNameField = {
+        {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {fake:'admin'}]}")},
+        {"listDatabases", fromjson("{ok:1, databases:[{fake:'admin'}, {name:'a'}, {name:'b'}]}")},
+    };
+
+    std::unique_ptr<DatabasesCloner> cloner = makeDummyDatabasesCloner();
+
+    for (auto&& resp : responsesWithoutDatabasesField) {
+        auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second);
+        ASSERT_EQ(parseResponseStatus.getStatus(), expectedResultForNoDatabasesField);
+    }
+
+    for (auto&& resp : responsesWithoutArrayOfDatabases) {
+        auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second);
+        ASSERT_EQ(parseResponseStatus.getStatus(), expectedResultForNoArrayOfDatabases);
+    }
+
+    for (auto&& resp : responsesWithInvalidAdminNameField) {
+        auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second);
+        ASSERT_TRUE(parseResponseStatus.isOK());
+        // We expect no elements to be swapped.
+        std::vector<BSONElement> dbNamesArray = parseResponseStatus.getValue();
+        std::string expectedResult = dbNamesArray[0].Obj().firstElement().str();
+        cloner->setAdminAsFirst_forTest(dbNamesArray);
+        ASSERT_EQUALS(expectedResult, dbNamesArray[0].Obj().firstElement().str());
+    }
+}
+
 TEST_F(DBsClonerTest, FailsOnListDatabases) {
     Status result{Status::OK()};
     Status expectedResult{ErrorCodes::BadValue, "foo"};