SERVER-34414 Create system indexes using the normal index creation and replication process.

             Do not create them directly on secondaries.
             Do create oplog entries for index creation.

7 files changed, 156 insertions, 16 deletions

diff --git a/jstests/replsets/buildindexes_false_with_system_indexes.js b/jstests/replsets/buildindexes_false_with_system_indexes.js
new file mode 100644
index 00000000000..2d39359ddbf
--- /dev/null
+++ b/jstests/replsets/buildindexes_false_with_system_indexes.js
@@ -0,0 +1,91 @@
+/*
+ * Tests that hidden nodes with buildIndexes: false behave correctly when system tables with
+ * default indexes are created.
+ *
+ */
+(function() {
+    'use strict';
+
+    load("jstests/replsets/rslib.js");
+
+    const testName = "buildindexes_false_with_system_indexes";
+
+    let rst = new ReplSetTest({
+        name: testName,
+        nodes: [
+            {},
+            {rsConfig: {priority: 0}},
+            {rsConfig: {priority: 0, hidden: true, buildIndexes: false}},
+        ],
+    });
+    const nodes = rst.startSet();
+    rst.initiate();
+
+    let primary = rst.getPrimary();
+    assert.eq(primary, nodes[0]);
+    let secondary = nodes[1];
+    const hidden = nodes[2];
+
+    rst.awaitReplication();
+    jsTestLog("Creating a role in the admin database");
+    let adminDb = primary.getDB("admin");
+    adminDb.createRole(
+        {role: 'test_role', roles: [{role: 'readWrite', db: 'test'}], privileges: []});
+    rst.awaitReplication();
+
+    jsTestLog("Creating a user in the admin database");
+    adminDb.createUser({user: 'test_user', pwd: 'test', roles: [{role: 'test_role', db: 'admin'}]});
+    rst.awaitReplication();
+
+    // Make sure the indexes we expect are present on all nodes.  The buildIndexes: false node
+    // should have only the _id_ index.
+    let secondaryAdminDb = secondary.getDB("admin");
+    const hiddenAdminDb = hidden.getDB("admin");
+
+    assert.eq(["_id_", "user_1_db_1"], adminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_", "role_1_db_1"], adminDb.system.roles.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_", "user_1_db_1"],
+              secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_", "role_1_db_1"],
+              secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+
+    // Drop the indexes and restart the secondary.  The indexes should not be re-created.
+    jsTestLog("Dropping system indexes and restarting secondary.");
+    adminDb.system.users.dropIndex("user_1_db_1");
+    adminDb.system.roles.dropIndex("role_1_db_1");
+    rst.awaitReplication();
+    assert.eq(["_id_"], adminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], adminDb.system.roles.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+
+    secondary = rst.restart(secondary, {}, true /* wait for node to become healthy */);
+    secondaryAdminDb = secondary.getDB("admin");
+    assert.eq(["_id_"], secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+
+    jsTestLog("Now restarting primary; indexes should be created.");
+    rst.restart(primary);
+    primary = rst.getPrimary();
+    adminDb = primary.getDB("admin");
+    assert.soonNoExcept(() => {
+        assert.eq(["_id_", "user_1_db_1"],
+                  adminDb.system.users.getIndexes().map(x => x.name).sort());
+        assert.eq(["_id_", "role_1_db_1"],
+                  adminDb.system.roles.getIndexes().map(x => x.name).sort());
+        return true;
+    });
+    rst.awaitReplication();
+    assert.eq(["_id_", "user_1_db_1"],
+              secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_", "role_1_db_1"],
+              secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort());
+    assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort());
+
+    rst.stopSet();
+}());
diff --git a/jstests/replsets/oplog_note_cmd.js b/jstests/replsets/oplog_note_cmd.js
index 0c92609535a..77757b457c2 100644
--- a/jstests/replsets/oplog_note_cmd.js
+++ b/jstests/replsets/oplog_note_cmd.js
@@ -13,9 +13,10 @@ var statusBefore = db.runCommand({replSetGetStatus: 1});
 assert.commandWorked(db.runCommand({appendOplogNote: 1, data: {a: 1}}));
 var statusAfter = db.runCommand({replSetGetStatus: 1});
 if (rs.getReplSetConfigFromNode().protocolVersion != 1) {
-    assert.lt(statusBefore.members[0].optime, statusAfter.members[0].optime);
+    assert.lt(bsonWoCompare(statusBefore.members[0].optime, statusAfter.members[0].optime), 0);
 } else {
-    assert.lt(statusBefore.members[0].optime.ts, statusAfter.members[0].optime.ts);
+    assert.lt(bsonWoCompare(statusBefore.members[0].optime.ts, statusAfter.members[0].optime.ts),
+              0);
 }
 
 // Make sure note written successfully
diff --git a/src/mongo/db/auth/auth_index_d.cpp b/src/mongo/db/auth/auth_index_d.cpp
index 47987ac40c4..9944a582f5c 100644
--- a/src/mongo/db/auth/auth_index_d.cpp
+++ b/src/mongo/db/auth/auth_index_d.cpp
@@ -47,6 +47,7 @@
 #include "mongo/db/db_raii.h"
 #include "mongo/db/index/index_descriptor.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/util/assert_util.h"
 #include "mongo/util/log.h"
@@ -105,6 +106,15 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx,
         return;
     }
 
+    // Do not try to generate any system indexes on a secondary.
+    auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+    uassert(ErrorCodes::NotMaster,
+            "Not primary while creating authorization index",
+            replCoord->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet ||
+                replCoord->canAcceptWritesForDatabase(ns.db()));
+
+    invariant(!opCtx->lockState()->inAWriteUnitOfWork());
+
     try {
         auto indexSpecStatus = index_key_validate::validateIndexSpec(
             spec.toBSON(), ns, serverGlobalParams.featureCompatibility);
@@ -115,8 +125,10 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx,
 
         MultiIndexBlock indexer(opCtx, collection);
 
+        std::vector<BSONObj> indexInfoObjs;
         MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
-            fassertStatusOK(40453, indexer.init(indexSpec));
+            indexInfoObjs = fassertStatusOK(40453, indexer.init(indexSpec));
+            invariant(indexInfoObjs.size() == 1);
         }
         MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "authorization index regeneration", ns.ns());
 
@@ -126,6 +138,8 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx,
             WriteUnitOfWork wunit(opCtx);
 
             indexer.commit();
+            opCtx->getServiceContext()->getOpObserver()->onCreateIndex(
+                opCtx, ns.getSystemIndexesCollection(), indexInfoObjs[0], false /* fromMigrate */);
 
             wunit.commit();
         }
@@ -205,22 +219,23 @@ Status verifySystemIndexes(OperationContext* txn) {
 void createSystemIndexes(OperationContext* txn, Collection* collection) {
     invariant(collection);
     const NamespaceString& ns = collection->ns();
+    BSONObj indexSpec;
     if (ns == AuthorizationManager::usersCollectionNamespace) {
-        auto indexSpec = fassertStatusOK(
+        indexSpec = fassertStatusOK(
             40455,
             index_key_validate::validateIndexSpec(
                 v3SystemUsersIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility));
-
-        fassertStatusOK(
-            40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec));
     } else if (ns == AuthorizationManager::rolesCollectionNamespace) {
-        auto indexSpec = fassertStatusOK(
+        indexSpec = fassertStatusOK(
             40457,
             index_key_validate::validateIndexSpec(
                 v3SystemRolesIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility));
-
+    }
+    if (!indexSpec.isEmpty()) {
+        txn->getServiceContext()->getOpObserver()->onCreateIndex(
+            txn, ns.getSystemIndexesCollection(), indexSpec, false /* fromMigrate */);
         fassertStatusOK(
-            40458, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec));
+            40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec));
     }
 }
 
diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp
index 23fdfb84e95..4b27ff0e48f 100644
--- a/src/mongo/db/catalog/database.cpp
+++ b/src/mongo/db/catalog/database.cpp
@@ -564,15 +564,26 @@ Collection* Database::createCollection(OperationContext* txn,
                                        : ic->getDefaultIdIndexSpec(featureCompatibilityVersion)));
             }
         }
-
-        if (nss.isSystem()) {
-            authindex::createSystemIndexes(txn, collection);
-        }
     }
 
     getGlobalServiceContext()->getOpObserver()->onCreateCollection(
         txn, nss, options, fullIdIndexSpec);
 
+    // It is necessary to create the system index *after* running the onCreateCollection so that
+    // the oplog timestamp for the index creation is after the oplog timestamp for the
+    // collection creation. This way both primary and any secondaries will see the index created
+    // after the collection is created.
+    if (createIdIndex && nss.isSystem()) {
+        // We only want to create the indexes here on the primary.  On secondaries, they will
+        // be created by the normal oplog application process.
+        auto coordinator = repl::ReplicationCoordinator::get(txn);
+        const bool canAcceptWrites =
+            (coordinator->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet) ||
+            coordinator->canAcceptWritesForDatabase(nss.db()) || nss.isSystemDotProfile();
+        if (canAcceptWrites) {
+            authindex::createSystemIndexes(txn, collection);
+        }
+    }
     return collection;
 }
 
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index a0850ef493e..05fc368e3b6 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -724,6 +724,9 @@ ExitCode _initAndListen(int listenPort) {
             log() << redact(status);
             if (status.code() == ErrorCodes::AuthSchemaIncompatible) {
                 exitCleanly(EXIT_NEED_UPGRADE);
+            } else if (status == ErrorCodes::NotMaster) {
+                // Try creating the indexes if we become master.  If we do not become master,
+                // the master will create the indexes and we will replicate them.
             } else {
                 quickExit(EXIT_FAILURE);
             }
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 05d070c08fe..c8737ba0090 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -37,6 +37,9 @@
 #include "mongo/base/init.h"
 #include "mongo/base/status_with.h"
 #include "mongo/bson/oid.h"
+#include "mongo/db/auth/auth_index_d.h"
+#include "mongo/db/auth/authorization_manager.h"
+#include "mongo/db/auth/authorization_manager_global.h"
 #include "mongo/db/catalog/database.h"
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/client.h"
@@ -454,6 +457,16 @@ OpTime ReplicationCoordinatorExternalStateImpl::onTransitionToPrimary(OperationC
     _shardingOnTransitionToPrimaryHook(txn);
     _dropAllTempCollections(txn);
 
+    // It is only necessary to check the system indexes on the first transition to master.
+    // On subsequent transitions to master the indexes will have already been created.
+    static std::once_flag verifySystemIndexesOnce;
+    std::call_once(verifySystemIndexesOnce, [txn] {
+        const auto globalAuthzManager = AuthorizationManager::get(txn->getServiceContext());
+        if (globalAuthzManager->shouldValidateAuthSchemaOnStartup()) {
+            fassert(65536, authindex::verifySystemIndexes(txn));
+        }
+    });
+
     serverGlobalParams.featureCompatibility.validateFeaturesAsMaster.store(true);
 
     return opTimeToReturn;
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index b0fdc935007..4ebbc0198d3 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -1290,6 +1290,9 @@ var ReplSetTest = function(opts) {
             // liveNodes must have been populated.
             var primary = rst.liveNodes.master;
             var combinedDBs = new Set(primary.getDBNames());
+            // replSetConfig will be undefined for master/slave passthrough.
+            const replSetConfig =
+                rst.getReplSetConfigFromNode ? rst.getReplSetConfigFromNode() : undefined;
 
             rst.getSecondaries().forEach(secondary => {
                 secondary.getDBNames().forEach(dbName => combinedDBs.add(dbName));
@@ -1378,8 +1381,10 @@ var ReplSetTest = function(opts) {
                     // Check that the following collection stats are the same across replica set
                     // members:
                     //  capped
-                    //  nindexes
+                    //  nindexes, except on nodes with buildIndexes: false
                     //  ns
+                    const hasSecondaryIndexes = !replSetConfig ||
+                        replSetConfig.members[rst.getNodeId(secondary)].buildIndexes !== false;
                     primaryCollections.forEach(collName => {
                         var primaryCollStats =
                             primary.getDB(dbName).runCommand({collStats: collName});
@@ -1389,7 +1394,8 @@ var ReplSetTest = function(opts) {
                         assert.commandWorked(secondaryCollStats);
 
                         if (primaryCollStats.capped !== secondaryCollStats.capped ||
-                            primaryCollStats.nindexes !== secondaryCollStats.nindexes ||
+                            (hasSecondaryIndexes &&
+                             primaryCollStats.nindexes !== secondaryCollStats.nindexes) ||
                             primaryCollStats.ns !== secondaryCollStats.ns) {
                             print(msgPrefix +
                                   ', the primary and secondary have different stats for the ' +