SERVER-40175 Rebuild any missing _id indexes on startup

1 files changed, 77 insertions, 32 deletions

diff --git a/jstests/multiVersion/collection_autoIndexId_false.js b/jstests/multiVersion/collection_autoIndexId_false.js
index 836db842c40..74fe0698435 100644
--- a/jstests/multiVersion/collection_autoIndexId_false.js
+++ b/jstests/multiVersion/collection_autoIndexId_false.js
@@ -1,66 +1,111 @@
-// Cannot upgrade a server to latest if a collection does not have an index on the _id field.
-
-// Create a collection with autoIndexId: false on a 3.6 server and ensure that an upgrade to latest
-// fails.
+/*
+ * If a collection does not have an index on the _id field (because of the use of autoIndexId: false
+ * in 3.6), the latest version of MongoDB will build the missing index.
+ *
+ * Ensure that if an index cannot be built, MongoDB shuts down cleanly and allows for any duplicate
+ * documents to be deleted in the previous version.
+ */
 (function() {
     'use strict';
     load('jstests/libs/get_index_helpers.js');
 
-    // Given a connection to a 3.6 server, attempt to upgrade to 4.0 and then to latest.
-    // shouldPass determines whether or not the upgrade should be successful.
-    function upgrade36ToLatest(conn, shouldPass) {
-        const dbpath = conn.dbpath;
+    const dbName = 'test';
+    const collName = 'collection_autoIndexId_false';
 
-        // Start with 4.0 because we can't skip more than one major version at a time.
-        MongoRunner.stopMongod(conn);
-        conn = MongoRunner.runMongod({binVersion: '4.0', dbpath: dbpath, noCleanData: true});
+    // Given a dbpath to a 3.6 server, attempt to upgrade to 4.0.
+    function upgrade36To40(dbpath) {
+        let conn = MongoRunner.runMongod({binVersion: '4.0', dbpath: dbpath, noCleanData: true});
         assert.neq(null, conn, 'mongod was unable to start with version 4.0');
         let adminDb = conn.getDB('admin');
         assert.commandWorked(adminDb.runCommand({'setFeatureCompatibilityVersion': '4.0'}));
 
-        // Start again with latest.
         MongoRunner.stopMongod(conn);
-        conn = MongoRunner.runMongod({binVersion: 'latest', dbpath: dbpath, noCleanData: true});
+    }
 
+    // Given a dbpath to a 3.6 server, attempt to upgrade to latest.
+    // shouldPass determines whether or not the upgrade should be successful.
+    function upgrade40ToLatest(dbpath, shouldPass) {
         if (shouldPass) {
+            let conn =
+                MongoRunner.runMongod({binVersion: 'latest', dbpath: dbpath, noCleanData: true});
             assert.neq(null, conn, 'mongod failed to start with latest version');
+
+            // Ensure the _id index exists.
+            let testDb = conn.getDB(dbName);
+            let coll = testDb.getCollection(collName);
+            let spec = GetIndexHelpers.findByKeyPattern(coll.getIndexes(), {_id: 1});
+            assert.neq(null, spec);
+
             MongoRunner.stopMongod(conn);
         } else {
-            assert.eq(null, conn, 'mongod started with latest version but should have failed');
-        }
+            let conn = MongoRunner.runMongod(
+                {binVersion: 'latest', dbpath: dbpath, noCleanData: true, waitForConnect: false});
 
-        resetDbpath(dbpath);
+            // This tests that the server shuts down cleanly despite the inability to build the _id
+            // index.
+            assert.eq(MongoRunner.EXIT_NEED_DOWNGRADE, waitProgram(conn.pid));
+        }
     }
 
-    // Create a collection with autoIndexId: false on a 3.6 server and asset that an upgrade to
-    // latest fails.
-    function cannotUpgradeWithoutIndex() {
+    // Create a collection with autoIndexId: false on a 3.6 server and assert that an upgrade to
+    // latest fails because there are duplicate values for the _id index.
+    function cannotUpgradeWithDuplicateIds() {
         let conn = MongoRunner.runMongod({binVersion: '3.6'});
         assert.neq(null, conn, 'mongod was unable able to start with version 3.6');
 
+        const dbpath = conn.dbpath;
+
         // Create a collection with autoIndexId: false.
-        let testDb = conn.getDB('test');
-        let coll = testDb.coll;
+        let testDb = conn.getDB(dbName);
+        let coll = testDb.getCollection(collName);
         assert.commandWorked(coll.runCommand('create', {autoIndexId: false}));
+        assert.commandWorked(coll.insert({_id: 0, a: 1}));
+        assert.commandWorked(coll.insert({_id: 0, a: 2}));
+        MongoRunner.stopMongod(conn);
 
-        upgrade36ToLatest(conn, false);
+        // The upgrade to 4.0 should always succeed because it does not care about the _id index.
+        upgrade36To40(dbpath);
+
+        let shouldPass = false;
+        upgrade40ToLatest(dbpath, shouldPass);
+
+        // Remove the duplicate (now in 4.0) and retry the upgrade to 4.2.
+        conn = MongoRunner.runMongod({binVersion: '4.0', dbpath: dbpath, noCleanData: true});
+        testDb = conn.getDB(dbName);
+        coll = testDb.getCollection(collName);
+        assert.commandWorked(coll.remove({_id: 0, a: 2}));
+        MongoRunner.stopMongod(conn);
+
+        shouldPass = true;
+        upgrade40ToLatest(dbpath, shouldPass);
+
+        resetDbpath(dbpath);
     }
 
-    // Create a collection with autoIndexId: false on a 3.6 server, manually create an index on the
-    // _id field, and ensure that an upgrade to latest passes.
-    function canUpgradeAfterCreatingIndex() {
+    // Create a collection with autoIndexId: false on a 3.6 server and assert that an upgrade to
+    // latest succeeds because the missing _id index is built.
+    function canUpgradeWithoutIndex() {
         let conn = MongoRunner.runMongod({binVersion: '3.6'});
         assert.neq(null, conn, 'mongod was unable able to start with version 3.6');
 
-        // Create a collection with autoIndexId: false and then manually create the _id index.
-        let testDb = conn.getDB('test');
-        let coll = testDb.coll;
+        const dbpath = conn.dbpath;
+
+        // Create a collection with autoIndexId: false.
+        let testDb = conn.getDB(dbName);
+        let coll = testDb.getCollection(collName);
         assert.commandWorked(coll.runCommand('create', {autoIndexId: false}));
-        assert.commandWorked(coll.ensureIndex({_id: 1}, {name: '_id_'}));
+        assert.commandWorked(coll.insert({_id: 0, a: 1}));
+        MongoRunner.stopMongod(conn);
 
-        upgrade36ToLatest(conn, true);
+        // The upgrade to 4.0 should always succeed because it does not care about the _id index.
+        upgrade36To40(dbpath);
+
+        const shouldPass = true;
+        upgrade40ToLatest(dbpath, shouldPass);
+
+        resetDbpath(dbpath);
     }
 
-    cannotUpgradeWithoutIndex();
-    canUpgradeAfterCreatingIndex();
+    cannotUpgradeWithDuplicateIds();
+    canUpgradeWithoutIndex();
 })();