SERVER-48054 On startup, upgrade all downgraded unique indexes to the latest format version

14 files changed, 288 insertions, 14 deletions

diff --git a/jstests/multiVersion/initsync_admin_system_unique_indexes.js b/jstests/multiVersion/initsync_admin_system_unique_indexes.js
new file mode 100644
index 00000000000..341e4a0e9be
--- /dev/null
+++ b/jstests/multiVersion/initsync_admin_system_unique_indexes.js
@@ -0,0 +1,65 @@
+/**
+ * Tests a scenario such that admin.system.* collections that sort lexicographically before
+ * admin.system.version are properly upgraded to the correct unique index version when upgraded to
+ * 4.4. Collections cloned before the admin.system.version collection will build unique indexes in a
+ * downgraded index format version. This test ensures that despite this, a 4.4 binary can
+ * self-correct the problem when starting up.
+ *
+ * See SERVER-48054.
+ */
+(function() {
+load('./jstests/multiVersion/libs/multi_rs.js');
+
+const nodes = {
+    n1: {binVersion: 'last-stable'},
+    n2: {binVersion: 'last-stable'},
+};
+
+jsTest.log("Starting replica set in version 4.2");
+const rst = new ReplSetTest({nodes: nodes});
+rst.startSet();
+rst.initiate();
+
+const primary = rst.getPrimary();
+const adminDB = primary.getDB('admin');
+
+// Create a custom role and a user to populate the admin.system.roles, and admin.system.users
+// collections.
+assert.commandWorked(adminDB.runCommand({createRole: "readRole", privileges: [], roles: ['read']}));
+assert.commandWorked(
+    adminDB.runCommand({createUser: "readUser", pwd: "password", roles: ['readRole']}));
+
+jsTest.log("Initial-syncing new node...");
+const newNode = rst.add({
+    rsConfig: {priority: 0, votes: 0},
+    binVersion: 'last-stable',
+});
+rst.reInitiate();
+rst.waitForState(newNode, ReplSetTest.State.SECONDARY);
+
+jsTest.log("Upgrading replica set to 4.4...");
+
+rst.upgradeSet({binVersion: "latest"});
+
+jsTest.log("Checking unique index format version");
+
+function checkFormatVersion(coll, indexName) {
+    const formatVersion = coll.aggregate({$collStats: {storageStats: {}}})
+                              .next()
+                              .storageStats.indexDetails[indexName]
+                              .metadata.formatVersion;
+    assert.eq(
+        formatVersion,
+        12,
+        `Expected ${coll.getFullName()} format version to be 12 for unique index '${indexName}'`);
+}
+
+const newNodeAdminDB = newNode.getDB('admin');
+reconnect(newNodeAdminDB);
+newNodeAdminDB.setSlaveOk(true);
+
+checkFormatVersion(newNodeAdminDB.getCollection('system.roles'), 'role_1_db_1');
+checkFormatVersion(newNodeAdminDB.getCollection('system.users'), 'user_1_db_1');
+
+rst.stopSet();
+})();
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 0d8cb1d752e..bdc7a3917a0 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -1176,6 +1176,7 @@ env.Library(
         'dbhelpers',
         'repair_database',
         'repl/repl_settings',
+        'storage/downgraded_unique_indexes',
         'storage/storage_repair_observer',
     ],
 )
diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp
index 7a19c8487ae..444c730207c 100644
--- a/src/mongo/db/repair_database_and_check_version.cpp
+++ b/src/mongo/db/repair_database_and_check_version.cpp
@@ -55,6 +55,7 @@
 #include "mongo/db/repl/replication_process.h"
 #include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/server_options.h"
+#include "mongo/db/storage/downgraded_unique_indexes.h"
 #include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_repair_observer.h"
 #include "mongo/logv2/log.h"
@@ -344,6 +345,47 @@ void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) {
 }
 
 /**
+ * Upgrades the metadata for every unique index.
+ */
+void upgradeAllUniqueIndexes(OperationContext* opCtx) {
+    invariant(opCtx->lockState()->isW());
+    invariant(serverGlobalParams.featureCompatibility.isVersionInitialized());
+
+    auto const storageEngine = opCtx->getServiceContext()->getStorageEngine();
+    auto dbNames = storageEngine->listDatabases();
+    for (auto dbName : dbNames) {
+        auto colls = CollectionCatalog::get(opCtx).getAllCollectionNamesFromDb(opCtx, dbName);
+        for (auto nss : colls) {
+            auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, nss);
+
+            // Updates the format version for all non-_id unique indexes.
+            std::vector<std::string> indexNames;
+            DurableCatalog::get(opCtx)->getAllUniqueIndexes(
+                opCtx, collection->getCatalogId(), &indexNames);
+            for (auto indexName : indexNames) {
+                const IndexDescriptor* desc =
+                    collection->getIndexCatalog()->findIndexByName(opCtx, indexName);
+                invariant(desc);
+
+                WriteUnitOfWork wuow(opCtx);
+                LOGV2(4805400,
+                      "Upgrading unique index metadata",
+                      "indexName"_attr = indexName,
+                      "collection"_attr = nss);
+
+                // Update index metadata in storage engine.
+                DurableCatalog::get(opCtx)->updateIndexMetadata(
+                    opCtx, collection->getCatalogId(), desc);
+
+                // Refresh the in-memory instance of the index.
+                collection->getIndexCatalog()->refreshEntry(opCtx, desc);
+                wuow.commit();
+            }
+        }
+    }
+}
+
+/**
  * Sets the appropriate flag on the service context decorable 'replSetMemberInStandaloneMode' to
  * 'true' if this is a replica set node running in standalone mode, otherwise 'false'.
  */
@@ -656,6 +698,15 @@ bool repairDatabasesAndCheckVersion(OperationContext* opCtx) {
                             "compatibility version is set to 4.4 or above");
     }
 
+    // If we have not upgraded all unique indexes, they must be upgraded before startup. To be
+    // conservative, we run this operation on all unique indexes, regardless of their state.
+    // See SERVER-48054.
+    if (hasDowngradedUniqueIndexes()) {
+        LOGV2(4805401, "Detected unique indexes that are not fully-upgraded");
+        upgradeAllUniqueIndexes(opCtx);
+    }
+
+
     LOGV2_DEBUG(21017, 1, "done repairDatabases");
     return nonLocalDatabases;
 }
diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index 366ce29d412..f22fde77f07 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -17,6 +17,16 @@ env.SConscript(
 )
 
 env.Library(
+    target='downgraded_unique_indexes',
+    source=[
+        'downgraded_unique_indexes.cpp',
+        ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        ],
+    )
+
+env.Library(
     target='index_entry_comparison',
     source=[
         'index_entry_comparison.cpp',
diff --git a/src/mongo/db/storage/downgraded_unique_indexes.cpp b/src/mongo/db/storage/downgraded_unique_indexes.cpp
new file mode 100644
index 00000000000..c19d4ca30f3
--- /dev/null
+++ b/src/mongo/db/storage/downgraded_unique_indexes.cpp
@@ -0,0 +1,46 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/storage/downgraded_unique_indexes.h"
+
+#include "mongo/platform/atomic_word.h"
+
+namespace mongo {
+namespace {
+AtomicWord<bool> _hasDowngradedUniqueIndexes{false};
+}
+
+bool hasDowngradedUniqueIndexes() {
+    return _hasDowngradedUniqueIndexes.load();
+}
+void setHasDowngradedUniqueIndexes() {
+    _hasDowngradedUniqueIndexes.store(true);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/downgraded_unique_indexes.h b/src/mongo/db/storage/downgraded_unique_indexes.h
new file mode 100644
index 00000000000..2fba0a1a467
--- /dev/null
+++ b/src/mongo/db/storage/downgraded_unique_indexes.h
@@ -0,0 +1,45 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+namespace mongo {
+
+/**
+ * Returns whether there is a unique index, detected at startup, that is not upgraded to the most
+ * recent version.
+ */
+bool hasDowngradedUniqueIndexes();
+
+/**
+ * Indicate that there is a unique index that is not upgraded to the most recent version.
+ */
+void setHasDowngradedUniqueIndexes();
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h
index 17860449ba6..ff1b2f924f1 100644
--- a/src/mongo/db/storage/durable_catalog.h
+++ b/src/mongo/db/storage/durable_catalog.h
@@ -188,6 +188,10 @@ public:
                                  StringData validationLevel,
                                  StringData validationAction) = 0;
 
+    virtual void updateIndexMetadata(OperationContext* opCtx,
+                                     RecordId catalogId,
+                                     const IndexDescriptor* desc) = 0;
+
     virtual Status removeIndex(OperationContext* opCtx,
                                RecordId catalogId,
                                StringData indexName) = 0;
@@ -266,6 +270,12 @@ public:
     virtual void getReadyIndexes(OperationContext* opCtx,
                                  RecordId catalogId,
                                  std::vector<std::string>* names) const = 0;
+    /**
+     * Returns names of all non-_id unique indexes on a collection, by catalogId.
+     */
+    virtual void getAllUniqueIndexes(OperationContext* opCtx,
+                                     RecordId catalogId,
+                                     std::vector<std::string>* names) const = 0;
 
     virtual bool isIndexPresent(OperationContext* opCtx,
                                 RecordId catalogId,
diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp
index 60a31a5a307..d36b2579751 100644
--- a/src/mongo/db/storage/durable_catalog_impl.cpp
+++ b/src/mongo/db/storage/durable_catalog_impl.cpp
@@ -975,6 +975,15 @@ void DurableCatalogImpl::updateValidator(OperationContext* opCtx,
     putMetaData(opCtx, catalogId, md);
 }
 
+void DurableCatalogImpl::updateIndexMetadata(OperationContext* opCtx,
+                                             RecordId catalogId,
+                                             const IndexDescriptor* desc) {
+    // Update any metadata Ident has for this index
+    const string ident = getIndexIdent(opCtx, catalogId, desc->indexName());
+    auto kvEngine = _engine->getEngine();
+    kvEngine->alterIdentMetadata(opCtx, ident, desc);
+}
+
 Status DurableCatalogImpl::removeIndex(OperationContext* opCtx,
                                        RecordId catalogId,
                                        StringData indexName) {
@@ -1202,6 +1211,19 @@ void DurableCatalogImpl::getReadyIndexes(OperationContext* opCtx,
     }
 }
 
+void DurableCatalogImpl::getAllUniqueIndexes(OperationContext* opCtx,
+                                             RecordId catalogId,
+                                             std::vector<std::string>* names) const {
+    BSONCollectionCatalogEntry::MetaData md = getMetaData(opCtx, catalogId);
+
+    for (unsigned i = 0; i < md.indexes.size(); i++) {
+        if (md.indexes[i].spec["unique"]) {
+            std::string indexName = md.indexes[i].spec["name"].String();
+            names->push_back(indexName);
+        }
+    }
+}
+
 bool DurableCatalogImpl::isIndexPresent(OperationContext* opCtx,
                                         RecordId catalogId,
                                         StringData indexName) const {
diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h
index 5c674af4824..26342094351 100644
--- a/src/mongo/db/storage/durable_catalog_impl.h
+++ b/src/mongo/db/storage/durable_catalog_impl.h
@@ -143,6 +143,10 @@ public:
                          StringData validationLevel,
                          StringData validationAction);
 
+    void updateIndexMetadata(OperationContext* opCtx,
+                             RecordId catalogId,
+                             const IndexDescriptor* desc);
+
     Status removeIndex(OperationContext* opCtx, RecordId catalogId, StringData indexName);
 
     Status prepareForIndexBuild(OperationContext* opCtx,
@@ -182,6 +186,9 @@ public:
     void getReadyIndexes(OperationContext* opCtx,
                          RecordId catalogId,
                          std::vector<std::string>* names) const;
+    void getAllUniqueIndexes(OperationContext* opCtx,
+                             RecordId catalogId,
+                             std::vector<std::string>* names) const;
 
     bool isIndexPresent(OperationContext* opCtx, RecordId catalogId, StringData indexName) const;
 
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 0935227f1d6..08c21af39c0 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -218,6 +218,10 @@ public:
      */
     virtual void flushAllFiles(OperationContext* opCtx, bool callerHoldsReadLock) {}
 
+    virtual void alterIdentMetadata(OperationContext* opCtx,
+                                    StringData ident,
+                                    const IndexDescriptor* desc){};
+
     /**
      * See StorageEngine::beginBackup for details
      */
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 6ea520aab53..73eeed396c7 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -88,6 +88,7 @@ if wiredtiger:
             '$BUILD_DIR/mongo/db/db_raii',
             '$BUILD_DIR/mongo/db/commands/server_status',
             '$BUILD_DIR/mongo/db/snapshot_window_options',
+            '$BUILD_DIR/mongo/db/storage/downgraded_unique_indexes',
             '$BUILD_DIR/mongo/db/storage/storage_repair_observer',
             '$BUILD_DIR/mongo/util/options_parser/options_parser',
             ],
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 7bdd4abab94..774277c25f9 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -44,6 +44,7 @@
 #include "mongo/db/json.h"
 #include "mongo/db/repl/repl_settings.h"
 #include "mongo/db/service_context.h"
+#include "mongo/db/storage/downgraded_unique_indexes.h"
 #include "mongo/db/storage/index_entry_comparison.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h"
@@ -515,20 +516,13 @@ KeyString::Version WiredTigerIndex::_handleVersionInfo(OperationContext* ctx,
     _dataFormatVersion = version.getValue();
 
     if (!desc->isIdIndex() && desc->unique()) {
-        Status versionStatus = _dataFormatVersion == kDataFormatV3KeyStringV0UniqueIndexVersionV1 ||
-                _dataFormatVersion == kDataFormatV4KeyStringV1UniqueIndexVersionV2
-            ? Status::OK()
-            : Status(ErrorCodes::UnsupportedFormat,
-                     str::stream()
-                         << "Index: {name: " << desc->indexName() << ", ns: " << desc->parentNS()
-                         << "} has incompatible format version: " << _dataFormatVersion
-                         << ". In MongoDB 4.2 onwards, WT secondary unique indexes use "
-                            "either format version 11 or 12. See "
-                            "https://dochub.mongodb.org/core/upgrade-4.2-procedures for "
-                            "detailed instructions on upgrading the index format. If this node is "
-                            "already upgraded to FCV 4.2, try restarting with a 4.2.4 binary to "
-                            "correct the unique index format version.");
-        fassertNoTrace(31179, versionStatus);
+        if (_dataFormatVersion != kDataFormatV3KeyStringV0UniqueIndexVersionV1 &&
+            _dataFormatVersion != kDataFormatV4KeyStringV1UniqueIndexVersionV2) {
+            invariant(_dataFormatVersion == kDataFormatV1KeyStringV0IndexVersionV1 ||
+                          _dataFormatVersion == kDataFormatV2KeyStringV1IndexVersionV2,
+                      str::stream() << _dataFormatVersion);
+            setHasDowngradedUniqueIndexes();
+        }
     }
 
     if (!isReadOnly) {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 23d134a728f..563ded97fbb 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -1748,6 +1748,20 @@ std::unique_ptr<RecordStore> WiredTigerKVEngine::makeTemporaryRecordStore(Operat
     return std::move(rs);
 }
 
+void WiredTigerKVEngine::alterIdentMetadata(OperationContext* opCtx,
+                                            StringData ident,
+                                            const IndexDescriptor* desc) {
+    WiredTigerSession session(_conn);
+    std::string uri = _uri(ident);
+
+    // Make the alter call to update metadata without taking exclusive lock to avoid conflicts with
+    // concurrent operations.
+    std::string alterString =
+        WiredTigerIndex::generateAppMetadataString(*desc) + "exclusive_refreshed=false,";
+    invariantWTOK(
+        session.getSession()->alter(session.getSession(), uri.c_str(), alterString.c_str()));
+}
+
 Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, RecoveryUnit* ru, StringData ident) {
     string uri = _uri(ident);
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 6a3074cdee9..f40d6059b5d 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -169,6 +169,10 @@ public:
 
     Status dropIdent(OperationContext* opCtx, RecoveryUnit* ru, StringData ident) override;
 
+    void alterIdentMetadata(OperationContext* opCtx,
+                            StringData ident,
+                            const IndexDescriptor* desc) override;
+
     Status okToRename(OperationContext* opCtx,
                       StringData fromNS,
                       StringData toNS,