SERVER-63578 Convert a unique index to a non-unique index via the collMod command

14 files changed, 195 insertions, 20 deletions

diff --git a/jstests/core/collmod_convert_to_unique.js b/jstests/core/collmod_convert_index_uniqueness.js
index 2d5871e69bd..30a11cb3a48 100644
--- a/jstests/core/collmod_convert_to_unique.js
+++ b/jstests/core/collmod_convert_index_uniqueness.js
@@ -1,5 +1,5 @@
 /**
- * Basic js tests for the collMod command converting regular indexes to unique indexes.
+ * Basic js tests for the collMod command converting between regular indexes and unique indexes.
  *
  * @tags: [
  *  # Cannot implicitly shard accessed collections because of collection existing when none
@@ -135,7 +135,7 @@ assert.commandFailedWithCode(
 assert.commandWorked(coll.remove({_id: 3}));
 
 // Successfully converts to a unique index.
-const result = assert.commandWorked(
+let result = assert.commandWorked(
     db.runCommand({collMod: collName, index: {keyPattern: {a: 1}, unique: true}}));
 
 // New index state should be reflected in 'unique_new' field in collMod response.
@@ -156,4 +156,35 @@ assert.eq(countUnique({a: 1}), 1, 'index should be unique now: ' + tojson(coll.g
 
 // Test uniqueness constraint.
 assert.commandFailedWithCode(coll.insert({_id: 100, a: 100}), ErrorCodes.DuplicateKey);
+
+//
+// Converting to non-unique index tests.
+//
+
+// Tries to modify with a false 'forceNonUnique' value.
+assert.commandFailedWithCode(db.runCommand({
+    collMod: collName,
+    index: {keyPattern: {a: 1}, forceNonUnique: false},
+}),
+                             ErrorCodes.BadValue);
+
+// Successfully converts to a regular index.
+result = assert.commandWorked(
+    db.runCommand({collMod: collName, index: {keyPattern: {a: 1}, forceNonUnique: true}}));
+
+// New index state should be reflected in 'forceNonUnique_new' field in collMod response.
+const assertForceNonUniqueNew = function(result) {
+    assert(result.hasOwnProperty('forceNonUnique_new'), tojson(result));
+    assert(result.forceNonUnique_new, tojson(result));
+};
+if (db.getMongo().isMongos()) {
+    // Checks the first shard's result from mongos.
+    assert(result.hasOwnProperty('raw'), tojson(result));
+    assertForceNonUniqueNew(Object.values(result.raw)[0]);
+} else {
+    assertForceNonUniqueNew(result);
+}
+
+// Tests the index now accepts duplicate keys.
+assert.commandWorked(coll.insert({_id: 100, a: 100}));
 })();
diff --git a/jstests/multiVersion/targetedTestsLastLtsFeatures/collMod_convert_to_non_unique.js b/jstests/multiVersion/targetedTestsLastLtsFeatures/collMod_convert_to_non_unique.js
new file mode 100644
index 00000000000..5c99098ba40
--- /dev/null
+++ b/jstests/multiVersion/targetedTestsLastLtsFeatures/collMod_convert_to_non_unique.js
@@ -0,0 +1,73 @@
+/**
+ * Tests that a unique index built in last-lts can be converted to a regular index after upgrading
+ * to latest version.
+ */
+
+(function() {
+"use strict";
+load('./jstests/multiVersion/libs/multi_rs.js');
+
+const lastLTSVersion = "last-lts";
+const latestVersion = "latest";
+
+const rst = new ReplSetTest({
+    nodes: 3,
+    nodeOptions: {binVersion: lastLTSVersion},
+});
+
+jsTestLog("Running test with version " + lastLTSVersion);
+rst.startSet();
+rst.initiate();
+
+let primary = rst.getPrimary();
+let testDB = primary.getDB('test');
+const collName = 'collMod_convert_to_non_unique';
+let coll = testDB.getCollection(collName);
+coll.drop();
+
+assert.commandWorked(testDB.createCollection(collName));
+
+// Creates a unique index and verifies it rejects duplicate keys.
+assert.commandWorked(coll.createIndex({a: 1}, {unique: true}));
+assert.commandWorked(coll.insertMany([{_id: 0, a: 0}, {_id: 1, a: 1}]));
+assert.commandFailedWithCode(coll.insert({_id: 2, a: 1}), ErrorCodes.DuplicateKey);
+
+jsTestLog("Upgrading version " + latestVersion);
+rst.upgradeSet({binVersion: latestVersion});
+
+primary = rst.getPrimary();
+testDB = primary.getDB('test');
+coll = testDB.getCollection(collName);
+
+let primaryAdminDB = primary.getDB("admin");
+assert.commandWorked(primaryAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+const collModIndexUniqueEnabled =
+    assert.commandWorked(primary.adminCommand({getParameter: 1, featureFlagCollModIndexUnique: 1}))
+        .featureFlagCollModIndexUnique.value;
+
+if (!collModIndexUniqueEnabled) {
+    jsTestLog('Skipping test because the collMod unique index feature flag is disabled');
+    rst.stopSet();
+    return;
+}
+
+// Converts the unique index back to non-unique and verifies it accepts duplicate keys.
+assert.commandWorked(
+    testDB.runCommand({collMod: collName, index: {keyPattern: {a: 1}, forceNonUnique: true}}));
+assert.commandWorked(coll.insert({_id: 2, a: 1}));
+
+// Attempts to convert the index to unique again but fails because of the duplicate.
+assert.commandWorked(
+    testDB.runCommand({collMod: collName, index: {keyPattern: {a: 1}, prepareUnique: true}}));
+assert.commandFailedWithCode(
+    testDB.runCommand({collMod: collName, index: {keyPattern: {a: 1}, unique: true}}),
+    ErrorCodes.CannotConvertIndexToUnique);
+
+// Removes the duplicate key and converts the index back to unique successfully.
+assert.commandWorked(coll.deleteOne({_id: 2}));
+assert.commandWorked(
+    testDB.runCommand({collMod: collName, index: {keyPattern: {a: 1}, unique: true}}));
+
+rst.stopSet();
+})();
diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp
index 0ffc15e0e60..747de3e044a 100644
--- a/src/mongo/db/catalog/coll_mod.cpp
+++ b/src/mongo/db/catalog/coll_mod.cpp
@@ -185,7 +185,8 @@ StatusWith<ParsedCollModRequest> parseCollModRequest(OperationContext* opCtx,
             }
 
             if (!cmdIndex.getExpireAfterSeconds() && !cmdIndex.getHidden() &&
-                !cmdIndex.getUnique() && !cmdIndex.getPrepareUnique()) {
+                !cmdIndex.getUnique() && !cmdIndex.getPrepareUnique() &&
+                !cmdIndex.getForceNonUnique()) {
                 return Status(ErrorCodes::InvalidOptions,
                               "no expireAfterSeconds, hidden, unique, or prepareUnique field");
             }
@@ -201,6 +202,13 @@ StatusWith<ParsedCollModRequest> parseCollModRequest(OperationContext* opCtx,
                             serverGlobalParams.featureCompatibility));
             }
 
+            if (cmdIndex.getUnique() && cmdIndex.getForceNonUnique()) {
+                return Status(
+                    ErrorCodes::InvalidOptions,
+                    "collMod does not support 'unique' and 'forceNonUnique' options at the "
+                    "same time");
+            }
+
             if (cmdIndex.getExpireAfterSeconds()) {
                 if (isTimeseries) {
                     return Status(ErrorCodes::InvalidOptions,
@@ -305,7 +313,7 @@ StatusWith<ParsedCollModRequest> parseCollModRequest(OperationContext* opCtx,
 
                 cmr.numModifications++;
                 if (bool unique = *cmdIndex.getUnique(); !unique) {
-                    return Status(ErrorCodes::BadValue, "Cannot make index non-unique");
+                    return Status(ErrorCodes::BadValue, "'Unique: false' option is not supported");
                 }
 
                 // Attempting to converting a unique index should be treated as a no-op.
@@ -350,6 +358,21 @@ StatusWith<ParsedCollModRequest> parseCollModRequest(OperationContext* opCtx,
                 }
             }
 
+            if (cmdIndex.getForceNonUnique()) {
+                cmr.numModifications++;
+                if (bool unique = *cmdIndex.getForceNonUnique(); !unique) {
+                    return Status(ErrorCodes::BadValue, "'forceNonUnique: false' is not supported");
+                }
+
+                // Attempting to convert a non-unique index should be treated as a no-op.
+                if (!cmrIndex->idx->unique()) {
+                    indexObjForOplog =
+                        indexObjForOplog.removeField(CollModIndex::kForceNonUniqueFieldName);
+                } else {
+                    cmrIndex->indexForceNonUnique = true;
+                }
+            }
+
             // The index options doc must contain either the name or key pattern, but not both.
             // If we have just one field, the index modifications requested matches the current
             // state in catalog and there is nothing further to do.
diff --git a/src/mongo/db/catalog/coll_mod_index.cpp b/src/mongo/db/catalog/coll_mod_index.cpp
index 7be401d110b..4a6a26237ea 100644
--- a/src/mongo/db/catalog/coll_mod_index.cpp
+++ b/src/mongo/db/catalog/coll_mod_index.cpp
@@ -128,8 +128,7 @@ void getKeysForIndex(OperationContext* opCtx,
 }
 
 /**
- * Adjusts unique setting on an index.
- * An index can be converted to unique but removing the uniqueness property is not allowed.
+ * Adjusts unique setting on an index to true.
  */
 void _processCollModIndexRequestUnique(OperationContext* opCtx,
                                        AutoGetCollection* autoColl,
@@ -150,7 +149,7 @@ void _processCollModIndexRequestUnique(OperationContext* opCtx,
     }
 
     *newUnique = true;
-    autoColl->getWritableCollection(opCtx)->updateUniqueSetting(opCtx, idx->indexName());
+    autoColl->getWritableCollection(opCtx)->updateUniqueSetting(opCtx, idx->indexName(), true);
     // Resets 'prepareUnique' to false after converting to unique index;
     autoColl->getWritableCollection(opCtx)->updatePrepareUniqueSetting(
         opCtx, idx->indexName(), false);
@@ -173,6 +172,19 @@ void _processCollModIndexRequestPrepareUnique(OperationContext* opCtx,
     }
 }
 
+/**
+ * Adjusts unique setting on an index to false.
+ */
+void _processCollModIndexRequestForceNonUnique(OperationContext* opCtx,
+                                               AutoGetCollection* autoColl,
+                                               const IndexDescriptor* idx,
+                                               boost::optional<bool>* newForceNonUnique) {
+    invariant(idx->unique(), str::stream() << "Index is already non-unique: " << idx->infoObj());
+
+    *newForceNonUnique = true;
+    autoColl->getWritableCollection(opCtx)->updateUniqueSetting(opCtx, idx->indexName(), false);
+}
+
 }  // namespace
 
 void processCollModIndexRequest(OperationContext* opCtx,
@@ -186,9 +198,11 @@ void processCollModIndexRequest(OperationContext* opCtx,
     auto indexHidden = collModIndexRequest.indexHidden;
     auto indexUnique = collModIndexRequest.indexUnique;
     auto indexPrepareUnique = collModIndexRequest.indexPrepareUnique;
+    auto indexForceNonUnique = collModIndexRequest.indexForceNonUnique;
 
     // Return early if there are no index modifications requested.
-    if (!indexExpireAfterSeconds && !indexHidden && !indexUnique && !indexPrepareUnique) {
+    if (!indexExpireAfterSeconds && !indexHidden && !indexUnique && !indexPrepareUnique &&
+        !indexForceNonUnique) {
         return;
     }
 
@@ -199,6 +213,7 @@ void processCollModIndexRequest(OperationContext* opCtx,
     boost::optional<bool> newUnique;
     boost::optional<bool> newPrepareUnique;
     boost::optional<bool> oldPrepareUnique;
+    boost::optional<bool> newForceNonUnique;
 
     // TTL Index
     if (indexExpireAfterSeconds) {
@@ -224,6 +239,11 @@ void processCollModIndexRequest(OperationContext* opCtx,
             opCtx, autoColl, idx, *indexPrepareUnique, &newPrepareUnique, &oldPrepareUnique);
     }
 
+    // User wants to convert an index back to be non-unique.
+    if (indexForceNonUnique) {
+        _processCollModIndexRequestForceNonUnique(opCtx, autoColl, idx, &newForceNonUnique);
+    }
+
     *indexCollModInfo =
         IndexCollModInfo{!newExpireSecs ? boost::optional<Seconds>() : Seconds(*newExpireSecs),
                          !oldExpireSecs ? boost::optional<Seconds>() : Seconds(*oldExpireSecs),
@@ -232,13 +252,14 @@ void processCollModIndexRequest(OperationContext* opCtx,
                          newUnique,
                          newPrepareUnique,
                          oldPrepareUnique,
+                         newForceNonUnique,
                          idx->indexName()};
 
     // This matches the default for IndexCatalog::refreshEntry().
     auto flags = CreateIndexEntryFlags::kIsReady;
 
     // Update data format version in storage engine metadata for index.
-    if (indexUnique) {
+    if (indexUnique || indexForceNonUnique) {
         flags = CreateIndexEntryFlags::kIsReady | CreateIndexEntryFlags::kUpdateMetadata;
     }
 
@@ -254,6 +275,7 @@ void processCollModIndexRequest(OperationContext* opCtx,
                                      newUnique,
                                      oldPrepareUnique,
                                      newPrepareUnique,
+                                     newForceNonUnique,
                                      result](boost::optional<Timestamp>) {
         // add the fields to BSONObjBuilder result
         if (oldExpireSecs) {
@@ -276,6 +298,10 @@ void processCollModIndexRequest(OperationContext* opCtx,
             result->append("prepareUnique_old", *oldPrepareUnique);
             result->append("prepareUnique_new", *newPrepareUnique);
         }
+        if (newForceNonUnique) {
+            invariant(*newForceNonUnique);
+            result->appendBool("forceNonUnique_new", true);
+        }
     });
 
     if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) {
diff --git a/src/mongo/db/catalog/coll_mod_index.h b/src/mongo/db/catalog/coll_mod_index.h
index acd7849d70e..ed6f24f4dd5 100644
--- a/src/mongo/db/catalog/coll_mod_index.h
+++ b/src/mongo/db/catalog/coll_mod_index.h
@@ -49,6 +49,7 @@ struct ParsedCollModIndexRequest {
     boost::optional<bool> indexHidden;
     boost::optional<bool> indexUnique;
     boost::optional<bool> indexPrepareUnique;
+    boost::optional<bool> indexForceNonUnique;
 };
 
 /**
diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h
index aa5bc579c4e..d2e76c12993 100644
--- a/src/mongo/db/catalog/collection.h
+++ b/src/mongo/db/catalog/collection.h
@@ -604,10 +604,9 @@ public:
     virtual void updateHiddenSetting(OperationContext* opCtx, StringData idxName, bool hidden) = 0;
 
     /*
-     * Converts the the given index to be unique.
-     * This is a one-way transformation - the uniqueness constraint cannot be removed.
+     * Converts the the given index to be unique or non-unique.
      */
-    virtual void updateUniqueSetting(OperationContext* opCtx, StringData idxName) = 0;
+    virtual void updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) = 0;
 
     /*
      * Disallows or allows new duplicates in the given index.
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index fa9a7d0dff4..92aec2083c5 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -1979,12 +1979,12 @@ void CollectionImpl::updateHiddenSetting(OperationContext* opCtx, StringData idx
     });
 }
 
-void CollectionImpl::updateUniqueSetting(OperationContext* opCtx, StringData idxName) {
+void CollectionImpl::updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) {
     int offset = _metadata->findIndexOffset(idxName);
     invariant(offset >= 0);
 
     _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) {
-        md.indexes[offset].updateUniqueSetting();
+        md.indexes[offset].updateUniqueSetting(unique);
     });
 }
 
diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h
index 5c21e188dc1..f89cdf96ede 100644
--- a/src/mongo/db/catalog/collection_impl.h
+++ b/src/mongo/db/catalog/collection_impl.h
@@ -443,7 +443,7 @@ public:
 
     void updateHiddenSetting(OperationContext* opCtx, StringData idxName, bool hidden) final;
 
-    void updateUniqueSetting(OperationContext* opCtx, StringData idxName) final;
+    void updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) final;
 
     void updatePrepareUniqueSetting(OperationContext* opCtx,
                                     StringData idxName,
diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h
index 65529dfd820..6f9f4736af4 100644
--- a/src/mongo/db/catalog/collection_mock.h
+++ b/src/mongo/db/catalog/collection_mock.h
@@ -411,7 +411,7 @@ public:
         std::abort();
     }
 
-    void updateUniqueSetting(OperationContext* opCtx, StringData idxName) {
+    void updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) {
         std::abort();
     }
 
diff --git a/src/mongo/db/coll_mod.idl b/src/mongo/db/coll_mod.idl
index a79677d9589..1b72151b12f 100644
--- a/src/mongo/db/coll_mod.idl
+++ b/src/mongo/db/coll_mod.idl
@@ -67,6 +67,10 @@ structs:
                 optional: true
                 type: safeBool
                 unstable: true
+            forceNonUnique:
+                optional: true
+                type: safeBool
+                unstable: true
 
     CollModReply:
         description: "The collMod command's reply."
@@ -100,6 +104,10 @@ structs:
                 optional: true
                 type: safeBool
                 unstable: true
+            forceNonUnique_new:
+                optional: true
+                type: safeBool
+                unstable: true
 
     CollModRequest:
         description: "The collMod command's request."
diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h
index 030521b9f4d..f1e8031c440 100644
--- a/src/mongo/db/op_observer.h
+++ b/src/mongo/db/op_observer.h
@@ -101,6 +101,7 @@ struct IndexCollModInfo {
     boost::optional<bool> unique;
     boost::optional<bool> prepareUnique;
     boost::optional<bool> oldPrepareUnique;
+    boost::optional<bool> forceNonUnique;
     std::string indexName;
 };
 
diff --git a/src/mongo/db/op_observer_util.cpp b/src/mongo/db/op_observer_util.cpp
index cd79324a2aa..60a5902cd82 100644
--- a/src/mongo/db/op_observer_util.cpp
+++ b/src/mongo/db/op_observer_util.cpp
@@ -67,6 +67,9 @@ BSONObj makeCollModCmdObj(const BSONObj& collModCmd,
             if (indexInfo->prepareUnique)
                 indexObjBuilder.append("prepareUnique", indexInfo->prepareUnique.get());
 
+            if (indexInfo->forceNonUnique)
+                indexObjBuilder.append("forceNonUnique", indexInfo->forceNonUnique.get());
+
             cmdObjBuilder.append(indexFieldName, indexObjBuilder.obj());
         } else {
             cmdObjBuilder.append(elem);
diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp
index 58cdc01fe39..2e45db1a989 100644
--- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp
+++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp
@@ -140,9 +140,19 @@ void BSONCollectionCatalogEntry::IndexMetaData::updateHiddenSetting(bool hidden)
 }
 
 
-void BSONCollectionCatalogEntry::IndexMetaData::updateUniqueSetting() {
-    BSONObjBuilder b(spec);
-    b.appendBool("unique", true);
+void BSONCollectionCatalogEntry::IndexMetaData::updateUniqueSetting(bool unique) {
+    // If unique == false, we remove this field from catalog rather than add a field with false.
+    BSONObjBuilder b;
+    for (BSONObjIterator bi(spec); bi.more();) {
+        BSONElement e = bi.next();
+        if (e.fieldNameStringData() != "unique") {
+            b.append(e);
+        }
+    }
+
+    if (unique) {
+        b.append("unique", unique);
+    }
     spec = b.obj();
 }
 
diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h
index 18bca894a71..b2afd733205 100644
--- a/src/mongo/db/storage/bson_collection_catalog_entry.h
+++ b/src/mongo/db/storage/bson_collection_catalog_entry.h
@@ -98,7 +98,7 @@ public:
 
         void updateHiddenSetting(bool hidden);
 
-        void updateUniqueSetting();
+        void updateUniqueSetting(bool unique);
 
         void updatePrepareUniqueSetting(bool prepareUnique);