SERVER-23761 make KVCatalog engines clear the NR feature bit for collation when possible

On startup, if the collation feature bit is set, we traverse the
metadata stored in the KVCatalog. If no collation metadata exists on a
collection or index, then we clear the feature bit and start up
successfully.

9 files changed, 134 insertions, 1 deletions

diff --git a/src/mongo/db/catalog/collection_options.cpp b/src/mongo/db/catalog/collection_options.cpp
index 69ef81e1aa8..b2aa3da4de4 100644
--- a/src/mongo/db/catalog/collection_options.cpp
+++ b/src/mongo/db/catalog/collection_options.cpp
@@ -105,6 +105,7 @@ void CollectionOptions::reset() {
     validator = BSONObj();
     validationLevel = "";
     validationAction = "";
+    collation = BSONObj();
 }
 
 bool CollectionOptions::isValid() const {
@@ -209,6 +210,16 @@ Status CollectionOptions::parse(const BSONObj& options) {
             }
 
             validationLevel = e.String();
+        } else if (fieldName == "collation") {
+            if (e.type() != mongo::Object) {
+                return Status(ErrorCodes::BadValue, "'collation' has to be a document.");
+            }
+
+            if (e.Obj().isEmpty()) {
+                return Status(ErrorCodes::BadValue, "'collation' cannot be an empty document.");
+            }
+
+            collation = e.Obj().getOwned();
         }
     }
 
@@ -259,6 +270,10 @@ BSONObj CollectionOptions::toBSON() const {
         b.append("validationAction", validationAction);
     }
 
+    if (!collation.isEmpty()) {
+        b.append("collation", collation);
+    }
+
     return b.obj();
 }
 }
diff --git a/src/mongo/db/catalog/collection_options.h b/src/mongo/db/catalog/collection_options.h
index 6dd40b2dd31..36f407e8985 100644
--- a/src/mongo/db/catalog/collection_options.h
+++ b/src/mongo/db/catalog/collection_options.h
@@ -105,5 +105,10 @@ struct CollectionOptions {
     BSONObj validator;
     std::string validationAction;
     std::string validationLevel;
+
+    // Collation information produced on a newer version, which supports the collation feature. We
+    // do the work to parse this information to ensure that we properly recognize when collation
+    // metadata is in the catalog, and downgrade must fail.
+    BSONObj collation;
 };
 }
diff --git a/src/mongo/db/catalog/collection_options_test.cpp b/src/mongo/db/catalog/collection_options_test.cpp
index b56b883e7a2..4e773f1b027 100644
--- a/src/mongo/db/catalog/collection_options_test.cpp
+++ b/src/mongo/db/catalog/collection_options_test.cpp
@@ -187,4 +187,51 @@ TEST(CollectionOptions, ModifyStorageEngineField) {
     BSONObj storageEngine1 = storageEngine.getObjectField("storageEngine1");
     ASSERT_EQUALS(1, storageEngine1.getIntField("x"));
 }
+
+TEST(CollectionOptions, FailToParseCollationThatIsNotAnObject) {
+    CollectionOptions options;
+    ASSERT_NOT_OK(options.parse(fromjson("{collation: 'notAnObject'}")));
+}
+
+TEST(CollectionOptions, FailToParseCollationThatIsAnEmptyObject) {
+    CollectionOptions options;
+    ASSERT_NOT_OK(options.parse(fromjson("{collation: {}}")));
+}
+
+TEST(CollectionOptions, CollationFieldParsesCorrectly) {
+    CollectionOptions options;
+    ASSERT_OK(options.parse(fromjson("{collation: {locale: 'en'}}")));
+    ASSERT_EQ(options.collation, fromjson("{locale: 'en'}"));
+    ASSERT_TRUE(options.isValid());
+    ASSERT_OK(options.validate());
+}
+
+TEST(CollectionOptions, ParsedCollationObjShouldBeOwned) {
+    CollectionOptions options;
+    ASSERT_OK(options.parse(fromjson("{collation: {locale: 'en'}}")));
+    ASSERT_EQ(options.collation, fromjson("{locale: 'en'}"));
+    ASSERT_TRUE(options.collation.isOwned());
+}
+
+TEST(CollectionOptions, ResetClearsCollationField) {
+    CollectionOptions options;
+    ASSERT_OK(options.parse(fromjson("{collation: {locale: 'en'}}")));
+    ASSERT_FALSE(options.collation.isEmpty());
+    options.reset();
+    ASSERT_TRUE(options.collation.isEmpty());
+}
+
+TEST(CollectionOptions, CollationFieldLeftEmptyWhenOmitted) {
+    CollectionOptions options;
+    ASSERT_OK(options.parse(fromjson("{validator: {a: 1}}")));
+    ASSERT_TRUE(options.collation.isEmpty());
+}
+
+TEST(CollectionOptions, CollationFieldNotDumpedToBSONWhenOmitted) {
+    CollectionOptions options;
+    ASSERT_OK(options.parse(fromjson("{validator: {a: 1}}")));
+    ASSERT_TRUE(options.collation.isEmpty());
+    BSONObj asBSON = options.toBSON();
+    ASSERT_FALSE(asBSON["collation"]);
+}
 }
diff --git a/src/mongo/db/storage/kv/kv_catalog_feature_tracker.h b/src/mongo/db/storage/kv/kv_catalog_feature_tracker.h
index 1024efa2a0f..55e77d9f1ed 100644
--- a/src/mongo/db/storage/kv/kv_catalog_feature_tracker.h
+++ b/src/mongo/db/storage/kv/kv_catalog_feature_tracker.h
@@ -60,7 +60,10 @@ public:
      * The next feature added to this enumeration should use the current value of 'kNextFeatureBit',
      * and 'kNextFeatureBit' should be changed to the next largest power of two.
      */
-    enum class NonRepairableFeature : std::uint64_t { kNextFeatureBit = 1 << 0 };
+    enum class NonRepairableFeature : std::uint64_t {
+        kCollation = 1 << 0,
+        kNextFeatureBit = 1 << 1
+    };
 
     using NonRepairableFeatureMask = std::underlying_type<NonRepairableFeature>::type;
 
diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp
index 4f2cd2e12f2..669a4d16da7 100644
--- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp
+++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp
@@ -114,6 +114,24 @@ void KVCollectionCatalogEntry::removePathLevelMultikeyInfoFromAllIndexes(Operati
     }
 }
 
+bool KVCollectionCatalogEntry::hasCollationMetadata(OperationContext* txn) const {
+    MetaData md = _getMetaData(txn);
+    if (!md.options.collation.isEmpty()) {
+        log() << "Collection " << md.ns << " has a default collation: " << md.options.collation;
+        return true;
+    }
+
+    bool indexWithCollationFound = false;
+    for (auto&& imd : md.indexes) {
+        if (imd.spec["collation"]) {
+            log() << "Collection " << md.ns << " has an index with a collation: " << imd.spec;
+            indexWithCollationFound = true;
+        }
+    }
+
+    return indexWithCollationFound;
+}
+
 void KVCollectionCatalogEntry::setIndexHead(OperationContext* txn,
                                             StringData indexName,
                                             const RecordId& newHead) {
diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h
index dc192180125..e8780e6ff4f 100644
--- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h
+++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h
@@ -56,6 +56,8 @@ public:
 
     void removePathLevelMultikeyInfoFromAllIndexes(OperationContext* txn) final;
 
+    bool hasCollationMetadata(OperationContext* txn) const;
+
     void setIndexHead(OperationContext* txn, StringData indexName, const RecordId& newHead) final;
 
     Status removeIndex(OperationContext* txn, StringData indexName) final;
diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
index 34f0166bd11..91afa400265 100644
--- a/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
+++ b/src/mongo/db/storage/kv/kv_database_catalog_entry.cpp
@@ -176,6 +176,18 @@ void KVDatabaseCatalogEntry::removePathLevelMultikeyInfoFromAllCollections(
     }
 }
 
+bool KVDatabaseCatalogEntry::hasCollationMetadata(OperationContext* opCtx) const {
+    for (auto&& collection : _collections) {
+        log() << "Checking collection '" << collection.first << "' for collation metadata...";
+        if (collection.second->hasCollationMetadata(opCtx)) {
+            return true;
+        }
+        log() << "Done checking collection '" << collection.first << "' for collation metadata";
+    }
+
+    return false;
+}
+
 Status KVDatabaseCatalogEntry::currentFilesCompatible(OperationContext* opCtx) const {
     // Delegate to the FeatureTracker as to whether the data files are compatible or not.
     return _engine->getCatalog()->getFeatureTracker()->isCompatibleWithCurrentCode(opCtx);
diff --git a/src/mongo/db/storage/kv/kv_database_catalog_entry.h b/src/mongo/db/storage/kv/kv_database_catalog_entry.h
index cba03001d73..240daac1c4a 100644
--- a/src/mongo/db/storage/kv/kv_database_catalog_entry.h
+++ b/src/mongo/db/storage/kv/kv_database_catalog_entry.h
@@ -66,6 +66,14 @@ public:
      */
     void removePathLevelMultikeyInfoFromAllCollections(OperationContext* opCtx);
 
+    /**
+     * Examines metadata for each collection and each index, and returns true if any
+     * collation-related metadata is found.
+     *
+     * This function is used to support downgrading from 3.4.
+     */
+    bool hasCollationMetadata(OperationContext* opCtx) const;
+
     virtual Status currentFilesCompatible(OperationContext* opCtx) const;
 
     virtual void getCollectionNamespaces(std::list<std::string>* out) const;
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index 588876140c4..14151bfcfb7 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -194,6 +194,29 @@ Status KVStorageEngine::requireDataFileCompatibilityWithPriorRelease(OperationCo
         }
     }
 
+    // If the collation feature is marked in use, we traverse the catalog to see if it actually has
+    // any collation-related metadata. The bit may have been set as in use but subsequently all
+    // indices or collections with collations were dropped in preparation for downgrade.
+    if (_catalog->getFeatureTracker()->isNonRepairableFeatureInUse(
+            opCtx, NonRepairableFeature::kCollation)) {
+        bool hasCollationMetadata = false;
+        {
+            stdx::lock_guard<stdx::mutex> lk(_dbsLock);
+            for (auto&& db : _dbs) {
+                const bool thisCollectionHasCollationMetadata =
+                    db.second->hasCollationMetadata(opCtx);
+                hasCollationMetadata = hasCollationMetadata || thisCollectionHasCollationMetadata;
+            }
+        }
+
+        if (!hasCollationMetadata) {
+            WriteUnitOfWork wuow(opCtx);
+            _catalog->getFeatureTracker()->markNonRepairableFeatureAsNotInUse(
+                opCtx, NonRepairableFeature::kCollation);
+            wuow.commit();
+        }
+    }
+
     auto status = _catalog->getFeatureTracker()->hasNoFeaturesMarkedAsInUse(opCtx);
     if (!status.isOK()) {
         return status;