SERVER-65933 Enable creation of columnstore index

Adds a new feature flag which must be enabled to create one.

7 files changed, 90 insertions, 27 deletions

diff --git a/jstests/noPassthrough/collmod_index_noop.js b/jstests/noPassthrough/collmod_index_noop.js
index e090fe87935..e10857e8dbe 100644
--- a/jstests/noPassthrough/collmod_index_noop.js
+++ b/jstests/noPassthrough/collmod_index_noop.js
@@ -15,7 +15,7 @@
 
 (function() {
 "use strict";
-load("jstests/libs/get_index_helpers.js");
+load("jstests/libs/index_catalog_helpers.js");
 
 const rst = new ReplSetTest({nodes: 2, nodeOpts: {binVersion: "latest"}});
 rst.startSet();
@@ -105,7 +105,7 @@ validateCollModOplogEntryCount({
                                1);
 
 // Test that the index was successfully modified.
-let idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "b_1");
+let idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "b_1");
 assert.eq(idxSpec.hidden, undefined);
 assert.eq(idxSpec.expireAfterSeconds, 10);
 
@@ -131,7 +131,7 @@ if (collModIndexUniqueEnabled) {
                                    1);
 
     // Test that the index was successfully modified.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "c_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "c_1");
     assert.eq(idxSpec.hidden, undefined);
     assert.eq(idxSpec.expireAfterSeconds, undefined);
     assert(idxSpec.unique, tojson(idxSpec));
@@ -149,7 +149,7 @@ if (collModIndexUniqueEnabled) {
                                    0);
 
     // Test that the index was unchanged.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "d_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "d_1");
     assert.eq(idxSpec.hidden, undefined);
     assert.eq(idxSpec.expireAfterSeconds, undefined);
     assert(idxSpec.unique, tojson(idxSpec));
@@ -167,7 +167,7 @@ if (collModIndexUniqueEnabled) {
                                    0);
 
     // Test that the index was unchanged.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "e_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "e_1");
     assert(idxSpec.hidden, tojson(idxSpec));
     assert.eq(idxSpec.expireAfterSeconds, undefined);
     assert(idxSpec.unique, tojson(idxSpec));
@@ -190,7 +190,7 @@ if (collModIndexUniqueEnabled) {
                                    1);
 
     // Test that the index was successfully modified.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "f_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "f_1");
     assert.eq(idxSpec.hidden, undefined);
     assert.eq(idxSpec.expireAfterSeconds, 20);
     assert(idxSpec.unique, tojson(idxSpec));
@@ -214,7 +214,7 @@ if (collModIndexUniqueEnabled) {
                                    1);
 
     // Test that the index was successfully modified.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "g_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "g_1");
     assert(idxSpec.hidden, tojson(idxSpec));
     assert.eq(idxSpec.expireAfterSeconds, 30);
     assert(idxSpec.unique, tojson(idxSpec));
@@ -232,7 +232,7 @@ if (collModIndexUniqueEnabled) {
                                    0);
 
     // Test that the index was unchanged.
-    idxSpec = GetIndexHelpers.findByName(primaryColl.getIndexes(), "h_1");
+    idxSpec = IndexCatalogHelpers.findByName(primaryColl.getIndexes(), "h_1");
     assert.eq(idxSpec.hidden, undefined);
     assert.eq(idxSpec.expireAfterSeconds, undefined);
     assert(idxSpec.prepareUnique, tojson(idxSpec));
diff --git a/jstests/noPassthrough/columnstore_index_persistence.js b/jstests/noPassthrough/columnstore_index_persistence.js
new file mode 100644
index 00000000000..ad9b4d77a2a
--- /dev/null
+++ b/jstests/noPassthrough/columnstore_index_persistence.js
@@ -0,0 +1,45 @@
+/**
+ * Tests that a columnstore index can be persisted and found in listIndexes after a server restart.
+ *
+ * @tags: [
+ *  requires_persistence,
+ *  # Replication requires journaling support so this tag also implies exclusion from
+ *  # --nojournal test configurations.
+ *  requires_replication,
+ * ]
+ */
+
+(function() {
+'use strict';
+
+load('jstests/libs/index_catalog_helpers.js');
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+let primary = rst.getPrimary();
+const columnstoreIndexesEnabled =
+    assert.commandWorked(primary.adminCommand({getParameter: 1, featureFlagColumnstoreIndexes: 1}))
+        .featureFlagColumnstoreIndexes.value;
+
+if (!columnstoreIndexesEnabled) {
+    jsTestLog('Skipping test because the columnstore index feature flag is disabled');
+    rst.stopSet();
+    return;
+}
+
+const collName = 'columnstore_index_persistence';
+let db_primary = primary.getDB('test');
+let coll_primary = db_primary.getCollection(collName);
+
+assert.commandWorked(coll_primary.createIndex({"$**": "columnstore"}));
+
+// Restarts the primary and checks the index spec is persisted.
+rst.restart(primary);
+rst.waitForPrimary();
+const indexList = rst.getPrimary().getDB('test').getCollection(collName).getIndexes();
+assert.neq(null, IndexCatalogHelpers.findByKeyPattern(indexList, {"$**": "columnstore"}));
+
+rst.stopSet();
+})();
diff --git a/jstests/noPassthrough/durable_hidden_index.js b/jstests/noPassthrough/durable_hidden_index.js
index 8f929575c4c..1552b2b94c5 100644
--- a/jstests/noPassthrough/durable_hidden_index.js
+++ b/jstests/noPassthrough/durable_hidden_index.js
@@ -11,12 +11,12 @@
 (function() {
 "use strict";
 
-load("jstests/libs/get_index_helpers.js");  // For GetIndexHelpers.findByName.
+load("jstests/libs/index_catalog_helpers.js");  // For IndexCatalogHelpers.findByName.
 
 const dbName = "test";
 
 function isIndexHidden(indexes, indexName) {
-    const idx = GetIndexHelpers.findByName(indexes, indexName);
+    const idx = IndexCatalogHelpers.findByName(indexes, indexName);
     return idx && idx.hidden;
 }
 
@@ -50,7 +50,7 @@ const secondaryDB = rst.getSecondary().getDB(dbName);
 assert(isIndexHidden(secondaryDB.coll.getIndexes(), "a_1"));
 
 // Test that 'hidden: false' shouldn't be written to the index catalog.
-let idxSpec = GetIndexHelpers.findByName(secondaryDB.coll.getIndexes(), "b_1");
+let idxSpec = IndexCatalogHelpers.findByName(secondaryDB.coll.getIndexes(), "b_1");
 assert.eq(idxSpec.hidden, undefined);
 
 rst.stopSet();
@@ -83,7 +83,7 @@ db = conn.getDB(dbName);
 assert(isIndexHidden(db.coll.getIndexes(), "a_1"));
 
 // Test that 'hidden: false' shouldn't be written to the index catalog.
-idxSpec = GetIndexHelpers.findByName(db.coll.getIndexes(), "b_1");
+idxSpec = IndexCatalogHelpers.findByName(db.coll.getIndexes(), "b_1");
 assert.eq(idxSpec.hidden, undefined);
 
 MongoRunner.stopMongod(conn);
diff --git a/jstests/noPassthrough/index_version_autoupgrade.js b/jstests/noPassthrough/index_version_autoupgrade.js
index 39b4a2f37bc..5f1c309c1ba 100644
--- a/jstests/noPassthrough/index_version_autoupgrade.js
+++ b/jstests/noPassthrough/index_version_autoupgrade.js
@@ -5,7 +5,7 @@
 (function() {
 "use strict";
 
-load("jstests/libs/get_index_helpers.js");
+load("jstests/libs/index_catalog_helpers.js");
 
 var conn = MongoRunner.runMongod({});
 assert.neq(null, conn, "mongod was unable to start up");
@@ -13,7 +13,7 @@ assert.neq(null, conn, "mongod was unable to start up");
 var testDB = conn.getDB("test");
 assert.commandWorked(testDB.runCommand({create: "index_version_autoupgrade"}));
 var allIndexes = testDB.index_version_autoupgrade.getIndexes();
-var spec = GetIndexHelpers.findByKeyPattern(allIndexes, {_id: 1});
+var spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, {_id: 1});
 assert.neq(null, spec, "Index with key pattern {_id: 1} not found: " + tojson(allIndexes));
 var defaultIndexVersion = spec.v;
 assert.lte(2, defaultIndexVersion, "Expected the defaultIndexVersion to be at least v=2");
@@ -39,13 +39,13 @@ function testIndexVersionAutoUpgrades(commandFn, doesAutoUpgrade) {
     assert.commandWorked(testDB.createCollection("index_version_autoupgrade",
                                                  {idIndex: {key: {_id: 1}, name: "_id_", v: 1}}));
     var allIndexes = coll.getIndexes();
-    var spec = GetIndexHelpers.findByKeyPattern(allIndexes, {_id: 1});
+    var spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, {_id: 1});
     assert.neq(null, spec, "Index with key pattern {_id: 1} not found: " + tojson(allIndexes));
     assert.eq(1, spec.v, "Expected a v=1 index to be built: " + tojson(spec));
 
     assert.commandWorked(coll.createIndex({withoutAnyOptions: 1}));
     allIndexes = coll.getIndexes();
-    spec = GetIndexHelpers.findByKeyPattern(allIndexes, {withoutAnyOptions: 1});
+    spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, {withoutAnyOptions: 1});
     assert.neq(null,
                spec,
                "Index with key pattern {withoutAnyOptions: 1} not found: " + tojson(allIndexes));
@@ -55,13 +55,13 @@ function testIndexVersionAutoUpgrades(commandFn, doesAutoUpgrade) {
 
     assert.commandWorked(coll.createIndex({withV1: 1}, {v: 1}));
     allIndexes = coll.getIndexes();
-    spec = GetIndexHelpers.findByKeyPattern(allIndexes, {withV1: 1});
+    spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, {withV1: 1});
     assert.neq(null, spec, "Index with key pattern {withV1: 1} not found: " + tojson(allIndexes));
     assert.eq(1, spec.v, "Expected a v=1 index to be built: " + tojson(spec));
 
     assert.commandWorked(coll.createIndex({withV2: 1}, {v: 2}));
     allIndexes = coll.getIndexes();
-    spec = GetIndexHelpers.findByKeyPattern(allIndexes, {withV2: 1});
+    spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, {withV2: 1});
     assert.neq(null, spec, "Index with key pattern {withV2: 1} not found: " + tojson(allIndexes));
     assert.eq(2, spec.v, "Expected a v=2 index to be built: " + tojson(spec));
 
@@ -87,7 +87,7 @@ function testIndexVersionAutoUpgrades(commandFn, doesAutoUpgrade) {
 
     expectedResults.forEach(function(expected) {
         var allIndexes = collToVerify.getIndexes();
-        var spec = GetIndexHelpers.findByKeyPattern(allIndexes, expected.keyPattern);
+        var spec = IndexCatalogHelpers.findByKeyPattern(allIndexes, expected.keyPattern);
         assert.neq(null,
                    spec,
                    "Index with key pattern " + tojson(expected.keyPattern) +
diff --git a/jstests/noPassthrough/internal_validate_features_as_master.js b/jstests/noPassthrough/internal_validate_features_as_master.js
index 7aca471e106..071ca153f73 100644
--- a/jstests/noPassthrough/internal_validate_features_as_master.js
+++ b/jstests/noPassthrough/internal_validate_features_as_master.js
@@ -5,7 +5,7 @@
 (function() {
 "use strict";
 
-load("jstests/libs/get_index_helpers.js");
+load("jstests/libs/index_catalog_helpers.js");
 
 // internalValidateFeaturesAsMaster can be set via startup parameter.
 let conn = MongoRunner.runMongod({setParameter: "internalValidateFeaturesAsMaster=1"});
diff --git a/jstests/noPassthrough/reindex_crash_rebuilds_id_index.js b/jstests/noPassthrough/reindex_crash_rebuilds_id_index.js
index 573810fd262..84993043ec6 100644
--- a/jstests/noPassthrough/reindex_crash_rebuilds_id_index.js
+++ b/jstests/noPassthrough/reindex_crash_rebuilds_id_index.js
@@ -9,7 +9,7 @@
  */
 (function() {
 
-load("jstests/libs/get_index_helpers.js");  // For GetIndexHelpers.
+load("jstests/libs/index_catalog_helpers.js");  // For IndexCatalogHelpers.
 
 // This test triggers an unclean shutdown, which may cause inaccurate fast counts.
 TestData.skipEnforceFastCountOnValidate = true;
@@ -30,7 +30,7 @@ let testColl = testDB.getCollection(collName);
 
 // Insert a single document and create the collection.
 testColl.insert({a: 1});
-let spec = GetIndexHelpers.findByKeyPattern(testColl.getIndexes(), {_id: 1});
+let spec = IndexCatalogHelpers.findByKeyPattern(testColl.getIndexes(), {_id: 1});
 assert.neq(null, spec, "_id index not found");
 assert.eq("_id_", spec.name, tojson(spec));
 
@@ -50,7 +50,7 @@ testColl = testDB.getCollection(collName);
 assert(testColl.exists());
 
 // The _id index should exist.
-spec = GetIndexHelpers.findByKeyPattern(testColl.getIndexes(), {_id: 1});
+spec = IndexCatalogHelpers.findByKeyPattern(testColl.getIndexes(), {_id: 1});
 assert.neq(null, spec, "_id index not found");
 assert.eq("_id_", spec.name, tojson(spec));
 
diff --git a/jstests/noPassthrough/serverstatus_index_stats.js b/jstests/noPassthrough/serverstatus_index_stats.js
index 300b71c6194..55a09653c64 100644
--- a/jstests/noPassthrough/serverstatus_index_stats.js
+++ b/jstests/noPassthrough/serverstatus_index_stats.js
@@ -26,6 +26,7 @@ const knownFeatures = [
     "2dsphere",
     "2dsphere_bucket",
     "collation",
+    "columnstore",
     "compound",
     "hashed",
     "id",
@@ -198,10 +199,9 @@ assertStats(db, (stats) => {
 
 lastStats = db.serverStatus().indexStats;
 
-const timeSeriesMetricIndexesEnabled = db.adminCommand({
-                                             getParameter: 1,
-                                             featureFlagTimeseriesMetricIndexes: 1
-                                         }).featureFlagTimeseriesMetricIndexes.value;
+const timeSeriesMetricIndexesEnabled =
+    assert.commandWorked(db.adminCommand({getParameter: 1, featureFlagTimeseriesMetricIndexes: 1}))
+        .featureFlagTimeseriesMetricIndexes.value;
 if (timeSeriesMetricIndexesEnabled) {
     assert.commandWorked(db.createCollection('ts', {timeseries: {timeField: 't'}}));
     assert.commandWorked(db.ts.createIndex({loc: '2dsphere'}));
@@ -232,6 +232,24 @@ if (timeSeriesMetricIndexesEnabled) {
 
 lastStats = db.serverStatus().indexStats;
 
+const columnstoreIndexesEnabled =
+    assert.commandWorked(db.adminCommand({getParameter: 1, featureFlagColumnstoreIndexes: 1}))
+        .featureFlagColumnstoreIndexes.value;
+if (columnstoreIndexesEnabled) {
+    // TODO SERVER-66021 should support having data.
+    // TODO SERVER-61644 (or sooner) should support accessing/using index and seeing that reflected.
+    assert.commandWorked(db.newCollection.createIndex({'$**': 'columnstore'}));
+    assertStats(db, (stats) => {
+        assertCountIncrease(lastStats, stats, 2);  // Includes _id index on new collection.
+        assertFeatureCountIncrease(lastStats, stats, 'columnstore', 1);
+
+        assertFeatureAccessIncrease(lastStats, stats, 'id', 0);
+        assertFeatureAccessIncrease(lastStats, stats, 'columnstore', 0);
+    });
+}
+
+lastStats = db.serverStatus().indexStats;
+
 // After restarting the server, we expect all of the access counters to reset to zero, but that the
 // feature counters remain the same as before startup.
 replSet.stopSet(undefined, /* restart */ true);