SERVER-67598 Add support for maxSpanSeconds and roundingSeconds arguments

5 files changed, 397 insertions, 47 deletions

diff --git a/jstests/change_streams/timeseries.js b/jstests/change_streams/timeseries.js
index 31984390dc9..2c08fec5253 100644
--- a/jstests/change_streams/timeseries.js
+++ b/jstests/change_streams/timeseries.js
@@ -291,6 +291,12 @@ let expectedChanges = [
     {"operationType": "drop", "ns": {"db": dbName, "coll": bucketsCollName}}
 ];
 
+if (FeatureFlagUtil.isEnabled(db, "TimeseriesScalabilityImprovements")) {
+    expectedChanges[0].operationDescription.timeseries.bucketRoundingSeconds = 60;
+    expectedChanges[8].stateBeforeChange.collectionOptions.timeseries.bucketRoundingSeconds = 60;
+    expectedChanges[9].stateBeforeChange.collectionOptions.timeseries.bucketRoundingSeconds = 86400;
+}
+
 if (!FeatureFlagUtil.isEnabled(db, "TimeseriesScalabilityImprovements")) {
     // Under this feature flag, buckets are created with the closed field set to false.
     // Remove closed field if the feature flag is not enabled.
diff --git a/jstests/core/timeseries/bucket_span_and_rounding_seconds.js b/jstests/core/timeseries/bucket_span_and_rounding_seconds.js
new file mode 100644
index 00000000000..51687928b64
--- /dev/null
+++ b/jstests/core/timeseries/bucket_span_and_rounding_seconds.js
@@ -0,0 +1,243 @@
+/**
+ * Tests timeseries collection creation with bucketRoundingSeconds and bucketMaxSpanSeconds
+ * parameters and checks that we correctly set their value (failing when parameters are
+ * not added correctly or are missing).
+ *
+ * @tags: [
+ *   # "Overriding safe failed response for :: create"
+ *   does_not_support_stepdowns,
+ *   # We need a timeseries collection.
+ *   requires_timeseries,
+ * ]
+ */
+(function() {
+'use strict';
+
+load("jstests/core/timeseries/libs/timeseries.js");
+
+if (!TimeseriesTest.timeseriesScalabilityImprovementsEnabled(db.getMongo())) {
+    jsTestLog(
+        "Skipped test as the featureFlagTimeseriesScalabilityImprovements feature flag is not enabled.");
+    return;
+}
+
+const testDB = db.getSiblingDB(jsTestName());
+assert.commandWorked(testDB.dropDatabase());
+
+const timeFieldName = 'time';
+const coll = testDB.t;
+const bucketRoundingSecondsTime = 4000;
+const bucketMaxSpanSecondsTime = 4000;
+const granularitySeconds = "seconds";
+const granularityMinutes = "minutes";
+const granularityHours = "hours";
+const bucketInvalidOptionsError = ErrorCodes.InvalidOptions;
+
+const granularityTimeOptionsArr = [granularitySeconds, granularityMinutes, granularityHours];
+
+const getBucketMaxSpanSecondsFromGranularity = function(granularity) {
+    switch (granularity) {
+        case 'seconds':
+            return 60 * 60;
+        case 'minutes':
+            return 60 * 60 * 24;
+        case 'hours':
+            return 60 * 60 * 24 * 30;
+        default:
+            assert(false, 'Invalid granularity: ' + granularity);
+    }
+};
+
+const getBucketRoundingSecondsFromGranularity = function(granularity) {
+    switch (granularity) {
+        case 'seconds':
+            return 60;
+        case 'minutes':
+            return 60 * 60;
+        case 'hours':
+            return 60 * 60 * 24;
+        default:
+            assert(false, 'Invalid granularity: ' + granularity);
+    }
+};
+
+const verifyCreateCommandFails = function(secondsOptions = {}, errorCode) {
+    coll.drop();
+    const fullTimeseriesOptions = Object.merge({timeField: timeFieldName}, secondsOptions);
+
+    if (errorCode) {
+        assert.commandFailedWithCode(
+            testDB.createCollection(coll.getName(), {timeseries: fullTimeseriesOptions}),
+            errorCode);
+    } else {
+        assert.commandFailed(
+            testDB.createCollection(coll.getName(), {timeseries: fullTimeseriesOptions}));
+    }
+
+    const collections =
+        assert.commandWorked(testDB.runCommand({listCollections: 1})).cursor.firstBatch;
+
+    assert.isnull(collections.find(entry => entry.name === 'system.buckets.' + coll.getName()));
+    assert.isnull(collections.find(entry => entry.name === coll.getName()));
+};
+
+(function createTimeseriesCollectionWithBucketSecondsOptions() {
+    jsTestLog("Create timeseries collection with bucketRoundingSeconds and bucketMaxSpanSeconds.");
+    // Create a timeseries collection with bucketRoundingSeconds, bucketMaxSpanSeconds and
+    // custom parameters. ListCollection should show view and bucket collection with the added
+    // properties.
+    assert.commandWorked(testDB.createCollection(coll.getName(), {
+        timeseries: {
+            timeField: timeFieldName,
+            bucketRoundingSeconds: bucketRoundingSecondsTime,
+            bucketMaxSpanSeconds: bucketMaxSpanSecondsTime
+        }
+    }));
+
+    let collections =
+        assert.commandWorked(testDB.runCommand({listCollections: 1})).cursor.firstBatch;
+
+    let collectionEntry =
+        collections.find(entry => entry.name === 'system.buckets.' + coll.getName());
+    assert(collectionEntry);
+    assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds, bucketRoundingSecondsTime);
+    assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds, bucketMaxSpanSecondsTime);
+
+    collectionEntry = collections.find(entry => entry.name === coll.getName());
+    assert(collectionEntry);
+    assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds, bucketRoundingSecondsTime);
+    assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds, bucketMaxSpanSecondsTime);
+
+    // Verify the create command succeeds with bucketRoundingSeconds, bucketMaxSpanSeconds set as
+    // their default granularity values.
+    for (const granularityTime of granularityTimeOptionsArr) {
+        coll.drop();
+        assert.commandWorked(testDB.createCollection(coll.getName(), {
+            timeseries: {
+                timeField: timeFieldName,
+                granularity: granularityTime,
+                bucketRoundingSeconds: getBucketRoundingSecondsFromGranularity(granularityTime),
+                bucketMaxSpanSeconds: getBucketMaxSpanSecondsFromGranularity(granularityTime)
+            }
+        }));
+        collections =
+            assert.commandWorked(testDB.runCommand({listCollections: 1})).cursor.firstBatch;
+
+        collectionEntry =
+            collections.find(entry => entry.name === 'system.buckets.' + coll.getName());
+        assert(collectionEntry);
+        assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+                  getBucketRoundingSecondsFromGranularity(granularityTime));
+        assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+                  getBucketMaxSpanSecondsFromGranularity(granularityTime));
+
+        collectionEntry = collections.find(entry => entry.name === coll.getName());
+        assert(collectionEntry);
+        assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+                  getBucketRoundingSecondsFromGranularity(granularityTime));
+        assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+                  getBucketMaxSpanSecondsFromGranularity(granularityTime));
+    }
+
+    // Verify the create command succeeds without setting bucketRoundingSeconds and
+    // bucketMaxSpanSeconds. This should set their default granularity values.
+    for (const granularityTime of granularityTimeOptionsArr) {
+        coll.drop();
+        assert.commandWorked(testDB.createCollection(coll.getName(), {
+            timeseries: {
+                timeField: timeFieldName,
+                granularity: granularityTime,
+            }
+        }));
+        collections =
+            assert.commandWorked(testDB.runCommand({listCollections: 1})).cursor.firstBatch;
+
+        collectionEntry =
+            collections.find(entry => entry.name === 'system.buckets.' + coll.getName());
+        assert(collectionEntry);
+        assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+                  getBucketRoundingSecondsFromGranularity(granularityTime));
+        assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+                  getBucketMaxSpanSecondsFromGranularity(granularityTime));
+
+        collectionEntry = collections.find(entry => entry.name === coll.getName());
+        assert(collectionEntry);
+        assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+                  getBucketRoundingSecondsFromGranularity(granularityTime));
+        assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+                  getBucketMaxSpanSecondsFromGranularity(granularityTime));
+    }
+
+    // Verify the create command succeeds without setting any field other than timeField, this
+    // should set granularity as seconds and bucketRoundingSeconds and bucketMaxSpanSeconds with
+    // their default granularity values.
+    coll.drop();
+    assert.commandWorked(testDB.createCollection(coll.getName(), {
+        timeseries: {
+            timeField: timeFieldName,
+        }
+    }));
+    collections = assert.commandWorked(testDB.runCommand({listCollections: 1})).cursor.firstBatch;
+
+    collectionEntry = collections.find(entry => entry.name === 'system.buckets.' + coll.getName());
+    assert(collectionEntry);
+    assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+              getBucketRoundingSecondsFromGranularity(granularitySeconds));
+    assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+              getBucketMaxSpanSecondsFromGranularity(granularitySeconds));
+
+    collectionEntry = collections.find(entry => entry.name === coll.getName());
+    assert(collectionEntry);
+    assert.eq(collectionEntry.options.timeseries.bucketRoundingSeconds,
+              getBucketRoundingSecondsFromGranularity(granularitySeconds));
+    assert.eq(collectionEntry.options.timeseries.bucketMaxSpanSeconds,
+              getBucketMaxSpanSecondsFromGranularity(granularitySeconds));
+})();
+
+(function createTimeseriesCollectionWithInvalidOptions() {
+    jsTestLog("Create timeseries collection with missing or extra arguments.");
+
+    // Verify the create command fails when the 'bucketRoundingSeconds' option is set but not the
+    // 'bucketMaxSpanSeconds' option.
+    verifyCreateCommandFails({bucketRoundingSeconds: bucketRoundingSecondsTime},
+                             bucketInvalidOptionsError);
+
+    // Verify the create command fails when the 'bucketMaxSpanSeconds' option is set but not the
+    // 'bucketRoundingSeconds' option.
+    verifyCreateCommandFails({bucketMaxSpanSeconds: bucketMaxSpanSecondsTime},
+                             bucketInvalidOptionsError);
+
+    // Verify the create command fails when the 'bucketMaxSpanSeconds' option is set but not the
+    // 'bucketRoundingSeconds' option (even if set to granularity default seconds value).
+    verifyCreateCommandFails({bucketMaxSpanSeconds: 3600}, bucketInvalidOptionsError);
+
+    // Verify the create command fails when bucketRoundingSeconds is different than
+    // bucketMaxSpanSeconds.
+    verifyCreateCommandFails({bucketRoundingSeconds: 100, bucketMaxSpanSeconds: 50},
+                             bucketInvalidOptionsError);
+
+    // Verify the create command fails when bucketRoundingSeconds or bucketMaxSpanSeconds is a
+    // negative value.
+    verifyCreateCommandFails({bucketRoundingSeconds: -1, bucketMaxSpanSeconds: -1});
+
+    // Verify the create command fails when granularity is set as minutes alongside
+    // bucketRoundingSeconds and bucketMaxSpanSeconds and they are not the default granularity
+    // values.
+    verifyCreateCommandFails({
+        granularity: granularityMinutes,
+        bucketRoundingSeconds: bucketRoundingSecondsTime,
+        bucketMaxSpanSeconds: bucketMaxSpanSecondsTime
+    },
+                             bucketInvalidOptionsError);
+
+    // Verify the create command fails when granularity is set as hours alongside
+    // bucketRoundingSeconds and bucketMaxSpanSeconds and they are not the default granularity
+    // values.
+    verifyCreateCommandFails({
+        granularity: granularityHours,
+        bucketRoundingSeconds: bucketRoundingSecondsTime,
+        bucketMaxSpanSeconds: bucketMaxSpanSecondsTime
+    },
+                             bucketInvalidOptionsError);
+})();
+})();
diff --git a/jstests/core/timeseries/timeseries_list_collections.js b/jstests/core/timeseries/timeseries_list_collections.js
index ee2d5dd143e..8364dca1fd9 100644
--- a/jstests/core/timeseries/timeseries_list_collections.js
+++ b/jstests/core/timeseries/timeseries_list_collections.js
@@ -9,6 +9,10 @@
 (function() {
 'use strict';
 
+load("jstests/core/timeseries/libs/timeseries.js");
+
+const testDB = db.getSiblingDB(jsTestName());
+
 const timeFieldName = 'time';
 const metaFieldName = 'meta';
 
@@ -28,6 +32,22 @@ const getBucketMaxSpanSeconds = function(granularity) {
     }
 };
 
+const getBucketRoundingSeconds = function(granularity) {
+    switch (granularity) {
+        case 'seconds':
+            return 60;
+        case 'minutes':
+            return 60 * 60;
+        case 'hours':
+            return 60 * 60 * 24;
+        default:
+            assert(false, 'Invalid granularity: ' + granularity);
+    }
+};
+
+const bucketMaxSpanSecondsFromMinutes = getBucketMaxSpanSeconds('minutes');
+const buckeRoundingSecondsFromMinutes = getBucketRoundingSeconds('minutes');
+
 const testOptions = function(options) {
     const coll = db.getCollection(collNamePrefix + collCount++);
     coll.drop();
@@ -43,6 +63,13 @@ const testOptions = function(options) {
             options.timeseries,
             {bucketMaxSpanSeconds: getBucketMaxSpanSeconds(options.timeseries.granularity)});
     }
+    if (TimeseriesTest.timeseriesScalabilityImprovementsEnabled(testDB)) {
+        if (!options.timeseries.hasOwnProperty('bucketRoundingSeconds')) {
+            Object.assign(
+                options.timeseries,
+                {bucketRoundingSeconds: getBucketRoundingSeconds(options.timeseries.granularity)});
+        }
+    }
 
     if (options.hasOwnProperty('collation')) {
         Object.assign(options.collation, {
@@ -78,13 +105,24 @@ testOptions({
         granularity: 'minutes',
     }
 });
-testOptions({
-    timeseries: {
-        timeField: timeFieldName,
-        granularity: 'minutes',
-        bucketMaxSpanSeconds: 60 * 60 * 24,
-    }
-});
+if (!TimeseriesTest.timeseriesScalabilityImprovementsEnabled(testDB)) {
+    testOptions({
+        timeseries: {
+            timeField: timeFieldName,
+            granularity: 'minutes',
+            bucketMaxSpanSeconds: bucketMaxSpanSecondsFromMinutes,
+        }
+    });
+} else {
+    testOptions({
+        timeseries: {
+            timeField: timeFieldName,
+            granularity: 'minutes',
+            bucketMaxSpanSeconds: bucketMaxSpanSecondsFromMinutes,
+            bucketRoundingSeconds: buckeRoundingSecondsFromMinutes,
+        }
+    });
+}
 testOptions({
     timeseries: {
         timeField: timeFieldName,
@@ -104,16 +142,32 @@ testOptions({
     collation: {locale: 'ja'},
 });
 testOptions({timeseries: {timeField: timeFieldName}, expireAfterSeconds: NumberLong(100)});
-testOptions({
-    timeseries: {
-        timeField: timeFieldName,
-        metaField: metaFieldName,
-        granularity: 'minutes',
-        bucketMaxSpanSeconds: 60 * 60 * 24,
-    },
-    storageEngine: {wiredTiger: {}},
-    indexOptionDefaults: {storageEngine: {wiredTiger: {}}},
-    collation: {locale: 'ja'},
-    expireAfterSeconds: NumberLong(100),
-});
+if (!TimeseriesTest.timeseriesScalabilityImprovementsEnabled(testDB)) {
+    testOptions({
+        timeseries: {
+            timeField: timeFieldName,
+            metaField: metaFieldName,
+            granularity: 'minutes',
+            bucketMaxSpanSeconds: bucketMaxSpanSecondsFromMinutes,
+        },
+        storageEngine: {wiredTiger: {}},
+        indexOptionDefaults: {storageEngine: {wiredTiger: {}}},
+        collation: {locale: 'ja'},
+        expireAfterSeconds: NumberLong(100),
+    });
+} else {
+    testOptions({
+        timeseries: {
+            timeField: timeFieldName,
+            metaField: metaFieldName,
+            granularity: 'minutes',
+            bucketMaxSpanSeconds: bucketMaxSpanSecondsFromMinutes,
+            bucketRoundingSeconds: buckeRoundingSecondsFromMinutes,
+        },
+        storageEngine: {wiredTiger: {}},
+        indexOptionDefaults: {storageEngine: {wiredTiger: {}}},
+        collation: {locale: 'ja'},
+        expireAfterSeconds: NumberLong(100),
+    });
+}
 })();
diff --git a/jstests/core/timeseries/timeseries_list_collections_filter_name.js b/jstests/core/timeseries/timeseries_list_collections_filter_name.js
index fb80b08400d..547558738a1 100644
--- a/jstests/core/timeseries/timeseries_list_collections_filter_name.js
+++ b/jstests/core/timeseries/timeseries_list_collections_filter_name.js
@@ -10,6 +10,8 @@
 (function() {
 'use strict';
 
+load("jstests/core/timeseries/libs/timeseries.js");
+
 const timeFieldName = 'time';
 
 const coll = db.timeseries_list_collections_filter_name;
@@ -20,13 +22,22 @@ assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField
 const collections =
     assert.commandWorked(db.runCommand({listCollections: 1, filter: {name: coll.getName()}}))
         .cursor.firstBatch;
-assert.eq(collections, [{
-              name: coll.getName(),
-              type: 'timeseries',
-              options: {
-                  timeseries:
-                      {timeField: timeFieldName, granularity: 'seconds', bucketMaxSpanSeconds: 3600}
-              },
-              info: {readOnly: false},
-          }]);
+
+const timeseriesOptions = {
+    timeField: timeFieldName,
+    granularity: 'seconds'
+};
+const extraBucketingParameters =
+    (TimeseriesTest.timeseriesScalabilityImprovementsEnabled(db.getMongo()))
+    ? {bucketRoundingSeconds: 60, bucketMaxSpanSeconds: 3600}
+    : {bucketMaxSpanSeconds: 3600};
+
+const collectionOptions = [{
+    name: coll.getName(),
+    type: 'timeseries',
+    options: {timeseries: Object.merge(timeseriesOptions, extraBucketingParameters)},
+    info: {readOnly: false},
+}];
+
+assert.eq(collections, collectionOptions);
 })();
diff --git a/jstests/noPassthrough/timeseries_create.js b/jstests/noPassthrough/timeseries_create.js
index 71e04e23b61..34d05c59d26 100644
--- a/jstests/noPassthrough/timeseries_create.js
+++ b/jstests/noPassthrough/timeseries_create.js
@@ -8,11 +8,17 @@
 (function() {
 "use strict";
 
+load("jstests/core/timeseries/libs/timeseries.js");
+
 const conn = MongoRunner.runMongod();
 
 const dbName = jsTestName();
+const testDB = conn.getDB(dbName);
 let collCount = 0;
 
+const bucketGranularityError = ErrorCodes.InvalidOptions;
+const bucketMaxSpanSecondsError = ErrorCodes.InvalidOptions;
+
 const testOptions = function(allowed,
                              createOptions,
                              timeseriesOptions = {
@@ -26,7 +32,6 @@ const testOptions = function(allowed,
                                  // passing all the test assertions.
                                  tearDown: (testDB, collName) => {},
                              }) {
-    const testDB = conn.getDB(dbName);
     const collName = 'timeseries_' + collCount++;
     const bucketsCollName = 'system.buckets.' + collName;
 
@@ -92,21 +97,53 @@ testValidTimeseriesOptions({timeField: "time"});
 testValidTimeseriesOptions({timeField: "time", metaField: "meta"});
 testValidTimeseriesOptions({timeField: "time", metaField: "meta", granularity: "seconds"});
 
-// A bucketMaxSpanSeconds may be provided, but only if they are the default for the granularity.
-testValidTimeseriesOptions(
-    {timeField: "time", metaField: "meta", granularity: "seconds", bucketMaxSpanSeconds: 60 * 60});
-testValidTimeseriesOptions({
-    timeField: "time",
-    metaField: "meta",
-    granularity: "minutes",
-    bucketMaxSpanSeconds: 60 * 60 * 24
-});
-testValidTimeseriesOptions({
-    timeField: "time",
-    metaField: "meta",
-    granularity: "hours",
-    bucketMaxSpanSeconds: 60 * 60 * 24 * 30
-});
+if (!TimeseriesTest.timeseriesScalabilityImprovementsEnabled(testDB)) {
+    // A bucketMaxSpanSeconds may be provided, but only if they are the default for the granularity.
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "seconds",
+        bucketMaxSpanSeconds: 60 * 60
+    });
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "minutes",
+        bucketMaxSpanSeconds: 60 * 60 * 24
+    });
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "hours",
+        bucketMaxSpanSeconds: 60 * 60 * 24 * 30
+    });
+} else {
+    // Granularity may be provided with bucketMaxSpanSeconds and bucketRoundingSeconds,
+    // but only if they are the default (seconds)).
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "seconds",
+        bucketMaxSpanSeconds: 60 * 60,
+        bucketRoundingSeconds: 60
+    });
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "minutes",
+        bucketMaxSpanSeconds: 60 * 60 * 24,
+        bucketRoundingSeconds: 60 * 60,
+    },
+                               bucketGranularityError);
+    testValidTimeseriesOptions({
+        timeField: "time",
+        metaField: "meta",
+        granularity: "hours",
+        bucketMaxSpanSeconds: 60 * 60 * 24 * 30,
+        bucketRoundingSeconds: 60 * 60 * 24,
+    },
+                               bucketGranularityError);
+}
 
 testValidTimeseriesOptions({timeField: "time", metaField: "meta", granularity: "minutes"});
 testValidTimeseriesOptions({timeField: "time", metaField: "meta", granularity: "hours"});
@@ -121,10 +158,10 @@ testInvalidTimeseriesOptions({timeField: "time", metaField: "sub.meta"}, ErrorCo
 testInvalidTimeseriesOptions({timeField: "time", metaField: "time"}, ErrorCodes.InvalidOptions);
 
 testInvalidTimeseriesOptions({timeField: "time", metaField: "meta", bucketMaxSpanSeconds: 10},
-                             5510500);
+                             bucketMaxSpanSecondsError);
 testInvalidTimeseriesOptions(
     {timeField: "time", metaField: "meta", granularity: 'minutes', bucketMaxSpanSeconds: 3600},
-    5510500);
+    bucketMaxSpanSecondsError);
 
 testCompatibleCreateOptions({expireAfterSeconds: NumberLong(100)});
 testCompatibleCreateOptions({storageEngine: {}});
@@ -158,7 +195,6 @@ testTimeseriesNamespaceExists((testDB, collName) => {
 
 // Tests that schema validation is enabled on the bucket collection.
 {
-    const testDB = conn.getDB(dbName);
     const coll = testDB.getCollection('timeseries_' + collCount++);
     coll.drop();
     assert.commandWorked(