diff options
| -rw-r--r-- | jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_continuous.js | 84 | ||||
| -rw-r--r-- | jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_lts.js (renamed from jstests/multiVersion/upgrade_timeseries_collection.js) | 37 | ||||
| -rw-r--r-- | jstests/noPassthrough/timeseries_measurement_indexes_downgrade.js | 59 | ||||
| -rw-r--r-- | src/mongo/db/catalog/coll_mod.cpp | 13 | ||||
| -rw-r--r-- | src/mongo/db/catalog/collection.h | 2 | ||||
| -rw-r--r-- | src/mongo/db/catalog/collection_impl.cpp | 12 | ||||
| -rw-r--r-- | src/mongo/db/commands/set_feature_compatibility_version_command.cpp | 60 |
7 files changed, 215 insertions, 52 deletions
diff --git a/jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_continuous.js b/jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_continuous.js new file mode 100644 index 00000000000..cfab82522e5 --- /dev/null +++ b/jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_continuous.js @@ -0,0 +1,84 @@ +/** + * Tests that upgrading time-series collections created using the last-continuous binary warns about + * potentially mixed-schema data when building secondary indexes on time-series measurements on the + * latest binary. Additionally, tests that downgrading FCV from 5.2 removes the + * 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag from time-series collections. + * + * TODO SERVER-60577: expand testing by checking that index builds will fail with mixed-schema data + * and succeed when there is no mixed-schema data in time-series collections. + */ +(function() { +"use strict"; + +load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/multiVersion/libs/multi_rs.js"); + +const oldVersion = "last-continuous"; +const nodes = { + n1: {binVersion: oldVersion}, + n2: {binVersion: oldVersion}, + n3: {binVersion: oldVersion} +}; + +const rst = new ReplSetTest({nodes: nodes}); +rst.startSet(); +rst.initiate(); + +const dbName = "test"; +const collName = jsTestName(); + +let primary = rst.getPrimary(); +let db = primary.getDB(dbName); + +// Create a time-series collection while using older binaries. +const timeField = "time"; +assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: timeField}})); + +jsTest.log("Upgrading replica set from last-continuous to latest"); +rst.upgradeSet({binVersion: "latest", setParameter: {logComponentVerbosity: tojson({storage: 1})}}); + +primary = rst.getPrimary(); +db = primary.getDB(dbName); + +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(primary)) { + jsTest.log("Skipping test as the featureFlagTimeseriesMetricIndexes feature flag is disabled"); + rst.stopSet(); + return; +} + +// Building indexes on time-series measurements is only supported in FCV >= 5.2. +jsTest.log("Setting FCV to 'latestFCV'"); +assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: latestFCV})); + +const bucketCollName = dbName + ".system.buckets." + collName; + +// The FCV upgrade process adds the catalog entry flag to time-series collections. +assert(checkLog.checkContainsWithCountJson(primary, 6057601, {setting: true}, /*expectedCount=*/1)); + +assert.commandWorked(db.getCollection(collName).createIndex({[timeField]: 1}, {name: "time_1"})); +assert(checkLog.checkContainsWithCountJson( + primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/0)); + +assert.commandWorked(db.getCollection(collName).createIndex({x: 1}, {name: "x_1"})); +assert(checkLog.checkContainsWithCountJson( + primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/1)); + +assert.commandWorked( + db.getCollection(collName).createIndex({[timeField]: 1, x: 1}, {name: "time_1_x_1"})); +assert(checkLog.checkContainsWithCountJson( + primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/2)); + +// Cannot downgrade when there are indexes on time-series measurements present. +assert.commandFailedWithCode( + primary.adminCommand({setFeatureCompatibilityVersion: lastContinuousFCV}), + ErrorCodes.CannotDowngrade); +assert.commandWorked(db.getCollection(collName).dropIndex("x_1")); +assert.commandWorked(db.getCollection(collName).dropIndex("time_1_x_1")); + +// The FCV downgrade process removes the catalog entry flag from time-series collections. +jsTest.log("Setting FCV to 'lastContinuousFCV'"); +assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: lastContinuousFCV})); +assert(checkLog.checkContainsWithCountJson(primary, 6057601, {setting: null}, /*expectedCount=*/1)); + +rst.stopSet(); +}());
\ No newline at end of file diff --git a/jstests/multiVersion/upgrade_timeseries_collection.js b/jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_lts.js index 1aa0b90e9ba..4d3e020b437 100644 --- a/jstests/multiVersion/upgrade_timeseries_collection.js +++ b/jstests/multiVersion/upgrade_downgrade_timeseries_collection_from_last_lts.js @@ -1,6 +1,8 @@ /** - * Tests that upgrading time-series collections created in earlier server versions warn about - * potentially mixed-schema data when building secondary indexes on time-series measurements. + * Tests that upgrading time-series collections created using the last-lts binary warns about + * potentially mixed-schema data when building secondary indexes on time-series measurements on the + * latest binary. Additionally, tests that downgrading FCV from 5.2 removes the + * 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag from time-series collections. * * TODO SERVER-60577: expand testing by checking that index builds will fail with mixed-schema data * and succeed when there is no mixed-schema data in time-series collections. @@ -33,34 +35,49 @@ const timeField = "time"; assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: timeField}})); jsTest.log("Upgrading replica set from last-lts to latest"); -rst.upgradeSet({binVersion: "latest"}); +rst.upgradeSet({binVersion: "latest", setParameter: {logComponentVerbosity: tojson({storage: 1})}}); primary = rst.getPrimary(); db = primary.getDB(dbName); -// Building indexes on time-series measurements is only supported in FCV >= 5.2. -jsTest.log("Setting FCV to 'latestFCV'"); -assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: latestFCV})); - if (!TimeseriesTest.timeseriesMetricIndexesEnabled(primary)) { jsTest.log("Skipping test as the featureFlagTimeseriesMetricIndexes feature flag is disabled"); rst.stopSet(); return; } +// Building indexes on time-series measurements is only supported in FCV >= 5.2. +jsTest.log("Setting FCV to 'latestFCV'"); +assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: latestFCV})); + const bucketCollName = dbName + ".system.buckets." + collName; -assert.commandWorked(db.getCollection(collName).createIndex({[timeField]: 1})); +// The FCV upgrade process adds the catalog entry flag to time-series collections. +assert(checkLog.checkContainsWithCountJson(primary, 6057601, {setting: true}, /*expectedCount=*/1)); + +assert.commandWorked(db.getCollection(collName).createIndex({[timeField]: 1}, {name: "time_1"})); assert(checkLog.checkContainsWithCountJson( primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/0)); -assert.commandWorked(db.getCollection(collName).createIndex({x: 1})); +assert.commandWorked(db.getCollection(collName).createIndex({x: 1}, {name: "x_1"})); assert(checkLog.checkContainsWithCountJson( primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/1)); -assert.commandWorked(db.getCollection(collName).createIndex({[timeField]: 1, x: 1})); +assert.commandWorked( + db.getCollection(collName).createIndex({[timeField]: 1, x: 1}, {name: "time_1_x_1"})); assert(checkLog.checkContainsWithCountJson( primary, 6057502, {namespace: bucketCollName}, /*expectedCount=*/2)); +// Cannot downgrade when there are indexes on time-series measurements present. +assert.commandFailedWithCode(primary.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}), + ErrorCodes.CannotDowngrade); +assert.commandWorked(db.getCollection(collName).dropIndex("x_1")); +assert.commandWorked(db.getCollection(collName).dropIndex("time_1_x_1")); + +// The FCV downgrade process removes the catalog entry flag from time-series collections. +jsTest.log("Setting FCV to 'lastLTSFCV'"); +assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV})); +assert(checkLog.checkContainsWithCountJson(primary, 6057601, {setting: null}, /*expectedCount=*/1)); + rst.stopSet(); }());
\ No newline at end of file diff --git a/jstests/noPassthrough/timeseries_measurement_indexes_downgrade.js b/jstests/noPassthrough/timeseries_measurement_indexes_downgrade.js index b8c457840d5..9a8593550d2 100644 --- a/jstests/noPassthrough/timeseries_measurement_indexes_downgrade.js +++ b/jstests/noPassthrough/timeseries_measurement_indexes_downgrade.js @@ -3,11 +3,7 @@ * measurements present. Additionally, this verifies that only indexes that are incompatible for * downgrade have the "originalSpec" field present on the buckets index definition. * - * TODO SERVER-60576: Re-enable this test. Downgrading FCV does not remove the - * 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag. When upgrading, an invariant will - * be triggered as the catalog entry flag is expected to be removed on downgrade. - * - * @tags: [__TEMPORARILY_DISABLED__] + * TODO SERVER-60912: Remove this test once kLastLTS is 6.0. */ (function() { "use strict"; @@ -37,7 +33,7 @@ assert.commandWorked(db.createCollection("system.buckets.abc")); assert.commandWorked(db.createCollection( coll.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}})); -function checkIndexForDowngrade(isCompatible, createdOnBucketsCollection) { +function checkIndexForDowngrade(withFCV, isCompatible, createdOnBucketsCollection) { const index = bucketsColl.getIndexes()[0]; if (isCompatible) { @@ -51,43 +47,72 @@ function checkIndexForDowngrade(isCompatible, createdOnBucketsCollection) { assert(index.hasOwnProperty("originalSpec")); } - assert.commandFailedWithCode(db.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}), + assert.commandFailedWithCode(db.adminCommand({setFeatureCompatibilityVersion: withFCV}), ErrorCodes.CannotDowngrade); assert.commandWorked(coll.dropIndexes("*")); } - assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV})); + assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: withFCV})); assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: latestFCV})); assert.commandWorked(coll.dropIndexes("*")); } +// TODO SERVER-60911: Remove downgrade checks for lastContinuousFCV once kLatest is 5.3. + assert.commandWorked(coll.createIndex({[timeFieldName]: 1})); -checkIndexForDowngrade(true, false); +checkIndexForDowngrade(lastLTSFCV, true, false); + +assert.commandWorked(coll.createIndex({[timeFieldName]: 1})); +checkIndexForDowngrade(lastContinuousFCV, true, false); + +assert.commandWorked(coll.createIndex({[metaFieldName]: 1})); +checkIndexForDowngrade(lastLTSFCV, true, false); assert.commandWorked(coll.createIndex({[metaFieldName]: 1})); -checkIndexForDowngrade(true, false); +checkIndexForDowngrade(lastContinuousFCV, true, false); + +assert.commandWorked(coll.createIndex({[metaFieldName]: 1, a: 1})); +checkIndexForDowngrade(lastLTSFCV, false, false); assert.commandWorked(coll.createIndex({[metaFieldName]: 1, a: 1})); -checkIndexForDowngrade(false, false); +checkIndexForDowngrade(lastContinuousFCV, false, false); + +assert.commandWorked(coll.createIndex({b: 1})); +checkIndexForDowngrade(lastLTSFCV, false, false); assert.commandWorked(coll.createIndex({b: 1})); -checkIndexForDowngrade(false, false); +checkIndexForDowngrade(lastContinuousFCV, false, false); assert.commandWorked(bucketsColl.createIndex({"control.min.c.d": 1, "control.max.c.d": 1})); -checkIndexForDowngrade(false, true); +checkIndexForDowngrade(lastLTSFCV, false, true); + +assert.commandWorked(bucketsColl.createIndex({"control.min.c.d": 1, "control.max.c.d": 1})); +checkIndexForDowngrade(lastContinuousFCV, false, true); assert.commandWorked(bucketsColl.createIndex({"control.min.e": 1, "control.min.f": 1})); -checkIndexForDowngrade(false, true); +checkIndexForDowngrade(lastLTSFCV, false, true); + +assert.commandWorked(bucketsColl.createIndex({"control.min.e": 1, "control.min.f": 1})); +checkIndexForDowngrade(lastContinuousFCV, false, true); + +assert.commandWorked(coll.createIndex({g: "2dsphere"})); +checkIndexForDowngrade(lastLTSFCV, false, false); assert.commandWorked(coll.createIndex({g: "2dsphere"})); -checkIndexForDowngrade(false, false); +checkIndexForDowngrade(lastContinuousFCV, false, false); + +assert.commandWorked(coll.createIndex({[metaFieldName]: "2d"})); +checkIndexForDowngrade(lastLTSFCV, true, false); assert.commandWorked(coll.createIndex({[metaFieldName]: "2d"})); -checkIndexForDowngrade(true, false); +checkIndexForDowngrade(lastContinuousFCV, true, false); + +assert.commandWorked(coll.createIndex({[metaFieldName]: "2dsphere"})); +checkIndexForDowngrade(lastLTSFCV, true, false); assert.commandWorked(coll.createIndex({[metaFieldName]: "2dsphere"})); -checkIndexForDowngrade(true, false); +checkIndexForDowngrade(lastContinuousFCV, true, false); MongoRunner.stopMongod(conn); }()); diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index dc02c016862..8ae88794bb7 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -639,8 +639,9 @@ Status _collModInternal(OperationContext* opCtx, opCtx, coll.getWritableCollection(), desc); } - // TODO SERVER-60911: When kLatest is 5.3, only check when upgrading from kLastLTS (5.0). - // TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated upgrade code. + // TODO SERVER-60911: When kLatest is 5.3, only check when upgrading from or downgrading to + // kLastLTS (5.0). + // TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated upgrade/downgrade code. if (coll->getTimeseriesOptions() && !coll->getTimeseriesBucketsMayHaveMixedSchemaData() && serverGlobalParams.featureCompatibility.isFCVUpgradingToOrAlreadyLatest()) { // While upgrading the FCV to 5.2+, collMod is called as part of the upgrade process to @@ -649,6 +650,14 @@ Status _collModInternal(OperationContext* opCtx, // time-series collection existed in earlier server versions and may have mixed-schema // data. coll.getWritableCollection()->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, true); + } else if (coll->getTimeseriesBucketsMayHaveMixedSchemaData() && + serverGlobalParams.featureCompatibility + .isFCVDowngradingOrAlreadyDowngradedFromLatest()) { + // While downgrading the FCV from 5.2, collMod is called as part of the downgrade + // process to remove the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry + // flag for time-series collections that have the flag. + coll.getWritableCollection()->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, + boost::none); } // Only observe non-view collMods, as view operations are observed as operations on the diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 051a1c46164..0ba948c2afe 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -536,7 +536,7 @@ public: * Sets the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag to 'setting' for this * collection. * - * Throws if FCV < 5.2 or if this is not a time-series collection. + * Throws if this is not a time-series collection. */ virtual void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx, boost::optional<bool> setting) = 0; diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 3c001c5af53..33426a8804b 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -1352,12 +1352,12 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext boost::optional<bool> setting) { uassert(6057500, "This is not a time-series collection", _metadata->options.timeseries); - // TODO SERVER-60911: When kLatest is 5.3, only check when upgrading from kLastLTS (5.0). - // TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated upgrade code. - uassert( - 6057501, - "Cannot set the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag if FCV < 5.2", - serverGlobalParams.featureCompatibility.isFCVUpgradingToOrAlreadyLatest()); + LOGV2_DEBUG(6057601, + 1, + "Setting 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag", + logAttrs(ns()), + logAttrs(uuid()), + "setting"_attr = setting); _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { md.timeseriesBucketsMayHaveMixedSchemaData = setting; diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 061faf02f6f..77d9800f8d0 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -489,16 +489,26 @@ private: _cancelTenantMigrations(opCtx); - // Secondary indexes on time-series measurements are only supported in 5.2 and up. If the - // user tries to downgrade the cluster to an earlier version, they must first remove all - // incompatible secondary indexes on time-series measurements. - if (requestedVersion < multiversion::FeatureCompatibilityVersion::kVersion_5_2) { + { + // Take the global lock in S mode to create a barrier for operations taking the global + // IX or X locks. This ensures that either + // - The global IX/X locked operation will start after the FCV change, see the + // downgrading to the last-lts or last-continuous FCV and act accordingly. + // - The global IX/X locked operation began prior to the FCV change, is acting on that + // assumption and will finish before downgrade procedures begin right after this. + Lock::GlobalLock lk(opCtx, MODE_S); + } + + // TODO SERVER-60911: When kLatest is 5.3, only check when downgrading to kLastLTS (5.0). + // TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated downgrade code. + if (serverGlobalParams.featureCompatibility + .isFCVDowngradingOrAlreadyDowngradedFromLatest()) { for (const auto& dbName : DatabaseHolder::get(opCtx)->getNames()) { - Lock::DBLock dbLock(opCtx, dbName, MODE_IS); + Lock::DBLock dbLock(opCtx, dbName, MODE_IX); catalog::forEachCollectionFromDb( opCtx, dbName, - MODE_IS, + MODE_X, [&](const CollectionPtr& collection) { invariant(collection->getTimeseriesOptions()); @@ -508,6 +518,10 @@ private: while (indexIt->more()) { auto indexEntry = indexIt->next(); + // Secondary indexes on time-series measurements are only supported + // in 5.2 and up. If the user tries to downgrade the cluster to an + // earlier version, they must first remove all incompatible secondary + // indexes on time-series measurements. uassert(ErrorCodes::CannotDowngrade, str::stream() << "Cannot downgrade the cluster when there are secondary " @@ -540,6 +554,30 @@ private: } } + if (!collection->getTimeseriesBucketsMayHaveMixedSchemaData()) { + // The catalog entry flag has already been removed. This can happen if + // the downgrade process was interrupted and is being run again. The + // downgrade process cannot be aborted at this point. + return true; + } + + BSONObjBuilder unusedBuilder; + Status status = collMod(opCtx, + collection->ns(), + BSON("collMod" << collection->ns().coll()), + &unusedBuilder); + + if (!status.isOK()) { + LOGV2_FATAL( + 6057600, + "Failed to remove catalog entry during downgrade", + "error"_attr = status, + "timeseriesBucketsMayHaveMixedSchemaData"_attr = + collection->getTimeseriesBucketsMayHaveMixedSchemaData(), + logAttrs(collection->ns()), + logAttrs(collection->uuid())); + } + return true; }, [&](const CollectionPtr& collection) { @@ -548,16 +586,6 @@ private: } } - { - // Take the global lock in S mode to create a barrier for operations taking the global - // IX or X locks. This ensures that either - // - The global IX/X locked operation will start after the FCV change, see the - // downgrading to the last-lts or last-continuous FCV and act accordingly. - // - The global IX/X locked operation began prior to the FCV change, is acting on that - // assumption and will finish before downgrade procedures begin right after this. - Lock::GlobalLock lk(opCtx, MODE_S); - } - uassert(ErrorCodes::Error(549181), "Failing upgrade due to 'failDowngrading' failpoint set", !failDowngrading.shouldFail()); |