diff options
34 files changed, 278 insertions, 19 deletions
diff --git a/jstests/sharding/timeseries_sharded_query.js b/jstests/sharding/timeseries_sharded_query.js index 4324d2c1dc9..5cdcdb1b406 100644 --- a/jstests/sharding/timeseries_sharded_query.js +++ b/jstests/sharding/timeseries_sharded_query.js @@ -13,11 +13,10 @@ load("jstests/sharding/libs/find_chunks_util.js"); Random.setRandomSeed(); -const st = new ShardingTest({shards: 2, rs: {nodes: 2}}); +const st = new ShardingTest({shards: 2}); const dbName = 'test'; const sDB = st.s.getDB(dbName); -const configDB = st.s0.getDB('config'); if (!TimeseriesTest.timeseriesCollectionsEnabled(st.shard0)) { jsTestLog("Skipping test because the time-series collection feature flag is disabled"); @@ -61,5 +60,70 @@ if (!TimeseriesTest.timeseriesCollectionsEnabled(st.shard0)) { assert.commandWorked(sDB.dropDatabase()); })(); +(function manuallyCraftedShardedTimeseriesCollectionCannotBeUsed() { + assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); + + assert.commandWorked(sDB.createCollection('coll')); + + st.ensurePrimaryShard(dbName, st.shard0.shardName); + for (let i = 0; i < 20; i++) { + assert.commandWorked(st.shard0.getDB(dbName).coll.insert({a: i})); + } + + assert.commandWorked(st.shard0.getDB(dbName).coll.createIndex({a: 1})); + assert.commandWorked(st.s.adminCommand({shardCollection: 'test.coll', key: {a: 1}})); + + // It modifies the time-series metadata on the CS and on the shards + let modifyTimeseriesMetadata = (opTimeseriesMetadata) => { + st.s.getDB('config').collections.update({_id: 'test.coll'}, opTimeseriesMetadata); + st.shard0.getDB('config').cache.collections.update({_id: 'test.coll'}, + opTimeseriesMetadata); + st.shard1.getDB('config').cache.collections.update({_id: 'test.coll'}, + opTimeseriesMetadata); + }; + + // It forces a bump of the collection version moving the {a: 0} chunk to destShardName + let bumpCollectionVersionThroughMoveChunk = (destShardName) => { + assert.commandWorked( + st.s.adminCommand({moveChunk: 'test.coll', find: {a: 0}, to: destShardName})); + }; + + // It forces a refresh of the routing info on the shards + let forceRefreshOnShards = () => { + assert.commandWorked(st.shard0.adminCommand( + {_flushRoutingTableCacheUpdates: 'test.coll', syncFromConfig: true})); + assert.commandWorked(st.shard1.adminCommand( + {_flushRoutingTableCacheUpdates: 'test.coll', syncFromConfig: true})); + }; + + // Hacky code to simulate that 'test.coll' is a sharded time-series collection + modifyTimeseriesMetadata({$set: {timeseriesFields: {timeField: "a"}}}); + bumpCollectionVersionThroughMoveChunk(st.shard1.shardName); + forceRefreshOnShards(); + + let check = (cmdRes) => { + assert.commandFailedWithCode(cmdRes, ErrorCodes.NotImplemented); + }; + + // CRUD ops & drop collection + check(st.s.getDB(dbName).runCommand({find: 'coll', filter: {a: 1}})); + check(st.s.getDB(dbName).runCommand({find: 'coll', filter: {}})); + check(st.s.getDB(dbName).runCommand({insert: 'coll', documents: [{a: 21}]})); + check(st.s.getDB(dbName).runCommand({insert: 'coll', documents: [{a: 21}, {a: 22}]})); + check(st.s.getDB(dbName).runCommand( + {update: 'coll', updates: [{q: {a: 1}, u: {$set: {b: 10}}}]})); + check(st.s.getDB(dbName).runCommand({update: 'coll', updates: [{q: {}, u: {$set: {b: 10}}}]})); + check(st.s.getDB(dbName).runCommand({delete: 'coll', deletes: [{q: {a: 1}, limit: 1}]})); + check(st.s.getDB(dbName).runCommand({delete: 'coll', deletes: [{q: {}, limit: 0}]})); + // check(st.s.getDB(dbName).runCommand({drop: 'coll'}), ErrorCodes.IllegalOperation); + + // Hacky code again to restore the previous environment to finish this test properly + modifyTimeseriesMetadata({$unset: {timeseriesFields: 1}}); + bumpCollectionVersionThroughMoveChunk(st.shard0.shardName); + forceRefreshOnShards(); + + assert.commandWorked(sDB.dropDatabase()); +})(); + st.stop(); })(); diff --git a/src/mongo/db/s/collection_metadata.h b/src/mongo/db/s/collection_metadata.h index a742b77726f..79f02080b19 100644 --- a/src/mongo/db/s/collection_metadata.h +++ b/src/mongo/db/s/collection_metadata.h @@ -261,6 +261,11 @@ public: return _cm->getReshardingFields(); } + const boost::optional<TypeCollectionTimeseriesFields>& getTimeseriesFields() const { + invariant(isSharded()); + return _cm->getTimeseriesFields(); + } + private: // The full routing table for the collection or boost::none if the collection is not sharded boost::optional<ChunkManager> _cm; diff --git a/src/mongo/db/s/collection_metadata_filtering_test.cpp b/src/mongo/db/s/collection_metadata_filtering_test.cpp index e0559141e43..1db44836b11 100644 --- a/src/mongo/db/s/collection_metadata_filtering_test.cpp +++ b/src/mongo/db/s/collection_metadata_filtering_test.cpp @@ -34,6 +34,7 @@ #include "mongo/db/s/operation_sharding_state.h" #include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/type_collection_timeseries_fields_gen.h" namespace mongo { namespace { @@ -59,7 +60,8 @@ protected: * time (75,25) shard0(chunk2, chunk4) shard1(chunk1, chunk3) * time (25,0) - no history */ - void prepareTestData() { + void prepareTestData( + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields = boost::none) { const OID epoch = OID::gen(); const ShardKeyPattern shardKeyPattern(BSON("_id" << 1)); @@ -71,6 +73,7 @@ protected: false, epoch, boost::none /* timestamp */, + timeseriesFields, boost::none, true, [&] { @@ -210,5 +213,32 @@ TEST_F(CollectionMetadataFilteringTest, FilterDocumentsTooFarInThePastThrowsStal operationContext(), CollectionShardingState::OrphanCleanupPolicy::kAllowOrphanCleanup)); } +TEST_F(CollectionMetadataFilteringTest, DisallowOpsOnShardedTimeseriesCollection) { + TypeCollectionTimeseriesFields timeseriesFields("fieldName"); + prepareTestData(timeseriesFields); + + BSONObj readConcern = BSON("readConcern" << BSON("level" + << "snapshot" + << "atClusterTime" << Timestamp(100, 0))); + + auto&& readConcernArgs = repl::ReadConcernArgs::get(operationContext()); + ASSERT_OK(readConcernArgs.initialize(readConcern["readConcern"])); + + AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); + auto* const css = CollectionShardingState::get(operationContext(), kNss); + + auto check = [&](const DBException& ex) { + ASSERT_EQ(ex.code(), ErrorCodes::NotImplemented) << ex.toString(); + ASSERT_STRING_CONTAINS(ex.reason(), + "Operations on sharded time-series collections are not supported"); + }; + + ASSERT_THROWS_WITH_CHECK( + css->getOwnershipFilter(operationContext(), + CollectionShardingState::OrphanCleanupPolicy::kAllowOrphanCleanup), + AssertionException, + check); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp index b5f9e77045d..30187003857 100644 --- a/src/mongo/db/s/collection_metadata_test.cpp +++ b/src/mongo/db/s/collection_metadata_test.cpp @@ -94,6 +94,7 @@ CollectionMetadata makeCollectionMetadataImpl( false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, std::move(reshardingFields), true, allChunks)), diff --git a/src/mongo/db/s/collection_sharding_runtime.cpp b/src/mongo/db/s/collection_sharding_runtime.cpp index 9edc8921fcd..21d5550a775 100644 --- a/src/mongo/db/s/collection_sharding_runtime.cpp +++ b/src/mongo/db/s/collection_sharding_runtime.cpp @@ -39,6 +39,7 @@ #include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/sharding_state.h" #include "mongo/logv2/log.h" +#include "mongo/s/type_collection_timeseries_fields_gen.h" #include "mongo/util/duration.h" namespace mongo { @@ -317,6 +318,11 @@ CollectionShardingRuntime::_getMetadataWithVersionCheckAt( optCurrentMetadata); const auto& currentMetadata = optCurrentMetadata->get(); + + uassert(ErrorCodes::NotImplemented, + "Operations on sharded time-series collections are not supported", + !currentMetadata.isSharded() || !currentMetadata.getTimeseriesFields()); + auto wantedShardVersion = currentMetadata.getShardVersion(); { diff --git a/src/mongo/db/s/collection_sharding_runtime_test.cpp b/src/mongo/db/s/collection_sharding_runtime_test.cpp index d18ba119720..cb7cbf2b6fc 100644 --- a/src/mongo/db/s/collection_sharding_runtime_test.cpp +++ b/src/mongo/db/s/collection_sharding_runtime_test.cpp @@ -66,6 +66,7 @@ protected: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {std::move(chunk)})), diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp index 106172047ab..b11ace72e72 100644 --- a/src/mongo/db/s/metadata_manager_test.cpp +++ b/src/mongo/db/s/metadata_manager_test.cpp @@ -86,6 +86,7 @@ protected: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {ChunkType{ diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp index ee7fd354c75..eeff0c7976b 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp @@ -163,6 +163,7 @@ protected: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {ChunkType{kNss, diff --git a/src/mongo/db/s/op_observer_sharding_test.cpp b/src/mongo/db/s/op_observer_sharding_test.cpp index 2e88a2de050..93b0a5b58f9 100644 --- a/src/mongo/db/s/op_observer_sharding_test.cpp +++ b/src/mongo/db/s/op_observer_sharding_test.cpp @@ -75,6 +75,7 @@ protected: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {std::move(chunk)}); diff --git a/src/mongo/db/s/range_deletion_util_test.cpp b/src/mongo/db/s/range_deletion_util_test.cpp index 77d14f9f53d..be8ed9635b8 100644 --- a/src/mongo/db/s/range_deletion_util_test.cpp +++ b/src/mongo/db/s/range_deletion_util_test.cpp @@ -103,6 +103,7 @@ public: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {ChunkType{kNss, diff --git a/src/mongo/db/s/resharding/resharding_data_replication_test.cpp b/src/mongo/db/s/resharding/resharding_data_replication_test.cpp index 10091b5f3df..dcbe12200b0 100644 --- a/src/mongo/db/s/resharding/resharding_data_replication_test.cpp +++ b/src/mongo/db/s/resharding/resharding_data_replication_test.cpp @@ -85,6 +85,7 @@ public: false /* unique */, std::move(epoch), boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none /* reshardingFields */, true /* allowMigrations */, chunks); diff --git a/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h b/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h index b8f4356bd17..bc0e4888e43 100644 --- a/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h +++ b/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h @@ -105,20 +105,21 @@ protected: auto range = ChunkRange(BSON(shardKey << MINKEY), BSON(shardKey << MAXKEY)); auto chunk = ChunkType( nss, std::move(range), ChunkVersion(1, 0, epoch, boost::none), shardThatChunkExistsOn); - ChunkManager cm( - kThisShard.getShardId(), - DatabaseVersion(uuid), - makeStandaloneRoutingTableHistory(RoutingTableHistory::makeNew(nss, - uuid, - shardKeyPattern, - nullptr, - false, - epoch, - boost::none, - boost::none, - true, - {std::move(chunk)})), - boost::none); + ChunkManager cm(kThisShard.getShardId(), + DatabaseVersion(uuid), + makeStandaloneRoutingTableHistory( + RoutingTableHistory::makeNew(nss, + uuid, + shardKeyPattern, + nullptr, + false, + epoch, + boost::none, + boost::none /* timeseriesFields */, + boost::none, + true, + {std::move(chunk)})), + boost::none); if (!OperationShardingState::isOperationVersioned(opCtx)) { const auto version = cm.getVersion(kThisShard.getShardId()); diff --git a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp index 786ab9e96de..d87a2164416 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp @@ -189,6 +189,7 @@ public: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, false, chunks); diff --git a/src/mongo/db/s/resharding/resharding_oplog_crud_application_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_crud_application_test.cpp index a8616102cdc..9a8c9231677 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_crud_application_test.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_crud_application_test.cpp @@ -258,6 +258,7 @@ private: false /* unique */, std::move(epoch), boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none /* reshardingFields */, true /* allowMigrations */, chunks); diff --git a/src/mongo/db/s/scoped_collection_metadata.h b/src/mongo/db/s/scoped_collection_metadata.h index 014f1a083da..6977e9733bf 100644 --- a/src/mongo/db/s/scoped_collection_metadata.h +++ b/src/mongo/db/s/scoped_collection_metadata.h @@ -101,6 +101,10 @@ public: return _impl->get().getReshardingFields(); } + const boost::optional<TypeCollectionTimeseriesFields>& getTimeseriesFields() const { + return _impl->get().getTimeseriesFields(); + } + protected: std::shared_ptr<Impl> _impl; }; diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp index f2c9a70828d..0c2addd5472 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp +++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp @@ -101,6 +101,7 @@ Status persistCollectionAndChangedChunks(OperationContext* opCtx, collAndChunks.shardKeyPattern, collAndChunks.shardKeyIsUnique); update.setDefaultCollation(collAndChunks.defaultCollation); + update.setTimeseriesFields(collAndChunks.timeseriesFields); update.setReshardingFields(collAndChunks.reshardingFields); update.setAllowMigrations(collAndChunks.allowMigrations); @@ -251,6 +252,7 @@ CollectionAndChangedChunks getPersistedMetadataSinceVersion(OperationContext* op shardCollectionEntry.getKeyPattern().toBSON(), shardCollectionEntry.getDefaultCollation(), shardCollectionEntry.getUnique(), + shardCollectionEntry.getTimeseriesFields(), shardCollectionEntry.getReshardingFields(), shardCollectionEntry.getAllowMigrations(), std::move(changedChunks)}; diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp index 026348146dd..2595b646e85 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp +++ b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp @@ -37,6 +37,7 @@ #include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_database.h" #include "mongo/s/catalog_cache_loader_mock.h" +#include "mongo/s/type_collection_timeseries_fields_gen.h" namespace mongo { namespace { @@ -208,6 +209,7 @@ vector<ChunkType> ShardServerCatalogCacheLoaderTest::setUpChunkLoaderWithFiveChu ASSERT_EQUALS(collAndChunksRes.epoch, collectionType.getEpoch()); ASSERT_EQUALS(collAndChunksRes.changedChunks.size(), 5UL); + ASSERT(!collAndChunksRes.timeseriesFields.is_initialized()); for (unsigned int i = 0; i < collAndChunksRes.changedChunks.size(); ++i) { ASSERT_BSONOBJ_EQ(collAndChunksRes.changedChunks[i].toShardBSON(), chunks[i].toShardBSON()); } @@ -506,5 +508,20 @@ TEST_F(ShardServerCatalogCacheLoaderTest, PrimaryLoadFromShardedAndFindDbMetadat ASSERT_EQUALS(dbType.getVersion().getTimestamp(), newDbType.getVersion().getTimestamp()); } +TEST_F(ShardServerCatalogCacheLoaderTest, TimeseriesFieldsAreProperlyPropagatedOnSSCCL) { + ChunkVersion collectionVersion(1, 0, OID::gen(), boost::none /* timestamp */); + + CollectionType collectionType = makeCollectionType(collectionVersion); + collectionType.setTimeseriesFields(TypeCollectionTimeseriesFields("fieldName")); + + vector<ChunkType> chunks = makeFiveChunks(collectionVersion); + + _remoteLoaderMock->setCollectionRefreshReturnValue(collectionType); + _remoteLoaderMock->setChunkRefreshReturnValue(chunks); + + auto collAndChunksRes = _shardLoader->getChunksSince(kNss, ChunkVersion::UNSHARDED()).get(); + ASSERT(collAndChunksRes.timeseriesFields.is_initialized()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/s/type_shard_collection.h b/src/mongo/db/s/type_shard_collection.h index df51ee8db3c..1c805a31c63 100644 --- a/src/mongo/db/s/type_shard_collection.h +++ b/src/mongo/db/s/type_shard_collection.h @@ -58,6 +58,7 @@ public: using ShardCollectionTypeBase::getNss; using ShardCollectionTypeBase::getRefreshing; using ShardCollectionTypeBase::getReshardingFields; + using ShardCollectionTypeBase::getTimeseriesFields; using ShardCollectionTypeBase::getTimestamp; using ShardCollectionTypeBase::getUnique; using ShardCollectionTypeBase::getUuid; @@ -69,6 +70,7 @@ public: using ShardCollectionTypeBase::setNss; using ShardCollectionTypeBase::setRefreshing; using ShardCollectionTypeBase::setReshardingFields; + using ShardCollectionTypeBase::setTimeseriesFields; using ShardCollectionTypeBase::setUnique; using ShardCollectionTypeBase::setUuid; diff --git a/src/mongo/db/s/type_shard_collection.idl b/src/mongo/db/s/type_shard_collection.idl index e0ce20f6087..71b1aac4f39 100644 --- a/src/mongo/db/s/type_shard_collection.idl +++ b/src/mongo/db/s/type_shard_collection.idl @@ -75,6 +75,7 @@ imports: - "mongo/db/keypattern.idl" - "mongo/s/chunk_version.idl" - "mongo/s/resharding/type_collection_fields.idl" + - "mongo/s/type_collection_timeseries_fields.idl" structs: ShardCollectionTypeBase: @@ -161,3 +162,8 @@ structs: It must be optional and not present when running in FCV 4.4, because binaries prior to 5.0 use strict parsing and will fail." optional: true + timeseriesFields: + type: TypeCollectionTimeseriesFields + description: "Time-series collection fields. Only set when this is a time-series + buckets collection." + optional: true diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index d6ff3a3e537..43989441b00 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -188,6 +188,7 @@ env.Library( 'shard_invalidated_for_targeting_exception.cpp', 'sharding_ddl_50_upgrade_downgrade.idl', 'stale_exception.cpp', + 'type_collection_timeseries_fields.idl', 'would_change_owning_shard_exception.cpp', ], LIBDEPS=[ diff --git a/src/mongo/s/catalog/type_collection.h b/src/mongo/s/catalog/type_collection.h index 9e3bf89e4a1..d723799d966 100644 --- a/src/mongo/s/catalog/type_collection.h +++ b/src/mongo/s/catalog/type_collection.h @@ -93,11 +93,13 @@ public: // Make getters and setters accessible. using CollectionTypeBase::getNss; using CollectionTypeBase::getReshardingFields; + using CollectionTypeBase::getTimeseriesFields; using CollectionTypeBase::getTimestamp; using CollectionTypeBase::getUnique; using CollectionTypeBase::getUpdatedAt; using CollectionTypeBase::setNss; using CollectionTypeBase::setReshardingFields; + using CollectionTypeBase::setTimeseriesFields; using CollectionTypeBase::setTimestamp; using CollectionTypeBase::setUnique; using CollectionTypeBase::setUpdatedAt; diff --git a/src/mongo/s/catalog/type_collection.idl b/src/mongo/s/catalog/type_collection.idl index 93458c0226e..5917c2faad5 100644 --- a/src/mongo/s/catalog/type_collection.idl +++ b/src/mongo/s/catalog/type_collection.idl @@ -34,6 +34,7 @@ imports: - "mongo/db/keypattern.idl" - "mongo/s/chunk_version.idl" - "mongo/s/resharding/type_collection_fields.idl" + - "mongo/s/type_collection_timeseries_fields.idl" structs: CollectionTypeBase: @@ -139,3 +140,8 @@ structs: It is optional for serialisation purposes, because in versions of MongoDB prior to 5.0, this value would be missing." optional: true + timeseriesFields: + type: TypeCollectionTimeseriesFields + description: "Time-series collection fields. Only set when this is a time-series + buckets collection." + optional: true diff --git a/src/mongo/s/catalog_cache.cpp b/src/mongo/s/catalog_cache.cpp index 8d635ce7caf..f6cbb4afbfc 100644 --- a/src/mongo/s/catalog_cache.cpp +++ b/src/mongo/s/catalog_cache.cpp @@ -670,6 +670,7 @@ CatalogCache::CollectionCache::LookupResult CatalogCache::CollectionCache::_look collectionAndChunks.shardKeyIsUnique, collectionAndChunks.epoch, collectionAndChunks.creationTime, + collectionAndChunks.timeseriesFields, std::move(collectionAndChunks.reshardingFields), collectionAndChunks.allowMigrations, collectionAndChunks.changedChunks); diff --git a/src/mongo/s/catalog_cache_loader.cpp b/src/mongo/s/catalog_cache_loader.cpp index d047def7fd2..a25aa552150 100644 --- a/src/mongo/s/catalog_cache_loader.cpp +++ b/src/mongo/s/catalog_cache_loader.cpp @@ -48,6 +48,7 @@ CatalogCacheLoader::CollectionAndChangedChunks::CollectionAndChangedChunks( const BSONObj& collShardKeyPattern, const BSONObj& collDefaultCollation, bool collShardKeyIsUnique, + boost::optional<TypeCollectionTimeseriesFields> collTimeseriesFields, boost::optional<TypeCollectionReshardingFields> collReshardingFields, bool allowMigrations, std::vector<ChunkType> chunks) @@ -57,6 +58,7 @@ CatalogCacheLoader::CollectionAndChangedChunks::CollectionAndChangedChunks( shardKeyPattern(collShardKeyPattern), defaultCollation(collDefaultCollation), shardKeyIsUnique(collShardKeyIsUnique), + timeseriesFields(std::move(collTimeseriesFields)), reshardingFields(std::move(collReshardingFields)), allowMigrations(allowMigrations), changedChunks(std::move(chunks)) {} diff --git a/src/mongo/s/catalog_cache_loader.h b/src/mongo/s/catalog_cache_loader.h index 76a6d47460e..528ff5519e3 100644 --- a/src/mongo/s/catalog_cache_loader.h +++ b/src/mongo/s/catalog_cache_loader.h @@ -72,6 +72,7 @@ public: const BSONObj& collShardKeyPattern, const BSONObj& collDefaultCollation, bool collShardKeyIsUnique, + boost::optional<TypeCollectionTimeseriesFields> collTimeseriesFields, boost::optional<TypeCollectionReshardingFields> collReshardingFields, bool allowMigrations, std::vector<ChunkType> chunks); @@ -85,6 +86,9 @@ public: BSONObj defaultCollation; bool shardKeyIsUnique; + // This information will be valid if the collection is a time-series buckets collection. + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields; + // If the collection is currently undergoing a resharding operation, the optional will be // populated. boost::optional<TypeCollectionReshardingFields> reshardingFields; diff --git a/src/mongo/s/catalog_cache_loader_mock.cpp b/src/mongo/s/catalog_cache_loader_mock.cpp index 3504fba10d1..96bdd409054 100644 --- a/src/mongo/s/catalog_cache_loader_mock.cpp +++ b/src/mongo/s/catalog_cache_loader_mock.cpp @@ -95,6 +95,7 @@ CollectionAndChangedChunks getCollectionRefresh( swCollectionReturnValue.getValue().getKeyPattern().toBSON(), swCollectionReturnValue.getValue().getDefaultCollation(), swCollectionReturnValue.getValue().getUnique(), + swCollectionReturnValue.getValue().getTimeseriesFields(), reshardingFields, swCollectionReturnValue.getValue().getAllowMigrations(), std::move(chunks)}; diff --git a/src/mongo/s/catalog_cache_test.cpp b/src/mongo/s/catalog_cache_test.cpp index d487fab8b0f..e1a0f3d24eb 100644 --- a/src/mongo/s/catalog_cache_test.cpp +++ b/src/mongo/s/catalog_cache_test.cpp @@ -38,6 +38,7 @@ #include "mongo/s/catalog_cache_loader_mock.h" #include "mongo/s/sharding_router_test_fixture.h" #include "mongo/s/stale_exception.h" +#include "mongo/s/type_collection_timeseries_fields_gen.h" namespace mongo { namespace { @@ -354,7 +355,6 @@ TEST_F(CatalogCacheTest, CheckEpochWithMatch) { _catalogCache->checkEpochOrThrow(kNss, collVersion, kShards[0]); } - TEST_F(CatalogCacheTest, GetDatabaseWithMetadataFormatChange) { const auto dbName = "testDB"; const auto uuid = UUID::gen(); @@ -382,7 +382,6 @@ TEST_F(CatalogCacheTest, GetDatabaseWithMetadataFormatChange) { getDatabaseWithRefreshAndCheckResults(versionWithoutTimestamp); } - TEST_F(CatalogCacheTest, GetCollectionWithMetadataFormatChange) { const auto dbVersion = DatabaseVersion(UUID::gen()); const auto epoch = OID::gen(); @@ -421,6 +420,26 @@ TEST_F(CatalogCacheTest, GetCollectionWithMetadataFormatChange) { getCollectionWithRefreshAndCheckResults(collVersionWithoutTimestamp); } +TEST_F(CatalogCacheTest, TimeseriesFieldsAreProperlyPropagatedOnCC) { + const auto dbVersion = DatabaseVersion(UUID::gen()); + const auto epoch = OID::gen(); + const auto version = ChunkVersion(1, 0, epoch, Timestamp(42)); + + loadDatabases({DatabaseType(kNss.db().toString(), kShards[0], true, dbVersion)}); + + auto coll = makeCollectionType(version); + coll.setTimeseriesFields(TypeCollectionTimeseriesFields("fieldName")); + + const auto scopedCollProv = scopedCollectionProvider(coll); + const auto scopedChunksProv = scopedChunksProvider(makeChunks(version)); + + const auto swChunkManager = + _catalogCache->getCollectionRoutingInfoWithRefresh(operationContext(), coll.getNss()); + ASSERT_OK(swChunkManager.getStatus()); + + const auto& chunkManager = swChunkManager.getValue(); + ASSERT(chunkManager.getTimeseriesFields().is_initialized()); +} } // namespace } // namespace mongo diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index 1bb16efcf2e..d5965db2cbc 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -314,6 +314,7 @@ RoutingTableHistory::RoutingTableHistory( KeyPattern shardKeyPattern, std::unique_ptr<CollatorInterface> defaultCollator, bool unique, + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields, boost::optional<TypeCollectionReshardingFields> reshardingFields, bool allowMigrations, ChunkMap chunkMap) @@ -322,6 +323,7 @@ RoutingTableHistory::RoutingTableHistory( _shardKeyPattern(shardKeyPattern), _defaultCollator(std::move(defaultCollator)), _unique(unique), + _timeseriesFields(std::move(timeseriesFields)), _reshardingFields(std::move(reshardingFields)), _allowMigrations(allowMigrations), _chunkMap(std::move(chunkMap)), @@ -736,6 +738,7 @@ RoutingTableHistory RoutingTableHistory::makeNew( bool unique, OID epoch, const boost::optional<Timestamp>& timestamp, + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields, boost::optional<TypeCollectionReshardingFields> reshardingFields, bool allowMigrations, const std::vector<ChunkType>& chunks) { @@ -744,6 +747,7 @@ RoutingTableHistory RoutingTableHistory::makeNew( std::move(shardKeyPattern), std::move(defaultCollator), std::move(unique), + std::move(timeseriesFields), boost::none, allowMigrations, ChunkMap{epoch, timestamp}) @@ -768,6 +772,7 @@ RoutingTableHistory RoutingTableHistory::makeUpdated( getShardKeyPattern().getKeyPattern(), CollatorInterface::cloneCollator(getDefaultCollator()), isUnique(), + _timeseriesFields, std::move(reshardingFields), allowMigrations, std::move(chunkMap)); @@ -798,6 +803,7 @@ RoutingTableHistory RoutingTableHistory::makeUpdatedReplacingTimestamp( getShardKeyPattern().getKeyPattern(), CollatorInterface::cloneCollator(getDefaultCollator()), _unique, + _timeseriesFields, _reshardingFields, _allowMigrations, std::move(newMap)); diff --git a/src/mongo/s/chunk_manager.h b/src/mongo/s/chunk_manager.h index f9e32bf6329..56a46bd4046 100644 --- a/src/mongo/s/chunk_manager.h +++ b/src/mongo/s/chunk_manager.h @@ -42,6 +42,7 @@ #include "mongo/s/database_version.h" #include "mongo/s/resharding/type_collection_fields_gen.h" #include "mongo/s/shard_key_pattern.h" +#include "mongo/s/type_collection_timeseries_fields_gen.h" #include "mongo/stdx/unordered_map.h" #include "mongo/util/concurrency/ticketholder.h" #include "mongo/util/read_through_cache.h" @@ -169,6 +170,7 @@ public: bool unique, OID epoch, const boost::optional<Timestamp>& timestamp, + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields, boost::optional<TypeCollectionReshardingFields> reshardingFields, bool allowMigrations, const std::vector<ChunkType>& chunks); @@ -301,6 +303,10 @@ public: return _uuid; } + const boost::optional<TypeCollectionTimeseriesFields>& getTimeseriesFields() const { + return _timeseriesFields; + } + const boost::optional<TypeCollectionReshardingFields>& getReshardingFields() const { return _reshardingFields; } @@ -317,6 +323,7 @@ private: KeyPattern shardKeyPattern, std::unique_ptr<CollatorInterface> defaultCollator, bool unique, + boost::optional<TypeCollectionTimeseriesFields> timeseriesFields, boost::optional<TypeCollectionReshardingFields> reshardingFields, bool allowMigrations, ChunkMap chunkMap); @@ -338,6 +345,9 @@ private: // Whether the sharding key is unique bool _unique; + // This information will be valid if the collection is a time-series buckets collection. + boost::optional<TypeCollectionTimeseriesFields> _timeseriesFields; + // The set of fields related to an ongoing resharding operation involving this collection. The // presence of the type inside the optional indicates that the collection is involved in a // resharding operation, and that these fields were present in the config.collections entry @@ -685,6 +695,10 @@ public: return _rt->optRt->getUUID(); } + const boost::optional<TypeCollectionTimeseriesFields>& getTimeseriesFields() const { + return _rt->optRt->getTimeseriesFields(); + } + const boost::optional<TypeCollectionReshardingFields>& getReshardingFields() const { return _rt->optRt->getReshardingFields(); } diff --git a/src/mongo/s/chunk_manager_query_test.cpp b/src/mongo/s/chunk_manager_query_test.cpp index 901da0d311e..b51886d4089 100644 --- a/src/mongo/s/chunk_manager_query_test.cpp +++ b/src/mongo/s/chunk_manager_query_test.cpp @@ -519,6 +519,7 @@ TEST_F(ChunkManagerQueryTest, SnapshotQueryWithMoreShardsThanLatestMetadata) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {chunk0, chunk1}); diff --git a/src/mongo/s/chunk_manager_refresh_bm.cpp b/src/mongo/s/chunk_manager_refresh_bm.cpp index bb1d6220257..c936ce9b85e 100644 --- a/src/mongo/s/chunk_manager_refresh_bm.cpp +++ b/src/mongo/s/chunk_manager_refresh_bm.cpp @@ -86,6 +86,7 @@ CollectionMetadata makeChunkManagerWithShardSelector(int nShards, true, collEpoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, chunks); @@ -175,6 +176,7 @@ auto BM_FullBuildOfChunkManager(benchmark::State& state, ShardSelectorFn selectS true, collEpoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, chunks); diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp index 488de79b089..18a2d7592c6 100644 --- a/src/mongo/s/config_server_catalog_cache_loader.cpp +++ b/src/mongo/s/config_server_catalog_cache_loader.cpp @@ -75,6 +75,7 @@ CollectionAndChangedChunks getChangedChunks(OperationContext* opCtx, coll.getKeyPattern().toBSON(), coll.getDefaultCollation(), coll.getUnique(), + coll.getTimeseriesFields(), coll.getReshardingFields(), coll.getAllowMigrations(), std::move(collAndChunks.second)}; diff --git a/src/mongo/s/routing_table_history_test.cpp b/src/mongo/s/routing_table_history_test.cpp index 07ac4872891..479c3610b08 100644 --- a/src/mongo/s/routing_table_history_test.cpp +++ b/src/mongo/s/routing_table_history_test.cpp @@ -167,6 +167,7 @@ public: false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {initChunk})); @@ -341,6 +342,7 @@ TEST_F(RoutingTableHistoryTest, TestSplits) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, {chunkAll}); @@ -394,6 +396,7 @@ TEST_F(RoutingTableHistoryTest, TestReplaceEmptyChunk) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, initialChunks); @@ -444,6 +447,7 @@ TEST_F(RoutingTableHistoryTest, TestUseLatestVersions) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, initialChunks); @@ -489,6 +493,7 @@ TEST_F(RoutingTableHistoryTest, TestOutOfOrderVersion) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, initialChunks); @@ -540,6 +545,7 @@ TEST_F(RoutingTableHistoryTest, TestMergeChunks) { epoch, boost::none, boost::none /* timestamp */, + boost::none /* timeseriesFields */, true, initialChunks); ASSERT_EQ(rt.numChunks(), 3); @@ -585,6 +591,7 @@ TEST_F(RoutingTableHistoryTest, TestMergeChunksOrdering) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, initialChunks); @@ -649,6 +656,7 @@ TEST_F(RoutingTableHistoryTest, TestFlatten) { false, epoch, boost::none /* timestamp */, + boost::none /* timeseriesFields */, boost::none, true, initialChunks); diff --git a/src/mongo/s/type_collection_timeseries_fields.idl b/src/mongo/s/type_collection_timeseries_fields.idl new file mode 100644 index 00000000000..a9a0da9f392 --- /dev/null +++ b/src/mongo/s/type_collection_timeseries_fields.idl @@ -0,0 +1,45 @@ +# Copyright (C) 2021-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +structs: + TypeCollectionTimeseriesFields: + description: "Fields for time-series buckets collection fields in config.collections." + strict: false + fields: + timeField: + description: "The name of the top-level field to be used for time." + type: string + metaField: + description: "The name of the top-level field describing the series." + type: string + optional: true |