diff options
-rw-r--r-- | jstests/noPassthroughWithMongod/timeseries_create.js | 7 | ||||
-rw-r--r-- | src/mongo/db/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/catalog/create_collection.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/namespace_string.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/namespace_string.h | 5 | ||||
-rw-r--r-- | src/mongo/db/timeseries/bucket_catalog.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/timeseries/bucket_catalog.h | 4 | ||||
-rw-r--r-- | src/mongo/db/timeseries/timeseries.idl | 19 | ||||
-rw-r--r-- | src/mongo/db/ttl.cpp | 34 |
9 files changed, 73 insertions, 16 deletions
diff --git a/jstests/noPassthroughWithMongod/timeseries_create.js b/jstests/noPassthroughWithMongod/timeseries_create.js index fabe63f87b9..6ff50d9df40 100644 --- a/jstests/noPassthroughWithMongod/timeseries_create.js +++ b/jstests/noPassthroughWithMongod/timeseries_create.js @@ -100,6 +100,9 @@ testValidTimeseriesOptions({timeField: "time", metaField: "meta"}); testValidTimeseriesOptions({timeField: "time", expireAfterSeconds: NumberLong(100)}); testValidTimeseriesOptions( {timeField: "time", metaField: "meta", expireAfterSeconds: NumberLong(100)}); +testValidTimeseriesOptions({timeField: "time", metaField: "meta", granularity: "seconds"}); +testValidTimeseriesOptions( + {timeField: "time", metaField: "meta", granularity: "seconds", bucketMaxSpanSeconds: 3600}); testInvalidTimeseriesOptions("", ErrorCodes.TypeMismatch); testInvalidTimeseriesOptions({timeField: 100}, ErrorCodes.TypeMismatch); @@ -114,6 +117,10 @@ testInvalidTimeseriesOptions({timeField: "time", invalidOption: {}}, 40415); testInvalidTimeseriesOptions({timeField: "sub.time"}, ErrorCodes.InvalidOptions); testInvalidTimeseriesOptions({timeField: "time", metaField: "sub.meta"}, ErrorCodes.InvalidOptions); testInvalidTimeseriesOptions({timeField: "time", metaField: "time"}, ErrorCodes.InvalidOptions); +testInvalidTimeseriesOptions({timeField: "time", metaField: "meta", granularity: "minutes"}, + ErrorCodes.InvalidOptions); +testInvalidTimeseriesOptions({timeField: "time", metaField: "meta", bucketMaxSpanSeconds: 10}, + ErrorCodes.InvalidOptions); testCompatibleCreateOptions({storageEngine: {}}); testCompatibleCreateOptions({indexOptionDefaults: {}}); diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 00520a3b284..865b287cda0 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1132,6 +1132,7 @@ env.Library( '$BUILD_DIR/mongo/db/commands/fsync_locked', '$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker', '$BUILD_DIR/mongo/idl/server_parameter', + 'catalog/database_holder', 'commands/server_status_core', 'service_context', 'write_ops', diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp index 66a8a62e799..5df1ec26d68 100644 --- a/src/mongo/db/catalog/create_collection.cpp +++ b/src/mongo/db/catalog/create_collection.cpp @@ -123,6 +123,16 @@ Status _createTimeseries(OperationContext* opCtx, options.viewOn = bucketsNs.coll().toString(); + auto granularity = options.timeseries->getGranularity(); + uassert(ErrorCodes::InvalidOptions, + "Time-series 'granularity' is required to be 'seconds'", + granularity == BucketGranularityEnum::Seconds); + + auto bucketMaxSpan = options.timeseries->getBucketMaxSpanSeconds(); + uassert(ErrorCodes::InvalidOptions, + "Time-series 'bucketMaxSpanSeconds' is required to be 3600", + bucketMaxSpan == 3600); + if (options.timeseries->getMetaField()) { options.pipeline = BSON_ARRAY(BSON("$_internalUnpackBucket" diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp index b8424d7361e..2414afacd6b 100644 --- a/src/mongo/db/namespace_string.cpp +++ b/src/mongo/db/namespace_string.cpp @@ -315,6 +315,11 @@ NamespaceString NamespaceString::makeTimeseriesBucketsNamespace() const { return {db(), kTimeseriesBucketsCollectionPrefix.toString() + coll()}; } +NamespaceString NamespaceString::bucketsNamespaceToTimeseries() const { + invariant(isTimeseriesBucketsCollection()); + return {db(), coll().substr(kTimeseriesBucketsCollectionPrefix.size())}; +} + bool NamespaceString::isReplicated() const { if (isLocal()) { return false; diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h index fdcc41739c6..f918180f567 100644 --- a/src/mongo/db/namespace_string.h +++ b/src/mongo/db/namespace_string.h @@ -357,6 +357,11 @@ public: NamespaceString makeTimeseriesBucketsNamespace() const; /** + * Returns the time-series view namespace for this buckets namespace. + */ + NamespaceString bucketsNamespaceToTimeseries() const; + + /** * Returns whether a namespace is replicated, based only on its string value. One notable * omission is that map reduce `tmp.mr` collections may or may not be replicated. Callers must * decide how to handle that case separately. diff --git a/src/mongo/db/timeseries/bucket_catalog.cpp b/src/mongo/db/timeseries/bucket_catalog.cpp index 4f25cb9cc92..ed86ce6441f 100644 --- a/src/mongo/db/timeseries/bucket_catalog.cpp +++ b/src/mongo/db/timeseries/bucket_catalog.cpp @@ -138,13 +138,13 @@ StatusWith<std::shared_ptr<BucketCatalog::WriteBatch>> BucketCatalog::insert( return true; } auto bucketTime = (*bucket).getTime(); - if (time - bucketTime >= kTimeseriesBucketMaxTimeRange) { + if (time - bucketTime >= Seconds(options.getBucketMaxSpanSeconds())) { stats->numBucketsClosedDueToTimeForward.fetchAndAddRelaxed(1); return true; } if (time < bucketTime) { if (!(*bucket)->hasBeenCommitted() && - (*bucket)->latestTime - time < kTimeseriesBucketMaxTimeRange) { + (*bucket)->latestTime - time < Seconds(options.getBucketMaxSpanSeconds())) { (*bucket).setTime(); } else { stats->numBucketsClosedDueToTimeBackward.fetchAndAddRelaxed(1); diff --git a/src/mongo/db/timeseries/bucket_catalog.h b/src/mongo/db/timeseries/bucket_catalog.h index a17d01f9d36..9f2b00befaa 100644 --- a/src/mongo/db/timeseries/bucket_catalog.h +++ b/src/mongo/db/timeseries/bucket_catalog.h @@ -47,10 +47,6 @@ class BucketCatalog { using IdleList = std::list<Bucket*>; public: - // This constant, together with parameters defined in timeseries.idl, defines limits on the - // measurements held in a bucket. - static constexpr auto kTimeseriesBucketMaxTimeRange = Hours(1); - class BucketId { friend class BucketCatalog; diff --git a/src/mongo/db/timeseries/timeseries.idl b/src/mongo/db/timeseries/timeseries.idl index 048a51f4e93..c8c65dd3e65 100644 --- a/src/mongo/db/timeseries/timeseries.idl +++ b/src/mongo/db/timeseries/timeseries.idl @@ -61,6 +61,16 @@ server_parameters: cpp_varname: gTimeseriesBucketsCollectionClusterById default: true +enums: + BucketGranularity: + description: "Describes a time-series collection's expected interval between subsequent + measurements" + type: string + values: + Seconds: "seconds" + Minutes: "minutes" + Hours: "hours" + structs: TimeseriesOptions: description: "The options that define a time-series collection." @@ -82,3 +92,12 @@ structs: deleted." type: safeInt64 optional: true + granularity: + description: "Describes the expected interval between subsequent measurements" + type: BucketGranularity + default: Seconds + bucketMaxSpanSeconds: + description: "The maximum range of time values for a bucket, in seconds" + type: safeInt + default: 3600 + validator: { gte: 1 } diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 68c7ed8a294..18f84480343 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -57,6 +57,7 @@ #include "mongo/db/timeseries/bucket_catalog.h" #include "mongo/db/ttl_collection_cache.h" #include "mongo/db/ttl_gen.h" +#include "mongo/db/views/view_catalog.h" #include "mongo/logv2/log.h" #include "mongo/util/background.h" #include "mongo/util/concurrency/idle_thread_block.h" @@ -294,15 +295,27 @@ private: * Generate the safe expiration date for a given collection and user-configured * expireAfterSeconds value. */ - Date_t safeExpirationDate(const CollectionPtr& coll, std::int64_t expireAfterSeconds) const { + Date_t safeExpirationDate(OperationContext* opCtx, + const CollectionPtr& coll, + std::int64_t expireAfterSeconds) const { if (coll->ns().isTimeseriesBucketsCollection()) { - // Don't delete data unless it is safely out of range of the bucket maximum time range. - // On time-series collections, the _id (and thus RecordId) is the minimum time value of - // a bucket. A bucket may have newer data, so we cannot safely delete the entire bucket - // yet until the maximum bucket range has passed, even if the minimum value can be - // expired. - return Date_t::now() - Seconds(expireAfterSeconds) - - Seconds(BucketCatalog::kTimeseriesBucketMaxTimeRange); + auto timeseriesNs = coll->ns().bucketsNamespaceToTimeseries(); + auto viewCatalog = DatabaseHolder::get(opCtx)->getViewCatalog(opCtx, timeseriesNs.db()); + invariant(viewCatalog); + auto viewDef = viewCatalog->lookup(opCtx, timeseriesNs.ns()); + uassert(ErrorCodes::NamespaceNotFound, + fmt::format("Could not find view definition for namespace: {}", + timeseriesNs.toString()), + viewDef); + + const auto bucketMaxSpan = Seconds(viewDef->timeseries()->getBucketMaxSpanSeconds()); + + // Don't delete data unless it is safely out of range of the bucket maximum time + // range. On time-series collections, the _id (and thus RecordId) is the minimum + // time value of a bucket. A bucket may have newer data, so we cannot safely delete + // the entire bucket yet until the maximum bucket range has passed, even if the + // minimum value can be expired. + return Date_t::now() - Seconds(expireAfterSeconds) - bucketMaxSpan; } return Date_t::now() - Seconds(expireAfterSeconds); @@ -375,7 +388,8 @@ private: const Date_t kDawnOfTime = Date_t::fromMillisSinceEpoch(std::numeric_limits<long long>::min()); - const auto expirationDate = safeExpirationDate(collection, secondsExpireElt.numberLong()); + const auto expirationDate = + safeExpirationDate(opCtx, collection, secondsExpireElt.numberLong()); const BSONObj startKey = BSON("" << kDawnOfTime); const BSONObj endKey = BSON("" << expirationDate); // The canonical check as to whether a key pattern element is "ascending" or @@ -459,7 +473,7 @@ private: "running TTL job for collection clustered by _id", logAttrs(collection->ns())); - const auto expirationDate = safeExpirationDate(collection, *expireAfterSeconds); + const auto expirationDate = safeExpirationDate(opCtx, collection, *expireAfterSeconds); // Generate upper bound ObjectId that compares greater than every ObjectId with a the same // timestamp or lower. |