diff options
18 files changed, 386 insertions, 106 deletions
diff --git a/jstests/core/timeseries/timeseries_internal_bounded_sort.js b/jstests/core/timeseries/timeseries_internal_bounded_sort.js index a158aa0bba3..67cdc4c1c31 100644 --- a/jstests/core/timeseries/timeseries_internal_bounded_sort.js +++ b/jstests/core/timeseries/timeseries_internal_bounded_sort.js @@ -47,9 +47,6 @@ const bucketMaxSpanSeconds = } assert.gt(buckets.aggregate([{$count: 'n'}]).next().n, 1, 'Expected more than one bucket'); } -// Create an index: we'll need this to scan the buckets in time order. -// TODO SERVER-60824 use the $natural / _id index instead. -assert.commandWorked(coll.createIndex({t: 1})); const unpackStage = getAggPlanStage(coll.explain().aggregate(), '$_internalUnpackBucket'); @@ -71,7 +68,7 @@ function assertSorted(result, ascending) { } function runTest(ascending) { - // Test sorting the whole collection. + // Test sorting the whole collection { const naive = buckets .aggregate([ @@ -82,21 +79,39 @@ function runTest(ascending) { .toArray(); assertSorted(naive, ascending); - const opt = buckets - .aggregate([ - {$sort: {'control.min.t': ascending ? 1 : -1}}, - unpackStage, - { - $_internalBoundedSort: { - sortKey: {t: ascending ? 1 : -1}, - bound: bucketMaxSpanSeconds, - } - }, - ]) - .toArray(); - assertSorted(opt, ascending); + const optFromMin = + buckets + .aggregate([ + {$sort: {'control.min.t': ascending ? 1 : -1}}, + unpackStage, + { + $_internalBoundedSort: { + sortKey: {t: ascending ? 1 : -1}, + bound: ascending ? {base: "min"} + : {base: "min", offset: bucketMaxSpanSeconds} + } + }, + ]) + .toArray(); + assertSorted(optFromMin, ascending); + assert.eq(naive, optFromMin); - assert.eq(naive, opt); + const optFromMax = + buckets + .aggregate([ + {$sort: {'control.max.t': ascending ? 1 : -1}}, + unpackStage, + { + $_internalBoundedSort: { + sortKey: {t: ascending ? 1 : -1}, + bound: ascending ? {base: "max", offset: -bucketMaxSpanSeconds} + : {base: "max"} + } + }, + ]) + .toArray(); + assertSorted(optFromMax, ascending); + assert.eq(naive, optFromMax); } // Test $sort + $limit. @@ -112,23 +127,43 @@ function runTest(ascending) { assertSorted(naive, ascending); assert.eq(100, naive.length); - const opt = buckets - .aggregate([ - {$sort: {'control.min.t': ascending ? 1 : -1}}, - unpackStage, - { - $_internalBoundedSort: { - sortKey: {t: ascending ? 1 : -1}, - bound: bucketMaxSpanSeconds, - } - }, - {$limit: 100}, - ]) - .toArray(); - assertSorted(opt, ascending); - assert.eq(100, opt.length); + const optFromMin = + buckets + .aggregate([ + {$sort: {'control.min.t': ascending ? 1 : -1}}, + unpackStage, + { + $_internalBoundedSort: { + sortKey: {t: ascending ? 1 : -1}, + bound: ascending ? {base: "min"} + : {base: "min", offset: bucketMaxSpanSeconds} + } + }, + {$limit: 100}, + ]) + .toArray(); + assertSorted(optFromMin, ascending); + assert.eq(100, optFromMin.length); + assert.eq(naive, optFromMin); - assert.eq(naive, opt); + const optFromMax = + buckets + .aggregate([ + {$sort: {'control.max.t': ascending ? 1 : -1}}, + unpackStage, + { + $_internalBoundedSort: { + sortKey: {t: ascending ? 1 : -1}, + bound: ascending ? {base: "max", offset: -bucketMaxSpanSeconds} + : {base: "max"} + } + }, + {$limit: 100}, + ]) + .toArray(); + assertSorted(optFromMax, ascending); + assert.eq(100, optFromMax.length); + assert.eq(naive, optFromMax); } } diff --git a/jstests/core/timeseries/timeseries_internal_bounded_sort_overflow.js b/jstests/core/timeseries/timeseries_internal_bounded_sort_overflow.js index 98911651c76..08191105d36 100644 --- a/jstests/core/timeseries/timeseries_internal_bounded_sort_overflow.js +++ b/jstests/core/timeseries/timeseries_internal_bounded_sort_overflow.js @@ -28,8 +28,6 @@ const buckets = db['system.buckets.' + coll.getName()]; coll.drop(); assert.commandWorked( db.createCollection(coll.getName(), {timeseries: {timeField: 't', metaField: 'm'}})); -const bucketMaxSpanSeconds = - db.getCollectionInfos({name: coll.getName()})[0].options.timeseries.bucketMaxSpanSeconds; const unpackStage = getAggPlanStage(coll.explain().aggregate(), '$_internalUnpackBucket'); assert(unpackStage.$_internalUnpackBucket); @@ -45,12 +43,7 @@ const result = buckets .aggregate([ {$sort: {'control.min.t': 1}}, unpackStage, - { - $_internalBoundedSort: { - sortKey: {t: 1}, - bound: bucketMaxSpanSeconds, - } - }, + {$_internalBoundedSort: {sortKey: {t: 1}, bound: {base: "min"}}}, ]) .toArray(); diff --git a/jstests/noPassthrough/timeseries_internal_bounded_sort_spilling.js b/jstests/noPassthrough/timeseries_internal_bounded_sort_spilling.js index e16d5d22b4b..07a592fb2e8 100644 --- a/jstests/noPassthrough/timeseries_internal_bounded_sort_spilling.js +++ b/jstests/noPassthrough/timeseries_internal_bounded_sort_spilling.js @@ -86,7 +86,7 @@ function assertSorted(result) { { $_internalBoundedSort: { sortKey: {t: 1}, - bound: bucketMaxSpanSeconds, + bound: {base: "min"}, } }, ], @@ -116,7 +116,7 @@ function assertSorted(result) { { $_internalBoundedSort: { sortKey: {t: 1}, - bound: bucketMaxSpanSeconds, + bound: {base: "min"}, } }, ], @@ -150,7 +150,7 @@ function assertSorted(result) { { $_internalBoundedSort: { sortKey: {t: 1}, - bound: bucketMaxSpanSeconds, + bound: {base: "min"}, } }, {$limit: 100}, diff --git a/src/mongo/db/cst/cst_pipeline_translation.cpp b/src/mongo/db/cst/cst_pipeline_translation.cpp index 36244d87fa3..ca85a8884a0 100644 --- a/src/mongo/db/cst/cst_pipeline_translation.cpp +++ b/src/mongo/db/cst/cst_pipeline_translation.cpp @@ -217,6 +217,12 @@ auto translateMeta(const CNode::ObjectChildren& object, ExpressionContext* expCt return make_intrusive<ExpressionMeta>(expCtx, DocumentMetadataFields::kSortKey); case KeyValue::textScore: return make_intrusive<ExpressionMeta>(expCtx, DocumentMetadataFields::kTextScore); + case KeyValue::timeseriesBucketMaxTime: + return make_intrusive<ExpressionMeta>(expCtx, + DocumentMetadataFields::kTimeseriesBucketMaxTime); + case KeyValue::timeseriesBucketMinTime: + return make_intrusive<ExpressionMeta>(expCtx, + DocumentMetadataFields::kTimeseriesBucketMinTime); default: MONGO_UNREACHABLE; } diff --git a/src/mongo/db/cst/key_value.h b/src/mongo/db/cst/key_value.h index 3750c94948b..cad3160b939 100644 --- a/src/mongo/db/cst/key_value.h +++ b/src/mongo/db/cst/key_value.h @@ -35,30 +35,32 @@ namespace mongo { -#define KEYVALUES(ENUMIFY) \ - ENUMIFY(absentKey) \ - ENUMIFY(decimalNegOneKey) \ - ENUMIFY(decimalOneKey) \ - ENUMIFY(decimalZeroKey) \ - ENUMIFY(doubleNegOneKey) \ - ENUMIFY(doubleOneKey) \ - ENUMIFY(doubleZeroKey) \ - ENUMIFY(falseKey) \ - ENUMIFY(geoNearDistance) \ - ENUMIFY(geoNearPoint) \ - ENUMIFY(indexKey) \ - ENUMIFY(intNegOneKey) \ - ENUMIFY(intOneKey) \ - ENUMIFY(intZeroKey) \ - ENUMIFY(longNegOneKey) \ - ENUMIFY(longOneKey) \ - ENUMIFY(longZeroKey) \ - ENUMIFY(randVal) \ - ENUMIFY(recordId) \ - ENUMIFY(searchHighlights) \ - ENUMIFY(searchScore) \ - ENUMIFY(sortKey) \ - ENUMIFY(textScore) \ +#define KEYVALUES(ENUMIFY) \ + ENUMIFY(absentKey) \ + ENUMIFY(decimalNegOneKey) \ + ENUMIFY(decimalOneKey) \ + ENUMIFY(decimalZeroKey) \ + ENUMIFY(doubleNegOneKey) \ + ENUMIFY(doubleOneKey) \ + ENUMIFY(doubleZeroKey) \ + ENUMIFY(falseKey) \ + ENUMIFY(geoNearDistance) \ + ENUMIFY(geoNearPoint) \ + ENUMIFY(indexKey) \ + ENUMIFY(intNegOneKey) \ + ENUMIFY(intOneKey) \ + ENUMIFY(intZeroKey) \ + ENUMIFY(longNegOneKey) \ + ENUMIFY(longOneKey) \ + ENUMIFY(longZeroKey) \ + ENUMIFY(randVal) \ + ENUMIFY(recordId) \ + ENUMIFY(searchHighlights) \ + ENUMIFY(searchScore) \ + ENUMIFY(sortKey) \ + ENUMIFY(textScore) \ + ENUMIFY(timeseriesBucketMaxTime) \ + ENUMIFY(timeseriesBucketMinTime) \ ENUMIFY(trueKey) QUERY_UTIL_NAMED_ENUM_DEFINE(KeyValue, KEYVALUES) diff --git a/src/mongo/db/exec/bucket_unpacker.cpp b/src/mongo/db/exec/bucket_unpacker.cpp index 3c8eecbae06..f8ed588f810 100644 --- a/src/mongo/db/exec/bucket_unpacker.cpp +++ b/src/mongo/db/exec/bucket_unpacker.cpp @@ -1112,6 +1112,14 @@ Document BucketUnpacker::getNext() { measurement.addField(name, Value{_computedMetaProjections[name]}); } + if (_includeMinTimeAsMetadata && _minTime) { + measurement.metadata().setTimeseriesBucketMinTime(*_minTime); + } + + if (_includeMaxTimeAsMetadata && _maxTime) { + measurement.metadata().setTimeseriesBucketMaxTime(*_maxTime); + } + return measurement.freeze(); } @@ -1175,12 +1183,41 @@ void BucketUnpacker::reset(BSONObj&& bucket) { _metaValue.missing()); } - auto&& controlField = _bucket[timeseries::kBucketControlFieldName]; uassert(5857902, "The $_internalUnpackBucket stage requires 'control' object to be present", controlField && controlField.type() == BSONType::Object); + if (_includeMinTimeAsMetadata) { + auto&& controlMin = controlField.Obj()[timeseries::kBucketControlMinFieldName]; + uassert(6460203, + str::stream() << "The $_internalUnpackBucket stage requires '" + << timeseries::kControlMinFieldNamePrefix << "' object to be present", + controlMin && controlMin.type() == BSONType::Object); + auto&& minTime = controlMin.Obj()[_spec.timeField()]; + uassert(6460204, + str::stream() << "The $_internalUnpackBucket stage requires '" + << timeseries::kControlMinFieldNamePrefix << "." << _spec.timeField() + << "' to be a date", + minTime && minTime.type() == BSONType::Date); + _minTime = minTime.date(); + } + + if (_includeMaxTimeAsMetadata) { + auto&& controlMax = controlField.Obj()[timeseries::kBucketControlMaxFieldName]; + uassert(6460205, + str::stream() << "The $_internalUnpackBucket stage requires '" + << timeseries::kControlMaxFieldNamePrefix << "' object to be present", + controlMax && controlMax.type() == BSONType::Object); + auto&& maxTime = controlMax.Obj()[_spec.timeField()]; + uassert(6460206, + str::stream() << "The $_internalUnpackBucket stage requires '" + << timeseries::kControlMaxFieldNamePrefix << "." << _spec.timeField() + << "' to be a date", + maxTime && maxTime.type() == BSONType::Date); + _maxTime = maxTime.date(); + } + auto&& versionField = controlField.Obj()[timeseries::kBucketControlVersionFieldName]; uassert(5857903, "The $_internalUnpackBucket stage requires 'control.version' field to be present", @@ -1305,6 +1342,17 @@ void BucketUnpacker::setBucketSpecAndBehavior(BucketSpec&& bucketSpec, Behavior eraseMetaFromFieldSetAndDetermineIncludeMeta(); determineIncludeTimeField(); eraseExcludedComputedMetaProjFields(); + + _includeMinTimeAsMetadata = _spec.includeMinTimeAsMetadata; + _includeMaxTimeAsMetadata = _spec.includeMaxTimeAsMetadata; +} + +void BucketUnpacker::setIncludeMinTimeAsMetadata() { + _includeMinTimeAsMetadata = true; +} + +void BucketUnpacker::setIncludeMaxTimeAsMetadata() { + _includeMaxTimeAsMetadata = true; } const std::set<std::string>& BucketUnpacker::fieldsToIncludeExcludeDuringUnpack() { diff --git a/src/mongo/db/exec/bucket_unpacker.h b/src/mongo/db/exec/bucket_unpacker.h index ac2cbf2f252..90465257415 100644 --- a/src/mongo/db/exec/bucket_unpacker.h +++ b/src/mongo/db/exec/bucket_unpacker.h @@ -177,6 +177,9 @@ public: bool assumeNoMixedSchemaData, IneligiblePredicatePolicy policy); + bool includeMinTimeAsMetadata = false; + bool includeMaxTimeAsMetadata = false; + private: // The set of field names in the data region that should be included or excluded. std::set<std::string> _fieldSet; @@ -278,7 +281,17 @@ public: return _numberOfMeasurements; } + bool includeMinTimeAsMetadata() const { + return _includeMinTimeAsMetadata; + } + + bool includeMaxTimeAsMetadata() const { + return _includeMaxTimeAsMetadata; + } + void setBucketSpecAndBehavior(BucketSpec&& bucketSpec, Behavior behavior); + void setIncludeMinTimeAsMetadata(); + void setIncludeMaxTimeAsMetadata(); // Add computed meta projection names to the bucket specification. void addComputedMetaProjFields(const std::vector<StringData>& computedFieldNames); @@ -313,6 +326,12 @@ private: // A flag used to mark that a bucket's metadata value should be materialized in measurements. bool _includeMetaField{false}; + // A flag used to mark that a bucket's min time should be materialized as metadata. + bool _includeMinTimeAsMetadata{false}; + + // A flag used to mark that a bucket's max time should be materialized as metadata. + bool _includeMaxTimeAsMetadata{false}; + // The bucket being unpacked. BSONObj _bucket; @@ -321,6 +340,16 @@ private: // measurement. Value _metaValue; + // Since the bucket min time is the same across all materialized measurements, we can cache the + // value in the reset phase and use it to materialize as a metadata field in each measurement + // if required by the pipeline. + boost::optional<Date_t> _minTime; + + // Since the bucket max time is the same across all materialized measurements, we can cache the + // value in the reset phase and use it to materialize as a metadata field in each measurement + // if required by the pipeline. + boost::optional<Date_t> _maxTime; + // Map <name, BSONElement> for the computed meta field projections. Updated for // every bucket upon reset(). stdx::unordered_map<std::string, BSONElement> _computedMetaProjections; diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.cpp b/src/mongo/db/exec/document_value/document_metadata_fields.cpp index aad877e8308..90e15ef5c2a 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.cpp +++ b/src/mongo/db/exec/document_value/document_metadata_fields.cpp @@ -85,6 +85,12 @@ void DocumentMetadataFields::mergeWith(const DocumentMetadataFields& other) { if (!hasSearchScoreDetails() && other.hasSearchScoreDetails()) { setSearchScoreDetails(other.getSearchScoreDetails()); } + if (!hasTimeseriesBucketMinTime() && other.hasTimeseriesBucketMinTime()) { + setTimeseriesBucketMinTime(other.getTimeseriesBucketMinTime()); + } + if (!hasTimeseriesBucketMaxTime() && other.hasTimeseriesBucketMaxTime()) { + setTimeseriesBucketMaxTime(other.getTimeseriesBucketMaxTime()); + } } void DocumentMetadataFields::copyFrom(const DocumentMetadataFields& other) { @@ -115,6 +121,12 @@ void DocumentMetadataFields::copyFrom(const DocumentMetadataFields& other) { if (other.hasSearchScoreDetails()) { setSearchScoreDetails(other.getSearchScoreDetails()); } + if (other.hasTimeseriesBucketMinTime()) { + setTimeseriesBucketMinTime(other.getTimeseriesBucketMinTime()); + } + if (other.hasTimeseriesBucketMaxTime()) { + setTimeseriesBucketMaxTime(other.getTimeseriesBucketMaxTime()); + } } size_t DocumentMetadataFields::getApproximateSize() const { @@ -184,6 +196,14 @@ void DocumentMetadataFields::serializeForSorter(BufBuilder& buf) const { buf.appendNum(static_cast<char>(MetaType::kSearchScoreDetails + 1)); getSearchScoreDetails().appendSelfToBufBuilder(buf); } + if (hasTimeseriesBucketMinTime()) { + buf.appendNum(static_cast<char>(MetaType::kTimeseriesBucketMinTime + 1)); + buf.appendNum(getTimeseriesBucketMinTime().toMillisSinceEpoch()); + } + if (hasTimeseriesBucketMaxTime()) { + buf.appendNum(static_cast<char>(MetaType::kTimeseriesBucketMaxTime + 1)); + buf.appendNum(getTimeseriesBucketMaxTime().toMillisSinceEpoch()); + } buf.appendNum(static_cast<char>(0)); } @@ -215,6 +235,10 @@ void DocumentMetadataFields::deserializeForSorter(BufReader& buf, DocumentMetada } else if (marker == static_cast<char>(MetaType::kSearchScoreDetails) + 1) { out->setSearchScoreDetails( BSONObj::deserializeForSorter(buf, BSONObj::SorterDeserializeSettings())); + } else if (marker == static_cast<char>(MetaType::kTimeseriesBucketMinTime) + 1) { + out->setTimeseriesBucketMinTime(Date_t::fromMillisSinceEpoch(buf.read<long long>())); + } else if (marker == static_cast<char>(MetaType::kTimeseriesBucketMaxTime) + 1) { + out->setTimeseriesBucketMaxTime(Date_t::fromMillisSinceEpoch(buf.read<long long>())); } else { uasserted(28744, "Unrecognized marker, unable to deserialize buffer"); } @@ -270,6 +294,10 @@ const char* DocumentMetadataFields::typeNameToDebugString(DocumentMetadataFields return "text score"; case DocumentMetadataFields::kSearchScoreDetails: return "$search score details"; + case DocumentMetadataFields::kTimeseriesBucketMinTime: + return "timeseries bucket min time"; + case DocumentMetadataFields::kTimeseriesBucketMaxTime: + return "timeseries bucket max time"; default: MONGO_UNREACHABLE; } diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.h b/src/mongo/db/exec/document_value/document_metadata_fields.h index 281d77ae671..12932c29686 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.h +++ b/src/mongo/db/exec/document_value/document_metadata_fields.h @@ -65,6 +65,8 @@ public: kSortKey, kTextScore, kSearchScoreDetails, + kTimeseriesBucketMinTime, + kTimeseriesBucketMaxTime, // New fields must be added before the kNumFields sentinel. kNumFields @@ -320,6 +322,39 @@ public: _holder->searchScoreDetails = details.getOwned(); } + bool hasTimeseriesBucketMinTime() const { + return _holder && _holder->metaFields.test(MetaType::kTimeseriesBucketMinTime); + } + + Date_t getTimeseriesBucketMinTime() const { + invariant(hasTimeseriesBucketMinTime()); + return _holder->timeseriesBucketMinTime; + } + + void setTimeseriesBucketMinTime(Date_t time) { + if (!_holder) { + _holder = std::make_unique<MetadataHolder>(); + } + _holder->metaFields.set(MetaType::kTimeseriesBucketMinTime); + _holder->timeseriesBucketMinTime = time; + } + + bool hasTimeseriesBucketMaxTime() const { + return _holder && _holder->metaFields.test(MetaType::kTimeseriesBucketMaxTime); + } + + Date_t getTimeseriesBucketMaxTime() const { + invariant(hasTimeseriesBucketMaxTime()); + return _holder->timeseriesBucketMaxTime; + } + + void setTimeseriesBucketMaxTime(Date_t time) { + if (!_holder) { + _holder = std::make_unique<MetadataHolder>(); + } + _holder->metaFields.set(MetaType::kTimeseriesBucketMaxTime); + _holder->timeseriesBucketMaxTime = time; + } void serializeForSorter(BufBuilder& buf) const; private: @@ -342,6 +377,8 @@ private: BSONObj indexKey; RecordId recordId; BSONObj searchScoreDetails; + Date_t timeseriesBucketMinTime; + Date_t timeseriesBucketMaxTime; }; // Null until the first setter is called, at which point a MetadataHolder struct is allocated. diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp index b4cf0e8be4d..c8f3a61c400 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp @@ -328,6 +328,18 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalUnpackBucket::createF field.find('.') == std::string::npos); bucketSpec.addComputedMetaProjFields(field); } + } else if (fieldName == kIncludeMinTimeAsMetadata) { + uassert(6460208, + str::stream() << kIncludeMinTimeAsMetadata + << " field must be a bool, got: " << elem.type(), + elem.type() == BSONType::Bool); + bucketSpec.includeMinTimeAsMetadata = elem.boolean(); + } else if (fieldName == kIncludeMaxTimeAsMetadata) { + uassert(6460209, + str::stream() << kIncludeMaxTimeAsMetadata + << " field must be a bool, got: " << elem.type(), + elem.type() == BSONType::Bool); + bucketSpec.includeMaxTimeAsMetadata = elem.boolean(); } else { uasserted(5346506, str::stream() @@ -436,6 +448,13 @@ void DocumentSourceInternalUnpackBucket::serializeToArray( return compFields; }()}); + if (_bucketUnpacker.includeMinTimeAsMetadata()) { + out.addField(kIncludeMinTimeAsMetadata, Value{_bucketUnpacker.includeMinTimeAsMetadata()}); + } + if (_bucketUnpacker.includeMaxTimeAsMetadata()) { + out.addField(kIncludeMaxTimeAsMetadata, Value{_bucketUnpacker.includeMaxTimeAsMetadata()}); + } + if (!explain) { array.push_back(Value(DOC(getSourceName() << out.freeze()))); if (_sampleSize) { @@ -991,6 +1010,14 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi // Keep going for next optimization. } + + if (deps.getNeedsMetadata(DocumentMetadataFields::MetaType::kTimeseriesBucketMinTime)) { + _bucketUnpacker.setIncludeMinTimeAsMetadata(); + } + + if (deps.getNeedsMetadata(DocumentMetadataFields::MetaType::kTimeseriesBucketMaxTime)) { + _bucketUnpacker.setIncludeMaxTimeAsMetadata(); + } } // Attempt to optimize last-point type queries. diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h index 9238d87176f..adcf44770aa 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h @@ -45,6 +45,8 @@ public: static constexpr StringData kExclude = "exclude"_sd; static constexpr StringData kAssumeNoMixedSchemaData = "assumeNoMixedSchemaData"_sd; static constexpr StringData kBucketMaxSpanSeconds = "bucketMaxSpanSeconds"_sd; + static constexpr StringData kIncludeMinTimeAsMetadata = "includeMinTimeAsMetadata"_sd; + static constexpr StringData kIncludeMaxTimeAsMetadata = "includeMaxTimeAsMetadata"_sd; static boost::intrusive_ptr<DocumentSource> createFromBsonInternal( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx); diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index b5ef461e697..a96c3876699 100644 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -36,6 +36,7 @@ #include "mongo/db/exec/document_value/document.h" #include "mongo/db/exec/document_value/document_comparator.h" +#include "mongo/db/exec/document_value/document_metadata_fields.h" #include "mongo/db/exec/document_value/value.h" #include "mongo/db/jsobj.h" #include "mongo/db/pipeline/expression.h" @@ -58,27 +59,34 @@ using std::unique_ptr; using std::vector; namespace { -struct BoundMakerAsc { - const Seconds bound; +constexpr StringData kMin = "min"_sd; +constexpr StringData kMax = "max"_sd; - DocumentSourceSort::SortableDate operator()(DocumentSourceSort::SortableDate key) const { - return {key.date - bound}; +struct BoundMakerMin { + const long long offset; // Offset in millis + + DocumentSourceSort::SortableDate operator()(DocumentSourceSort::SortableDate key, + const Document& doc) const { + return {Date_t::fromMillisSinceEpoch( + doc.metadata().getTimeseriesBucketMinTime().toMillisSinceEpoch() + offset)}; } - long long serialize() const { - return duration_cast<Seconds>(bound).count(); + Document serialize() const { + return Document{{{"base"_sd, kMin}, {"offset"_sd, offset}}}; } }; -struct BoundMakerDesc { - const Seconds bound; +struct BoundMakerMax { + const long long offset; - DocumentSourceSort::SortableDate operator()(DocumentSourceSort::SortableDate key) const { - return {key.date + bound}; + DocumentSourceSort::SortableDate operator()(DocumentSourceSort::SortableDate key, + const Document& doc) const { + return {Date_t::fromMillisSinceEpoch( + doc.metadata().getTimeseriesBucketMaxTime().toMillisSinceEpoch() + offset)}; } - long long serialize() const { - return duration_cast<Seconds>(bound).count(); + Document serialize() const { + return Document{{{"base"_sd, kMax}, {"offset"_sd, offset}}}; } }; struct CompAsc { @@ -102,10 +110,14 @@ struct CompDesc { } }; -using TimeSorterAsc = - BoundedSorter<DocumentSourceSort::SortableDate, Document, CompAsc, BoundMakerAsc>; -using TimeSorterDesc = - BoundedSorter<DocumentSourceSort::SortableDate, Document, CompDesc, BoundMakerDesc>; +using TimeSorterAscMin = + BoundedSorter<DocumentSourceSort::SortableDate, Document, CompAsc, BoundMakerMin>; +using TimeSorterAscMax = + BoundedSorter<DocumentSourceSort::SortableDate, Document, CompAsc, BoundMakerMax>; +using TimeSorterDescMin = + BoundedSorter<DocumentSourceSort::SortableDate, Document, CompDesc, BoundMakerMin>; +using TimeSorterDescMax = + BoundedSorter<DocumentSourceSort::SortableDate, Document, CompDesc, BoundMakerMax>; } // namespace constexpr StringData DocumentSourceSort::kStageName; @@ -299,9 +311,8 @@ Pipeline::SourceContainer::iterator DocumentSourceSort::doOptimizeAt( DepsTracker::State DocumentSourceSort::getDependencies(DepsTracker* deps) const { _sortExecutor->sortPattern().addDependencies(deps); - if (pExpCtx->needsMerge) { - // Include the sort key if we will merge several sorted streams later. - deps->setNeedsMetadata(DocumentMetadataFields::kSortKey, true); + if (_requiredMetadata.any()) { + deps->requestMetadata(_requiredMetadata); } return DepsTracker::State::SEE_NEXT; @@ -345,7 +356,25 @@ intrusive_ptr<DocumentSourceSort> DocumentSourceSort::parseBoundedSort( pat[0].fieldPath->getPathLength() == 1); BSONElement bound = args["bound"]; - int boundN = uassertStatusOK(bound.parseIntegerElementToNonNegativeInt()); + uassert( + 6460200, "$_internalBoundedSort bound must be an object", bound && bound.type() == Object); + + BSONElement boundOffsetElem = bound.Obj()["offset"]; + long long boundOffset = 0; + if (boundOffsetElem && boundOffsetElem.isNumber()) { + boundOffset = uassertStatusOK(boundOffsetElem.parseIntegerElementToLong()) * + 1000; // convert to millis + } + + BSONElement boundBaseElem = bound.Obj()["base"]; + uassert(6460201, + "$_internalBoundedSort bound.base must be a string", + boundBaseElem && boundBaseElem.type() == String); + StringData boundBase = boundBaseElem.valueStringData(); + uassert(6460202, + str::stream() << "$_internalBoundedSort bound.base must be '" << kMin << "' or '" + << kMax << "'", + boundBase == kMin || boundBase == kMax); SortOptions opts; opts.maxMemoryUsageBytes = internalQueryMaxBlockingSortMemoryUsageBytes.load(); @@ -355,11 +384,26 @@ intrusive_ptr<DocumentSourceSort> DocumentSourceSort::parseBoundedSort( } auto ds = DocumentSourceSort::create(expCtx, pat); - if (pat[0].isAscending) { - ds->_timeSorter.reset(new TimeSorterAsc{opts, CompAsc{}, BoundMakerAsc{Seconds{boundN}}}); + if (boundBase == kMin) { + if (pat[0].isAscending) { + ds->_timeSorter.reset( + new TimeSorterAscMin{opts, CompAsc{}, BoundMakerMin{boundOffset}}); + } else { + ds->_timeSorter.reset( + new TimeSorterDescMin{opts, CompDesc{}, BoundMakerMin{boundOffset}}); + } + ds->_requiredMetadata.set(DocumentMetadataFields::MetaType::kTimeseriesBucketMinTime); + } else if (boundBase == kMax) { + if (pat[0].isAscending) { + ds->_timeSorter.reset( + new TimeSorterAscMax{opts, CompAsc{}, BoundMakerMax{boundOffset}}); + } else { + ds->_timeSorter.reset( + new TimeSorterDescMax{opts, CompDesc{}, BoundMakerMax{boundOffset}}); + } + ds->_requiredMetadata.set(DocumentMetadataFields::MetaType::kTimeseriesBucketMaxTime); } else { - ds->_timeSorter.reset( - new TimeSorterDesc{opts, CompDesc{}, BoundMakerDesc{Seconds{boundN}}}); + MONGO_UNREACHABLE; } return ds; } @@ -463,8 +507,16 @@ std::string nextFileName() { template class ::mongo::BoundedSorter<::mongo::DocumentSourceSort::SortableDate, ::mongo::Document, ::mongo::CompAsc, - ::mongo::BoundMakerAsc>; + ::mongo::BoundMakerMin>; +template class ::mongo::BoundedSorter<::mongo::DocumentSourceSort::SortableDate, + ::mongo::Document, + ::mongo::CompAsc, + ::mongo::BoundMakerMax>; +template class ::mongo::BoundedSorter<::mongo::DocumentSourceSort::SortableDate, + ::mongo::Document, + ::mongo::CompDesc, + ::mongo::BoundMakerMin>; template class ::mongo::BoundedSorter<::mongo::DocumentSourceSort::SortableDate, ::mongo::Document, ::mongo::CompDesc, - ::mongo::BoundMakerDesc>; + ::mongo::BoundMakerMax>; diff --git a/src/mongo/db/pipeline/document_source_sort.h b/src/mongo/db/pipeline/document_source_sort.h index cd669051e67..84213ce9fd2 100644 --- a/src/mongo/db/pipeline/document_source_sort.h +++ b/src/mongo/db/pipeline/document_source_sort.h @@ -55,6 +55,9 @@ public: int memUsageForSorter() const { return sizeof(SortableDate); } + std::string toString() const { + return date.toString(); + } }; static constexpr StringData kStageName = "$sort"_sd; @@ -213,6 +216,7 @@ private: using TimeSorterInterface = BoundedSorterInterface<SortableDate, Document>; std::unique_ptr<TimeSorterInterface> _timeSorter; + QueryMetadataBitSet _requiredMetadata; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index b3c1f55856b..c2ddc5959b5 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -2990,6 +2990,8 @@ const std::string recordIdName = "recordId"; const std::string indexKeyName = "indexKey"; const std::string sortKeyName = "sortKey"; const std::string searchScoreDetailsName = "searchScoreDetails"; +const std::string timeseriesBucketMinTimeName = "timeseriesBucketMinTime"; +const std::string timeseriesBucketMaxTimeName = "timeseriesBucketMaxTime"; using MetaType = DocumentMetadataFields::MetaType; const StringMap<DocumentMetadataFields::MetaType> kMetaNameToMetaType = { @@ -3003,6 +3005,8 @@ const StringMap<DocumentMetadataFields::MetaType> kMetaNameToMetaType = { {searchScoreDetailsName, MetaType::kSearchScoreDetails}, {sortKeyName, MetaType::kSortKey}, {textScoreName, MetaType::kTextScore}, + {timeseriesBucketMinTimeName, MetaType::kTimeseriesBucketMinTime}, + {timeseriesBucketMaxTimeName, MetaType::kTimeseriesBucketMaxTime}, }; const stdx::unordered_map<DocumentMetadataFields::MetaType, StringData> kMetaTypeToMetaName = { @@ -3016,6 +3020,8 @@ const stdx::unordered_map<DocumentMetadataFields::MetaType, StringData> kMetaTyp {MetaType::kSearchScoreDetails, searchScoreDetailsName}, {MetaType::kSortKey, sortKeyName}, {MetaType::kTextScore, textScoreName}, + {MetaType::kTimeseriesBucketMinTime, timeseriesBucketMinTimeName}, + {MetaType::kTimeseriesBucketMaxTime, timeseriesBucketMaxTimeName}, }; } // namespace @@ -3081,6 +3087,14 @@ Value ExpressionMeta::evaluate(const Document& root, Variables* variables) const case MetaType::kSearchScoreDetails: return metadata.hasSearchScoreDetails() ? Value(metadata.getSearchScoreDetails()) : Value(); + case MetaType::kTimeseriesBucketMinTime: + return metadata.hasTimeseriesBucketMinTime() + ? Value(metadata.getTimeseriesBucketMinTime()) + : Value(); + case MetaType::kTimeseriesBucketMaxTime: + return metadata.hasTimeseriesBucketMaxTime() + ? Value(metadata.getTimeseriesBucketMaxTime()) + : Value(); default: MONGO_UNREACHABLE; } diff --git a/src/mongo/db/sorter/SConscript b/src/mongo/db/sorter/SConscript index 26084eca010..ae8184f222d 100644 --- a/src/mongo/db/sorter/SConscript +++ b/src/mongo/db/sorter/SConscript @@ -11,6 +11,7 @@ sorterEnv.CppUnitTest( 'sorter_test.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/exec/document_value/document_value', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/storage/encryption_hooks', '$BUILD_DIR/mongo/db/storage/storage_options', diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp index 65eac71aba4..c2a2711199a 100644 --- a/src/mongo/db/sorter/sorter.cpp +++ b/src/mongo/db/sorter/sorter.cpp @@ -1371,7 +1371,7 @@ void BoundedSorter<Key, Value, Comparator, BoundMaker>::add(Key key, Value value !_checkInput || !_min || compare(*_min, key) <= 0); // Each new item can potentially give us a tighter bound (a higher min). - Key newMin = makeBound(key); + Key newMin = makeBound(key, value); if (!_min || compare(*_min, newMin) < 0) _min = newMin; diff --git a/src/mongo/db/sorter/sorter.h b/src/mongo/db/sorter/sorter.h index d628777ad8b..645715a7666 100644 --- a/src/mongo/db/sorter/sorter.h +++ b/src/mongo/db/sorter/sorter.h @@ -41,6 +41,7 @@ #include <vector> #include "mongo/bson/util/builder.h" +#include "mongo/db/exec/document_value/document.h" #include "mongo/db/sorter/sorter_gen.h" #include "mongo/util/assert_util.h" #include "mongo/util/bufreader.h" @@ -404,7 +405,7 @@ public: virtual std::pair<Key, Value> next() = 0; // Serialize the bound for explain output - virtual long long serializeBound() const = 0; + virtual Document serializeBound() const = 0; virtual size_t numSpills() const = 0; @@ -478,8 +479,8 @@ public: std::pair<Key, Value> next(); // Serialize the bound for explain output - long long serializeBound() const { - return makeBound.serialize(); + Document serializeBound() const { + return {makeBound.serialize()}; }; size_t numSpills() const { diff --git a/src/mongo/db/sorter/sorter_test.cpp b/src/mongo/db/sorter/sorter_test.cpp index edbda871916..c5e5ab8b850 100644 --- a/src/mongo/db/sorter/sorter_test.cpp +++ b/src/mongo/db/sorter/sorter_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/db/pipeline/document_source.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest #include "mongo/platform/basic.h" @@ -1005,19 +1006,19 @@ public: } }; struct BoundMakerAsc { - Key operator()(Key k) const { + Key operator()(Key k, const Doc&) const { return k - 10; } - long long serialize() const { - return 10; + Document serialize() const { + MONGO_UNREACHABLE; } }; struct BoundMakerDesc { - Key operator()(Key k) const { + Key operator()(Key k, const Doc&) const { return k + 10; } - long long serialize() const { - return 10; + Document serialize() const { + MONGO_UNREACHABLE; } }; |