diff options
author | David Storch <david.storch@mongodb.com> | 2020-03-23 17:53:58 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-03-25 15:03:59 +0000 |
commit | 5b8b35ecebeb019cd09fb0623a407b08ec2f3a09 (patch) | |
tree | b8ce6b1a5c6fe91df171ca91bb10ddb69e21b129 | |
parent | 5d5ca410a2a8666a071f3e76f3ae4de17de0b322 (diff) | |
download | mongo-5b8b35ecebeb019cd09fb0623a407b08ec2f3a09.tar.gz |
SERVER-43361 Remove compatibility with 4.2 sort key format
This logic was necessary in 4.4 to support the 4.2/4.4
upgrade/downgrade process. A 4.6 node can always use the new
array-based sort key format, since it never communicates
with a 4.2 mongos or mongod.
24 files changed, 87 insertions, 294 deletions
diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 5d5fcca7e8f..6046efd2364 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -116,17 +116,6 @@ boost::intrusive_ptr<ExpressionContext> makeExpressionContext( boost::none // uuid ); expCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; - // TODO (SERVER-43361): We will not need to set 'sortKeyFormat' after branching for 4.5. - auto assumeInternalClient = !opCtx->getClient()->session() || - (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient); - if (assumeInternalClient) { - expCtx->sortKeyFormat = - queryRequest.use44SortKeys() ? SortKeyFormat::k44SortKey : SortKeyFormat::k42SortKey; - } else { - // The client is not a mongoS, so we will not need to use an older sort key format to - // support it. Use default value for the ExpressionContext's 'sortKeyFormat' member - // variable, which is the newest format. - } return expCtx; } diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index 3c35ee3dcda..2c8d43652b3 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -294,12 +294,9 @@ public: try { while (!FindCommon::enoughForGetMore(request.batchSize.value_or(0), *numResults) && PlanExecutor::ADVANCED == (*state = exec->getNext(&doc, nullptr))) { - auto* expCtx = exec->getExpCtx().get(); // Note that "needsMerge" implies a find or aggregate operation, which should // always have a non-NULL 'expCtx' value. - BSONObj obj = cursor->needsMerge() - ? doc.toBsonWithMetaData(expCtx->sortKeyFormat) - : doc.toBson(); + BSONObj obj = cursor->needsMerge() ? doc.toBsonWithMetaData() : doc.toBson(); // If adding this object will cause us to exceed the message size limit, then we // stash it for later. diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index 1a3dd2404c3..73ecf092365 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -216,8 +216,7 @@ bool handleCursorCommand(OperationContext* opCtx, // for later. auto* expCtx = exec->getExpCtx().get(); - BSONObj next = expCtx->needsMerge ? nextDoc.toBsonWithMetaData(expCtx->sortKeyFormat) - : nextDoc.toBson(); + BSONObj next = expCtx->needsMerge ? nextDoc.toBsonWithMetaData() : nextDoc.toBson(); if (!FindCommon::haveSpaceForNext(next, objCount, responseBuilder.bytesUsed())) { exec->enqueue(nextDoc); stashedResult = true; @@ -618,30 +617,6 @@ Status runAggregate(OperationContext* opCtx, invariant(collatorToUse); expCtx = makeExpressionContext(opCtx, request, std::move(*collatorToUse), uuid); - // If this is a shard server, we need to use the correct sort key format for the mongoS that - // generated this query. We determine the version by checking for the "use44SortKeys" flag - // in the aggregation request. - // TODO (SERVER-43361): This check will be unnecessary after branching for 4.5. - if (expCtx->fromMongos) { - if (request.getUse44SortKeys()) { - // This request originated with 4.4-or-newer mongoS, which can understand the new - // sort key format. Note: it's possible that merging will actually occur on a mongoD - // (for pipelines that merge on a shard), but if the mongoS is 4.4 or newer, all - // shard servers must also be 4.4 or newer. - expCtx->sortKeyFormat = SortKeyFormat::k44SortKey; - } else { - // This request originated with an older mongoS that will not understand the new - // sort key format. We must use the older format, which differs depending on whether - // or not the pipeline is a change stream. Non-$changeStream pipelines may still - // merge on a mongoD, but a 4.4 mongoD can still understand the 4.2 sort key format. - expCtx->sortKeyFormat = liteParsedPipeline.hasChangeStream() - ? SortKeyFormat::k42ChangeStreamSortKey - : SortKeyFormat::k42SortKey; - } - } else { - // Use default value for the ExpressionContext's 'sortKeyFormat' member variable. - } - auto pipeline = Pipeline::parse(request.getPipeline(), expCtx); // Check that the view's collation matches the collation of any views involved in the diff --git a/src/mongo/db/exec/document_value/document.cpp b/src/mongo/db/exec/document_value/document.cpp index 2e384442fcb..a59c755d373 100644 --- a/src/mongo/db/exec/document_value/document.cpp +++ b/src/mongo/db/exec/document_value/document.cpp @@ -470,7 +470,7 @@ constexpr StringData Document::metaFieldGeoNearPoint; constexpr StringData Document::metaFieldSearchScore; constexpr StringData Document::metaFieldSearchHighlights; -BSONObj Document::toBsonWithMetaData(SortKeyFormat sortKeyFormat) const { +BSONObj Document::toBsonWithMetaData() const { BSONObjBuilder bb; toBson(&bb); if (!metadata()) { @@ -481,24 +481,10 @@ BSONObj Document::toBsonWithMetaData(SortKeyFormat sortKeyFormat) const { bb.append(metaFieldTextScore, metadata().getTextScore()); if (metadata().hasRandVal()) bb.append(metaFieldRandVal, metadata().getRandVal()); - if (metadata().hasSortKey()) { - switch (sortKeyFormat) { - case SortKeyFormat::k42ChangeStreamSortKey: - invariant(metadata().isSingleElementKey()); - metadata().getSortKey().addToBsonObj(&bb, metaFieldSortKey); - break; - case SortKeyFormat::k42SortKey: - bb.append(metaFieldSortKey, - DocumentMetadataFields::serializeSortKeyAsObject( - metadata().isSingleElementKey(), metadata().getSortKey())); - break; - case SortKeyFormat::k44SortKey: - bb.append(metaFieldSortKey, - DocumentMetadataFields::serializeSortKeyAsArray( - metadata().isSingleElementKey(), metadata().getSortKey())); - break; - } - } + if (metadata().hasSortKey()) + bb.append(metaFieldSortKey, + DocumentMetadataFields::serializeSortKey(metadata().isSingleElementKey(), + metadata().getSortKey())); if (metadata().hasGeoNearDistance()) bb.append(metaFieldGeoNearDistance, metadata().getGeoNearDistance()); if (metadata().hasGeoNearPoint()) diff --git a/src/mongo/db/exec/document_value/document.h b/src/mongo/db/exec/document_value/document.h index cb3b7421f05..7719958330c 100644 --- a/src/mongo/db/exec/document_value/document.h +++ b/src/mongo/db/exec/document_value/document.h @@ -59,26 +59,6 @@ class MutableDocument; */ class Position; -/** - * "Sort keys" are stored in memory as a Value with Array type (with an exception for singleton sort - * keys). When serializing a sort key for storage in document metadata or as part of a - * {$meta: "sortKey"} projection, there are three possible storage formats: - */ -enum class SortKeyFormat { - // We expect the in-memory sort key to have one object, which has the format: - // {_data: ..., _typeBits:...}. This same object becomes the serialized sort key. This format - // exists for compatibility with 4.2 and will be deleted in 4.6 (SERVER-43361). - k42ChangeStreamSortKey, - - // A sort key with values "a" and "b" would get translated to an object that looks like: - // {"": "a", "": "b"}. Also scheduled for deletion in 4.6. - k42SortKey, - - // A sort key with values "a" and "b" would get translated to an array that looks like: - // ["a", "b"]. - k44SortKey, -}; - /** A Document is similar to a BSONObj but with a different in-memory representation. * * A Document can be treated as a const std::map<std::string, const Value> that is @@ -262,16 +242,9 @@ public: boost::optional<BSONObj> toBsonIfTriviallyConvertible() const; /** - * Like the 'toBson()' method, but includes metadata at the top-level. When the metadata - * includes a sort key, the 'sortKeyFormat' parameter controls how it gets converted from its - * in-memory representation as a Value to its serialized representation as either a BSONObj or - * BSONArray. The possible formats are described in the comment above the 'SortKeyFormat' enum. - * - * Note that the 'fromBsonWithMetaData()' function does not need a corresponding 'sortKeyFormat' - * parameter, because sort key deserialization is able to infer the sort key format based on the - * layout of the object. + * Like the 'toBson()' method, but includes metadata as top-level fields. */ - BSONObj toBsonWithMetaData(SortKeyFormat sortKeyFormat) const; + BSONObj toBsonWithMetaData() const; /** * Like Document(BSONObj) but treats top-level fields with special names as metadata. diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.cpp b/src/mongo/db/exec/document_value/document_metadata_fields.cpp index 16bda6ce44a..6d58f4ce7f4 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.cpp +++ b/src/mongo/db/exec/document_value/document_metadata_fields.cpp @@ -207,23 +207,7 @@ void DocumentMetadataFields::deserializeForSorter(BufReader& buf, DocumentMetada } } -BSONObj DocumentMetadataFields::serializeSortKeyAsObject(bool isSingleElementKey, - const Value& value) { - // Missing values don't serialize correctly in this format, so use nulls instead, since they are - // considered equivalent with woCompare(). - if (isSingleElementKey) { - return BSON("" << missingToNull(value)); - } - invariant(value.isArray()); - BSONObjBuilder bb; - for (auto&& val : value.getArray()) { - bb << "" << missingToNull(val); - } - return bb.obj(); -} - -BSONArray DocumentMetadataFields::serializeSortKeyAsArray(bool isSingleElementKey, - const Value& value) { +BSONArray DocumentMetadataFields::serializeSortKey(bool isSingleElementKey, const Value& value) { // Missing values don't serialize correctly in this format, so use nulls instead, since they are // considered equivalent with woCompare(). if (isSingleElementKey) { diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.h b/src/mongo/db/exec/document_value/document_metadata_fields.h index 83482a32abc..71497516716 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.h +++ b/src/mongo/db/exec/document_value/document_metadata_fields.h @@ -77,26 +77,12 @@ public: static void deserializeForSorter(BufReader& buf, DocumentMetadataFields* out); /** - * Converts a Value representing an in-memory sort key to a BSONObj representing a serialized - * sort key. If 'isSingleElementKey' is true, returns a BSON object with 'value' as its only - * value - and an empty field name. Otherwise returns a BSONObj with one field for each value in - * the array, each field using the empty string as the key name. - * - * Note that this format for serializing a sort key is deprecated and will be removed as part of - * SERVER-43361. The serialized sort key is a BSONObj with a field for each key component. Each - * field name is the empty string, meaning that the fields have the same name - * (e.g., {"": 1, "": 2}). The new preferred way to serialize a sort key is as a BSONArray - * (e.g.: [1, 2]), which can be done with the 'serializeSortKeyAsArray()' function. - */ - static BSONObj serializeSortKeyAsObject(bool isSingleElementKey, const Value& value); - - /** * Converts a Value representing an in-memory sort key to a BSONArray representing a serialized * sort key. If 'isSingleElementKey' is true, returns a BSONArray with 'value' as its only * element. Otherwise, converts 'value' (which is expected to be an Array) to a BSONArray. Any * Value elements whose value is "missing" get converted to BSONNull. */ - static BSONArray serializeSortKeyAsArray(bool isSingleElementKey, const Value& value); + static BSONArray serializeSortKey(bool isSingleElementKey, const Value& value); /** * Converts a BSONObj representing a serialized sort key into a Value, which we use for diff --git a/src/mongo/db/exec/document_value/document_value_test.cpp b/src/mongo/db/exec/document_value/document_value_test.cpp index 86fb3f17686..2eb26618e24 100644 --- a/src/mongo/db/exec/document_value/document_value_test.cpp +++ b/src/mongo/db/exec/document_value/document_value_test.cpp @@ -598,7 +598,7 @@ TEST(MetaFields, IndexKeyMetadataSerializesCorrectly) { ASSERT_TRUE(doc.metadata().hasIndexKey()); ASSERT_BSONOBJ_EQ(doc.metadata().getIndexKey(), BSON("b" << 1)); - auto serialized = doc.toBsonWithMetaData(SortKeyFormat::k42SortKey); + auto serialized = doc.toBsonWithMetaData(); ASSERT_BSONOBJ_EQ(serialized, BSON("a" << 1 << "$indexKey" << BSON("b" << 1))); } @@ -712,7 +712,7 @@ TEST(MetaFields, ToAndFromBson) { docBuilder.metadata().setSearchHighlights(DOC_ARRAY("abc"_sd << "def"_sd)); Document doc = docBuilder.freeze(); - BSONObj obj = doc.toBsonWithMetaData(SortKeyFormat::k42SortKey); + BSONObj obj = doc.toBsonWithMetaData(); ASSERT_EQ(10.0, obj[Document::metaFieldTextScore].Double()); ASSERT_EQ(20, obj[Document::metaFieldRandVal].numberLong()); ASSERT_EQ(30.0, obj[Document::metaFieldSearchScore].Double()); diff --git a/src/mongo/db/exec/return_key.cpp b/src/mongo/db/exec/return_key.cpp index 9dd71625924..8691f336b0e 100644 --- a/src/mongo/db/exec/return_key.cpp +++ b/src/mongo/db/exec/return_key.cpp @@ -97,22 +97,10 @@ Status ReturnKeyStage::_extractIndexKey(WorkingSetMember* member) { continue; } - switch (_sortKeyFormat) { - case SortKeyFormat::k44SortKey: - md.setNestedField( - fieldPath, - Value(DocumentMetadataFields::serializeSortKeyAsArray( - member->metadata().isSingleElementKey(), member->metadata().getSortKey()))); - break; - case SortKeyFormat::k42SortKey: - md.setNestedField( - fieldPath, - Value(DocumentMetadataFields::serializeSortKeyAsObject( - member->metadata().isSingleElementKey(), member->metadata().getSortKey()))); - break; - default: - MONGO_UNREACHABLE; - } + md.setNestedField( + fieldPath, + Value(DocumentMetadataFields::serializeSortKey(member->metadata().isSingleElementKey(), + member->metadata().getSortKey()))); } member->keyData.clear(); diff --git a/src/mongo/db/exec/return_key.h b/src/mongo/db/exec/return_key.h index c9981dced1c..b33a9cd0c31 100644 --- a/src/mongo/db/exec/return_key.h +++ b/src/mongo/db/exec/return_key.h @@ -49,12 +49,10 @@ public: ReturnKeyStage(ExpressionContext* expCtx, std::vector<FieldPath> sortKeyMetaFields, WorkingSet* ws, - SortKeyFormat sortKeyFormat, std::unique_ptr<PlanStage> child) : PlanStage(expCtx, std::move(child), kStageName.rawData()), _ws(*ws), - _sortKeyMetaFields(std::move(sortKeyMetaFields)), - _sortKeyFormat(sortKeyFormat) {} + _sortKeyMetaFields(std::move(sortKeyMetaFields)) {} StageType stageType() const final { return STAGE_RETURN_KEY; @@ -81,7 +79,5 @@ private: // The field names associated with any sortKey meta-projection(s). Empty if there is no sortKey // meta-projection. std::vector<FieldPath> _sortKeyMetaFields; - - const SortKeyFormat _sortKeyFormat; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/aggregation_request.cpp b/src/mongo/db/pipeline/aggregation_request.cpp index f0e230ef37a..57fd1715aca 100644 --- a/src/mongo/db/pipeline/aggregation_request.cpp +++ b/src/mongo/db/pipeline/aggregation_request.cpp @@ -196,17 +196,15 @@ StatusWith<AggregationRequest> AggregationRequest::parseFromBSON( return ex.toStatus(); } } else if (fieldName == kUse44SortKeys) { - // TODO (SERVER-43361): After branching for 4.5, we will accept this option but ignore - // it, as we will be able to assume that any supported mongoS will be recent enough to - // understand the 4.4 sort key format. In the version that follows, we will be able to - // completely remove this option. if (elem.type() != BSONType::Bool) { return {ErrorCodes::TypeMismatch, str::stream() << kUse44SortKeys << " must be a boolean, not a " << typeName(elem.type())}; } - request.setUse44SortKeys(elem.boolean()); + // TODO SERVER-47065: A 4.6 node still has to accept the 'use44SortKeys' field, since it + // could be included in a command sent from a 4.4 mongos or 4.4 mongod. In 4.7, this + // code to tolerate the 'use44SortKeys' field can be deleted. } else if (fieldName == "useNewUpsert"_sd) { // TODO SERVER-46751: we must retain the ability to ingest the 'useNewUpsert' field for // 4.6 upgrade purposes, since a 4.4 mongoS will always send {useNewUpsert:true} to the @@ -317,7 +315,6 @@ Document AggregationRequest::serializeToCommandObj() const { _writeConcern ? Value(_writeConcern->toBSON()) : Value()}, // Only serialize runtime constants if any were specified. {kRuntimeConstants, _runtimeConstants ? Value(_runtimeConstants->toBSON()) : Value()}, - {kUse44SortKeys, _use44SortKeys ? Value(true) : Value()}, {kIsMapReduceCommand, _isMapReduceCommand ? Value(true) : Value()}, }; } diff --git a/src/mongo/db/pipeline/aggregation_request.h b/src/mongo/db/pipeline/aggregation_request.h index c6f2b2aee40..2bd81f9ec02 100644 --- a/src/mongo/db/pipeline/aggregation_request.h +++ b/src/mongo/db/pipeline/aggregation_request.h @@ -218,10 +218,6 @@ public: return _runtimeConstants; } - bool getUse44SortKeys() const { - return _use44SortKeys; - } - bool getIsMapReduceCommand() const { return _isMapReduceCommand; } @@ -290,10 +286,6 @@ public: _runtimeConstants = std::move(runtimeConstants); } - void setUse44SortKeys(bool use44SortKeys) { - _use44SortKeys = use44SortKeys; - } - void setIsMapReduceCommand(bool isMapReduce) { _isMapReduceCommand = isMapReduce; } @@ -348,10 +340,6 @@ private: // $$NOW). boost::optional<RuntimeConstants> _runtimeConstants; - // All aggregation requests from mongos-4.4 set this flag, indicating that shard results should - // use the updated sort key format when returning change stream results. - bool _use44SortKeys = false; - // True when an aggregation was invoked by the MapReduce command. bool _isMapReduceCommand = false; }; diff --git a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp index e5f51c1883d..84282489c5e 100644 --- a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp +++ b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp @@ -364,14 +364,14 @@ TEST_F(DocumentSourceMergeCursorsTest, ShouldEnforceSortSpecifiedViaARMParams) { onCommand([&](const auto& request) { return cursorResponseObj(expCtx->ns, kExhaustedCursorID, - {BSON("x" << 1 << "$sortKey" << BSON("" << 1)), - BSON("x" << 3 << "$sortKey" << BSON("" << 3))}); + {BSON("x" << 1 << "$sortKey" << BSON_ARRAY(1)), + BSON("x" << 3 << "$sortKey" << BSON_ARRAY(3))}); }); onCommand([&](const auto& request) { return cursorResponseObj(expCtx->ns, kExhaustedCursorID, - {BSON("x" << 2 << "$sortKey" << BSON("" << 2)), - BSON("x" << 4 << "$sortKey" << BSON("" << 4))}); + {BSON("x" << 2 << "$sortKey" << BSON_ARRAY(2)), + BSON("x" << 4 << "$sortKey" << BSON_ARRAY(4))}); }); future.default_timed_get(); diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 60ec5b0d711..6d8769e02c7 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -2700,24 +2700,10 @@ Value ExpressionMeta::evaluate(const Document& root, Variables* variables) const case MetaType::kIndexKey: return metadata.hasIndexKey() ? Value(metadata.getIndexKey()) : Value(); case MetaType::kSortKey: - if (metadata.hasSortKey()) { - switch (getExpressionContext()->sortKeyFormat) { - case SortKeyFormat::k42ChangeStreamSortKey: - invariant(metadata.isSingleElementKey()); - return Value(metadata.getSortKey()); - case SortKeyFormat::k42SortKey: - return Value(DocumentMetadataFields::serializeSortKeyAsObject( - metadata.isSingleElementKey(), metadata.getSortKey())); - case SortKeyFormat::k44SortKey: - return Value(DocumentMetadataFields::serializeSortKeyAsArray( - metadata.isSingleElementKey(), metadata.getSortKey())); - break; - default: - MONGO_UNREACHABLE; - } - } else { - return Value(); - } + return metadata.hasSortKey() + ? Value(DocumentMetadataFields::serializeSortKey(metadata.isSingleElementKey(), + metadata.getSortKey())) + : Value(); default: MONGO_UNREACHABLE; } diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index de9442dd6b3..96f9f538837 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -346,12 +346,6 @@ public: boost::optional<ServerGlobalParams::FeatureCompatibility::Version> maxFeatureCompatibilityVersion; - // In a shard mongoD, we use this to determine the format for "sortKey" data. This format has to - // match the format that the mongoS expects. - // TODO (SERVER-43361): After branching for 4.5, there will be no need to support any format - // other than the "k44SortKey" format, so this will be removed. - SortKeyFormat sortKeyFormat = SortKeyFormat::k44SortKey; - // True if this ExpressionContext is used to parse a view definition pipeline. bool isParsingViewDefinition = false; diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.cpp b/src/mongo/db/pipeline/sharded_agg_helpers.cpp index e957c4940d3..f8144df09ba 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.cpp +++ b/src/mongo/db/pipeline/sharded_agg_helpers.cpp @@ -104,7 +104,6 @@ RemoteCursor openChangeStreamNewShardMonitor(const boost::intrusive_ptr<Expressi << BSON(DocumentSourceChangeStreamSpec::kStartAtOperationTimeFieldName << startMonitoringAtTime << DocumentSourceChangeStreamSpec::kAllowToRunOnConfigDBFieldName << true))}); - aggReq.setUse44SortKeys(true); aggReq.setFromMongos(true); aggReq.setNeedsMerge(true); aggReq.setBatchSize(0); @@ -151,14 +150,6 @@ BSONObj genericTransformForShards(MutableDocument&& cmdForShards, Value(static_cast<long long>(*expCtx->opCtx->getTxnNumber())); } - if (expCtx->inMongos) { - // TODO (SERVER-43361): We set this flag to indicate to the shards that the mongos will be - // able to understand change stream sort keys in the new format. After branching for 4.5, - // there will only be one sort key format for changes streams, so there will be no need to - // set this flag anymore. This flag has no effect on pipelines without a change stream. - cmdForShards[AggregationRequest::kUse44SortKeys] = Value(true); - } - return cmdForShards.freeze().toBson(); } diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index dc5d3434462..b18a1702d1e 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -441,7 +441,6 @@ StatusWith<PrepareExecutionResult> prepareExecution(OperationContext* opCtx, ? QueryPlannerCommon::extractSortKeyMetaFieldsFromProjection(*cqProjection) : std::vector<FieldPath>{}, ws, - canonicalQuery->getExpCtx()->sortKeyFormat, std::move(root)); } else if (cqProjection) { // There might be a projection. The idhack stage will always fetch the full diff --git a/src/mongo/db/query/query_request.cpp b/src/mongo/db/query/query_request.cpp index dd80516535c..75d327de215 100644 --- a/src/mongo/db/query/query_request.cpp +++ b/src/mongo/db/query/query_request.cpp @@ -417,7 +417,10 @@ StatusWith<unique_ptr<QueryRequest>> QueryRequest::parseFromFindCommand(unique_p if (!status.isOK()) { return status; } - qr->_use44SortKeys = el.boolean(); + + // TODO SERVER-47065: A 4.6 node still has to accept the '_use44SortKeys' field, since + // it could be included in a command sent from a 4.4 mongos. In 4.7 development, this + // code to tolerate the '_use44SortKeys' field can be deleted. } else if (!isGenericArgument(fieldName)) { return Status(ErrorCodes::FailedToParse, str::stream() << "Failed to parse: " << cmdObj.toString() << ". " @@ -602,10 +605,6 @@ void QueryRequest::asFindCommandInternal(BSONObjBuilder* cmdBuilder) const { if (!_resumeAfter.isEmpty()) { cmdBuilder->append(kResumeAfterField, _resumeAfter); } - - if (_use44SortKeys) { - cmdBuilder->append(kUse44SortKeys, true); - } } void QueryRequest::addShowRecordIdMetaProj() { diff --git a/src/mongo/db/query/query_request.h b/src/mongo/db/query/query_request.h index 3593489cec8..02c0aae29f1 100644 --- a/src/mongo/db/query/query_request.h +++ b/src/mongo/db/query/query_request.h @@ -450,14 +450,6 @@ public: _resumeAfter = resumeAfter; } - bool use44SortKeys() const { - return _use44SortKeys; - } - - void setUse44SortKeys(bool use44SortKeys) { - _use44SortKeys = use44SortKeys; - } - /** * Return options as a bit vector. */ @@ -595,8 +587,6 @@ private: // The Timestamp that RecoveryUnit::setTimestampReadSource() should be called with. The optional // should only ever be engaged when testing commands are enabled. boost::optional<Timestamp> _internalReadAtClusterTime; - - bool _use44SortKeys = false; }; } // namespace mongo diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp index e9965b836aa..33507b71ba7 100644 --- a/src/mongo/db/query/query_request_test.cpp +++ b/src/mongo/db/query/query_request_test.cpp @@ -448,7 +448,6 @@ TEST(QueryRequestTest, ParseFromCommandAllFlagsTrue) { "awaitData: true," "allowPartialResults: true," "readOnce: true," - "_use44SortKeys: true," "allowSpeculativeMajorityRead: true}"); const NamespaceString nss("test.testns"); bool isExplain = false; @@ -463,7 +462,6 @@ TEST(QueryRequestTest, ParseFromCommandAllFlagsTrue) { ASSERT(qr->isTailableAndAwaitData()); ASSERT(qr->isAllowPartialResults()); ASSERT(qr->isReadOnce()); - ASSERT(qr->use44SortKeys()); ASSERT(qr->allowSpeculativeMajorityRead()); } @@ -476,15 +474,6 @@ TEST(QueryRequestTest, ParseFromCommandReadOnceDefaultsToFalse) { ASSERT(!qr->isReadOnce()); } -TEST(QueryRequestTest, ParseFromCommandUse44SortKeysDefaultsToFalse) { - BSONObj cmdObj = fromjson("{find: 'testns'}"); - const NamespaceString nss("test.testns"); - bool isExplain = false; - unique_ptr<QueryRequest> qr( - assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); - ASSERT(!qr->use44SortKeys()); -} - TEST(QueryRequestTest, ParseFromCommandValidMinMax) { BSONObj cmdObj = fromjson( "{find: 'testns'," @@ -864,17 +853,6 @@ TEST(QueryRequestTest, ParseFromCommandRuntimeConstantsSubfieldsWrongType) { ErrorCodes::TypeMismatch); } -TEST(QueryRequestTest, ParseFromCommandUse44SortKeysWrongType) { - BSONObj cmdObj = BSON("find" - << "testns" - << "_use44SortKeys" - << "shouldBeBool"); - const NamespaceString nss("test.testns"); - bool isExplain = false; - auto result = QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain); - ASSERT_NOT_OK(result.getStatus()); -} - // // Parsing errors where a field has the right type but a bad value. // diff --git a/src/mongo/db/query/stage_builder.cpp b/src/mongo/db/query/stage_builder.cpp index 01005defd04..75ed2509be1 100644 --- a/src/mongo/db/query/stage_builder.cpp +++ b/src/mongo/db/query/stage_builder.cpp @@ -156,11 +156,8 @@ std::unique_ptr<PlanStage> buildStages(OperationContext* opCtx, auto returnKeyNode = static_cast<const ReturnKeyNode*>(root); auto childStage = buildStages(opCtx, collection, cq, qsol, returnKeyNode->children[0], ws); - return std::make_unique<ReturnKeyStage>(expCtx, - std::move(returnKeyNode->sortKeyMetaFields), - ws, - cq.getExpCtx()->sortKeyFormat, - std::move(childStage)); + return std::make_unique<ReturnKeyStage>( + expCtx, std::move(returnKeyNode->sortKeyMetaFields), ws, std::move(childStage)); } case STAGE_PROJECTION_DEFAULT: { auto pn = static_cast<const ProjectionNodeDefault*>(root); diff --git a/src/mongo/s/query/async_results_merger.cpp b/src/mongo/s/query/async_results_merger.cpp index 4c2d86bd828..9b300cb8eac 100644 --- a/src/mongo/s/query/async_results_merger.cpp +++ b/src/mongo/s/query/async_results_merger.cpp @@ -54,8 +54,22 @@ namespace { const int kMaxNumFailedHostRetryAttempts = 3; /** - * Returns the sort key out of the $sortKey metadata field in 'obj'. This object is of the form - * {'': 'firstSortKey', '': 'secondSortKey', ...}. + * Returns the sort key out of the $sortKey metadata field in 'obj'. The sort key should be + * formatted as an array with one value per field of the sort pattern: + * {..., $sortKey: [<firstSortKeyComponent>, <secondSortKeyComponent>, ...], ...} + * + * This function returns the sort key not as an array, but as the equivalent BSONObj: + * {"0": <firstSortKeyComponent>, "1": <secondSortKeyComponent>} + * + * The return value is allowed to omit the key names, so the caller should not rely on the key names + * being present. That is, the return value could consist of an object such as + * {"": <firstSortKeyComponent>, "": <secondSortKeyComponent>} + * + * If 'compareWholeSortKey' is true, then the value inside the $sortKey is directly interpreted as a + * single-element sort key. For example, given the document + * {..., $sortKey: <value>, ...} + * and 'compareWholeSortKey'=true, this function will return + * {"": <value>} */ BSONObj extractSortKey(BSONObj obj, bool compareWholeSortKey) { auto key = obj[AsyncResultsMerger::kSortKeyField]; @@ -63,12 +77,8 @@ BSONObj extractSortKey(BSONObj obj, bool compareWholeSortKey) { if (compareWholeSortKey) { return key.wrap(); } - // TODO (SERVER-43361): We expect the sort key to be an array, but if the sort key originated - // from a 4.2 mongod, it will be a document, instead. Either way, 'isABSONObj()' will return - // true, and 'compareSortKeys()' will behave the same way. After branching for 4.5, we can - // tighten this invariant to specifically require a 'BSONArray' type. - invariant(key.isABSONObj()); - return key.Obj(); + invariant(key.type() == BSONType::Array); + return key.embeddedObject(); } /** diff --git a/src/mongo/s/query/async_results_merger_test.cpp b/src/mongo/s/query/async_results_merger_test.cpp index 653a8c103c3..49ca39fed20 100644 --- a/src/mongo/s/query/async_results_merger_test.cpp +++ b/src/mongo/s/query/async_results_merger_test.cpp @@ -132,7 +132,7 @@ TEST_F(AsyncResultsMergerTest, SingleShardSorted) { // Shard responds; the handleBatchResponse callbacks are run and ARM's remotes get updated. std::vector<CursorResponse> responses; - std::vector<BSONObj> batch = {fromjson("{$sortKey: {'': 5}}"), fromjson("{$sortKey: {'': 6}}")}; + std::vector<BSONObj> batch = {fromjson("{$sortKey: [5]}"), fromjson("{$sortKey: [6]}")}; responses.emplace_back(kTestNss, CursorId(0), batch); scheduleNetworkResponses(std::move(responses)); @@ -144,10 +144,10 @@ TEST_F(AsyncResultsMergerTest, SingleShardSorted) { // ARM returns all results in order. executor()->waitForEvent(readyEvent); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 5}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [5]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 6}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [6]}"), *unittest::assertGet(arm->nextReady()).getResult()); // After returning all the buffered results, ARM returns EOF immediately because the cursor was @@ -249,8 +249,7 @@ TEST_F(AsyncResultsMergerTest, MultiShardSorted) { // First shard responds; the handleBatchResponse callback is run and ARM's remote gets updated. std::vector<CursorResponse> responses; - std::vector<BSONObj> batch1 = {fromjson("{$sortKey: {'': 5}}"), - fromjson("{$sortKey: {'': 6}}")}; + std::vector<BSONObj> batch1 = {fromjson("{$sortKey: [5]}"), fromjson("{$sortKey: [6]}")}; responses.emplace_back(kTestNss, CursorId(0), batch1); scheduleNetworkResponses(std::move(responses)); @@ -262,8 +261,7 @@ TEST_F(AsyncResultsMergerTest, MultiShardSorted) { // Second shard responds; the handleBatchResponse callback is run and ARM's remote gets updated. responses.clear(); - std::vector<BSONObj> batch2 = {fromjson("{$sortKey: {'': 3}}"), - fromjson("{$sortKey: {'': 9}}")}; + std::vector<BSONObj> batch2 = {fromjson("{$sortKey: [3]}"), fromjson("{$sortKey: [9]}")}; responses.emplace_back(kTestNss, CursorId(0), batch2); scheduleNetworkResponses(std::move(responses)); @@ -273,16 +271,16 @@ TEST_F(AsyncResultsMergerTest, MultiShardSorted) { // ARM returns all results in sorted order. executor()->waitForEvent(readyEvent); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 3}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [3]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 5}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [5]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 6}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [6]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 9}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [9]}"), *unittest::assertGet(arm->nextReady()).getResult()); // After returning all the buffered results, the ARM returns EOF immediately because both shards @@ -404,14 +402,13 @@ TEST_F(AsyncResultsMergerTest, CompoundSortKey) { // Deliver responses. std::vector<CursorResponse> responses; - std::vector<BSONObj> batch1 = {fromjson("{$sortKey: {'': 5, '': 9}}"), - fromjson("{$sortKey: {'': 4, '': 20}}")}; + std::vector<BSONObj> batch1 = {fromjson("{$sortKey: [5, 9]}"), fromjson("{$sortKey: [4, 20]}")}; responses.emplace_back(kTestNss, CursorId(0), batch1); - std::vector<BSONObj> batch2 = {fromjson("{$sortKey: {'': 10, '': 11}}"), - fromjson("{$sortKey: {'': 4, '': 4}}")}; + std::vector<BSONObj> batch2 = {fromjson("{$sortKey: [10, 11]}"), + fromjson("{$sortKey: [4, 4]}")}; responses.emplace_back(kTestNss, CursorId(0), batch2); - std::vector<BSONObj> batch3 = {fromjson("{$sortKey: {'': 10, '': 12}}"), - fromjson("{$sortKey: {'': 5, '': 9}}")}; + std::vector<BSONObj> batch3 = {fromjson("{$sortKey: [10, 12]}"), + fromjson("{$sortKey: [5, 9]}")}; responses.emplace_back(kTestNss, CursorId(0), batch3); scheduleNetworkResponses(std::move(responses)); executor()->waitForEvent(readyEvent); @@ -419,22 +416,22 @@ TEST_F(AsyncResultsMergerTest, CompoundSortKey) { // ARM returns all results in sorted order. ASSERT_TRUE(arm->ready()); ASSERT_TRUE(arm->remotesExhausted()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 10, '': 11}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [10, 11]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 10, '': 12}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [10, 12]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 5, '': 9}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [5, 9]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 5, '': 9}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [5, 9]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 4, '': 4}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [4, 4]}"), *unittest::assertGet(arm->nextReady()).getResult()); ASSERT_TRUE(arm->ready()); - ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: {'': 4, '': 20}}"), + ASSERT_BSONOBJ_EQ(fromjson("{$sortKey: [4, 20]}"), *unittest::assertGet(arm->nextReady()).getResult()); // After returning all the buffered results, the ARM returns EOF immediately because both shards @@ -1346,8 +1343,8 @@ DEATH_TEST_REGEX_F( auto firstDocSortKey = makeResumeToken(Timestamp(1, 4), uuid, BSON("_id" << 1)); auto firstCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 4)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 1}}, $sortKey: {'': {_data: '" - << firstDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 1}}, $sortKey: [{_data: '" + << firstDocSortKey.firstElement().String() << "'}]}"); cursors.push_back(makeRemoteCursor( kTestShardIds[0], kTestShardHosts[0], @@ -1379,8 +1376,8 @@ DEATH_TEST_REGEX_F(AsyncResultsMergerTest, auto firstDocSortKey = makeResumeToken(Timestamp(1, 4), uuid, BSON("_id" << 1)); auto firstCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 4)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 1}}, $sortKey: {'': {_data: '" - << firstDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 1}}, $sortKey: [{_data: '" + << firstDocSortKey.firstElement().String() << "'}]}"); cursors.push_back(makeRemoteCursor( kTestShardIds[0], kTestShardHosts[0], @@ -1407,8 +1404,8 @@ TEST_F(AsyncResultsMergerTest, SortedTailableCursorNotReadyIfRemoteHasLowerPostB auto firstDocSortKey = makeResumeToken(Timestamp(1, 4), uuid, BSON("_id" << 1)); auto firstCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 4)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 1}}, $sortKey: {'': {data: '" - << firstDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 1}}, $sortKey: [{data: '" + << firstDocSortKey.firstElement().String() << "'}]}"); cursors.push_back(makeRemoteCursor( kTestShardIds[0], kTestShardHosts[0], @@ -1458,8 +1455,8 @@ TEST_F(AsyncResultsMergerTest, SortedTailableCursorNewShardOrderedAfterExisting) auto pbrtFirstCursor = makePostBatchResumeToken(Timestamp(1, 6)); auto firstCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 4)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 1}}, $sortKey: {'': {_data: '" - << firstDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 1}}, $sortKey: [{_data: '" + << firstDocSortKey.firstElement().String() << "'}]}"); std::vector<BSONObj> batch1 = {firstCursorResponse}; auto firstDoc = batch1.front(); responses.emplace_back(kTestNss, CursorId(123), batch1, boost::none, pbrtFirstCursor); @@ -1487,8 +1484,8 @@ TEST_F(AsyncResultsMergerTest, SortedTailableCursorNewShardOrderedAfterExisting) auto pbrtSecondCursor = makePostBatchResumeToken(Timestamp(1, 6)); auto secondCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 5)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 2}}, $sortKey: {'': {_data: '" - << secondDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 2}}, $sortKey: [{_data: '" + << secondDocSortKey.firstElement().String() << "'}]}"); std::vector<BSONObj> batch2 = {secondCursorResponse}; auto secondDoc = batch2.front(); responses.emplace_back(kTestNss, CursorId(456), batch2, boost::none, pbrtSecondCursor); @@ -1536,8 +1533,8 @@ TEST_F(AsyncResultsMergerTest, SortedTailableCursorNewShardOrderedBeforeExisting auto pbrtFirstCursor = makePostBatchResumeToken(Timestamp(1, 5)); auto firstCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 4)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 1}}, $sortKey: {'': {_data: '" - << firstDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 1}}, $sortKey: [{_data: '" + << firstDocSortKey.firstElement().String() << "'}]}"); std::vector<BSONObj> batch1 = {firstCursorResponse}; responses.emplace_back(kTestNss, CursorId(123), batch1, boost::none, pbrtFirstCursor); scheduleNetworkResponses(std::move(responses)); @@ -1564,8 +1561,8 @@ TEST_F(AsyncResultsMergerTest, SortedTailableCursorNewShardOrderedBeforeExisting auto pbrtSecondCursor = makePostBatchResumeToken(Timestamp(1, 5)); auto secondCursorResponse = fromjson( str::stream() << "{_id: {clusterTime: {ts: Timestamp(1, 3)}, uuid: '" << uuid.toString() - << "', documentKey: {_id: 2}}, $sortKey: {'': {_data: '" - << secondDocSortKey.firstElement().String() << "'}}}"); + << "', documentKey: {_id: 2}}, $sortKey: [{_data: '" + << secondDocSortKey.firstElement().String() << "'}]}"); std::vector<BSONObj> batch2 = {secondCursorResponse}; // The last observed time should still be later than the first shard, so we can get the data // from it. diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp index 4febc8a5ba3..9a65ce9e3cc 100644 --- a/src/mongo/s/query/cluster_find.cpp +++ b/src/mongo/s/query/cluster_find.cpp @@ -162,10 +162,6 @@ StatusWith<std::unique_ptr<QueryRequest>> transformQueryForShards( // Any expansion of the 'showRecordId' flag should have already happened on mongos. newQR->setShowRecordId(false); - // Indicate to shard servers that this is a 4.4 or newer mongoS, and they should serialize sort - // keys in the new format. - newQR->setUse44SortKeys(true); - invariant(newQR->validate()); return std::move(newQR); } @@ -189,9 +185,6 @@ std::vector<std::pair<ShardId, BSONObj>> constructRequestsForShards( // Forwards the QueryRequest as is to a single shard so that limit and skip can // be applied on mongod. qrToForward = std::make_unique<QueryRequest>(query.getQueryRequest()); - // Indicate to shard servers that this is a 4.4 or newer mongoS, and they should serialize - // sort keys in the new format. - qrToForward->setUse44SortKeys(true); } auto shardRegistry = Grid::get(opCtx)->shardRegistry(); |