summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/mongo/db/s/SConscript3
-rw-r--r--src/mongo/db/s/auto_split_vector.cpp358
-rw-r--r--src/mongo/db/s/auto_split_vector.h134
-rw-r--r--src/mongo/db/s/auto_split_vector_command.cpp98
-rw-r--r--src/mongo/db/s/auto_split_vector_test.cpp262
-rw-r--r--src/mongo/db/s/balancer/balancer.cpp3
-rw-r--r--src/mongo/db/s/chunk_splitter.cpp23
-rw-r--r--src/mongo/db/s/config/initial_split_policy.cpp3
-rw-r--r--src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp14
-rw-r--r--src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp2
-rw-r--r--src/mongo/s/SConscript1
-rw-r--r--src/mongo/s/request_types/auto_split_vector.idl71
-rw-r--r--src/mongo/s/shard_util.cpp78
-rw-r--r--src/mongo/s/shard_util.h3
-rw-r--r--src/mongo/s/write_ops/cluster_write.cpp3
15 files changed, 999 insertions, 57 deletions
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 75fa982f880..7f96b5499ff 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -33,6 +33,7 @@ env.Library(
'active_migrations_registry.cpp',
'active_move_primaries_registry.cpp',
'active_shard_collection_registry.cpp',
+ 'auto_split_vector.cpp',
'chunk_move_write_concern_options.cpp',
'chunk_splitter.cpp',
'collection_range_deleter.cpp',
@@ -220,6 +221,7 @@ env.Library(
env.Library(
target='sharding_commands_d',
source=[
+ 'auto_split_vector_command.cpp',
'check_sharding_index_command.cpp',
'cleanup_orphaned_cmd.cpp',
'clone_catalog_data_command.cpp',
@@ -304,6 +306,7 @@ env.CppUnitTest(
'active_migrations_registry_test.cpp',
'active_move_primaries_registry_test.cpp',
'active_shard_collection_registry_test.cpp',
+ 'auto_split_vector_test.cpp',
'catalog_cache_loader_mock.cpp',
'implicit_create_collection_test.cpp',
'migration_chunk_cloner_source_legacy_test.cpp',
diff --git a/src/mongo/db/s/auto_split_vector.cpp b/src/mongo/db/s/auto_split_vector.cpp
new file mode 100644
index 00000000000..a430d84593c
--- /dev/null
+++ b/src/mongo/db/s/auto_split_vector.cpp
@@ -0,0 +1,358 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/auto_split_vector.h"
+
+#include "mongo/base/status_with.h"
+#include "mongo/db/bson/dotted_path_support.h"
+#include "mongo/db/catalog/index_catalog.h"
+#include "mongo/db/catalog_raii.h"
+#include "mongo/db/dbhelpers.h"
+#include "mongo/db/exec/working_set_common.h"
+#include "mongo/db/index/index_descriptor.h"
+#include "mongo/db/keypattern.h"
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/query/internal_plans.h"
+#include "mongo/db/query/plan_executor.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+
+constexpr int estimatedAdditionalBytesPerItemInBSONArray{2};
+
+constexpr int kMaxSplitPointsToReposition{3};
+
+BSONObj prettyKey(const BSONObj& keyPattern, const BSONObj& key) {
+ return key.replaceFieldNames(keyPattern).clientReadable();
+}
+
+/*
+ * Takes the given min/max BSON objects that are a prefix of the shardKey and return two new BSON
+ * object extended to cover the entire shardKey. See KeyPattern::extendRangeBound documentation for
+ * some examples.
+ */
+const std::tuple<BSONObj, BSONObj> getMinMaxExtendedBounds(const IndexDescriptor* shardKeyIdx,
+ const BSONObj& min,
+ const BSONObj& max) {
+ KeyPattern kp(shardKeyIdx->keyPattern());
+
+ // Extend min to get (min, MinKey, MinKey, ....)
+ BSONObj minKey = Helpers::toKeyFormat(kp.extendRangeBound(min, false /* upperInclusive */));
+ BSONObj maxKey;
+ if (max.isEmpty()) {
+ // if max not specified, make it (MaxKey, Maxkey, MaxKey...)
+ maxKey = Helpers::toKeyFormat(kp.extendRangeBound(max, true /* upperInclusive */));
+ } else {
+ // otherwise make it (max,MinKey,MinKey...) so that bound is non-inclusive
+ maxKey = Helpers::toKeyFormat(kp.extendRangeBound(max, false /* upperInclusive*/));
+ }
+
+ return std::tuple<BSONObj, BSONObj>(minKey, maxKey);
+}
+
+/*
+ * Returns true if the final key in the range is the same as the first key, false otherwise.
+ */
+bool maxKeyEqualToMinKey(OperationContext* opCtx,
+ const Collection* collection,
+ const IndexDescriptor* shardKeyIdx,
+ const BSONObj& minBound,
+ const BSONObj& maxBound,
+ const BSONObj& minKeyInChunk) {
+ BSONObj maxKeyInChunk;
+ {
+ auto backwardIdxScanner = InternalPlanner::indexScan(opCtx,
+ collection,
+ shardKeyIdx,
+ maxBound,
+ minBound,
+ BoundInclusion::kIncludeEndKeyOnly,
+ PlanExecutor::YIELD_AUTO,
+ InternalPlanner::BACKWARD);
+
+ PlanExecutor::ExecState state = backwardIdxScanner->getNext(&maxKeyInChunk, nullptr);
+ uassert(ErrorCodes::OperationFailed,
+ "can't open a cursor to find final key in range (desired range is possibly empty)",
+ state == PlanExecutor::ADVANCED);
+ }
+
+ if (minKeyInChunk.woCompare(maxKeyInChunk) == 0) {
+ // Range contains only documents with a single key value. So we cannot possibly find a
+ // split point, and there is no need to scan any further.
+ warning() << "possible low cardinality key detected in " << collection->ns().toString()
+ << " - range " << redact(prettyKey(shardKeyIdx->keyPattern(), minKeyInChunk))
+ << " -->> " << redact(prettyKey(shardKeyIdx->keyPattern(), maxKeyInChunk))
+ << " contains only the key "
+ << redact(prettyKey(shardKeyIdx->keyPattern(), minKeyInChunk));
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Reshuffle fields according to the shard key pattern.
+ */
+auto orderShardKeyFields(const BSONObj& keyPattern, BSONObj& key) {
+ return dotted_path_support::extractElementsBasedOnTemplate(
+ prettyKey(keyPattern, key.getOwned()), keyPattern);
+}
+
+} // namespace
+
+std::vector<BSONObj> autoSplitVector(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const BSONObj& keyPattern,
+ const BSONObj& min,
+ const BSONObj& max,
+ long long maxChunkSizeBytes) {
+ std::vector<BSONObj> splitKeys;
+
+ int elapsedMillisToFindSplitPoints;
+
+ // Contains each key appearing multiple times and estimated to be able to fill-in a chunk alone
+ auto tooFrequentKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
+
+ {
+ AutoGetCollection autoGetColl(opCtx, nss, MODE_IS);
+ auto* collection = autoGetColl.getCollection();
+
+ uassert(ErrorCodes::NamespaceNotFound, "ns not found", collection);
+
+ // Get the size estimate for this namespace
+ const long long totalLocalCollDocuments = collection->numRecords(opCtx);
+ const long long dataSize = collection->dataSize(opCtx);
+
+ // Return empty vector if current estimated data size is less than max chunk size
+ if (dataSize < maxChunkSizeBytes || totalLocalCollDocuments == 0) {
+ return {};
+ }
+
+ // Allow multiKey based on the invariant that shard keys must be single-valued. Therefore,
+ // any multi-key index prefixed by shard key cannot be multikey over the shard key fields.
+ auto catalog = collection->getIndexCatalog();
+ auto shardKeyIdx =
+ catalog->findShardKeyPrefixedIndex(opCtx, keyPattern, false /* requireSingleKey */);
+ uassert(ErrorCodes::IndexNotFound,
+ str::stream() << "couldn't find index over splitting key "
+ << keyPattern.clientReadable().toString(),
+ shardKeyIdx);
+
+ const auto minKeyAndMaxKey = getMinMaxExtendedBounds(shardKeyIdx, min, max);
+ const auto minKey = std::get<0>(minKeyAndMaxKey);
+ const auto maxKey = std::get<1>(minKeyAndMaxKey);
+
+ // Setup the index scanner that will be used to find the split points
+ auto forwardIdxScanner = InternalPlanner::indexScan(opCtx,
+ &(*collection),
+ shardKeyIdx,
+ minKey,
+ maxKey,
+ BoundInclusion::kIncludeStartKeyOnly,
+ PlanExecutor::YIELD_AUTO,
+ InternalPlanner::FORWARD);
+
+ // Get minimum key belonging to the chunk
+ BSONObj minKeyInOriginalChunk;
+ {
+ PlanExecutor::ExecState state =
+ forwardIdxScanner->getNext(&minKeyInOriginalChunk, nullptr);
+ uassert(ErrorCodes::OperationFailed,
+ "can't open a cursor to scan the range (desired range is possibly empty)",
+ state == PlanExecutor::ADVANCED);
+ }
+
+ // Return empty vector if chunk's min and max keys are the same.
+ if (maxKeyEqualToMinKey(
+ opCtx, collection, shardKeyIdx, minKey, maxKey, minKeyInOriginalChunk)) {
+ return {};
+ }
+
+ log() << "request split points lookup for chunk " << nss.toString() << " " << redact(minKey)
+ << " -->> " << redact(maxKey);
+
+ // Use the average document size and number of documents to find the approximate number of
+ // keys each chunk should contain
+ const long long avgDocSize = dataSize / totalLocalCollDocuments;
+
+ // Split at max chunk size
+ long long maxDocsPerChunk = maxChunkSizeBytes / avgDocSize;
+
+ BSONObj currentKey; // Last key seen during the index scan
+ long long numScannedKeys = 1; // minKeyInOriginalChunk has already been scanned
+ std::size_t resultArraySize = 0; // Approximate size in bytes of the split points array
+ bool reachedMaxBSONSize = false; // True if the split points vector becomes too big
+
+ // Lambda to check whether the split points vector would exceed BSONObjMaxUserSize in case
+ // of additional split key of the specified size.
+ auto checkMaxBSONSize = [&resultArraySize](const int additionalKeySize) {
+ return resultArraySize + additionalKeySize > BSONObjMaxUserSize;
+ };
+
+ // Reference to last split point that needs to be checked in order to avoid adding duplicate
+ // split points. Initialized to the min of the first chunk being split.
+ auto minKeyElement = orderShardKeyFields(keyPattern, minKeyInOriginalChunk);
+ auto lastSplitPoint = minKeyElement;
+
+ Timer timer; // To measure time elapsed while searching split points
+
+ // Traverse the index and add the maxDocsPerChunk-th key to the result vector
+ while (forwardIdxScanner->getNext(&currentKey, nullptr) == PlanExecutor::ADVANCED) {
+ if (++numScannedKeys >= maxDocsPerChunk) {
+ currentKey = orderShardKeyFields(keyPattern, currentKey);
+
+ if (currentKey.woCompare(lastSplitPoint) == 0) {
+ // Do not add again the same split point in case of frequent shard key.
+ tooFrequentKeys.insert(currentKey.getOwned());
+ continue;
+ }
+
+ const auto additionalKeySize =
+ currentKey.objsize() + estimatedAdditionalBytesPerItemInBSONArray;
+ if (checkMaxBSONSize(additionalKeySize)) {
+ if (splitKeys.empty()) {
+ // Keep trying until finding at least one split point that isn't above
+ // the max object user size. Very improbable corner case: the shard key
+ // size for the chosen split point is exactly 16MB.
+ continue;
+ }
+ reachedMaxBSONSize = true;
+ break;
+ }
+
+ resultArraySize += additionalKeySize;
+ splitKeys.push_back(currentKey.getOwned());
+ lastSplitPoint = splitKeys.back();
+ numScannedKeys = 0;
+
+ LOG(4) << "picked a split key: " << redact(currentKey);
+ }
+ }
+
+ // Avoid creating small chunks by fairly recalculating the last split points if the last
+ // chunk would be too small (containing less than `80% maxDocsPerChunk` documents).
+ bool lastChunk80PercentFull = numScannedKeys >= maxDocsPerChunk * 0.8;
+ if (!lastChunk80PercentFull && !splitKeys.empty() && !reachedMaxBSONSize) {
+ // Eventually recalculate the last split points (at most `kMaxSplitPointsToReposition`).
+ int nSplitPointsToReposition = splitKeys.size() > kMaxSplitPointsToReposition
+ ? kMaxSplitPointsToReposition
+ : splitKeys.size();
+
+ // Equivalent to: (nSplitPointsToReposition * maxDocsPerChunk + numScannedKeys) divided
+ // by the number of reshuffled chunks (nSplitPointsToReposition + 1).
+ const auto maxDocsPerNewChunk = maxDocsPerChunk -
+ ((maxDocsPerChunk - numScannedKeys) / (nSplitPointsToReposition + 1));
+
+ if (numScannedKeys < maxDocsPerChunk - maxDocsPerNewChunk) {
+ // If the surplus is not too much, simply keep a bigger last chunk.
+ // The surplus is considered enough if repositioning the split points would imply
+ // generating chunks with a number of documents lower than `67% maxDocsPerChunk`.
+ splitKeys.pop_back();
+ } else {
+ // Fairly recalculate the last `nSplitPointsToReposition` split points.
+ splitKeys.erase(splitKeys.end() - nSplitPointsToReposition, splitKeys.end());
+
+ auto forwardIdxScanner =
+ InternalPlanner::indexScan(opCtx,
+ collection,
+ shardKeyIdx,
+ splitKeys.empty() ? minKeyElement : splitKeys.back(),
+ maxKey,
+ BoundInclusion::kIncludeStartKeyOnly,
+ PlanExecutor::YIELD_AUTO,
+ InternalPlanner::FORWARD);
+
+ numScannedKeys = 0;
+
+ auto previousSplitPoint = splitKeys.empty() ? minKeyElement : splitKeys.back();
+ while (forwardIdxScanner->getNext(&currentKey, nullptr) == PlanExecutor::ADVANCED) {
+ if (++numScannedKeys >= maxDocsPerNewChunk) {
+ currentKey = orderShardKeyFields(keyPattern, currentKey);
+
+ const auto compareWithPreviousSplitPoint =
+ currentKey.woCompare(previousSplitPoint);
+ if (compareWithPreviousSplitPoint > 0) {
+ const auto additionalKeySize =
+ currentKey.objsize() + estimatedAdditionalBytesPerItemInBSONArray;
+ if (checkMaxBSONSize(additionalKeySize)) {
+ reachedMaxBSONSize = true;
+ break;
+ }
+
+ splitKeys.push_back(currentKey.getOwned());
+ previousSplitPoint = splitKeys.back();
+ numScannedKeys = 0;
+
+ if (--nSplitPointsToReposition == 0) {
+ break;
+ }
+ } else if (compareWithPreviousSplitPoint == 0) {
+ // Don't add again the same split point in case of frequent shard key.
+ tooFrequentKeys.insert(currentKey.getOwned());
+ }
+ }
+ }
+ }
+ }
+
+ elapsedMillisToFindSplitPoints = timer.millis();
+
+ if (reachedMaxBSONSize) {
+ log() << "Max BSON response size reached for split vector before the end of chunk for "
+ "namespace "
+ << nss.toString() << " - range "
+ << redact(prettyKey(shardKeyIdx->keyPattern(), minKey)) << " --> "
+ << redact(prettyKey(shardKeyIdx->keyPattern(), maxKey));
+ }
+ }
+
+ // Emit a warning for each frequent key
+ for (const auto& frequentKey : tooFrequentKeys) {
+ warning() << "possible low cardinality key detected in " << nss.toString()
+ << " - key: " << redact(prettyKey(keyPattern, frequentKey));
+ }
+
+ if (elapsedMillisToFindSplitPoints > serverGlobalParams.slowMS) {
+ warning() << "Finding the auto split vector for " << nss.toString() << " completed over "
+ << redact(keyPattern) << " - numSplits: " << splitKeys.size()
+ << " - duration: " << elapsedMillisToFindSplitPoints << "ms";
+ }
+
+ std::sort(
+ splitKeys.begin(), splitKeys.end(), SimpleBSONObjComparator::kInstance.makeLessThan());
+
+ return splitKeys;
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/auto_split_vector.h b/src/mongo/db/s/auto_split_vector.h
new file mode 100644
index 00000000000..559c1f814d6
--- /dev/null
+++ b/src/mongo/db/s/auto_split_vector.h
@@ -0,0 +1,134 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+
+/**
+ * Given a chunk, determines whether it satisfies the requisites to be auto-splitted and - if so -
+ * returns the split points (shard keys representing the lower bounds of the new chunks to create).
+ *
+ * The logic implemented can be summarized as follows: given a `maxChunkSize` of `x` MB, the
+ * algorithm aims to choose the split points so that the resulting chunks' size would be around
+ * `maxChunkSize`. As it is too expensive to precisely determine the dimension of a chunk, it is
+ * assumed a uniform distribution of document sizes, hence the aim is to balance the number of
+ * documents per chunk.
+ *
+ * ======= ALGORITHM DESCRIPTION =======
+ *
+ * The split points for a chunk `C` belonging to a collection `coll` are calculated as follows:
+ * - `averageDocumentSize` = `totalCollSizeOnShard / numberOfCollDocs`
+ * - `maxNumberOfDocsPerChunk` = `maxChunkSize / averageDocumentSize`
+ * - Scan forward the shard key index entries for `coll` that are belonging to chunk `C`:
+ * - (1) Choose a split point every `maxNumberOfDocsPerChunk` scanned keys.
+ * - (2) As it needs to be avoided the creation of small chunks, consider the number of documents
+ * `S` that the right-most chunk would contain given the calculated split points:
+ * --- (2.1) IF `S >= 80% maxNumberOfDocsPerChunk`, return the list of calculated split points.
+ * --- (2.2) ELSE IF `S` documents could be fairly redistributed in the last chunks so that their
+ * size would be at least `67% maxNumberOfDocsPerChunk`: recalculate the last split points (max 3).
+ * --- (2.3) ELSE simply remove the last split point and keep a bigger last chunk.
+ *
+ *
+ * ============== EXAMPLES =============
+ *
+ * ========= EXAMPLE (CASE 2.1) ========
+ * `maxChunkSize` = 100MB
+ * `averageDocumentSize` = 1MB
+ * `maxNumberOfDocsPerChunk` = 100
+ *
+ * Shard key type: integer
+ * Chunk `C` bounds: `[0, maxKey)` . Chunk `C` contains 190 documents with shard keys [0...189].
+ *
+ * (1) Initially calculated split points: [99].
+ * (2) The last chunk would contain the interval `[99-189]` so `S = 90`
+ * (2.1) `S >= 80% maxChunkSize`, so keep the current split points.
+ *
+ * Returned split points: [99].
+ *
+ * ========= EXAMPLE (CASE 2.2) ========
+ * `maxChunkSize` = 100MB
+ * `averageDocumentSize` = 1MB
+ * `maxNumberOfDocsPerChunk` = 100
+ *
+ * Shard key type: integer
+ * Chunk `C` bounds: `[0, maxKey)` . Chunk `C` contains 140 documents with shard keys [0...139].
+ *
+ * (1) Initially calculated split points: [99].
+ * (2) The last chunk would contain the interval `[99-139]` so `S = 40`
+ * (2.2) `S` documents can be redistributed on the last split point by generating chunks of size >=
+ * 67% maxChunkSize. Recalculate.
+ *
+ * Returned split points: [69].
+ *
+ * ========= EXAMPLE (CASE 2.3) ========
+ * `maxChunkSize` = 100MB
+ * `averageDocumentSize` = 1MB
+ * `maxNumberOfDocsPerChunk` = 100
+ *
+ * Shard key type: integer
+ * Chunk `C` bounds: `[0, maxKey)` . Chunk `C` contains 120 documents with shard keys [0...119].
+ *
+ * (1) Initially calculated split points: [99].
+ * (2) The last chunk would contain the interval `[99-119]` so `S = 20`
+ * (2.3) `S` documents can't be redistributed on the last split point by generating chunks of size
+ * >= 67% maxChunkSize. So remove the last split point.
+ *
+ * Returned split points: [].
+ */
+std::vector<BSONObj> autoSplitVector(OperationContext* opCtx,
+ const NamespaceString& nss,
+ const BSONObj& keyPattern,
+ const BSONObj& min,
+ const BSONObj& max,
+ long long maxChunkSizeBytes);
+
+/*
+ * Utility function for deserializing autoSplitVector/splitVector responses.
+ */
+static std::vector<BSONObj> parseSplitKeys(const BSONElement& splitKeysArray) {
+ uassert(ErrorCodes::TypeMismatch,
+ "The split keys vector must be represented as a BSON array",
+ !splitKeysArray.eoo() && splitKeysArray.type() == BSONType::Array);
+
+ std::vector<BSONObj> splitKeys;
+ for (const auto& elem : splitKeysArray.Obj()) {
+ uassert(ErrorCodes::TypeMismatch,
+ "Each element of the split keys array must be an object",
+ elem.type() == BSONType::Object);
+ splitKeys.push_back(elem.embeddedObject().getOwned());
+ }
+
+ return splitKeys;
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/s/auto_split_vector_command.cpp b/src/mongo/db/s/auto_split_vector_command.cpp
new file mode 100644
index 00000000000..b789fb86855
--- /dev/null
+++ b/src/mongo/db/s/auto_split_vector_command.cpp
@@ -0,0 +1,98 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/db/s/auto_split_vector.h"
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/s/sharding_state.h"
+#include "mongo/s/request_types/auto_split_vector_gen.h"
+
+namespace mongo {
+namespace {
+
+class AutoSplitVectorCommand final : public TypedCommand<AutoSplitVectorCommand> {
+public:
+ AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+ return Command::AllowedOnSecondary::kNever;
+ }
+
+ std::string help() const override {
+ return "Internal command returning the split points for a chunk, given the maximum chunk "
+ "size.";
+ }
+
+ using Request = AutoSplitVectorRequest;
+ using Response = AutoSplitVectorResponse;
+
+ class Invocation final : public InvocationBase {
+ public:
+ using InvocationBase::InvocationBase;
+
+ Response typedRun(OperationContext* opCtx) {
+ uassert(ErrorCodes::IllegalOperation,
+ "The autoSplitVector command can only be invoked on shards (no CSRS).",
+ serverGlobalParams.clusterRole == ClusterRole::ShardServer);
+
+ uassertStatusOK(ShardingState::get(opCtx)->canAcceptShardedCommands());
+
+ const auto& req = request();
+
+ auto splitKeys = autoSplitVector(opCtx,
+ ns(),
+ req.getKeyPattern(),
+ req.getMin(),
+ req.getMax(),
+ req.getMaxChunkSizeBytes());
+
+ AutoSplitVectorResponse res;
+ res.setSplitKeys(splitKeys);
+ return res;
+ }
+
+ private:
+ NamespaceString ns() const override {
+ return request().getNamespace();
+ }
+
+ bool supportsWriteConcern() const override {
+ return false;
+ }
+
+ void doCheckAuthorization(OperationContext* opCtx) const override {
+ uassert(ErrorCodes::Unauthorized,
+ "Unauthorized",
+ AuthorizationSession::get(opCtx->getClient())
+ ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
+ ActionType::splitVector));
+ }
+ };
+} autoSplitVectorCommand;
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/s/auto_split_vector_test.cpp b/src/mongo/db/s/auto_split_vector_test.cpp
new file mode 100644
index 00000000000..88825cc25f3
--- /dev/null
+++ b/src/mongo/db/s/auto_split_vector_test.cpp
@@ -0,0 +1,262 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/db/s/auto_split_vector.h"
+#include "mongo/db/catalog/create_collection.h"
+#include "mongo/db/db_raii.h"
+#include "mongo/db/dbdirectclient.h"
+#include "mongo/db/s/collection_sharding_runtime.h"
+#include "mongo/db/s/operation_sharding_state.h"
+#include "mongo/db/s/split_vector.h"
+#include "mongo/platform/random.h"
+#include "mongo/s/shard_server_test_fixture.h"
+#include "mongo/util/log.h"
+
+namespace mongo {
+namespace {
+
+const NamespaceString kNss = NamespaceString("autosplitDB", "coll");
+const std::string kPattern = "_id";
+
+/*
+ * Call the autoSplitVector function of the test collection on a chunk with bounds [0, 100) and with
+ * the specified `maxChunkSizeMB`.
+ */
+std::vector<BSONObj> autoSplit(OperationContext* opCtx, int maxChunkSizeMB) {
+ return autoSplitVector(opCtx,
+ kNss,
+ BSON(kPattern << 1) /* shard key pattern */,
+ BSON(kPattern << 0) /* min */,
+ BSON(kPattern << 1000) /* max */,
+ maxChunkSizeMB * 1024 * 1024 /* max chunk size in bytes*/);
+}
+
+class AutoSplitVectorTest : public ShardServerTestFixture {
+public:
+ /*
+ * Before each test case:
+ * - Creates a sharded collection with shard key `_id`
+ */
+ void setUp() {
+ ShardServerTestFixture::setUp();
+
+ auto opCtx = operationContext();
+
+ {
+ uassertStatusOK(createCollection(
+ operationContext(), kNss.db().toString(), BSON("create" << kNss.coll())));
+ }
+
+ DBDirectClient client(opCtx);
+ client.createIndex(kNss.ns(), BSON(kPattern << 1));
+ }
+
+ /*
+ * Insert the specified number of documents in the test collection, with incremental shard key
+ * `_id` starting from `_nextShardKey`.
+ */
+ void insertNDocsOf1MB(OperationContext* opCtx, int nDocs) {
+ DBDirectClient client(opCtx);
+
+ std::string s(1024 * 1024 - 24, 'a'); // To get a 1MB document
+ for (int i = 0; i < nDocs; i++) {
+ BSONObjBuilder builder;
+ builder.append(kPattern, _nextShardKey++);
+ builder.append("str", s);
+ BSONObj obj = builder.obj();
+ ASSERT(obj.objsize() == 1024 * 1024); // 1 MB document
+ client.insert(kNss.toString(), obj);
+ }
+ }
+
+ /*
+ * Get the number of documents inserted until now.
+ */
+ int getInsertedSize() {
+ return _nextShardKey;
+ }
+
+private:
+ int _nextShardKey = 0;
+};
+
+class AutoSplitVectorTest10MB : public AutoSplitVectorTest {
+ /*
+ * Before each test case:
+ * - Creates a sharded collection with shard key `_id`
+ * - Inserts `10` documents of ~1MB size (shard keys [0...9])
+ */
+ void setUp() {
+ AutoSplitVectorTest::setUp();
+
+ auto opCtx = operationContext();
+
+ DBDirectClient client(opCtx);
+ client.createIndex(kNss.ns(), BSON(kPattern << 1));
+
+ insertNDocsOf1MB(opCtx, 10 /* nDocs */);
+ ASSERT_EQUALS(10UL, client.count(kNss.ns()));
+ }
+};
+
+// Throw exception upon calling autoSplitVector on dropped/unexisting collection
+TEST_F(AutoSplitVectorTest10MB, NoCollection) {
+ ASSERT_THROWS_CODE(autoSplitVector(operationContext(),
+ NamespaceString("dummy", "collection"),
+ BSON(kPattern << 1) /* shard key pattern */,
+ BSON(kPattern << 0) /* min */,
+ BSON(kPattern << 100) /* max */,
+ 1 * 1024 * 1024 /* max chunk size in bytes*/),
+ DBException,
+ ErrorCodes::NamespaceNotFound);
+}
+
+// No split points if estimated `data size < max chunk size`
+TEST_F(AutoSplitVectorTest10MB, NoSplitIfDataLessThanMaxChunkSize) {
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 11 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 0UL);
+}
+
+// Do not split in case of `chunk size == maxChunkSize`
+TEST_F(AutoSplitVectorTest10MB, NoSplitIfDataEqualMaxChunkSize) {
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 10 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 0UL);
+}
+
+// No split points if `chunk size > max chunk size` but threshold not reached
+TEST_F(AutoSplitVectorTest10MB, NoSplitIfDataLessThanThreshold) {
+ const auto surplus = 2;
+ {
+ // Increase collection size so that the auto splitter can actually be triggered. Use a
+ // different range to don't interfere with the chunk getting splitted.
+ insertNDocsOf1MB(operationContext(), surplus /* nDocs */);
+ }
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 10 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 0UL);
+}
+
+// One split point if `chunk size > max chunk size` and threshold reached
+TEST_F(AutoSplitVectorTest10MB, SplitIfDataSlightlyMoreThanThreshold) {
+ const auto surplus = 4;
+ insertNDocsOf1MB(operationContext(), surplus /* nDocs */);
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 10 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 1UL);
+ ASSERT_EQ(6, splitKeys.front().getIntField(kPattern));
+}
+
+// Split points if `data size > max chunk size * 2` and threshold reached
+TEST_F(AutoSplitVectorTest10MB, SplitIfDataMoreThanThreshold) {
+ const auto surplus = 14;
+ insertNDocsOf1MB(operationContext(), surplus /* nDocs */);
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 10 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 2UL);
+ ASSERT_EQ(7, splitKeys.front().getIntField(kPattern));
+ ASSERT_EQ(15, splitKeys.back().getIntField(kPattern));
+}
+
+// Split points are not recalculated if the right-most chunk is at least `80% maxChunkSize`
+TEST_F(AutoSplitVectorTest10MB, NoRecalculateIfBigLastChunk) {
+ const auto surplus = 8;
+ insertNDocsOf1MB(operationContext(), surplus /* nDocs */);
+ std::vector<BSONObj> splitKeys = autoSplit(operationContext(), 10 /* maxChunkSizeMB */);
+ ASSERT_EQ(splitKeys.size(), 1UL);
+ ASSERT_EQ(9, splitKeys.front().getIntField(kPattern));
+}
+
+class RepositionLastSplitPointsTest : public AutoSplitVectorTest {
+public:
+ /*
+ * Tests that last split points are properly repositioned in case the surplus allows so or not
+ * repositioned otherwise.
+ */
+ void checkRepositioning(int maxDocsPerChunk, int surplus, int nSplitPoints) {
+ ASSERT(surplus >= 0 && surplus < maxDocsPerChunk);
+
+ const auto maxDocsPerNewChunk =
+ maxDocsPerChunk - ((maxDocsPerChunk - surplus) / (nSplitPoints + 1));
+ bool mustReposition =
+ surplus >= maxDocsPerChunk - maxDocsPerNewChunk && surplus < maxDocsPerChunk * 0.8;
+
+ int toInsert = (maxDocsPerChunk * nSplitPoints) - getInsertedSize() + surplus;
+ insertNDocsOf1MB(operationContext(), toInsert);
+
+ int expectedChunkSize =
+ mustReposition ? getInsertedSize() / (nSplitPoints + 1) : maxDocsPerChunk;
+ std::vector<BSONObj> splitKeys =
+ autoSplit(operationContext(), maxDocsPerChunk /* maxChunkSizeMB */);
+
+ int approximateNextMin = expectedChunkSize;
+ for (const auto& splitKey : splitKeys) {
+ int _id = splitKey.getIntField(kPattern);
+ // Expect an approximate match due to integers rounding in the split points algorithm.
+ ASSERT(_id >= approximateNextMin - 2 && _id <= approximateNextMin + 2)
+ << BSON("approximateNextMin" << approximateNextMin << "splitKeys" << splitKeys
+ << "maxDocsPerChunk"
+ << maxDocsPerChunk
+ << "surplus"
+ << surplus
+ << "nSplitPoints"
+ << nSplitPoints
+ << "maxDocsPerNewChunk"
+ << maxDocsPerNewChunk
+ << "mustReposition"
+ << mustReposition
+ << "toInsert"
+ << toInsert
+ << "expectedChunkSize"
+ << expectedChunkSize);
+ approximateNextMin = _id + expectedChunkSize;
+ }
+ }
+};
+
+
+// Test that last split points are recalculated fairly (if the surplus allows so)
+TEST_F(RepositionLastSplitPointsTest, RandomRepositioningTest) {
+ PseudoRandom random(SecureRandom::create()->nextInt64());
+ // Avoid small sizes already checked in other test cases.
+ // Random maxDocsPerChunk in interval: [10, 110).
+ int maxDocsPerChunk = random.nextInt32(100) + 10;
+ // Random surplus in interval: [0, maxDocsPerChunk).
+ int surplus = random.nextInt32(maxDocsPerChunk);
+
+ log() << "RandomRepositioningTest parameters - maxDocsPerChunk: " << maxDocsPerChunk
+ << " - surplus: " << surplus;
+
+ for (int nSplitPointsToReposition = 1; nSplitPointsToReposition < 4;
+ nSplitPointsToReposition++) {
+ checkRepositioning(maxDocsPerChunk, surplus, nSplitPointsToReposition);
+ }
+}
+
+
+} // namespace
+} // namespace mongo
diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp
index 4f7caa95c1f..ae9b5e12f77 100644
--- a/src/mongo/db/s/balancer/balancer.cpp
+++ b/src/mongo/db/s/balancer/balancer.cpp
@@ -639,8 +639,7 @@ void Balancer::_splitOrMarkJumbo(OperationContext* opCtx,
nss,
cm->getShardKeyPattern(),
ChunkRange(chunk.getMin(), chunk.getMax()),
- Grid::get(opCtx)->getBalancerConfiguration()->getMaxChunkSizeBytes(),
- boost::none));
+ Grid::get(opCtx)->getBalancerConfiguration()->getMaxChunkSizeBytes()));
uassert(ErrorCodes::CannotSplit, "No split points found", !splitPoints.empty());
diff --git a/src/mongo/db/s/chunk_splitter.cpp b/src/mongo/db/s/chunk_splitter.cpp
index bd4c8878544..73eb4266aa5 100644
--- a/src/mongo/db/s/chunk_splitter.cpp
+++ b/src/mongo/db/s/chunk_splitter.cpp
@@ -39,9 +39,9 @@
#include "mongo/db/client.h"
#include "mongo/db/dbdirectclient.h"
#include "mongo/db/namespace_string.h"
+#include "mongo/db/s/auto_split_vector.h"
#include "mongo/db/s/sharding_state.h"
#include "mongo/db/s/split_chunk.h"
-#include "mongo/db/s/split_vector.h"
#include "mongo/db/service_context.h"
#include "mongo/s/balancer_configuration.h"
#include "mongo/s/catalog/type_chunk.h"
@@ -320,18 +320,15 @@ void ChunkSplitter::_runAutosplit(const NamespaceString& nss,
<< " dataWritten since last check: " << dataWritten
<< " maxChunkSizeBytes: " << maxChunkSizeBytes;
- auto splitPoints = uassertStatusOK(splitVector(opCtx.get(),
- nss,
- cm->getShardKeyPattern().toBSON(),
- chunk.getMin(),
- chunk.getMax(),
- false,
- boost::none,
- boost::none,
- boost::none,
- maxChunkSizeBytes));
-
- if (splitPoints.size() <= 1) {
+ const auto& shardKeyPattern = cm->getShardKeyPattern();
+ auto splitPoints = autoSplitVector(opCtx.get(),
+ nss,
+ shardKeyPattern.toBSON(),
+ chunk.getMin(),
+ chunk.getMax(),
+ maxChunkSizeBytes);
+
+ if (splitPoints.empty()) {
// No split points means there isn't enough data to split on; 1 split point means we
// have between half the chunk size to full chunk size so there is no need to split yet
return;
diff --git a/src/mongo/db/s/config/initial_split_policy.cpp b/src/mongo/db/s/config/initial_split_policy.cpp
index 9d1c8a7eee4..86d73344e4a 100644
--- a/src/mongo/db/s/config/initial_split_policy.cpp
+++ b/src/mongo/db/s/config/initial_split_policy.cpp
@@ -394,8 +394,7 @@ InitialSplitPolicy::ShardCollectionConfig InitialSplitPolicy::createFirstChunksU
nss,
shardKeyPattern,
ChunkRange(keyPattern.globalMin(), keyPattern.globalMax()),
- balancerConfig->getMaxChunkSizeBytes(),
- 0));
+ balancerConfig->getMaxChunkSizeBytes()));
return generateShardCollectionInitialChunks(nss,
shardKeyPattern,
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp
index 3b9645eb03a..62f0a9633a5 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp
@@ -68,21 +68,21 @@ class ShardCollectionTestBase : public ConfigServerTestFixture {
protected:
void expectSplitVector(const HostAndPort& shardHost,
const ShardKeyPattern& keyPattern,
- const BSONObj& splitPoints) {
+ const BSONArray& splitPoints) {
onCommand([&](const RemoteCommandRequest& request) {
ASSERT_EQUALS(shardHost, request.target);
string cmdName = request.cmdObj.firstElement().fieldName();
- ASSERT_EQUALS("splitVector", cmdName);
- ASSERT_EQUALS(kNamespace.ns(),
- request.cmdObj["splitVector"].String()); // splitVector uses full ns
+ ASSERT_EQUALS("autoSplitVector", cmdName);
+ // autoSplitVector concatenates the collection name to the command's db
+ const auto receivedNs =
+ request.dbname + '.' + request.cmdObj["autoSplitVector"].String();
+ ASSERT_EQUALS(kNamespace.ns(), receivedNs);
ASSERT_BSONOBJ_EQ(keyPattern.toBSON(), request.cmdObj["keyPattern"].Obj());
ASSERT_BSONOBJ_EQ(keyPattern.getKeyPattern().globalMin(), request.cmdObj["min"].Obj());
ASSERT_BSONOBJ_EQ(keyPattern.getKeyPattern().globalMax(), request.cmdObj["max"].Obj());
ASSERT_EQUALS(64 * 1024 * 1024ULL,
static_cast<uint64_t>(request.cmdObj["maxChunkSizeBytes"].numberLong()));
- ASSERT_EQUALS(0, request.cmdObj["maxSplitPoints"].numberLong());
- ASSERT_EQUALS(0, request.cmdObj["maxChunkObjects"].numberLong());
ASSERT_BSONOBJ_EQ(
ReadPreferenceSetting(ReadPreference::PrimaryPreferred).toContainingBSON(),
@@ -338,7 +338,7 @@ TEST_F(ConfigServerShardCollectionTest, RangeSharding_NoInitialSplitPoints_NoSpl
});
// Respond to the splitVector command sent to the shard to figure out initial split points.
- expectSplitVector(shardHost, keyPattern, BSONObj());
+ expectSplitVector(shardHost, keyPattern, BSONArray());
// Expect the set shard version for that namespace.
// We do not check for a specific ChunkVersion, because we cannot easily know the OID that was
diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp
index e07695ac82f..c8300788eaf 100644
--- a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp
+++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp
@@ -737,7 +737,7 @@ Status MigrationChunkClonerSourceLegacy::_storeCurrentLocs(OperationContext* opC
if (totalRecs > 0) {
avgRecSize = collection->dataSize(opCtx) / totalRecs;
maxRecsWhenFull = _args.getMaxChunkSizeBytes() / avgRecSize;
- maxRecsWhenFull = 130 * maxRecsWhenFull / 100; // pad some slack
+ maxRecsWhenFull = 2 * maxRecsWhenFull; // pad some slack
} else {
avgRecSize = 0;
maxRecsWhenFull = kMaxObjectPerChunk + 1;
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index 5c409c2b9e5..e0cdbe622f4 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -145,6 +145,7 @@ env.Library(
'stale_exception.cpp',
env.Idlc('catalog/type_chunk_base.idl')[0],
env.Idlc('database_version.idl')[0],
+ env.Idlc('request_types/auto_split_vector.idl')[0],
env.Idlc('request_types/clone_catalog_data.idl')[0],
env.Idlc('request_types/clear_jumbo_flag.idl')[0],
env.Idlc('request_types/create_collection.idl')[0],
diff --git a/src/mongo/s/request_types/auto_split_vector.idl b/src/mongo/s/request_types/auto_split_vector.idl
new file mode 100644
index 00000000000..42e55424c4e
--- /dev/null
+++ b/src/mongo/s/request_types/auto_split_vector.idl
@@ -0,0 +1,71 @@
+# Copyright(C) 2021 - present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# This IDL file describes the BSON format for the autoSplitVector command.
+
+global:
+ cpp_namespace: "mongo"
+ cpp_includes:
+ - "mongo/db/s/auto_split_vector.h"
+
+imports:
+ - "mongo/idl/basic_types.idl"
+
+types:
+ bson_vector:
+ bson_serialization_type: any
+ description: "An array of objects representing the split keys."
+ cpp_type: "std::vector<mongo::BSONObj>"
+ deserializer: ::mongo::parseSplitKeys
+
+structs:
+ AutoSplitVectorResponse:
+ description: "The reply of an autoSplitVector command."
+ strict: false
+ fields:
+ splitKeys: bson_vector
+
+commands:
+ autoSplitVector:
+ cpp_name: AutoSplitVectorRequest
+ description: "Internal autoSplitVector command"
+ strict: false
+ namespace: concatenate_with_db
+ fields:
+ keyPattern:
+ type: object
+ description: "Shard key pattern of the collection"
+ min:
+ type: object
+ description: "Min key of the chunk"
+ max:
+ type: object
+ description: "Max key of the chunk"
+ maxChunkSizeBytes:
+ type: safeInt64
+ description: "Max chunk size of the collection expressed in bytes"
diff --git a/src/mongo/s/shard_util.cpp b/src/mongo/s/shard_util.cpp
index a861e69e9e7..116d636ef92 100644
--- a/src/mongo/s/shard_util.cpp
+++ b/src/mongo/s/shard_util.cpp
@@ -39,8 +39,10 @@
#include "mongo/client/read_preference.h"
#include "mongo/client/remote_command_targeter.h"
#include "mongo/db/namespace_string.h"
+#include "mongo/db/s/auto_split_vector.h"
#include "mongo/s/client/shard_registry.h"
#include "mongo/s/grid.h"
+#include "mongo/s/request_types/auto_split_vector_gen.h"
#include "mongo/s/shard_key_pattern.h"
#include "mongo/util/log.h"
#include "mongo/util/mongoutils/str.h"
@@ -94,45 +96,65 @@ StatusWith<std::vector<BSONObj>> selectChunkSplitPoints(OperationContext* opCtx,
const NamespaceString& nss,
const ShardKeyPattern& shardKeyPattern,
const ChunkRange& chunkRange,
- long long chunkSizeBytes,
- boost::optional<int> maxObjs) {
- BSONObjBuilder cmd;
- cmd.append("splitVector", nss.ns());
- cmd.append("keyPattern", shardKeyPattern.toBSON());
- chunkRange.append(&cmd);
- cmd.append("maxChunkSizeBytes", chunkSizeBytes);
- if (maxObjs) {
- cmd.append("maxChunkObjects", *maxObjs);
- }
-
+ long long chunkSizeBytes) {
auto shardStatus = Grid::get(opCtx)->shardRegistry()->getShard(opCtx, shardId);
if (!shardStatus.isOK()) {
return shardStatus.getStatus();
}
- auto cmdStatus = shardStatus.getValue()->runCommandWithFixedRetryAttempts(
- opCtx,
- ReadPreferenceSetting{ReadPreference::PrimaryPreferred},
- "admin",
- cmd.obj(),
- Shard::RetryPolicy::kIdempotent);
- if (!cmdStatus.isOK()) {
- return std::move(cmdStatus.getStatus());
- }
- if (!cmdStatus.getValue().commandStatus.isOK()) {
- return std::move(cmdStatus.getValue().commandStatus);
+ auto invokeSplitCommand = [&](const BSONObj& command, const StringData db) {
+ return shardStatus.getValue()->runCommandWithFixedRetryAttempts(
+ opCtx,
+ ReadPreferenceSetting{ReadPreference::PrimaryPreferred},
+ db.toString(),
+ command,
+ Shard::RetryPolicy::kIdempotent);
+ };
+
+ AutoSplitVectorRequest req(nss);
+ req.setKeyPattern(shardKeyPattern.toBSON());
+ req.setMin(chunkRange.getMin());
+ req.setMax(chunkRange.getMax());
+ req.setMaxChunkSizeBytes(chunkSizeBytes);
+
+
+ auto cmdStatus = invokeSplitCommand(req.toBSON({}), nss.db());
+
+ // Fallback to splitVector command in case of mixed binaries not supporting autoSplitVector
+ bool fallback = [&]() {
+ auto status = Shard::CommandResponse::getEffectiveStatus(cmdStatus);
+ return !status.isOK() && status.code() == ErrorCodes::CommandNotFound;
+ }();
+
+ if (fallback) {
+ BSONObjBuilder cmd;
+ cmd.append("splitVector", nss.ns());
+ cmd.append("keyPattern", shardKeyPattern.toBSON());
+ chunkRange.append(&cmd);
+ cmd.append("maxChunkSizeBytes", chunkSizeBytes);
+ cmdStatus = invokeSplitCommand(cmd.obj(), NamespaceString::kAdminDb);
}
- const auto response = std::move(cmdStatus.getValue().response);
- std::vector<BSONObj> splitPoints;
+ auto status = Shard::CommandResponse::getEffectiveStatus(cmdStatus);
+ if (!status.isOK()) {
+ return status;
+ }
+
+ if (fallback) {
+ const auto response = std::move(cmdStatus.getValue().response);
+ std::vector<BSONObj> splitPoints;
- BSONObjIterator it(response.getObjectField("splitKeys"));
- while (it.more()) {
- splitPoints.push_back(it.next().Obj().getOwned());
+ BSONObjIterator it(response.getObjectField("splitKeys"));
+ while (it.more()) {
+ splitPoints.push_back(it.next().Obj().getOwned());
+ }
+ return std::move(splitPoints);
}
- return std::move(splitPoints);
+ const auto response = AutoSplitVectorResponse::parse(
+ IDLParserErrorContext("AutoSplitVectorResponse"), std::move(cmdStatus.getValue().response));
+ return response.getSplitKeys();
}
StatusWith<boost::optional<ChunkRange>> splitChunkAtMultiplePoints(
diff --git a/src/mongo/s/shard_util.h b/src/mongo/s/shard_util.h
index a44b3fae919..dba785df22c 100644
--- a/src/mongo/s/shard_util.h
+++ b/src/mongo/s/shard_util.h
@@ -78,8 +78,7 @@ StatusWith<std::vector<BSONObj>> selectChunkSplitPoints(OperationContext* opCtx,
const NamespaceString& nss,
const ShardKeyPattern& shardKeyPattern,
const ChunkRange& chunkRange,
- long long chunkSizeBytes,
- boost::optional<int> maxObjs);
+ long long chunkSizeBytes);
/**
* Asks the specified shard to split the chunk described by min/maxKey into the respective split
diff --git a/src/mongo/s/write_ops/cluster_write.cpp b/src/mongo/s/write_ops/cluster_write.cpp
index 1e9af832516..493c3ef7f28 100644
--- a/src/mongo/s/write_ops/cluster_write.cpp
+++ b/src/mongo/s/write_ops/cluster_write.cpp
@@ -302,8 +302,7 @@ void updateChunkWriteStatsAndSplitIfNeeded(OperationContext* opCtx,
nss,
manager->getShardKeyPattern(),
chunkRange,
- chunkSizeToUse,
- boost::none));
+ chunkSizeToUse));
if (splitPoints.size() <= 1) {
// No split points means there isn't enough data to split on; 1 split point means we
View Lorry Controller Status