Revert "SERVER-71771 Defragmenter should cap chunk size estimation"

This reverts commit 43221bcda3d39d58b8ef648185c63d3840791b4f.

10 files changed, 31 insertions, 95 deletions

diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp
index 4a613199e2e..a9b4aa0cb6f 100644
--- a/src/mongo/db/s/balancer/balancer.cpp
+++ b/src/mongo/db/s/balancer/balancer.cpp
@@ -584,8 +584,7 @@ void Balancer::_consumeActionStreamLoop() {
                                               dataSizeAction.chunkRange,
                                               dataSizeAction.version,
                                               dataSizeAction.keyPattern,
-                                              dataSizeAction.estimatedValue,
-                                              dataSizeAction.maxSize)
+                                              dataSizeAction.estimatedValue)
                             .thenRunOn(*executor)
                             .onCompletion([this,
                                            selectedStream,
diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler.h b/src/mongo/db/s/balancer/balancer_commands_scheduler.h
index 2a16209a2c0..a75e18e944e 100644
--- a/src/mongo/db/s/balancer/balancer_commands_scheduler.h
+++ b/src/mongo/db/s/balancer/balancer_commands_scheduler.h
@@ -148,8 +148,7 @@ public:
                                                          const ChunkRange& chunkRange,
                                                          const ChunkVersion& version,
                                                          const KeyPattern& keyPattern,
-                                                         bool estimatedValue,
-                                                         int64_t maxSize) = 0;
+                                                         bool estimatedValue) = 0;
 
     virtual SemiFuture<void> requestMoveRange(OperationContext* opCtx,
                                               const ShardsvrMoveRange& request,
diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp
index ca15b3b420f..955ede552ef 100644
--- a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp
+++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp
@@ -184,7 +184,6 @@ const std::string DataSizeCommandInfo::kKeyPattern = "keyPattern";
 const std::string DataSizeCommandInfo::kMinValue = "min";
 const std::string DataSizeCommandInfo::kMaxValue = "max";
 const std::string DataSizeCommandInfo::kEstimatedValue = "estimate";
-const std::string DataSizeCommandInfo::kMaxSizeValue = "maxSize";
 
 const std::string SplitChunkCommandInfo::kCommandName = "splitChunk";
 const std::string SplitChunkCommandInfo::kShardName = "from";
@@ -360,15 +359,13 @@ SemiFuture<DataSizeResponse> BalancerCommandsSchedulerImpl::requestDataSize(
     const ChunkRange& chunkRange,
     const ChunkVersion& version,
     const KeyPattern& keyPattern,
-    bool estimatedValue,
-    int64_t maxSize) {
+    bool estimatedValue) {
     auto commandInfo = std::make_shared<DataSizeCommandInfo>(nss,
                                                              shardId,
                                                              keyPattern.toBSON(),
                                                              chunkRange.getMin(),
                                                              chunkRange.getMax(),
                                                              estimatedValue,
-                                                             maxSize,
                                                              version);
 
     return _buildAndEnqueueNewRequest(opCtx, std::move(commandInfo))
@@ -380,8 +377,7 @@ SemiFuture<DataSizeResponse> BalancerCommandsSchedulerImpl::requestDataSize(
             }
             long long sizeBytes = remoteResponse.data["size"].number();
             long long numObjects = remoteResponse.data["numObjects"].number();
-            bool maxSizeReached = remoteResponse.data["maxReached"].trueValue();
-            return DataSizeResponse(sizeBytes, numObjects, maxSizeReached);
+            return DataSizeResponse(sizeBytes, numObjects);
         })
         .semi();
 }
diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h
index 5803a880029..92f9f074441 100644
--- a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h
+++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h
@@ -336,14 +336,12 @@ public:
                         const BSONObj& lowerBoundKey,
                         const BSONObj& upperBoundKey,
                         bool estimatedValue,
-                        int64_t maxSize,
                         const ChunkVersion& version)
         : CommandInfo(shardId, nss, boost::none),
           _shardKeyPattern(shardKeyPattern),
           _lowerBoundKey(lowerBoundKey),
           _upperBoundKey(upperBoundKey),
           _estimatedValue(estimatedValue),
-          _maxSize(maxSize),
           _version(version) {}
 
     BSONObj serialise() const override {
@@ -352,8 +350,7 @@ public:
             .append(kKeyPattern, _shardKeyPattern)
             .append(kMinValue, _lowerBoundKey)
             .append(kMaxValue, _upperBoundKey)
-            .append(kEstimatedValue, _estimatedValue)
-            .append(kMaxSizeValue, _maxSize);
+            .append(kEstimatedValue, _estimatedValue);
 
         _version.serializeToBSON(ChunkVersion::kShardVersionField, &commandBuilder);
 
@@ -365,7 +362,6 @@ private:
     BSONObj _lowerBoundKey;
     BSONObj _upperBoundKey;
     bool _estimatedValue;
-    int64_t _maxSize;
     ChunkVersion _version;
 
     static const std::string kCommandName;
@@ -373,7 +369,6 @@ private:
     static const std::string kMinValue;
     static const std::string kMaxValue;
     static const std::string kEstimatedValue;
-    static const std::string kMaxSizeValue;
 };
 
 class SplitChunkCommandInfo : public CommandInfo {
@@ -594,8 +589,7 @@ public:
                                                  const ChunkRange& chunkRange,
                                                  const ChunkVersion& version,
                                                  const KeyPattern& keyPattern,
-                                                 bool estimatedValue,
-                                                 int64_t maxSize) override;
+                                                 bool estimatedValue) override;
 
 private:
     enum class SchedulerState { Recovering, Running, Stopping, Stopped };
diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp b/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp
index fdf01b9eebd..678e5f63f9f 100644
--- a/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp
+++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp
@@ -319,15 +319,13 @@ TEST_F(BalancerCommandsSchedulerTest, SuccessfulRequestChunkDataSizeCommand) {
     _scheduler.start(operationContext(), getMigrationRecoveryDefaultValues());
     ChunkType chunk = makeChunk(0, kShardId0);
 
-    auto futureResponse =
-        _scheduler.requestDataSize(operationContext(),
-                                   kNss,
-                                   chunk.getShard(),
-                                   chunk.getRange(),
-                                   chunk.getVersion(),
-                                   KeyPattern(BSON("x" << 1)),
-                                   false /* issuedByRemoteUser */,
-                                   (kDefaultMaxChunkSizeBytes / 100) * 25 /* maxSize */);
+    auto futureResponse = _scheduler.requestDataSize(operationContext(),
+                                                     kNss,
+                                                     chunk.getShard(),
+                                                     chunk.getRange(),
+                                                     chunk.getVersion(),
+                                                     KeyPattern(BSON("x" << 1)),
+                                                     false /* issuedByRemoteUser */);
     auto swReceivedDataSize = futureResponse.getNoThrow();
     ASSERT_OK(swReceivedDataSize.getStatus());
     auto receivedDataSize = swReceivedDataSize.getValue();
diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp
index 99cb2498ad6..d52421edc9e 100644
--- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp
+++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp
@@ -166,11 +166,6 @@ public:
         auto collectionChunks = getCollectionChunks(opCtx, coll);
         const auto collectionZones = getCollectionZones(opCtx, coll);
 
-        // Calculate small chunk threshold to limit dataSize commands
-        const auto maxChunkSizeBytes = getCollectionMaxChunkSizeBytes(opCtx, coll);
-        const int64_t smallChunkSizeThreshold =
-            (maxChunkSizeBytes / 100) * kSmallChunkSizeThresholdPctg;
-
         stdx::unordered_map<ShardId, PendingActions> pendingActionsByShards;
         // Find ranges of chunks; for single-chunk ranges, request DataSize; for multi-range, issue
         // merge
@@ -197,7 +192,6 @@ public:
             new MergeAndMeasureChunksPhase(coll.getNss(),
                                            coll.getUuid(),
                                            coll.getKeyPattern().toBSON(),
-                                           smallChunkSizeThreshold,
                                            std::move(pendingActionsByShards)));
     }
 
@@ -223,15 +217,14 @@ public:
 
             if (pendingActions.rangesWithoutDataSize.size() > pendingActions.rangesToMerge.size()) {
                 const auto& rangeToMeasure = pendingActions.rangesWithoutDataSize.back();
-                nextAction = boost::optional<DefragmentationAction>(
-                    DataSizeInfo(shardId,
-                                 _nss,
-                                 _uuid,
-                                 rangeToMeasure,
-                                 shardVersion,
-                                 _shardKey,
-                                 true /* estimate */,
-                                 _smallChunkSizeThresholdBytes /* maxSize */));
+                nextAction =
+                    boost::optional<DefragmentationAction>(DataSizeInfo(shardId,
+                                                                        _nss,
+                                                                        _uuid,
+                                                                        rangeToMeasure,
+                                                                        shardVersion,
+                                                                        _shardKey,
+                                                                        true /* estimate */));
                 pendingActions.rangesWithoutDataSize.pop_back();
             } else if (!pendingActions.rangesToMerge.empty()) {
                 const auto& rangeToMerge = pendingActions.rangesToMerge.back();
@@ -304,17 +297,10 @@ public:
                                             dataSizeAction.version,
                                             dataSizeAction.shardId);
                             auto catalogManager = ShardingCatalogManager::get(opCtx);
-                            // Max out the chunk size if it has has been estimated as
-                            // bigger than _smallChunkSizeThresholdBytes; this will exlude
-                            // the chunk from the list of candidates considered by
-                            // MoveAndMergeChunksPhase
-                            auto estimatedSize = dataSizeResponse.getValue().maxSizeReached
-                                ? std::numeric_limits<int64_t>::max()
-                                : dataSizeResponse.getValue().sizeBytes;
                             catalogManager->setChunkEstimatedSize(
                                 opCtx,
                                 chunk,
-                                estimatedSize,
+                                dataSizeResponse.getValue().sizeBytes,
                                 ShardingCatalogClient::kMajorityWriteConcern);
                         },
                         [&]() {
@@ -367,12 +353,10 @@ private:
         const NamespaceString& nss,
         const UUID& uuid,
         const BSONObj& shardKey,
-        const int64_t smallChunkSizeThresholdBytes,
         stdx::unordered_map<ShardId, PendingActions>&& pendingActionsByShards)
         : _nss(nss),
           _uuid(uuid),
           _shardKey(shardKey),
-          _smallChunkSizeThresholdBytes(smallChunkSizeThresholdBytes),
           _pendingActionsByShards(std::move(pendingActionsByShards)) {}
 
     void _abort(const DefragmentationPhaseEnum nextPhase) {
@@ -384,7 +368,6 @@ private:
     const NamespaceString _nss;
     const UUID _uuid;
     const BSONObj _shardKey;
-    const int64_t _smallChunkSizeThresholdBytes;
     stdx::unordered_map<ShardId, PendingActions> _pendingActionsByShards;
     boost::optional<ShardId> _shardToProcess;
     size_t _outstandingActions{0};
@@ -785,6 +768,8 @@ private:
         bool _isChunkToMergeLeftSibling;
     };
 
+    static constexpr uint64_t kSmallChunkSizeThresholdPctg = 25;
+
     const NamespaceString _nss;
 
     const UUID _uuid;
diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h
index 12a8b90c7d9..c5174d9944b 100644
--- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h
+++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h
@@ -64,9 +64,6 @@ public:
     virtual bool isComplete() const = 0;
 
     virtual void userAbort() = 0;
-
-protected:
-    static constexpr uint64_t kSmallChunkSizeThresholdPctg = 25;
 };
 
 class BalancerDefragmentationPolicyImpl : public BalancerDefragmentationPolicy {
diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp b/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp
index 068df07cdc8..c5b71710c24 100644
--- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp
+++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp
@@ -270,7 +270,7 @@ TEST_F(BalancerDefragmentationPolicyTest,
     ASSERT_TRUE(nextAction.is_initialized());
     DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction);
 
-    auto resp = StatusWith(DataSizeResponse(2000, 4, false));
+    auto resp = StatusWith(DataSizeResponse(2000, 4));
     _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp);
 
     // 1. The outcome of the data size has been stored in the expected document...
@@ -287,33 +287,6 @@ TEST_F(BalancerDefragmentationPolicyTest,
     verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMoveAndMergeChunks);
 }
 
-TEST_F(BalancerDefragmentationPolicyTest,
-       TestPhaseOneDataSizeResponsesWithMaxSizeReachedCausesChunkToBeSkippedByPhaseTwo) {
-    auto coll = setupCollectionWithPhase({makeConfigChunkEntry()});
-    setDefaultClusterStats();
-    _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll);
-    auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext());
-    ASSERT_TRUE(nextAction.has_value());
-    DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction);
-
-    auto resp = StatusWith(DataSizeResponse(2000, 4, true));
-    _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp);
-
-    // 1. The outcome of the data size has been stored in the expected document...
-    auto chunkQuery = BSON(ChunkType::collectionUUID()
-                           << kUuid << ChunkType::min(kKeyAtMin) << ChunkType::max(kKeyAtMax));
-    auto configChunkDoc =
-        findOneOnConfigCollection(operationContext(), ChunkType::ConfigNS, chunkQuery).getValue();
-    ASSERT_EQ(configChunkDoc.getField("estimatedDataSizeBytes").safeNumberLong(),
-              std::numeric_limits<int64_t>::max());
-
-    // No new action is expected - and the algorithm should converge
-    nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext());
-    ASSERT_TRUE(nextAction == boost::none);
-    ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid()));
-    verifyExpectedDefragmentationPhaseOndisk(boost::none);
-}
-
 TEST_F(BalancerDefragmentationPolicyTest, TestRetriableFailedDataSizeActionGetsReissued) {
     auto coll = setupCollectionWithPhase({makeConfigChunkEntry()});
     _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll);
@@ -348,7 +321,7 @@ TEST_F(BalancerDefragmentationPolicyTest, TestRemoveCollectionEndsDefragmentatio
     auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext());
     DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction);
 
-    auto resp = StatusWith(DataSizeResponse(2000, 4, false));
+    auto resp = StatusWith(DataSizeResponse(2000, 4));
     _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp);
 
     // Remove collection entry from config.collections
diff --git a/src/mongo/db/s/balancer/balancer_policy.cpp b/src/mongo/db/s/balancer/balancer_policy.cpp
index ab3fd5660cc..0cf71995d04 100644
--- a/src/mongo/db/s/balancer/balancer_policy.cpp
+++ b/src/mongo/db/s/balancer/balancer_policy.cpp
@@ -1031,15 +1031,13 @@ DataSizeInfo::DataSizeInfo(const ShardId& shardId,
                            const ChunkRange& chunkRange,
                            const ChunkVersion& version,
                            const KeyPattern& keyPattern,
-                           bool estimatedValue,
-                           int64_t maxSize)
+                           bool estimatedValue)
     : shardId(shardId),
       nss(nss),
       uuid(uuid),
       chunkRange(chunkRange),
       version(version),
       keyPattern(keyPattern),
-      estimatedValue(estimatedValue),
-      maxSize(maxSize) {}
+      estimatedValue(estimatedValue) {}
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/balancer/balancer_policy.h b/src/mongo/db/s/balancer/balancer_policy.h
index 4dbfdb797c3..35f0c8ada30 100644
--- a/src/mongo/db/s/balancer/balancer_policy.h
+++ b/src/mongo/db/s/balancer/balancer_policy.h
@@ -186,8 +186,7 @@ struct DataSizeInfo {
                  const ChunkRange& chunkRange,
                  const ChunkVersion& version,
                  const KeyPattern& keyPattern,
-                 bool estimatedValue,
-                 int64_t maxSize);
+                 bool estimatedValue);
 
     ShardId shardId;
     NamespaceString nss;
@@ -196,16 +195,14 @@ struct DataSizeInfo {
     ChunkVersion version;
     KeyPattern keyPattern;
     bool estimatedValue;
-    int64_t maxSize;
 };
 
 struct DataSizeResponse {
-    DataSizeResponse(long long sizeBytes, long long numObjects, bool maxSizeReached)
-        : sizeBytes(sizeBytes), numObjects(numObjects), maxSizeReached(maxSizeReached) {}
+    DataSizeResponse(long long sizeBytes, long long numObjects)
+        : sizeBytes(sizeBytes), numObjects(numObjects) {}
 
     long long sizeBytes;
     long long numObjects;
-    bool maxSizeReached;
 };
 
 typedef stdx::