SERVER-64406 apply dynamic throttling to balancer rounds

2 files changed, 32 insertions, 18 deletions

diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp
index dc40cd77e5d..1b832d0f79e 100644
--- a/src/mongo/db/s/balancer/balancer.cpp
+++ b/src/mongo/db/s/balancer/balancer.cpp
@@ -84,7 +84,7 @@ const Seconds kBalanceRoundDefaultInterval(10);
 // be balanced. This value should be set sufficiently low so that imbalanced clusters will quickly
 // reach balanced state, but setting it too low may cause CRUD operations to start failing due to
 // not being able to establish a stable shard version.
-const Seconds kShortBalanceRoundInterval(1);
+const Seconds kBalancerMigrationsThrottling(1);
 
 /**
  * Balancer status response
@@ -240,7 +240,7 @@ Balancer* Balancer::get(OperationContext* operationContext) {
 }
 
 Balancer::Balancer()
-    : _balancedLastTime(0),
+    : _numMigrationsInRound(0),
       _random(std::random_device{}()),
       _clusterStats(std::make_unique<ClusterStatisticsImpl>(_random)),
       _chunkSelectionPolicy(
@@ -666,6 +666,7 @@ void Balancer::_mainThread() {
     LOGV2(6036606, "Balancer worker thread initialised. Entering main loop.");
 
     // Main balancer loop
+    auto prevRoundEndTime = Date_t::fromMillisSinceEpoch(0);
     while (!_stopRequested()) {
         BalanceRoundDetails roundDetails;
 
@@ -740,35 +741,48 @@ void Balancer::_mainThread() {
 
                 if (chunksToRebalance.empty() && chunksToDefragment.empty()) {
                     LOGV2_DEBUG(21862, 1, "No need to move any chunk");
-                    _balancedLastTime = 0;
+                    _numMigrationsInRound = 0;
                 } else {
-                    _balancedLastTime =
+                    _numMigrationsInRound =
                         _moveChunks(opCtx.get(), chunksToRebalance, chunksToDefragment);
 
                     roundDetails.setSucceeded(
                         static_cast<int>(chunksToRebalance.size() + chunksToDefragment.size()),
-                        _balancedLastTime);
+                        _numMigrationsInRound);
 
                     ShardingLogging::get(opCtx.get())
                         ->logAction(opCtx.get(), "balancer.round", "", roundDetails.toBSON())
                         .ignore();
                 }
-
-                LOGV2_DEBUG(21863, 1, "End balancing round");
             }
 
-            Milliseconds balancerInterval =
-                _balancedLastTime ? kShortBalanceRoundInterval : kBalanceRoundDefaultInterval;
+            LOGV2_DEBUG(21863, 1, "End balancing round");
+            auto currRoundEndTime = Date_t::now();
+            auto balancerInterval = [&]() -> Milliseconds {
+                // If a test is setting a value, or no migrations have been performed, wait for a
+                // fixed amount of time
+                boost::optional<Milliseconds> forcedValue(boost::none);
+                overrideBalanceRoundInterval.execute([&](const BSONObj& data) {
+                    forcedValue = Milliseconds(data["intervalMs"].numberInt());
+                    LOGV2(21864,
+                          "overrideBalanceRoundInterval: using customized balancing interval",
+                          "balancerInterval"_attr = *forcedValue);
+                });
+                if (forcedValue) {
+                    return *forcedValue;
+                }
+                if (_numMigrationsInRound == 0) {
+                    return kBalanceRoundDefaultInterval;
+                }
 
-            overrideBalanceRoundInterval.execute([&](const BSONObj& data) {
-                balancerInterval = Milliseconds(data["intervalMs"].numberInt());
-                LOGV2(21864,
-                      "overrideBalanceRoundInterval: using shorter balancing interval: "
-                      "{balancerInterval}",
-                      "overrideBalanceRoundInterval: using shorter balancing interval",
-                      "balancerInterval"_attr = balancerInterval);
-            });
+                // Otherwise, apply the throttling constraint
+                auto timeSinceLastEndOfRound = currRoundEndTime - prevRoundEndTime;
+                return timeSinceLastEndOfRound < kBalancerMigrationsThrottling
+                    ? kBalancerMigrationsThrottling - timeSinceLastEndOfRound
+                    : Milliseconds(0);
+            }();
 
+            prevRoundEndTime = currRoundEndTime;
             _endRound(opCtx.get(), balancerInterval);
         } catch (const DBException& e) {
             LOGV2(21865,
diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h
index be31a053d0a..4271974f7ba 100644
--- a/src/mongo/db/s/balancer/balancer.h
+++ b/src/mongo/db/s/balancer/balancer.h
@@ -292,7 +292,7 @@ private:
     stdx::condition_variable _defragmentationCondVar;
 
     // Number of moved chunks in last round
-    int _balancedLastTime;
+    int _numMigrationsInRound;
 
     // Source of randomness when metadata needs to be randomized.
     BalancerRandomSource _random;