SERVER-62022 Reduce dbCheck info logging in prod, log start/stop

3 files changed, 66 insertions, 17 deletions

diff --git a/src/mongo/db/repl/dbcheck.cpp b/src/mongo/db/repl/dbcheck.cpp
index 028df1c3469..cf208dad077 100644
--- a/src/mongo/db/repl/dbcheck.cpp
+++ b/src/mongo/db/repl/dbcheck.cpp
@@ -110,6 +110,10 @@ std::string renderForHealthLog(OplogEntriesEnum op) {
             return "dbCheckBatch";
         case OplogEntriesEnum::Collection:
             return "dbCheckCollection";
+        case OplogEntriesEnum::Start:
+            return "dbCheckStart";
+        case OplogEntriesEnum::Stop:
+            return "dbCheckStop";
     }
 
     MONGO_UNREACHABLE;
@@ -119,30 +123,35 @@ std::string renderForHealthLog(OplogEntriesEnum op) {
 /**
  * Fills in the timestamp and scope, which are always the same for dbCheck's entries.
  */
-std::unique_ptr<HealthLogEntry> dbCheckHealthLogEntry(const NamespaceString& nss,
+std::unique_ptr<HealthLogEntry> dbCheckHealthLogEntry(const boost::optional<NamespaceString>& nss,
                                                       SeverityEnum severity,
                                                       const std::string& msg,
                                                       OplogEntriesEnum operation,
-                                                      const BSONObj& data) {
+                                                      const boost::optional<BSONObj>& data) {
     auto entry = std::make_unique<HealthLogEntry>();
-    entry->setNss(nss);
+    if (nss) {
+        entry->setNss(*nss);
+    }
     entry->setTimestamp(Date_t::now());
     entry->setSeverity(severity);
     entry->setScope(ScopeEnum::Cluster);
     entry->setMsg(msg);
     entry->setOperation(renderForHealthLog(operation));
-    entry->setData(data);
+    if (data) {
+        entry->setData(*data);
+    }
     return entry;
 }
 
 /**
  * Get an error message if the check fails.
  */
-std::unique_ptr<HealthLogEntry> dbCheckErrorHealthLogEntry(const NamespaceString& nss,
-                                                           const std::string& msg,
-                                                           OplogEntriesEnum operation,
-                                                           const Status& err,
-                                                           const BSONObj& context) {
+std::unique_ptr<HealthLogEntry> dbCheckErrorHealthLogEntry(
+    const boost::optional<NamespaceString>& nss,
+    const std::string& msg,
+    OplogEntriesEnum operation,
+    const Status& err,
+    const BSONObj& context) {
     return dbCheckHealthLogEntry(
         nss,
         SeverityEnum::Error,
@@ -340,6 +349,10 @@ bool DbCheckHasher::_canHash(const BSONObj& obj) {
 
 namespace {
 
+// Cumulative number of batches processed. Can wrap around; it's not guaranteed to be in lockstep
+// with other replica set members.
+unsigned int batchesProcessed = 0;
+
 Status dbCheckBatchOnSecondary(OperationContext* opCtx,
                                const repl::OpTime& optime,
                                const DbCheckOplogBatch& entry) {
@@ -386,7 +399,13 @@ Status dbCheckBatchOnSecondary(OperationContext* opCtx,
                                           optime,
                                           collection->getCollectionOptions());
 
-        HealthLog::get(opCtx).log(*logEntry);
+        batchesProcessed++;
+        if (kDebugBuild || logEntry->getSeverity() != SeverityEnum::Info ||
+            (batchesProcessed % gDbCheckHealthLogEveryNBatches.load() == 0)) {
+            // On debug builds, health-log every batch result; on release builds, health-log
+            // every N batches.
+            HealthLog::get(opCtx).log(*logEntry);
+        }
     } catch (const DBException& exception) {
         // In case of an error, report it to the health log,
         auto logEntry = dbCheckErrorHealthLogEntry(
@@ -424,6 +443,14 @@ Status dbCheckOplogCommand(OperationContext* opCtx,
             // TODO SERVER-61963.
             return Status::OK();
         }
+        case OplogEntriesEnum::Start:
+            // fallthrough
+        case OplogEntriesEnum::Stop:
+            const auto entry = mongo::dbCheckHealthLogEntry(
+                boost::none /*nss*/, SeverityEnum::Info, "", type, boost::none /*data*/
+            );
+            HealthLog::get(Client::getCurrent()->getServiceContext()).log(*entry);
+            return Status::OK();
     }
 
     MONGO_UNREACHABLE;
diff --git a/src/mongo/db/repl/dbcheck.h b/src/mongo/db/repl/dbcheck.h
index 73c9bef29f1..0c9368b9ad5 100644
--- a/src/mongo/db/repl/dbcheck.h
+++ b/src/mongo/db/repl/dbcheck.h
@@ -51,20 +51,21 @@ class OpTime;
 /**
  * Logs an entry into 'local.system.healthLog'.
  */
-std::unique_ptr<HealthLogEntry> dbCheckHealthLogEntry(const NamespaceString& nss,
+std::unique_ptr<HealthLogEntry> dbCheckHealthLogEntry(const boost::optional<NamespaceString>& nss,
                                                       SeverityEnum severity,
                                                       const std::string& msg,
                                                       OplogEntriesEnum operation,
-                                                      const BSONObj& data);
+                                                      const boost::optional<BSONObj>& data);
 
 /**
  * Logs an error into 'local.system.healthLog'.
  */
-std::unique_ptr<HealthLogEntry> dbCheckErrorHealthLogEntry(const NamespaceString& nss,
-                                                           const std::string& msg,
-                                                           OplogEntriesEnum operation,
-                                                           const Status& err,
-                                                           const BSONObj& context = BSONObj());
+std::unique_ptr<HealthLogEntry> dbCheckErrorHealthLogEntry(
+    const boost::optional<NamespaceString>& nss,
+    const std::string& msg,
+    OplogEntriesEnum operation,
+    const Status& err,
+    const BSONObj& context = BSONObj());
 
 std::unique_ptr<HealthLogEntry> dbCheckWarningHealthLogEntry(const NamespaceString& nss,
                                                              const std::string& msg,
diff --git a/src/mongo/db/repl/dbcheck.idl b/src/mongo/db/repl/dbcheck.idl
index c1e7fedfacc..48e71f32c31 100644
--- a/src/mongo/db/repl/dbcheck.idl
+++ b/src/mongo/db/repl/dbcheck.idl
@@ -73,6 +73,15 @@ server_parameters:
         validator:
           gte: 20
           lte: 120000
+    dbCheckHealthLogEveryNBatches:
+        description: 'Emit an info-severity health log batch every N batches processed'
+        set_at: [ startup, runtime ]
+        cpp_vartype: 'AtomicWord<int>'
+        cpp_varname: gDbCheckHealthLogEveryNBatches
+        default: 25
+        validator:
+          gte: 1
+          lte: 10000
 
 types:
   _id_key:
@@ -89,6 +98,8 @@ enums:
     values:
       Batch: "batch"
       Collection: "collection"
+      Start: "start"
+      Stop: "stop"
 
 structs:
   DbCheckSingleInvocation:
@@ -216,3 +227,13 @@ structs:
       options:
         type: object
         cpp_name: options
+
+  DbCheckOplogStartStop:
+    description: "Oplog entry for dbCheck start and stop"
+    fields:
+      dbCheck:
+        type: namespacestring
+        cpp_name: nss
+      type:
+        type: OplogEntries
+        cpp_name: type