SERVER-67814 Fix detection of and report the number of time-series collections requiring extended range support in serverStatus

11 files changed, 142 insertions, 67 deletions

diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript
index c07e0f32a2b..4787b6901ae 100644
--- a/src/mongo/db/catalog/SConscript
+++ b/src/mongo/db/catalog/SConscript
@@ -341,6 +341,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/server_base',
         '$BUILD_DIR/mongo/db/service_context',
         '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range',
+        'catalog_stats',
         'collection',
         'collection_catalog',
         'database_holder',
diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp
index 59e0769ff9f..98219a55b95 100644
--- a/src/mongo/db/catalog/catalog_control.cpp
+++ b/src/mongo/db/catalog/catalog_control.cpp
@@ -34,6 +34,7 @@
 
 #include "mongo/db/catalog/catalog_control.h"
 
+#include "mongo/db/catalog/catalog_stats.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/catalog/database.h"
@@ -228,6 +229,10 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) {
     LOGV2(20272, "closeCatalog: closing storage engine catalog");
     opCtx->getServiceContext()->getStorageEngine()->closeCatalog(opCtx);
 
+    // Reset the stats counter for extended range time-series collections. This is maintained
+    // outside the catalog itself.
+    catalog_stats::requiresTimeseriesExtendedRangeSupport.store(0);
+
     reopenOnFailure.dismiss();
     return previousCatalogState;
 }
diff --git a/src/mongo/db/catalog/catalog_stats.cpp b/src/mongo/db/catalog/catalog_stats.cpp
index 36446c6096a..9b1c0f075a3 100644
--- a/src/mongo/db/catalog/catalog_stats.cpp
+++ b/src/mongo/db/catalog/catalog_stats.cpp
@@ -28,7 +28,7 @@
  */
 
 
-#include "mongo/platform/basic.h"
+#include "mongo/db/catalog/catalog_stats.h"
 
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/catalog/database_holder.h"
@@ -38,8 +38,10 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
 
+namespace mongo::catalog_stats {
 
-namespace mongo {
+// Number of time-series collections requiring extended range support
+AtomicWord<int> requiresTimeseriesExtendedRangeSupport;
 
 namespace {
 class CatalogStatsSSS : public ServerStatusSection {
@@ -60,6 +62,7 @@ public:
         int timeseries = 0;
         int internalCollections = 0;
         int internalViews = 0;
+        int timeseriesExtendedRange = 0;
 
         void toBson(BSONObjBuilder* builder) const {
             builder->append("collections", collections);
@@ -69,6 +72,9 @@ public:
             builder->append("views", views);
             builder->append("internalCollections", internalCollections);
             builder->append("internalViews", internalViews);
+            if (timeseriesExtendedRange > 0) {
+                builder->append("timeseriesExtendedRange", timeseriesExtendedRange);
+            }
         }
     };
 
@@ -82,6 +88,7 @@ public:
         stats.capped = catalogStats.userCapped;
         stats.clustered = catalogStats.userClustered;
         stats.internalCollections = catalogStats.internal;
+        stats.timeseriesExtendedRange = requiresTimeseriesExtendedRangeSupport.load();
 
         const auto viewCatalogDbNames = catalog->getViewCatalogDbNames(opCtx);
         for (const auto& dbName : viewCatalogDbNames) {
@@ -105,4 +112,4 @@ public:
 
 } catalogStatsSSS;
 }  // namespace
-}  // namespace mongo
+}  // namespace mongo::catalog_stats
diff --git a/src/mongo/db/catalog/catalog_stats.h b/src/mongo/db/catalog/catalog_stats.h
new file mode 100644
index 00000000000..54cc04f0bb6
--- /dev/null
+++ b/src/mongo/db/catalog/catalog_stats.h
@@ -0,0 +1,38 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/platform/atomic_word.h"
+
+namespace mongo::catalog_stats {
+
+extern AtomicWord<int> requiresTimeseriesExtendedRangeSupport;
+
+}  // namespace mongo::catalog_stats
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index b9ed0bb2f83..57ac56ebc4b 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -33,6 +33,7 @@
 #include "mongo/bson/simple_bsonelement_comparator.h"
 #include "mongo/bson/simple_bsonobj_comparator.h"
 #include "mongo/crypto/fle_crypto.h"
+#include "mongo/db/catalog/catalog_stats.h"
 #include "mongo/db/catalog/document_validation.h"
 #include "mongo/db/catalog/index_catalog_impl.h"
 #include "mongo/db/catalog/index_key_validate.h"
@@ -1032,14 +1033,17 @@ void CollectionImpl::setRequiresTimeseriesExtendedRangeSupport(OperationContext*
 
     bool expected = false;
     bool set = _shared->_requiresTimeseriesExtendedRangeSupport.compareAndSwap(&expected, true);
-    if (set && !timeseries::collectionHasTimeIndex(opCtx, *this)) {
-        LOGV2_WARNING(
-            6679402,
-            "Time-series collection contains dates outside the standard range. Some query "
-            "optimizations may be disabled. Please consider building an index on timeField to "
-            "re-enable them.",
-            "nss"_attr = ns().getTimeseriesViewNamespace(),
-            "timeField"_attr = _metadata->options.timeseries->getTimeField());
+    if (set) {
+        catalog_stats::requiresTimeseriesExtendedRangeSupport.fetchAndAdd(1);
+        if (!timeseries::collectionHasTimeIndex(opCtx, *this)) {
+            LOGV2_WARNING(
+                6679402,
+                "Time-series collection contains dates outside the standard range. Some query "
+                "optimizations may be disabled. Please consider building an index on timeField to "
+                "re-enable them.",
+                "nss"_attr = ns().getTimeseriesViewNamespace(),
+                "timeField"_attr = _metadata->options.timeseries->getTimeField());
+        }
     }
 }
 
diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp
index 1576d9e93c0..0ddf1c96316 100644
--- a/src/mongo/db/commands/write_commands.cpp
+++ b/src/mongo/db/commands/write_commands.cpp
@@ -1340,33 +1340,6 @@ public:
                 curOp.debug().additiveMetrics.ninserted = 0;
             }
 
-            {
-                // Check if any of the measurements have dates outside the standard range. We need
-                // to do this before we start doing any inserts so we can mark the collection as
-                // requiring extended range support before the measurements land. (Other threads
-                // could potentially commit our inserts for us.)
-                //
-                // If our writes end up erroring out or getting rolled back, then this flag will
-                // stay set. This is okay though, as it only disables some query optimizations and
-                // won't result in any correctness issues if the flag is set when it doesn't need to
-                // be (as opposed to NOT being set when it DOES need to be -- that will cause
-                // correctness issues). Additionally, if the user tried to insert measurements with
-                // dates outside the standard range, chances are they will do so again, and we will
-                // have just set the flag a little early.
-                auto bucketsNs = makeTimeseriesBucketsNamespace(ns());
-                auto bucketsColl =
-                    CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx,
-                                                                                      bucketsNs);
-                auto timeSeriesOptions = *bucketsColl->getTimeseriesOptions();
-
-                if (auto currentSetting = bucketsColl->getRequiresTimeseriesExtendedRangeSupport();
-                    !currentSetting &&
-                    timeseries::measurementsHaveDateOutsideStandardRange(
-                        timeSeriesOptions, request().getDocuments())) {
-                    bucketsColl->setRequiresTimeseriesExtendedRangeSupport(opCtx);
-                }
-            }
-
             std::vector<write_ops::WriteError> errors;
             boost::optional<repl::OpTime> opTime;
             boost::optional<OID> electionId;
diff --git a/src/mongo/db/op_observer/SConscript b/src/mongo/db/op_observer/SConscript
index 07dbc1d7ad3..36cb605336f 100644
--- a/src/mongo/db/op_observer/SConscript
+++ b/src/mongo/db/op_observer/SConscript
@@ -81,6 +81,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/server_feature_flags',
         '$BUILD_DIR/mongo/db/session/session_catalog',
         '$BUILD_DIR/mongo/db/timeseries/bucket_catalog',
+        '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range',
         '$BUILD_DIR/mongo/db/transaction/transaction',
         '$BUILD_DIR/mongo/db/views/views_mongod',
         '$BUILD_DIR/mongo/s/coreshard',
diff --git a/src/mongo/db/op_observer/op_observer_impl.cpp b/src/mongo/db/op_observer/op_observer_impl.cpp
index 37c5e438bc2..d065b89e932 100644
--- a/src/mongo/db/op_observer/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer/op_observer_impl.cpp
@@ -71,6 +71,7 @@
 #include "mongo/db/server_options.h"
 #include "mongo/db/session/session_catalog_mongod.h"
 #include "mongo/db/timeseries/bucket_catalog.h"
+#include "mongo/db/timeseries/timeseries_extended_range.h"
 #include "mongo/db/transaction/transaction_participant.h"
 #include "mongo/db/transaction/transaction_participant_gen.h"
 #include "mongo/db/views/durable_view_catalog.h"
@@ -692,6 +693,25 @@ void OpObserverImpl::onInserts(OperationContext* opCtx,
                     }
                 });
         }
+    } else if (nss.isTimeseriesBucketsCollection()) {
+        // Check if the bucket _id is sourced from a date outside the standard range. If our writes
+        // end up erroring out or getting rolled back, then this flag will stay set. This is okay
+        // though, as it only disables some query optimizations and won't result in any correctness
+        // issues if the flag is set when it doesn't need to be (as opposed to NOT being set when it
+        // DOES need to be -- that will cause correctness issues). Additionally, if the user tried
+        // to insert measurements with dates outside the standard range, chances are they will do so
+        // again, and we will have only set the flag a little early.
+        auto bucketsColl =
+            CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx, nss);
+        auto timeSeriesOptions = bucketsColl->getTimeseriesOptions();
+        if (timeSeriesOptions.has_value()) {
+            if (auto currentSetting = bucketsColl->getRequiresTimeseriesExtendedRangeSupport();
+                !currentSetting &&
+                timeseries::bucketsHaveDateOutsideStandardRange(
+                    timeSeriesOptions.value(), first, last)) {
+                bucketsColl->setRequiresTimeseriesExtendedRangeSupport(opCtx);
+            }
+        }
     }
 }
 
diff --git a/src/mongo/db/timeseries/timeseries_extended_range.cpp b/src/mongo/db/timeseries/timeseries_extended_range.cpp
index c1c85813b1e..44a7ff060d9 100644
--- a/src/mongo/db/timeseries/timeseries_extended_range.cpp
+++ b/src/mongo/db/timeseries/timeseries_extended_range.cpp
@@ -39,16 +39,24 @@ bool dateOutsideStandardRange(Date_t date) {
     return timeSeconds < 0 || timeSeconds > kMaxNormalRangeTimestamp;
 }
 
-bool measurementsHaveDateOutsideStandardRange(const TimeseriesOptions& options,
-                                              const std::vector<BSONObj>& measurements) {
-    return std::any_of(measurements.begin(), measurements.end(), [&](const BSONObj& doc) -> bool {
-        auto timeElem = doc[options.getTimeField()];
-        if (!timeElem || BSONType::Date != timeElem.type()) {
-            // This measurement is missing a valid time element.
-            return false;
-        }
-        auto date = timeElem.Date();
+bool bucketsHaveDateOutsideStandardRange(const TimeseriesOptions& options,
+                                         std::vector<InsertStatement>::const_iterator first,
+                                         std::vector<InsertStatement>::const_iterator last) {
+    return std::any_of(first, last, [&](const InsertStatement& stmt) -> bool {
+        auto controlElem = stmt.doc.getField(timeseries::kBucketControlFieldName);
+        uassert(6781400,
+                "Time series bucket document is missing 'control' field",
+                controlElem.isABSONObj());
+        auto minElem = controlElem.Obj().getField(timeseries::kBucketControlMinFieldName);
+        uassert(6781401,
+                "Time series bucket document is missing 'control.min' field",
+                minElem.isABSONObj());
+        auto timeElem = minElem.Obj().getField(options.getTimeField());
+        uassert(6781402,
+                "Time series bucket document does not have a valid min time element",
+                timeElem && BSONType::Date == timeElem.type());
 
+        auto date = timeElem.Date();
         return dateOutsideStandardRange(date);
     });
 }
diff --git a/src/mongo/db/timeseries/timeseries_extended_range.h b/src/mongo/db/timeseries/timeseries_extended_range.h
index 717e76ff923..f37ac278c26 100644
--- a/src/mongo/db/timeseries/timeseries_extended_range.h
+++ b/src/mongo/db/timeseries/timeseries_extended_range.h
@@ -43,11 +43,12 @@ namespace mongo::timeseries {
 bool dateOutsideStandardRange(Date_t date);
 
 /**
- * Determines whether any of the given measurements have timeField values that lie outside the
- * standard range.
+ * Determines whether any of the given buckets have control.min.timeField values that lie outside
+ * the standard range.
  */
-bool measurementsHaveDateOutsideStandardRange(const TimeseriesOptions& options,
-                                              const std::vector<BSONObj>& measurements);
+bool bucketsHaveDateOutsideStandardRange(const TimeseriesOptions& options,
+                                         std::vector<InsertStatement>::const_iterator first,
+                                         std::vector<InsertStatement>::const_iterator last);
 
 /**
  * Uses a heuristic to determine whether a given time-series collection may contain measurements
@@ -62,4 +63,5 @@ bool collectionMayRequireExtendedRangeSupport(OperationContext* opCtx,
  * is either control.min.<timeField> or control.max.<timeField>.
  */
 bool collectionHasTimeIndex(OperationContext* opCtx, const Collection& collection);
+
 }  // namespace mongo::timeseries
diff --git a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp
index 6803842bb1a..a6392aabd1b 100644
--- a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp
+++ b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp
@@ -49,33 +49,49 @@ TEST(TimeseriesExtendedRangeSupport, DateOutsideStandardRange) {
     ASSERT_TRUE(timeseries::dateOutsideStandardRange(extendedHigh));
 }
 
-TEST(TimeseriesExtendedRangeSupport, MeasurementsHaveDateOutsideStandardRange) {
+TEST(TimeseriesExtendedRangeSupport, BucketsHaveDateOutsideStandardRange) {
     TimeseriesOptions options;
     options.setTimeField("time"_sd);
 
-    std::vector<BSONObj> standardRange = {
-        mongo::fromjson(R"({"time": {"$date": "1970-01-01T00:00:00.000Z"}})"),
-        mongo::fromjson(R"({"time": {"$date": "2000-01-01T00:00:00.000Z"}})"),
-        mongo::fromjson(R"({"time": {"$date": "2038-01-01T00:00:00.000Z"}})"),
+    std::vector<InsertStatement> standardRange = {
+        {0,
+         mongo::fromjson(
+             R"({"control": {"min": {"time": {"$date": "1970-01-01T00:00:00.000Z"}}}})")},
+        {1,
+         mongo::fromjson(
+             R"({"control": {"min": {"time": {"$date": "2000-01-01T00:00:00.000Z"}}}})")},
+        {2,
+         mongo::fromjson(
+             R"({"control": {"min": {"time": {"$date": "2038-01-01T00:00:00.000Z"}}}})")},
     };
 
-    std::vector<BSONObj> extendedRangeLow = {
+    std::vector<InsertStatement> extendedRangeLow = {
         // Dates before Unix epoch have to be hard-coded as seconds-offset rather than date strings
-        mongo::fromjson(R"({"time": {"$date": -2147483649000}})"),  // -((1 << 31) + 1) seconds
-        mongo::fromjson(R"({"time": {"$date": -1000}})"),
+        {3,
+         // -((1 << 31) + 1) seconds
+         mongo::fromjson(R"({"control": {"min": {"time": {"$date": -2147483649000}}}})")},
+        {4, mongo::fromjson(R"({"control": {"min": {"time": {"$date": -1000}}}})")},
     };
 
-    std::vector<BSONObj> extendedRangeHigh = {
-        mongo::fromjson(R"({"time": {"$date": "2039-01-01T00:00:00.000Z"}})"),
-        mongo::fromjson(R"({"time": {"$date": "2110-01-01T00:00:00.000Z"}})"),
+    std::vector<InsertStatement> extendedRangeHigh = {
+        {5,
+         mongo::fromjson(
+             R"({"control": {"min": {"time": {"$date": "2039-01-01T00:00:00.000Z"}}}})")},
+        {6,
+         mongo::fromjson(
+             R"({"control": {"min": {"time": {"$date": "2110-01-01T00:00:00.000Z"}}}})")},
     };
 
-    ASSERT_FALSE(timeseries::measurementsHaveDateOutsideStandardRange(options, standardRange));
-    ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, extendedRangeLow));
-    ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, extendedRangeHigh));
+    ASSERT_FALSE(timeseries::bucketsHaveDateOutsideStandardRange(
+        options, standardRange.begin(), standardRange.end()));
+    ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange(
+        options, extendedRangeLow.begin(), extendedRangeLow.end()));
+    ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange(
+        options, extendedRangeHigh.begin(), extendedRangeHigh.end()));
 
-    std::vector<BSONObj> mixed = {standardRange[0], standardRange[1], extendedRangeLow[0]};
-    ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, mixed));
+    std::vector<InsertStatement> mixed = {standardRange[0], standardRange[1], extendedRangeLow[0]};
+    ASSERT_TRUE(
+        timeseries::bucketsHaveDateOutsideStandardRange(options, mixed.begin(), mixed.end()));
 }
 
 }  // namespace