diff options
author | Dan Larkin-York <dan.larkin-york@mongodb.com> | 2022-08-18 18:35:51 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-18 20:21:52 +0000 |
commit | 65002ae2d4bacf9413383faa4b38d3480a2a8328 (patch) | |
tree | 01b4e46e18968a9ecaa6037db004761ba69a0f2e /src/mongo | |
parent | 44c6926a700486de3c5f2c320f1d92c8c2cdbff9 (diff) | |
download | mongo-65002ae2d4bacf9413383faa4b38d3480a2a8328.tar.gz |
SERVER-67814 Fix detection of and report the number of time-series collections requiring extended range support in serverStatus
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/catalog/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/catalog/catalog_control.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/catalog/catalog_stats.cpp | 13 | ||||
-rw-r--r-- | src/mongo/db/catalog/catalog_stats.h | 38 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_impl.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/commands/write_commands.cpp | 27 | ||||
-rw-r--r-- | src/mongo/db/op_observer/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/op_observer/op_observer_impl.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/timeseries/timeseries_extended_range.cpp | 26 | ||||
-rw-r--r-- | src/mongo/db/timeseries/timeseries_extended_range.h | 10 | ||||
-rw-r--r-- | src/mongo/db/timeseries/timeseries_extended_range_test.cpp | 48 |
11 files changed, 142 insertions, 67 deletions
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index c07e0f32a2b..4787b6901ae 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -341,6 +341,7 @@ env.Library( '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range', + 'catalog_stats', 'collection', 'collection_catalog', 'database_holder', diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp index 59e0769ff9f..98219a55b95 100644 --- a/src/mongo/db/catalog/catalog_control.cpp +++ b/src/mongo/db/catalog/catalog_control.cpp @@ -34,6 +34,7 @@ #include "mongo/db/catalog/catalog_control.h" +#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/database.h" @@ -228,6 +229,10 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) { LOGV2(20272, "closeCatalog: closing storage engine catalog"); opCtx->getServiceContext()->getStorageEngine()->closeCatalog(opCtx); + // Reset the stats counter for extended range time-series collections. This is maintained + // outside the catalog itself. + catalog_stats::requiresTimeseriesExtendedRangeSupport.store(0); + reopenOnFailure.dismiss(); return previousCatalogState; } diff --git a/src/mongo/db/catalog/catalog_stats.cpp b/src/mongo/db/catalog/catalog_stats.cpp index 36446c6096a..9b1c0f075a3 100644 --- a/src/mongo/db/catalog/catalog_stats.cpp +++ b/src/mongo/db/catalog/catalog_stats.cpp @@ -28,7 +28,7 @@ */ -#include "mongo/platform/basic.h" +#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/database_holder.h" @@ -38,8 +38,10 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand +namespace mongo::catalog_stats { -namespace mongo { +// Number of time-series collections requiring extended range support +AtomicWord<int> requiresTimeseriesExtendedRangeSupport; namespace { class CatalogStatsSSS : public ServerStatusSection { @@ -60,6 +62,7 @@ public: int timeseries = 0; int internalCollections = 0; int internalViews = 0; + int timeseriesExtendedRange = 0; void toBson(BSONObjBuilder* builder) const { builder->append("collections", collections); @@ -69,6 +72,9 @@ public: builder->append("views", views); builder->append("internalCollections", internalCollections); builder->append("internalViews", internalViews); + if (timeseriesExtendedRange > 0) { + builder->append("timeseriesExtendedRange", timeseriesExtendedRange); + } } }; @@ -82,6 +88,7 @@ public: stats.capped = catalogStats.userCapped; stats.clustered = catalogStats.userClustered; stats.internalCollections = catalogStats.internal; + stats.timeseriesExtendedRange = requiresTimeseriesExtendedRangeSupport.load(); const auto viewCatalogDbNames = catalog->getViewCatalogDbNames(opCtx); for (const auto& dbName : viewCatalogDbNames) { @@ -105,4 +112,4 @@ public: } catalogStatsSSS; } // namespace -} // namespace mongo +} // namespace mongo::catalog_stats diff --git a/src/mongo/db/catalog/catalog_stats.h b/src/mongo/db/catalog/catalog_stats.h new file mode 100644 index 00000000000..54cc04f0bb6 --- /dev/null +++ b/src/mongo/db/catalog/catalog_stats.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/platform/atomic_word.h" + +namespace mongo::catalog_stats { + +extern AtomicWord<int> requiresTimeseriesExtendedRangeSupport; + +} // namespace mongo::catalog_stats diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index b9ed0bb2f83..57ac56ebc4b 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/simple_bsonelement_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/crypto/fle_crypto.h" +#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/document_validation.h" #include "mongo/db/catalog/index_catalog_impl.h" #include "mongo/db/catalog/index_key_validate.h" @@ -1032,14 +1033,17 @@ void CollectionImpl::setRequiresTimeseriesExtendedRangeSupport(OperationContext* bool expected = false; bool set = _shared->_requiresTimeseriesExtendedRangeSupport.compareAndSwap(&expected, true); - if (set && !timeseries::collectionHasTimeIndex(opCtx, *this)) { - LOGV2_WARNING( - 6679402, - "Time-series collection contains dates outside the standard range. Some query " - "optimizations may be disabled. Please consider building an index on timeField to " - "re-enable them.", - "nss"_attr = ns().getTimeseriesViewNamespace(), - "timeField"_attr = _metadata->options.timeseries->getTimeField()); + if (set) { + catalog_stats::requiresTimeseriesExtendedRangeSupport.fetchAndAdd(1); + if (!timeseries::collectionHasTimeIndex(opCtx, *this)) { + LOGV2_WARNING( + 6679402, + "Time-series collection contains dates outside the standard range. Some query " + "optimizations may be disabled. Please consider building an index on timeField to " + "re-enable them.", + "nss"_attr = ns().getTimeseriesViewNamespace(), + "timeField"_attr = _metadata->options.timeseries->getTimeField()); + } } } diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp index 1576d9e93c0..0ddf1c96316 100644 --- a/src/mongo/db/commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands.cpp @@ -1340,33 +1340,6 @@ public: curOp.debug().additiveMetrics.ninserted = 0; } - { - // Check if any of the measurements have dates outside the standard range. We need - // to do this before we start doing any inserts so we can mark the collection as - // requiring extended range support before the measurements land. (Other threads - // could potentially commit our inserts for us.) - // - // If our writes end up erroring out or getting rolled back, then this flag will - // stay set. This is okay though, as it only disables some query optimizations and - // won't result in any correctness issues if the flag is set when it doesn't need to - // be (as opposed to NOT being set when it DOES need to be -- that will cause - // correctness issues). Additionally, if the user tried to insert measurements with - // dates outside the standard range, chances are they will do so again, and we will - // have just set the flag a little early. - auto bucketsNs = makeTimeseriesBucketsNamespace(ns()); - auto bucketsColl = - CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx, - bucketsNs); - auto timeSeriesOptions = *bucketsColl->getTimeseriesOptions(); - - if (auto currentSetting = bucketsColl->getRequiresTimeseriesExtendedRangeSupport(); - !currentSetting && - timeseries::measurementsHaveDateOutsideStandardRange( - timeSeriesOptions, request().getDocuments())) { - bucketsColl->setRequiresTimeseriesExtendedRangeSupport(opCtx); - } - } - std::vector<write_ops::WriteError> errors; boost::optional<repl::OpTime> opTime; boost::optional<OID> electionId; diff --git a/src/mongo/db/op_observer/SConscript b/src/mongo/db/op_observer/SConscript index 07dbc1d7ad3..36cb605336f 100644 --- a/src/mongo/db/op_observer/SConscript +++ b/src/mongo/db/op_observer/SConscript @@ -81,6 +81,7 @@ env.Library( '$BUILD_DIR/mongo/db/server_feature_flags', '$BUILD_DIR/mongo/db/session/session_catalog', '$BUILD_DIR/mongo/db/timeseries/bucket_catalog', + '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range', '$BUILD_DIR/mongo/db/transaction/transaction', '$BUILD_DIR/mongo/db/views/views_mongod', '$BUILD_DIR/mongo/s/coreshard', diff --git a/src/mongo/db/op_observer/op_observer_impl.cpp b/src/mongo/db/op_observer/op_observer_impl.cpp index 37c5e438bc2..d065b89e932 100644 --- a/src/mongo/db/op_observer/op_observer_impl.cpp +++ b/src/mongo/db/op_observer/op_observer_impl.cpp @@ -71,6 +71,7 @@ #include "mongo/db/server_options.h" #include "mongo/db/session/session_catalog_mongod.h" #include "mongo/db/timeseries/bucket_catalog.h" +#include "mongo/db/timeseries/timeseries_extended_range.h" #include "mongo/db/transaction/transaction_participant.h" #include "mongo/db/transaction/transaction_participant_gen.h" #include "mongo/db/views/durable_view_catalog.h" @@ -692,6 +693,25 @@ void OpObserverImpl::onInserts(OperationContext* opCtx, } }); } + } else if (nss.isTimeseriesBucketsCollection()) { + // Check if the bucket _id is sourced from a date outside the standard range. If our writes + // end up erroring out or getting rolled back, then this flag will stay set. This is okay + // though, as it only disables some query optimizations and won't result in any correctness + // issues if the flag is set when it doesn't need to be (as opposed to NOT being set when it + // DOES need to be -- that will cause correctness issues). Additionally, if the user tried + // to insert measurements with dates outside the standard range, chances are they will do so + // again, and we will have only set the flag a little early. + auto bucketsColl = + CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx, nss); + auto timeSeriesOptions = bucketsColl->getTimeseriesOptions(); + if (timeSeriesOptions.has_value()) { + if (auto currentSetting = bucketsColl->getRequiresTimeseriesExtendedRangeSupport(); + !currentSetting && + timeseries::bucketsHaveDateOutsideStandardRange( + timeSeriesOptions.value(), first, last)) { + bucketsColl->setRequiresTimeseriesExtendedRangeSupport(opCtx); + } + } } } diff --git a/src/mongo/db/timeseries/timeseries_extended_range.cpp b/src/mongo/db/timeseries/timeseries_extended_range.cpp index c1c85813b1e..44a7ff060d9 100644 --- a/src/mongo/db/timeseries/timeseries_extended_range.cpp +++ b/src/mongo/db/timeseries/timeseries_extended_range.cpp @@ -39,16 +39,24 @@ bool dateOutsideStandardRange(Date_t date) { return timeSeconds < 0 || timeSeconds > kMaxNormalRangeTimestamp; } -bool measurementsHaveDateOutsideStandardRange(const TimeseriesOptions& options, - const std::vector<BSONObj>& measurements) { - return std::any_of(measurements.begin(), measurements.end(), [&](const BSONObj& doc) -> bool { - auto timeElem = doc[options.getTimeField()]; - if (!timeElem || BSONType::Date != timeElem.type()) { - // This measurement is missing a valid time element. - return false; - } - auto date = timeElem.Date(); +bool bucketsHaveDateOutsideStandardRange(const TimeseriesOptions& options, + std::vector<InsertStatement>::const_iterator first, + std::vector<InsertStatement>::const_iterator last) { + return std::any_of(first, last, [&](const InsertStatement& stmt) -> bool { + auto controlElem = stmt.doc.getField(timeseries::kBucketControlFieldName); + uassert(6781400, + "Time series bucket document is missing 'control' field", + controlElem.isABSONObj()); + auto minElem = controlElem.Obj().getField(timeseries::kBucketControlMinFieldName); + uassert(6781401, + "Time series bucket document is missing 'control.min' field", + minElem.isABSONObj()); + auto timeElem = minElem.Obj().getField(options.getTimeField()); + uassert(6781402, + "Time series bucket document does not have a valid min time element", + timeElem && BSONType::Date == timeElem.type()); + auto date = timeElem.Date(); return dateOutsideStandardRange(date); }); } diff --git a/src/mongo/db/timeseries/timeseries_extended_range.h b/src/mongo/db/timeseries/timeseries_extended_range.h index 717e76ff923..f37ac278c26 100644 --- a/src/mongo/db/timeseries/timeseries_extended_range.h +++ b/src/mongo/db/timeseries/timeseries_extended_range.h @@ -43,11 +43,12 @@ namespace mongo::timeseries { bool dateOutsideStandardRange(Date_t date); /** - * Determines whether any of the given measurements have timeField values that lie outside the - * standard range. + * Determines whether any of the given buckets have control.min.timeField values that lie outside + * the standard range. */ -bool measurementsHaveDateOutsideStandardRange(const TimeseriesOptions& options, - const std::vector<BSONObj>& measurements); +bool bucketsHaveDateOutsideStandardRange(const TimeseriesOptions& options, + std::vector<InsertStatement>::const_iterator first, + std::vector<InsertStatement>::const_iterator last); /** * Uses a heuristic to determine whether a given time-series collection may contain measurements @@ -62,4 +63,5 @@ bool collectionMayRequireExtendedRangeSupport(OperationContext* opCtx, * is either control.min.<timeField> or control.max.<timeField>. */ bool collectionHasTimeIndex(OperationContext* opCtx, const Collection& collection); + } // namespace mongo::timeseries diff --git a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp index 6803842bb1a..a6392aabd1b 100644 --- a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp +++ b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp @@ -49,33 +49,49 @@ TEST(TimeseriesExtendedRangeSupport, DateOutsideStandardRange) { ASSERT_TRUE(timeseries::dateOutsideStandardRange(extendedHigh)); } -TEST(TimeseriesExtendedRangeSupport, MeasurementsHaveDateOutsideStandardRange) { +TEST(TimeseriesExtendedRangeSupport, BucketsHaveDateOutsideStandardRange) { TimeseriesOptions options; options.setTimeField("time"_sd); - std::vector<BSONObj> standardRange = { - mongo::fromjson(R"({"time": {"$date": "1970-01-01T00:00:00.000Z"}})"), - mongo::fromjson(R"({"time": {"$date": "2000-01-01T00:00:00.000Z"}})"), - mongo::fromjson(R"({"time": {"$date": "2038-01-01T00:00:00.000Z"}})"), + std::vector<InsertStatement> standardRange = { + {0, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "1970-01-01T00:00:00.000Z"}}}})")}, + {1, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "2000-01-01T00:00:00.000Z"}}}})")}, + {2, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "2038-01-01T00:00:00.000Z"}}}})")}, }; - std::vector<BSONObj> extendedRangeLow = { + std::vector<InsertStatement> extendedRangeLow = { // Dates before Unix epoch have to be hard-coded as seconds-offset rather than date strings - mongo::fromjson(R"({"time": {"$date": -2147483649000}})"), // -((1 << 31) + 1) seconds - mongo::fromjson(R"({"time": {"$date": -1000}})"), + {3, + // -((1 << 31) + 1) seconds + mongo::fromjson(R"({"control": {"min": {"time": {"$date": -2147483649000}}}})")}, + {4, mongo::fromjson(R"({"control": {"min": {"time": {"$date": -1000}}}})")}, }; - std::vector<BSONObj> extendedRangeHigh = { - mongo::fromjson(R"({"time": {"$date": "2039-01-01T00:00:00.000Z"}})"), - mongo::fromjson(R"({"time": {"$date": "2110-01-01T00:00:00.000Z"}})"), + std::vector<InsertStatement> extendedRangeHigh = { + {5, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "2039-01-01T00:00:00.000Z"}}}})")}, + {6, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "2110-01-01T00:00:00.000Z"}}}})")}, }; - ASSERT_FALSE(timeseries::measurementsHaveDateOutsideStandardRange(options, standardRange)); - ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, extendedRangeLow)); - ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, extendedRangeHigh)); + ASSERT_FALSE(timeseries::bucketsHaveDateOutsideStandardRange( + options, standardRange.begin(), standardRange.end())); + ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( + options, extendedRangeLow.begin(), extendedRangeLow.end())); + ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( + options, extendedRangeHigh.begin(), extendedRangeHigh.end())); - std::vector<BSONObj> mixed = {standardRange[0], standardRange[1], extendedRangeLow[0]}; - ASSERT_TRUE(timeseries::measurementsHaveDateOutsideStandardRange(options, mixed)); + std::vector<InsertStatement> mixed = {standardRange[0], standardRange[1], extendedRangeLow[0]}; + ASSERT_TRUE( + timeseries::bucketsHaveDateOutsideStandardRange(options, mixed.begin(), mixed.end())); } } // namespace |