diff options
Diffstat (limited to 'src/mongo/rpc')
| -rw-r--r-- | src/mongo/rpc/SConscript | 19 | ||||
| -rw-r--r-- | src/mongo/rpc/rewrite_state_change_errors.cpp | 235 | ||||
| -rw-r--r-- | src/mongo/rpc/rewrite_state_change_errors.h | 95 | ||||
| -rw-r--r-- | src/mongo/rpc/rewrite_state_change_errors_server_parameter.idl | 40 | ||||
| -rw-r--r-- | src/mongo/rpc/rewrite_state_change_errors_test.cpp | 181 |
5 files changed, 570 insertions, 0 deletions
diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript index c16d1e116c4..dd7d6205c84 100644 --- a/src/mongo/rpc/SConscript +++ b/src/mongo/rpc/SConscript @@ -74,6 +74,23 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/idl/server_parameter', + 'rewrite_state_change_errors', + ], +) + +env.Library( + target='rewrite_state_change_errors', + source=[ + 'rewrite_state_change_errors.cpp', + 'rewrite_state_change_errors_server_parameter.idl', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/mutable/mutable_bson', + '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/rpc/protocol', + '$BUILD_DIR/mongo/s/is_mongos', + '$BUILD_DIR/third_party/shim_pcrecpp', ], ) @@ -170,12 +187,14 @@ if wiredtiger: 'op_msg_test.cpp', 'protocol_test.cpp', 'reply_builder_test.cpp', + 'rewrite_state_change_errors_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/client/clientdriver_minimal', '$BUILD_DIR/third_party/wiredtiger/wiredtiger_checksum', 'client_metadata', 'metadata', + 'rewrite_state_change_errors', 'rpc', ] ) diff --git a/src/mongo/rpc/rewrite_state_change_errors.cpp b/src/mongo/rpc/rewrite_state_change_errors.cpp new file mode 100644 index 00000000000..9f0fea0c294 --- /dev/null +++ b/src/mongo/rpc/rewrite_state_change_errors.cpp @@ -0,0 +1,235 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kNetwork + +#include "mongo/rpc/rewrite_state_change_errors.h" + +#include "mongo/platform/basic.h" + +#include <array> +#include <string> + +#include <boost/optional.hpp> +#include <fmt/format.h> +#include <pcrecpp.h> + +#include "mongo/bson/mutable/document.h" +#include "mongo/bson/mutable/element.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" +#include "mongo/platform/atomic_word.h" +#include "mongo/rpc/message.h" +#include "mongo/rpc/op_msg.h" +#include "mongo/rpc/rewrite_state_change_errors_server_parameter_gen.h" +#include "mongo/s/is_mongos.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/static_immortal.h" + +namespace mongo::rpc { +namespace { + +struct RewriteEnabled { + bool enabled = rewriteStateChangeErrors; +}; + +/** Enable for the entire service. */ +auto enabledForService = ServiceContext::declareDecoration<RewriteEnabled>(); + +/** Enable for a single operation. */ +auto enabledForOperation = OperationContext::declareDecoration<RewriteEnabled>(); + + +/** + * We must replace key phrases in `errmsg` with sensible strings that + * drivers will ignore. Return scrubbed `val`, or no value if it + * doesn't need scrubbing. + * + * See + * https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring.rst#not-master-and-node-is-recovering + */ +boost::optional<std::string> scrubErrmsg(StringData val) { + struct Scrub { + pcrecpp::RE pat; + std::string sub; + }; + static const StaticImmortal scrubs = std::array{ + Scrub{pcrecpp::RE("not master"), "(NOT_PRIMARY)"}, + Scrub{pcrecpp::RE("node is recovering"), "(NODE_IS_RECOVERING)"}, + }; + // Fast scan for the common case that no key phrase is present. + static const StaticImmortal fastScan = [] { + std::string pat; + StringData sep; + auto out = std::back_inserter(pat); + for (const auto& scrub : *scrubs) { + out = format_to(out, FMT_STRING("{}({})"), sep, scrub.pat.pattern()); + sep = "|"_sd; + } + return pcrecpp::RE(pat); + }(); + + pcrecpp::StringPiece pcreVal(val.rawData(), val.size()); + + if (fastScan->PartialMatch(pcreVal)) { + std::string s{val}; + bool didSub = false; + for (auto&& scrub : *scrubs) { + bool subOk = scrub.pat.GlobalReplace(scrub.sub, &s); + didSub = (didSub || subOk); + } + if (didSub) + return s; + } + return {}; +} + +/** + * Change the {code, codeName, errmsg} fields to cloak a proxied state change + * error. We choose `HostUnreachable` as it is retryable but doesn't induce an + * SDAM state change. + */ +void editErrorNode(mutablebson::Element&& node) { + if (auto codeElement = node["code"]; codeElement.ok()) { + static constexpr auto newCode = ErrorCodes::HostUnreachable; + uassertStatusOK(codeElement.setValueInt(newCode)); + // If there's a corresponding `codeName`, replace it to match. + if (auto codeName = node["codeName"]; codeName.ok()) + uassertStatusOK(codeName.setValueString(ErrorCodes::errorString(newCode))); + } + if (auto errmsg = node["errmsg"]; errmsg.ok() && errmsg.isType(String)) + if (auto scrubbed = scrubErrmsg(errmsg.getValueString())) + uassertStatusOK(errmsg.setValueString(*scrubbed)); +} + +/** + * If `node` contains a numeric "code" field that indicates a state change, + * returns the value of that code field as an `ErrorCodes::Error`. Otherwise + * returns a disengaged optional. + * + * There are two categories of state change errors: ShutdownError and + * NotPrimaryError. + */ +boost::optional<ErrorCodes::Error> needsRewrite(ServiceContext* sc, const BSONObj& node) { + int32_t intCode{}; + if (!node["code"].coerce(&intCode)) + return {}; + ErrorCodes::Error ec{intCode}; + + if (ErrorCodes::isA<ErrorCategory::NotPrimaryError>(ec)) { + return ec; + } + + // ShutdownError codes are correct if this server is also in shutdown. If + // this server is shutting down, then even if the shutdown error didn't + // originate from this server, it might as well have. + if (ErrorCodes::isA<ErrorCategory::ShutdownError>(ec) && !sc->getKillAllOperations()) { + return ec; + } + + return {}; +} + +/** + * Returns a copy of doc with errors rewritten to cloak state change errors if + * necessary. Returns disengaged optional if no changes were necessary. + */ +boost::optional<BSONObj> rewriteDocument(const BSONObj& doc, OperationContext* opCtx) { + boost::optional<mutablebson::Document> mutableDoc; + auto lazyMutableRoot = [&] { + if (!mutableDoc) + mutableDoc.emplace(doc); + return mutableDoc->root(); + }; + + ServiceContext* sc = opCtx->getServiceContext(); + + // Skip unless there's an "ok" element value equivalent to expected 0 or 1. + // "ok" is conventionally a NumberDouble, but coerce since this is unspecified. + double okValue = 0; + if (!doc.getField("ok").coerce(&okValue) || (okValue != 0 && okValue != 1)) + return {}; // Skip: missing or unusable "ok" field. + + boost::optional<ErrorCodes::Error> oldCode; + + // The root of the doc is an error-bearing node if not ok. + if (okValue == 0 && (oldCode = needsRewrite(sc, doc))) + editErrorNode(lazyMutableRoot()); + + // The `writeErrors` and `writeConcernError` nodes might need editing. + // `writeErrors` is an array of error-bearing nodes like the doc root. + if (const auto& we = doc["writeErrors"]; we.type() == Array) { + size_t idx = 0; + BSONObj bArr = we.Obj(); + for (auto ai = bArr.begin(); ai != bArr.end(); ++ai, ++idx) + if (ai->type() == Object && (oldCode = needsRewrite(sc, ai->Obj()))) + editErrorNode(lazyMutableRoot()["writeErrors"][idx]); + } + + // `writeConcernError` is a single error-bearing node. + if (const auto& wce = doc["writeConcernError"]; wce.type() == Object) { + if ((oldCode = needsRewrite(sc, wce.Obj()))) + editErrorNode(lazyMutableRoot()["writeConcernError"]); + } + + if (mutableDoc) { + LOGV2_DEBUG(1, 5054900, "Rewrote state change error", "code"_attr = oldCode); + return mutableDoc->getObject(); + } + return {}; +} + +} // namespace + + +bool RewriteStateChangeErrors::getEnabled(OperationContext* opCtx) { + return enabledForOperation(opCtx).enabled; +} + +void RewriteStateChangeErrors::setEnabled(OperationContext* opCtx, bool e) { + enabledForOperation(opCtx).enabled = e; +} + +bool RewriteStateChangeErrors::getEnabled(ServiceContext* sc) { + return enabledForService(sc).enabled; +} + +void RewriteStateChangeErrors::setEnabled(ServiceContext* sc, bool e) { + enabledForService(sc).enabled = e; +} + +boost::optional<BSONObj> RewriteStateChangeErrors::rewrite(BSONObj doc, OperationContext* opCtx) { + auto sc = opCtx->getServiceContext(); + if (!isMongos() || (sc && !getEnabled(sc)) || !getEnabled(opCtx)) + return {}; + return rewriteDocument(doc, opCtx); +} + +} // namespace mongo::rpc diff --git a/src/mongo/rpc/rewrite_state_change_errors.h b/src/mongo/rpc/rewrite_state_change_errors.h new file mode 100644 index 00000000000..9edccd7e999 --- /dev/null +++ b/src/mongo/rpc/rewrite_state_change_errors.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <boost/optional.hpp> + +#include "mongo/db/operation_context.h" +#include "mongo/db/service_context.h" +#include "mongo/rpc/message.h" + +namespace mongo::rpc { + +class RewriteStateChangeErrors { +public: + /** + * Enable/disable for an entire server. + * The default is determined by server parameter "rewriteStateChangeErrors". + */ + static bool getEnabled(ServiceContext* sc); + static void setEnabled(ServiceContext* sc, bool e); + + /** Enable/disable only for a single operation. */ + static bool getEnabled(OperationContext* opCtx); + static void setEnabled(OperationContext* opCtx, bool e); + + /** + * Transforms an outgoing message to conditionally mask "state change" errors, + * which are errors that cause a client to change its connection to the host + * that sent it, such as marking it "Unknown". A shutdown error that's emitted + * by a proxy server (e.g. mongos) can be misinterpreted by a naive client to + * be indicative of the proxy server shutting down. So this simple rewrite + * scheme attempts to mask those errors on the way out. + * + * Return a disengaged optional if no rewrite is needed. + * Otherwise, returns a copy of doc with the errors rewritten. + * + * Error-bearing subobjects are places in the document where a `code` element + * can indicate an error status. When rewriting, any error-bearing subobjects + * are examined for a `code` element. + * + * In an error document (with "ok" mapped to 0.0), only the root element is + * error-bearing. In a success document (where "ok" is mapped to 1.0), the + * `writeConcernError` subobject and all the subobjects in the `writeErrors` + * BSON array are error-bearing subobjects. + * + * Rewriting occurs only if all of these conditions are met: + * + * - This feature wasn't deselected with a `setEnabled` call, either on the + * `opCtx` or for more widely for its associated ServiceContext. + * This will happen with failpoint-induced errors or with "hello" commands, + * both of which are exempt from state change error rewriting. + * + * - doc contains no error nodes that need rewriting. + * - NotPrimaryError codes always need rewriting. + * - ShutdownError codes need rewriting unless this server is + * shutting down. + * + * Any rewritten `code` is replaced by `HostUnreachable`, and associated + * `codeName` replaced to be consistent with the new code. Additionally, the + * `errmsg` in the subobject has all occurrences of the key phrases "not + * master" and "node is recovering" replaced with "(NOT_PRIMARY)" and + * "(NODE_IS_RECOVERING)" respectively so that client state changes based on + * the presence of these legacy strings are suppressed. + */ + static boost::optional<BSONObj> rewrite(BSONObj doc, OperationContext* opCtx); +}; + +} // namespace mongo::rpc diff --git a/src/mongo/rpc/rewrite_state_change_errors_server_parameter.idl b/src/mongo/rpc/rewrite_state_change_errors_server_parameter.idl new file mode 100644 index 00000000000..cdb8fe44669 --- /dev/null +++ b/src/mongo/rpc/rewrite_state_change_errors_server_parameter.idl @@ -0,0 +1,40 @@ +# Copyright (C) 2021-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo::rpc" + +server_parameters: + # Kill switch just in case this behavior is slow or breaks something. + rewriteStateChangeErrors: + description: >- + A failsafe to disable mongos state change error rewriting. + set_at: startup + cpp_vartype: bool + cpp_varname: rewriteStateChangeErrors + default: true diff --git a/src/mongo/rpc/rewrite_state_change_errors_test.cpp b/src/mongo/rpc/rewrite_state_change_errors_test.cpp new file mode 100644 index 00000000000..66cda0cd0c2 --- /dev/null +++ b/src/mongo/rpc/rewrite_state_change_errors_test.cpp @@ -0,0 +1,181 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/rpc/rewrite_state_change_errors.h" + +#include "mongo/base/error_codes.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/service_context.h" +#include "mongo/rpc/message.h" +#include "mongo/rpc/op_msg.h" +#include "mongo/s/is_mongos.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::rpc { +namespace { + +class RewriteStateChangeErrorsTest : public unittest::Test { +public: + void setUp() override { + _savedIsMongos = isMongos(); + setMongos(true); // whole feature only happens on mongos + RewriteStateChangeErrors::setEnabled(&*sc, true); + } + + void tearDown() override { + setMongos(_savedIsMongos); + } + + /** Run rewrite on `obj` and return what it was remapped to if anything. */ + BSONObj rewriteObj(const BSONObj& obj) { + if (auto newDoc = RewriteStateChangeErrors::rewrite(obj, &*opCtx)) + return *newDoc; + return obj; + } + + /** Make an error node corresponding to `ec`. */ + BSONObj errorObject(ErrorCodes::Error ec) { + BSONObjBuilder bob; + if (ec == ErrorCodes::OK) + bob.append("ok", 1.); + else + bob.append("ok", 0.) + .append("code", static_cast<int>(ec)) + .append("codeName", ErrorCodes::errorString(ec)); + return bob.obj(); + } + + /** Make an error node corresponding to `ec` with `errmsg`. */ + BSONObj errorObject(ErrorCodes::Error ec, std::string errmsg) { + return BSONObjBuilder(errorObject(ec)).append("errmsg", errmsg).obj(); + } + + /** A few codes and what we expect them to be rewritten to. */ + struct InOutCode { + ErrorCodes::Error in; + ErrorCodes::Error out; + }; + static constexpr InOutCode errorCodeScenarios[] = { + {ErrorCodes::InterruptedAtShutdown, ErrorCodes::HostUnreachable}, + {ErrorCodes::ShutdownInProgress, ErrorCodes::HostUnreachable}, + {ErrorCodes::OK, ErrorCodes::OK}, + {ErrorCodes::BadValue, ErrorCodes::BadValue}, + }; + + ServiceContext::UniqueServiceContext sc = ServiceContext::make(); + ServiceContext::UniqueClient cc = sc->makeClient("test", nullptr); + ServiceContext::UniqueOperationContext opCtx = sc->makeOperationContext(cc.get()); + +private: + bool _savedIsMongos; +}; + +// Rewrite Shutdown errors received from proxied commands. +TEST_F(RewriteStateChangeErrorsTest, Enabled) { + for (auto&& [in, out] : errorCodeScenarios) { + ASSERT_BSONOBJ_EQ(rewriteObj(errorObject(in)), errorObject(out)); + } +} + +// Check that rewrite behavior can be disabled per-ServiceContext. +TEST_F(RewriteStateChangeErrorsTest, Disabled) { + RewriteStateChangeErrors::setEnabled(&*sc, false); + ASSERT_BSONOBJ_EQ(rewriteObj(errorObject(ErrorCodes::InterruptedAtShutdown)), + errorObject(ErrorCodes::InterruptedAtShutdown)); +} + +// Check that rewrite behavior can be disabled per-opCtx. +TEST_F(RewriteStateChangeErrorsTest, DisabledOpCtx) { + RewriteStateChangeErrors::setEnabled(&*opCtx, false); + ASSERT_BSONOBJ_EQ(rewriteObj(errorObject(ErrorCodes::InterruptedAtShutdown)), + errorObject(ErrorCodes::InterruptedAtShutdown)); +} + +// If locally shutting down, then shutdown errors must not be rewritten. +TEST_F(RewriteStateChangeErrorsTest, LocalShutdown) { + sc->setKillAllOperations(); + ASSERT_BSONOBJ_EQ(rewriteObj(errorObject(ErrorCodes::InterruptedAtShutdown)), + errorObject(ErrorCodes::InterruptedAtShutdown)); +} + +TEST_F(RewriteStateChangeErrorsTest, RewriteErrmsg) { + const std::pair<std::string, std::string> scenarios[] = { + {"not master", "(NOT_PRIMARY)"}, + {"node is recovering", "(NODE_IS_RECOVERING)"}, + {"NOT master", "NOT master"}, + {"", ""}, + {" not masternot master ", " (NOT_PRIMARY)(NOT_PRIMARY) "}, + {"not masternode is recovering", "(NOT_PRIMARY)(NODE_IS_RECOVERING)"}, + }; + for (auto&& io : scenarios) { + ASSERT_BSONOBJ_EQ(rewriteObj(errorObject(ErrorCodes::InterruptedAtShutdown, io.first)), + errorObject(ErrorCodes::HostUnreachable, io.second)); + } +} + +// Can find and rewrite the `writeConcernError` in an `ok:1` response. +TEST_F(RewriteStateChangeErrorsTest, WriteConcernError) { + // Make an OK object, and append a `writeConcernError` subobject bearing the `ec` error. + auto wceObject = [&](ErrorCodes::Error ec) { + return BSONObjBuilder(errorObject(ErrorCodes::OK)) + .append("writeConcernError", errorObject(ec)) + .obj(); + }; + for (auto&& [in, out] : errorCodeScenarios) { + ASSERT_BSONOBJ_EQ(rewriteObj(wceObject(in)), wceObject(out)); + } +} + +// Can find and rewrite the `writeErrors` array elements in an `ok:1` response. +TEST_F(RewriteStateChangeErrorsTest, WriteErrors) { + // Make an OK object, and append a `writeErrors` subobject bearing the `ec` errors. + auto weObject = [&](std::vector<ErrorCodes::Error> ecVec) { + BSONObjBuilder bob(errorObject(ErrorCodes::OK)); + { + BSONArrayBuilder bab(bob.subarrayStart("writeErrors")); + for (ErrorCodes::Error ec : ecVec) + bab.append(errorObject(ec)); + } + return bob.obj(); + }; + for (auto&& [in, out] : errorCodeScenarios) { + ASSERT_BSONOBJ_EQ(rewriteObj(weObject({in})), weObject({out})); + } + // Now try all the errorCodeScenarios as a single array of `writeErrors`. + std::vector<ErrorCodes::Error> allIn, allOut; + for (auto&& [in, out] : errorCodeScenarios) { + allIn.push_back(in); + allOut.push_back(out); + } + ASSERT_BSONOBJ_EQ(rewriteObj(weObject(allIn)), weObject(allOut)); +} + +} // namespace +} // namespace mongo::rpc |