SERVER-26506 add tracking for commands

5 files changed, 419 insertions, 0 deletions

diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript
index fcd14efee87..b049a69e2ff 100644
--- a/src/mongo/rpc/SConscript
+++ b/src/mongo/rpc/SConscript
@@ -150,6 +150,7 @@ env.Library(
         'metadata/server_selection_metadata.cpp',
         'metadata/sharding_metadata.cpp',
         'metadata/repl_set_metadata.cpp',
+        'metadata/tracking_metadata.cpp',
     ],
     LIBDEPS=[
         'client_metadata',
@@ -168,6 +169,7 @@ env.CppUnitTest(
     source=[
         'metadata/server_selection_metadata_test.cpp',
         'metadata/sharding_metadata_test.cpp',
+        'metadata/tracking_metadata_test.cpp',
     ],
     LIBDEPS=[
         'metadata',
diff --git a/src/mongo/rpc/metadata.cpp b/src/mongo/rpc/metadata.cpp
index d286ece26ca..00a9d58dc9f 100644
--- a/src/mongo/rpc/metadata.cpp
+++ b/src/mongo/rpc/metadata.cpp
@@ -37,6 +37,7 @@
 #include "mongo/rpc/metadata/config_server_metadata.h"
 #include "mongo/rpc/metadata/server_selection_metadata.h"
 #include "mongo/rpc/metadata/sharding_metadata.h"
+#include "mongo/rpc/metadata/tracking_metadata.h"
 
 namespace mongo {
 namespace rpc {
@@ -49,6 +50,7 @@ Status readRequestMetadata(OperationContext* txn, const BSONObj& metadataObj) {
     BSONElement ssmElem;
     BSONElement auditElem;
     BSONElement configSvrElem;
+    BSONElement trackingElem;
     BSONElement clientElem;
 
     for (const auto& metadataElem : metadataObj) {
@@ -61,6 +63,8 @@ Status readRequestMetadata(OperationContext* txn, const BSONObj& metadataObj) {
             configSvrElem = metadataElem;
         } else if (fieldName == ClientMetadata::fieldName()) {
             clientElem = metadataElem;
+        } else if (fieldName == TrackingMetadata::fieldName()) {
+            trackingElem = metadataElem;
         }
     }
 
@@ -88,6 +92,12 @@ Status readRequestMetadata(OperationContext* txn, const BSONObj& metadataObj) {
     }
     ConfigServerMetadata::get(txn) = std::move(configServerMetadata.getValue());
 
+    auto trackingMetadata = TrackingMetadata::readFromMetadata(trackingElem);
+    if (!trackingMetadata.isOK()) {
+        return trackingMetadata.getStatus();
+    }
+    TrackingMetadata::get(txn) = std::move(trackingMetadata.getValue());
+
     return Status::OK();
 }
 
diff --git a/src/mongo/rpc/metadata/tracking_metadata.cpp b/src/mongo/rpc/metadata/tracking_metadata.cpp
new file mode 100644
index 00000000000..efe2cbf50e0
--- /dev/null
+++ b/src/mongo/rpc/metadata/tracking_metadata.cpp
@@ -0,0 +1,156 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/rpc/metadata/tracking_metadata.h"
+
+#include "mongo/bson/util/bson_check.h"
+#include "mongo/bson/util/bson_extract.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/rpc/metadata.h"
+
+namespace mongo {
+namespace rpc {
+
+namespace {
+
+const char kOperIdFieldName[] = "operId";
+const char kOperNameFieldName[] = "operName";
+const char kParentOperIdFieldName[] = "parentOperId";
+
+}  // unnamed namespace
+
+const OperationContext::Decoration<TrackingMetadata> TrackingMetadata::get =
+    OperationContext::declareDecoration<TrackingMetadata>();
+
+TrackingMetadata::TrackingMetadata(OID operId, std::string operName)
+    : _operId(std::move(operId)), _operName(std::move(operName)) {}
+
+TrackingMetadata::TrackingMetadata(OID operId, std::string operName, std::string parentOperId)
+    : _operId(std::move(operId)),
+      _operName(std::move(operName)),
+      _parentOperId(std::move(parentOperId)) {}
+
+StatusWith<TrackingMetadata> TrackingMetadata::readFromMetadata(const BSONObj& metadataObj) {
+    return readFromMetadata(metadataObj.getField(fieldName()));
+}
+
+void TrackingMetadata::initWithOperName(const std::string& name) {
+    // _operId to be already initialized if it was created with constructChildMetadata.
+    if (!_operId) {
+        OID operId;
+        operId.init();
+        _operId = operId;
+    }
+    _operName = name;
+}
+
+std::string TrackingMetadata::toString() const {
+    invariant(_operId);
+    invariant(_operName);
+    std::ostringstream stream;
+    if (_parentOperId) {
+        stream << "Cmd: " << *_operName << ", TrackingId: " << *_parentOperId << "|" << *_operId;
+    } else {
+        stream << "Cmd: " << *_operName << ", TrackingId: " << *_operId;
+    }
+    return stream.str();
+}
+
+TrackingMetadata TrackingMetadata::constructChildMetadata() const {
+    OID newOperId;
+    newOperId.init();
+    std::string newParentOperId =
+        _parentOperId ? *_parentOperId + "|" + _operId->toString() : _operId->toString();
+
+    return TrackingMetadata(newOperId, std::string(), newParentOperId);
+}
+
+StatusWith<TrackingMetadata> TrackingMetadata::readFromMetadata(const BSONElement& metadataElem) {
+    if (metadataElem.eoo()) {
+        return TrackingMetadata{};
+    } else if (metadataElem.type() != mongo::Object) {
+        return {ErrorCodes::TypeMismatch,
+                str::stream() << "TrackingMetadata element has incorrect type: expected"
+                              << mongo::Object
+                              << " but got "
+                              << metadataElem.type()};
+    }
+
+    BSONObj metadataObj = metadataElem.Obj();
+
+    OID operId;
+    auto status = bsonExtractOIDField(metadataObj, kOperIdFieldName, &operId);
+    if (!status.isOK()) {
+        return status;
+    }
+
+    std::string operName;
+    status = bsonExtractStringField(metadataObj, kOperNameFieldName, &operName);
+    if (!status.isOK()) {
+        return status;
+    }
+
+    std::string parentOperId;
+    status = bsonExtractStringField(metadataObj, kParentOperIdFieldName, &parentOperId);
+    if (!status.isOK()) {
+        if (status != ErrorCodes::NoSuchKey) {
+            return status;
+        }
+        return TrackingMetadata(std::move(operId), std::move(operName));
+    }
+
+    return TrackingMetadata(std::move(operId), std::move(operName), std::move(parentOperId));
+}
+
+void TrackingMetadata::writeToMetadata(BSONObjBuilder* builder) const {
+    BSONObjBuilder metadataBuilder(builder->subobjStart(fieldName()));
+
+    invariant(_operId);
+    invariant(_operName);
+    metadataBuilder.append(kOperIdFieldName, *_operId);
+    metadataBuilder.append(kOperNameFieldName, *_operName);
+
+    if (_parentOperId) {
+        metadataBuilder.append(kParentOperIdFieldName, *_parentOperId);
+    }
+}
+
+BSONObj TrackingMetadata::removeTrackingData(BSONObj metadata) {
+    BSONObjBuilder builder;
+    for (auto elem : metadata) {
+        if (elem.fieldNameStringData() != rpc::TrackingMetadata::fieldName()) {
+            builder.append(elem);
+        }
+    }
+    return builder.obj();
+}
+
+}  // namespace rpc
+}  // namespace mongo
diff --git a/src/mongo/rpc/metadata/tracking_metadata.h b/src/mongo/rpc/metadata/tracking_metadata.h
new file mode 100644
index 00000000000..3fdbac60c37
--- /dev/null
+++ b/src/mongo/rpc/metadata/tracking_metadata.h
@@ -0,0 +1,142 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/jsobj.h"
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+
+class BSONObj;
+class BSONObjBuilder;
+
+namespace rpc {
+
+/**
+ * This class encapsulates the metadata sent on every command request and response,
+ * containing data to track command execution. The current implementation only applies to commands
+ * that go through ShardRemote, which is most sharding catalog operations. It excludes the entire
+ * replication subsystem, query-type client commands, and write-type client commands.
+ *
+ * format:
+ * tracking_info: {
+ *     operId: ObjectId("") // unique ID for the current operation
+ *     operName: string // command name of the current operation
+ *     parentOperId: string  // '|' separated chain of the ancestor commands, oldest first
+ * }
+ */
+class TrackingMetadata {
+public:
+    static const OperationContext::Decoration<TrackingMetadata> get;
+
+    TrackingMetadata() = default;
+    explicit TrackingMetadata(OID operId, std::string operName);
+    explicit TrackingMetadata(OID operId, std::string operName, std::string parentOperId);
+
+    /**
+     * Parses the metadata from the given metadata object.
+     * Returns a NoSuchKey error status if it does not have operId or operName set.
+     * Returns a TypeMismatch error if operId is not OID and operName or parentOperId are not String
+     * If no metadata is found, returns a default-constructed TrackingMetadata.
+     */
+    static StatusWith<TrackingMetadata> readFromMetadata(const BSONObj& metadataObj);
+
+    /**
+     * Parses TrackingMetadata from a pre-extracted BSONElement. When reading a metadata object,
+     * this form is more efficient as it permits parsing the metadata in one pass.
+     */
+    static StatusWith<TrackingMetadata> readFromMetadata(const BSONElement& metadataElem);
+
+    /**
+     * Writes the metadata to the given BSONObjBuilder for building a command request or response
+     * metadata. Only valid to call if operId and operName are set.
+     */
+    void writeToMetadata(BSONObjBuilder* builder) const;
+
+    /**
+     * Returns the Id of this operation.
+     */
+    boost::optional<OID> getOperId() const {
+        return _operId;
+    }
+
+    /**
+     * Returns the name of this operation.
+     */
+    boost::optional<std::string> getOperName() const {
+        return _operName;
+    }
+
+    /**
+     * Returns the parent operId of this operation.
+     */
+    boost::optional<std::string> getParentOperId() const {
+        return _parentOperId;
+    }
+
+    static StringData fieldName() {
+        return "tracking_info";
+    }
+
+    /**
+     * Sets operName to name argument. Intended to initialize the metadata when command name is
+     * known.
+     */
+    void initWithOperName(const std::string& name);
+
+    /*
+     *  get|set isLogged are used to avoid logging parent metadata more than once.
+     */
+    bool getIsLogged() const {
+        return _isLogged;
+    }
+
+    void setIsLogged(bool isLogged) {
+        _isLogged = isLogged;
+    }
+
+    /*
+     *  Builds metadata for child command by updating parentOperId with current operId and
+     *  setting operId to a new value.
+     */
+    TrackingMetadata constructChildMetadata() const;
+
+    std::string toString() const;
+
+    static BSONObj removeTrackingData(BSONObj metadata);
+
+private:
+    boost::optional<OID> _operId;
+    boost::optional<std::string> _operName;
+    boost::optional<std::string> _parentOperId;
+    bool _isLogged{false};
+};
+
+}  // namespace rpc
+}  // namespace mongo
diff --git a/src/mongo/rpc/metadata/tracking_metadata_test.cpp b/src/mongo/rpc/metadata/tracking_metadata_test.cpp
new file mode 100644
index 00000000000..f2f43f6b138
--- /dev/null
+++ b/src/mongo/rpc/metadata/tracking_metadata_test.cpp
@@ -0,0 +1,109 @@
+/*
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/base/status.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/db/repl/optime.h"
+#include "mongo/rpc/metadata/tracking_metadata.h"
+#include "mongo/stdx/chrono.h"
+#include "mongo/unittest/unittest.h"
+
+namespace {
+
+using namespace mongo;
+using namespace mongo::rpc;
+using mongo::unittest::assertGet;
+
+TrackingMetadata checkParse(const BSONObj& metadata) {
+    return assertGet(TrackingMetadata::readFromMetadata(metadata));
+}
+
+const auto kOperId = OID{"541b1a00e8a23afa832b218e"};
+const auto kOperName = "testCmd";
+const auto kParentOperId = "541b1a00e8a23afa832b2016";
+
+TEST(TrackingMetadata, ReadFromMetadata) {
+    {
+        auto metadata = checkParse(BSON(
+            "tracking_info" << BSON("operId" << kOperId << "operName" << kOperName << "parentOperId"
+                                             << kParentOperId)));
+        ASSERT_EQ(*metadata.getOperId(), kOperId);
+        ASSERT_EQ(*metadata.getParentOperId(), kParentOperId);
+        ASSERT_EQ(*metadata.getOperName(), kOperName);
+    }
+}
+
+void checkParseFails(const BSONObj& metadata, ErrorCodes::Error error) {
+    auto tm = TrackingMetadata::readFromMetadata(metadata);
+    ASSERT_NOT_OK(tm.getStatus());
+    ASSERT_EQ(tm.getStatus(), error);
+}
+
+TEST(TrackingMetadata, ReadFromInvalidMetadata) {
+    { checkParseFails(BSON("tracking_info" << 1), ErrorCodes::TypeMismatch); }
+    { checkParseFails(BSON("tracking_info" << BSON("o" << 111)), ErrorCodes::NoSuchKey); }
+    { checkParseFails(BSON("tracking_info" << BSON("operId" << 111)), ErrorCodes::TypeMismatch); }
+    { checkParseFails(BSON("tracking_info" << BSON("operId" << kOperId)), ErrorCodes::NoSuchKey); }
+    {
+        checkParseFails(BSON("tracking_info" << BSON("operId" << kOperId << "operName" << 111)),
+                        ErrorCodes::TypeMismatch);
+    }
+    {
+        checkParseFails(BSON("tracking_info" << BSON("operId" << kOperId << "operName" << kOperName
+                                                              << "parentOperId"
+                                                              << 111)),
+                        ErrorCodes::TypeMismatch);
+    }
+}
+
+TEST(TrackingMetadata, Roundtrip1) {
+    BSONObjBuilder bob;
+    TrackingMetadata tmBefore{kOperId, kOperName, kParentOperId};
+    tmBefore.writeToMetadata(&bob);
+    auto tmAfter = TrackingMetadata::readFromMetadata(bob.obj());
+    ASSERT_OK(tmAfter.getStatus());
+    auto metadata = tmAfter.getValue();
+    ASSERT_EQ(*metadata.getOperId(), kOperId);
+    ASSERT_EQ(*metadata.getParentOperId(), kParentOperId);
+    ASSERT_EQ(*metadata.getOperName(), kOperName);
+}
+
+TEST(TrackingMetadata, Roundtrip2) {
+    BSONObjBuilder bob;
+    TrackingMetadata tmBefore{kOperId, kOperName};
+    tmBefore.writeToMetadata(&bob);
+    auto tmAfter = TrackingMetadata::readFromMetadata(bob.obj());
+    ASSERT_OK(tmAfter.getStatus());
+    auto metadata = tmAfter.getValue();
+    ASSERT_EQ(*metadata.getOperId(), kOperId);
+    ASSERT_EQ(*metadata.getOperName(), kOperName);
+}
+
+}  // namespace