SERVER-18236 send GLEStats over OP_COMMAND metadata object

16 files changed, 558 insertions, 45 deletions

diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp
index e9ff7a92bb0..ba2ef712ff9 100644
--- a/src/mongo/db/dbcommands.cpp
+++ b/src/mongo/db/dbcommands.cpp
@@ -97,6 +97,7 @@
 #include "mongo/rpc/reply_builder_interface.h"
 #include "mongo/rpc/metadata.h"
 #include "mongo/rpc/metadata/server_selection_metadata.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
 #include "mongo/s/d_state.h"
 #include "mongo/s/stale_exception.h"  // for SendStaleConfigException
 #include "mongo/scripting/engine.h"
@@ -1153,16 +1154,6 @@ namespace mongo {
         bool _impersonation;
     };
 
-namespace {
-    // TODO remove as part of SERVER-18236
-    void appendGLEHelperData(BSONObjBuilder& bob, const Timestamp& opTime, const OID& oid) {
-        BSONObjBuilder subobj(bob.subobjStart(kGLEStatsFieldName));
-        subobj.append(kGLEStatsLastOpTimeFieldName, opTime);
-        subobj.appendOID(kGLEStatsElectionIdFieldName, const_cast<OID*>(&oid));
-        subobj.done();
-    }
-}  // namespace
-
     /**
      * this handles
      - auth
@@ -1373,18 +1364,19 @@ namespace {
         }
 
         bool result = this->run(txn, db, interposedCmd, queryFlags, errmsg, replyBuilderBob);
+        BSONObjBuilder metadataBob;
 
         // For commands from mongos, append some info to help getLastError(w) work.
         // TODO: refactor out of here as part of SERVER-18326
         if (replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet &&
             shardingState.enabled()) {
-            appendGLEHelperData(
-                    replyBuilderBob,
-                    repl::ReplClientInfo::forClient(txn->getClient()).getLastOp().getTimestamp(),
-                    replCoord->getElectionId());
+            rpc::ShardingMetadata(
+                repl::ReplClientInfo::forClient(txn->getClient()).getLastOp().getTimestamp(),
+                replCoord->getElectionId()
+            ).writeToMetadata(&metadataBob);
         }
 
-        replyBuilder->setMetadata(rpc::makeEmptyMetadata());
+        replyBuilder->setMetadata(metadataBob.done());
 
         auto cmdResponse = replyBuilderBob.done();
 
diff --git a/src/mongo/db/lasterror.h b/src/mongo/db/lasterror.h
index 55a04c8334d..cc1d0616927 100644
--- a/src/mongo/db/lasterror.h
+++ b/src/mongo/db/lasterror.h
@@ -38,10 +38,6 @@ namespace mongo {
     class BSONObjBuilder;
 
     static const char kUpsertedFieldName[] = "upserted";
-    static const char kGLEStatsFieldName[] = "$gleStats";
-    static const char kGLEStatsLastOpTimeFieldName[] = "lastOpTime";
-    static const char kGLEStatsLastOpTimeTermFieldName[] = "lastOpTimeTerm";
-    static const char kGLEStatsElectionIdFieldName[] = "electionId";
 
     class LastError {
     public:
diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript
index e59e959f6ea..23a682b8b8b 100644
--- a/src/mongo/rpc/SConscript
+++ b/src/mongo/rpc/SConscript
@@ -115,6 +115,7 @@ env.Library(
     source=[
         'metadata.cpp',
         'metadata/server_selection_metadata.cpp',
+        'metadata/sharding_metadata.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/client/read_preference',
@@ -128,6 +129,7 @@ env.CppUnitTest(
     ],
     source=[
         'metadata/server_selection_metadata_test.cpp',
+        'metadata/sharding_metadata_test.cpp',
     ],
     LIBDEPS=[
         'metadata',
diff --git a/src/mongo/rpc/legacy_reply.cpp b/src/mongo/rpc/legacy_reply.cpp
index 1dc40c83cd6..31923818160 100644
--- a/src/mongo/rpc/legacy_reply.cpp
+++ b/src/mongo/rpc/legacy_reply.cpp
@@ -31,9 +31,11 @@
 #include "mongo/rpc/legacy_reply.h"
 
 #include <utility>
+#include <tuple>
 
 #include "mongo/util/assert_util.h"
 #include "mongo/util/mongoutils/str.h"
+#include "mongo/rpc/metadata.h"
 
 namespace mongo {
 namespace rpc {
@@ -61,12 +63,15 @@ namespace rpc {
                 str::stream() << "Got legacy command reply with a bad startingFrom field,"
                               << " expected a value of 0 but got " << qr.getStartingFrom(),
                 qr.getStartingFrom() == 0);
+
         // TODO bson validation
-        _commandReply = BSONObj(qr.data());
+        std::tie(_commandReply, _metadata) = uassertStatusOK(
+            rpc::upconvertReplyMetadata(BSONObj(qr.data()))
+        );
     }
 
     const BSONObj& LegacyReply::getMetadata() const {
-        return _metadataPlaceholder;
+        return _metadata;
     }
 
     const BSONObj& LegacyReply::getCommandReply() const {
diff --git a/src/mongo/rpc/legacy_reply.h b/src/mongo/rpc/legacy_reply.h
index fddc050a71b..f4182c1d3a0 100644
--- a/src/mongo/rpc/legacy_reply.h
+++ b/src/mongo/rpc/legacy_reply.h
@@ -79,7 +79,7 @@ namespace rpc {
         const Message* _message;
 
         // TODO: SERVER-18236
-        BSONObj _metadataPlaceholder{};
+        BSONObj _metadata{};
         BSONObj _commandReply{}; // will hold unowned
     };
 
diff --git a/src/mongo/rpc/legacy_reply_builder.cpp b/src/mongo/rpc/legacy_reply_builder.cpp
index 4ed4ff88886..343e6874d4f 100644
--- a/src/mongo/rpc/legacy_reply_builder.cpp
+++ b/src/mongo/rpc/legacy_reply_builder.cpp
@@ -31,6 +31,7 @@
 #include "mongo/db/dbmessage.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/rpc/legacy_reply_builder.h"
+#include "mongo/rpc/metadata.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/util/assert_util.h"
 
@@ -50,14 +51,17 @@ namespace rpc {
 
     LegacyReplyBuilder& LegacyReplyBuilder::setMetadata(BSONObj metadata) {
         invariant(_state == State::kMetadata);
-        // no op for now: SERVER-18236
+        _metadata = std::move(metadata);
         _state = State::kCommandReply;
         return *this;
     }
 
     LegacyReplyBuilder& LegacyReplyBuilder::setRawCommandReply(BSONObj commandReply) {
         invariant(_state == State::kCommandReply);
-        commandReply.appendSelfToBufBuilder(_builder);
+        BSONObj downconvertedCommandReply = uassertStatusOK(
+            rpc::downconvertReplyMetadata(std::move(commandReply), std::move(_metadata))
+        );
+        downconvertedCommandReply.appendSelfToBufBuilder(_builder);
         _state = State::kOutputDocs;
         return *this;
     }
diff --git a/src/mongo/rpc/legacy_reply_builder.h b/src/mongo/rpc/legacy_reply_builder.h
index 35dc49a911e..cb8a52cae45 100644
--- a/src/mongo/rpc/legacy_reply_builder.h
+++ b/src/mongo/rpc/legacy_reply_builder.h
@@ -57,6 +57,7 @@ namespace rpc {
 
     private:
         BufBuilder _builder{};
+        BSONObj _metadata{};
         std::unique_ptr<Message> _message;
         State _state{State::kMetadata};
     };
diff --git a/src/mongo/rpc/metadata.cpp b/src/mongo/rpc/metadata.cpp
index 595ca571f16..43e4cf37eff 100644
--- a/src/mongo/rpc/metadata.cpp
+++ b/src/mongo/rpc/metadata.cpp
@@ -32,6 +32,8 @@
 
 #include "mongo/client/dbclientinterface.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/rpc/metadata/audit_metadata.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
 #include "mongo/rpc/metadata/server_selection_metadata.h"
 
 namespace mongo {
@@ -81,10 +83,27 @@ namespace rpc {
         int legacyQueryFlags = 0;
         BSONObjBuilder legacyCommandBob;
 
-        auto downconvertStatus = ServerSelectionMetadata::downconvert(cmdObj,
-                                                                      metadata,
-                                                                      &legacyCommandBob,
-                                                                      &legacyQueryFlags);
+
+    StatusWith<CommandReplyWithMetadata> upconvertReplyMetadata(BSONObj legacyReply) {
+        BSONObjBuilder commandReplyBob;
+        BSONObjBuilder metadataBob;
+
+        auto upconvertStatus = ShardingMetadata::upconvert(legacyReply,
+                                                           &commandReplyBob,
+                                                           &metadataBob);
+        if (!upconvertStatus.isOK()) {
+            return upconvertStatus;
+        }
+
+        return std::make_tuple(commandReplyBob.obj(), metadataBob.obj());
+    }
+
+    StatusWith<BSONObj> downconvertReplyMetadata(BSONObj commandReply, BSONObj replyMetadata) {
+        BSONObjBuilder legacyCommandReplyBob;
+
+        auto downconvertStatus = ShardingMetadata::downconvert(commandReply,
+                                                               replyMetadata,
+                                                               &legacyCommandReplyBob);
         if (!downconvertStatus.isOK()) {
             return downconvertStatus;
         }
diff --git a/src/mongo/rpc/metadata/sharding_metadata.cpp b/src/mongo/rpc/metadata/sharding_metadata.cpp
new file mode 100644
index 00000000000..e33a4e075bd
--- /dev/null
+++ b/src/mongo/rpc/metadata/sharding_metadata.cpp
@@ -0,0 +1,155 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/rpc/metadata/sharding_metadata.h"
+
+#include <utility>
+
+#include "mongo/bson/util/bson_extract.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/util/mongoutils/str.h"
+
+namespace mongo {
+namespace rpc {
+
+namespace {
+
+    const char kGLEStatsFieldName[] = "$gleStats";
+    const char kGLEStatsLastOpTimeFieldName[] = "lastOpTime";
+    const char kGLEStatsElectionIdFieldName[] = "electionId";
+
+}  // namespace
+
+    StatusWith<ShardingMetadata> ShardingMetadata::readFromMetadata(const BSONObj& metadataObj) {
+        BSONElement smElem;
+        auto smExtractStatus = bsonExtractTypedField(metadataObj,
+                                                     kGLEStatsFieldName,
+                                                     mongo::Object,
+                                                     &smElem);
+        if (!smExtractStatus.isOK()) {
+            return smExtractStatus;
+        }
+
+        if (smElem.embeddedObject().nFields() != 2) {
+            return Status(ErrorCodes::InvalidOptions,
+                          str::stream() << "The $gleStats object can only have 2 fields, but got "
+                                        << smElem.embeddedObject().toString());
+        }
+
+        BSONElement lastOpTimeElem;
+        auto lastOpTimeExtractStatus = bsonExtractTypedField(smElem.embeddedObject(),
+                                                             kGLEStatsLastOpTimeFieldName,
+                                                             mongo::bsonTimestamp,
+                                                             &lastOpTimeElem);
+        if (!lastOpTimeExtractStatus.isOK()) {
+            return lastOpTimeExtractStatus;
+        }
+
+        BSONElement lastElectionIdElem;
+        auto lastElectionIdExtractStatus = bsonExtractTypedField(smElem.embeddedObject(),
+                                                                 kGLEStatsElectionIdFieldName,
+                                                                 mongo::jstOID,
+                                                                 &lastElectionIdElem);
+        if (!lastElectionIdExtractStatus.isOK()) {
+            return lastElectionIdExtractStatus;
+        }
+
+        return ShardingMetadata(lastOpTimeElem.timestamp(), lastElectionIdElem.OID());
+    }
+
+    Status ShardingMetadata::writeToMetadata(BSONObjBuilder* metadataBob) const {
+        BSONObjBuilder subobj(metadataBob->subobjStart(kGLEStatsFieldName));
+        subobj.append(kGLEStatsLastOpTimeFieldName, getLastOpTime());
+        subobj.append(kGLEStatsElectionIdFieldName, getLastElectionId());
+        return Status::OK();
+    }
+
+    Status ShardingMetadata::downconvert(const BSONObj& commandReply,
+                                         const BSONObj& replyMetadata,
+                                         BSONObjBuilder* legacyCommandReplyBob) {
+
+        legacyCommandReplyBob->appendElements(commandReply);
+
+        auto swShardingMetadata = readFromMetadata(replyMetadata);
+        if (swShardingMetadata.isOK()) {
+            // We can reuse the same logic to write the sharding metadata out to the legacy
+            // command as the element has the same format whether it is there or on the metadata
+            // object.
+            swShardingMetadata.getValue().writeToMetadata(legacyCommandReplyBob);
+        }
+        else if (swShardingMetadata.getStatus() == ErrorCodes::NoSuchKey) {
+            // It is valid to not have a $gleStats field.
+        }
+        else {
+            return swShardingMetadata.getStatus();
+        }
+        return Status::OK();
+    }
+
+    Status ShardingMetadata::upconvert(const BSONObj& legacyCommand,
+                                       BSONObjBuilder* commandBob,
+                                       BSONObjBuilder* metadataBob) {
+        // We can reuse the same logic to read the sharding metadata out from the legacy command
+        // as it has the same format whether it is there or on the metadata object.
+        auto swShardingMetadata = readFromMetadata(legacyCommand);
+        if (swShardingMetadata.isOK()) {
+            swShardingMetadata.getValue().writeToMetadata(metadataBob);
+
+            // Write out the command excluding the $gleStats subobject.
+            for (const auto& elem : legacyCommand) {
+                if (elem.fieldNameStringData() != StringData(kGLEStatsFieldName)) {
+                    commandBob->append(elem);
+                }
+            }
+        } else if (swShardingMetadata.getStatus() == ErrorCodes::NoSuchKey) {
+            // it is valid to not have a $gleStats field
+            commandBob->appendElements(legacyCommand);
+        }
+        else {
+            return swShardingMetadata.getStatus();
+        }
+        return Status::OK();
+    }
+
+    ShardingMetadata::ShardingMetadata(Timestamp lastOpTime, OID lastElectionId)
+        : _lastOpTime(std::move(lastOpTime))
+        , _lastElectionId(std::move(lastElectionId))
+    {}
+
+    const Timestamp& ShardingMetadata::getLastOpTime() const {
+        return _lastOpTime;
+    }
+
+    const OID& ShardingMetadata::getLastElectionId() const {
+        return _lastElectionId;
+    }
+
+}  // namespace rpc
+}  // namespace mongo
diff --git a/src/mongo/rpc/metadata/sharding_metadata.h b/src/mongo/rpc/metadata/sharding_metadata.h
new file mode 100644
index 00000000000..00114624996
--- /dev/null
+++ b/src/mongo/rpc/metadata/sharding_metadata.h
@@ -0,0 +1,90 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/jsobj.h"
+
+namespace mongo {
+    class BSONObj;
+    class BSONObjBuilder;
+    class Status;
+    template <typename T> class StatusWith;
+namespace rpc {
+
+    /**
+     * This class compromises the reply metadata fields that concern sharding. MongoD attaches
+     * this information to a command reply, which MongoS uses to process getLastError.
+     */
+    class ShardingMetadata {
+    public:
+
+        /**
+         * Reads ShardingMetadata from a metadata object.
+         */
+        static StatusWith<ShardingMetadata> readFromMetadata(const BSONObj& metadataObj);
+
+        /**
+         * Writes ShardingMetadata to a metadata builder.
+         */
+        Status writeToMetadata(BSONObjBuilder* metadataBob) const;
+
+        /**
+         * Rewrites the ShardingMetadata from the legacy OP_QUERY format to the metadata object
+         * format.
+         */
+        static Status downconvert(const BSONObj& commandReply,
+                                  const BSONObj& replyMetadata,
+                                  BSONObjBuilder* legacyCommandReply);
+
+        /**
+         * Rewrites the ShardingMetadata from the legacy OP_QUERY format to the metadata object
+         * format.
+         */
+        static Status upconvert(const BSONObj& legacyCommandReply,
+                                BSONObjBuilder* commandReplyBob,
+                                BSONObjBuilder* metadataBob);
+
+        /**
+         * Gets the OpTime of the oplog entry of the last succssful write operation executed by the
+         * server that produced the metadata.
+         */
+        const Timestamp& getLastOpTime() const;
+
+        /**
+         * Gets the most recent election id observed by the server that produced the metadata.
+         */
+        const OID& getLastElectionId() const;
+
+        ShardingMetadata(Timestamp lastOpTime, OID lastElectionId);
+
+    private:
+        Timestamp _lastOpTime;
+        OID _lastElectionId;
+    };
+
+}  // namespace rpc
+}  // namespace mongo
diff --git a/src/mongo/rpc/metadata/sharding_metadata_test.cpp b/src/mongo/rpc/metadata/sharding_metadata_test.cpp
new file mode 100644
index 00000000000..19df5a59bcc
--- /dev/null
+++ b/src/mongo/rpc/metadata/sharding_metadata_test.cpp
@@ -0,0 +1,226 @@
+/*
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/base/status.h"
+#include "mongo/db/jsobj.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
+#include "mongo/stdx/chrono.h"
+#include "mongo/unittest/unittest.h"
+
+namespace {
+
+    using namespace mongo;
+    using namespace mongo::rpc;
+    using mongo::unittest::assertGet;
+
+    ShardingMetadata checkParse(const BSONObj& metadata) {
+        return assertGet(ShardingMetadata::readFromMetadata(metadata));
+    }
+
+    const auto kElectionId = OID{"541b1a00e8a23afa832b218e"};
+    const auto kLastOpTime = Timestamp(stdx::chrono::seconds{1337}, 800u);
+
+    TEST(ShardingMetadata, ReadFromMetadata) {
+        {
+            auto sm = checkParse(BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                          "electionId" << kElectionId)));
+            ASSERT_EQ(sm.getLastElectionId(), kElectionId);
+            ASSERT_EQ(sm.getLastOpTime(), kLastOpTime);
+        }
+        {
+            // We don't care about order.
+            auto sm = checkParse(BSON("$gleStats" << BSON("electionId" << kElectionId <<
+                                                          "lastOpTime" << kLastOpTime)));
+
+            ASSERT_EQ(sm.getLastElectionId(), kElectionId);
+            ASSERT_EQ(sm.getLastOpTime(), kLastOpTime);
+        }
+    }
+
+    void checkParseFails(const BSONObj& metadata, ErrorCodes::Error error) {
+        auto sm = ShardingMetadata::readFromMetadata(metadata);
+        ASSERT_NOT_OK(sm.getStatus());
+        ASSERT_EQ(sm.getStatus(), error);
+    }
+
+    TEST(ShardingMetadata, ReadFromInvalidMetadata) {
+        {
+            checkParseFails(BSONObj(),
+                            ErrorCodes::NoSuchKey);
+        }
+        {
+            checkParseFails(BSON("$gleStats" << 1),
+                            ErrorCodes::TypeMismatch);
+
+        }
+        {
+            checkParseFails(BSON("$gleStats" << BSONObj()),
+                            ErrorCodes::InvalidOptions);
+        }
+        {
+            checkParseFails(BSON("$gleStats" << BSON("lastOpTime" << 3 <<
+                                                     "electionId" << kElectionId)),
+                            ErrorCodes::TypeMismatch);
+        }
+        {
+            checkParseFails(BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                     "electionId" << 3)),
+                            ErrorCodes::TypeMismatch);
+        }
+        {
+            checkParseFails(BSON("$gleStats" << BSON("lastOpTime" << kElectionId <<
+                                                     "electionId" << kLastOpTime)),
+                            ErrorCodes::TypeMismatch);
+        }
+        {
+            checkParseFails(BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                     "electionId" << kElectionId <<
+                                                     "extra" << "this should not be here")),
+                            ErrorCodes::InvalidOptions);
+        }
+    }
+
+    void checkUpconvert(const BSONObj& legacyCommandReply,
+                        const BSONObj& upconvertedCommandReply,
+                        const BSONObj& upconvertedReplyMetadata) {
+        {
+            BSONObjBuilder commandReplyBob;
+            BSONObjBuilder metadataBob;
+            ASSERT_OK(ShardingMetadata::upconvert(legacyCommandReply,
+                                                  &commandReplyBob,
+                                                  &metadataBob));
+            ASSERT_EQ(commandReplyBob.done(), upconvertedCommandReply);
+            ASSERT_EQ(metadataBob.done(), upconvertedReplyMetadata);
+        }
+    }
+
+    TEST(ShardingMetadata, UpconvertValidMetadata) {
+        {
+            checkUpconvert(BSON("ok" << 1),
+
+                           BSON("ok" << 1),
+
+                           BSONObj());
+        }
+        {
+            checkUpconvert(BSON("ok" << 1 <<
+                                "$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                    "electionId" << kElectionId)),
+
+                           BSON("ok" << 1),
+
+                           BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                    "electionId" << kElectionId)));
+        }
+        {
+            checkUpconvert(BSON("ok" << 1 <<
+                                "somestuff" << "some other stuff" <<
+                                "$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                    "electionId" << kElectionId) <<
+                                "morestuff" << "more other stuff"),
+
+                           BSON("ok" << 1 <<
+                                "somestuff" << "some other stuff" <<
+                                "morestuff" << "more other stuff"),
+
+                           BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                    "electionId" << kElectionId)));
+        }
+    }
+
+    void checkUpconvertFails(const BSONObj& legacyCommandReply,
+                             ErrorCodes::Error why) {
+        BSONObjBuilder ignoredCommand;
+        BSONObjBuilder ignoredMetadata;
+        auto status = ShardingMetadata::upconvert(legacyCommandReply,
+                                                  &ignoredCommand,
+                                                  &ignoredMetadata);
+        ASSERT_NOT_OK(status);
+        ASSERT_EQ(status, why);
+    }
+
+    TEST(ShardingMetadata, UpconvertInvalidMetadata) {
+        {
+            checkUpconvertFails(BSON("ok" << 1 <<
+                                     "$gleStats" << 1),
+                                ErrorCodes::TypeMismatch);
+        }
+        {
+            checkUpconvertFails(BSON("ok" << 1 <<
+                                     "$gleStats" << BSON("lastOpTime" << 1)),
+                                ErrorCodes::InvalidOptions);
+        }
+        {
+            checkUpconvertFails(BSON("ok" << 1 <<
+                                     "$gleStats" << BSON("lastOpTime" << 2 <<
+                                                         "foo" << 1)),
+                                ErrorCodes::TypeMismatch);
+        }
+        {
+            checkUpconvertFails(BSON("ok" << 1 <<
+                                     "$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                         "electionId" << kElectionId <<
+                                                         "krandom" << "shouldnotbehere")),
+                                ErrorCodes::InvalidOptions);
+        }
+    }
+
+    void checkDownconvert(const BSONObj& commandReply,
+                          const BSONObj& metadata,
+                          const BSONObj& downconvertedCommand) {
+        BSONObjBuilder downconvertedCommandBob;
+        ASSERT_OK(ShardingMetadata::downconvert(commandReply,
+                                                metadata,
+                                                &downconvertedCommandBob));
+        ASSERT_EQ(downconvertedCommandBob.done(), downconvertedCommand);
+    }
+
+    TEST(ShardingMetadata, Downconvert) {
+        {
+            checkDownconvert(BSON("ok" << 1),
+                             BSON("$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                      "electionId" << kElectionId)),
+                             BSON("ok" << 1 <<
+                                  "$gleStats" << BSON("lastOpTime" << kLastOpTime <<
+                                                      "electionId" << kElectionId)));
+        }
+        {
+            checkDownconvert(BSON("ok" << 1),
+                             BSONObj(),
+                             BSON("ok" << 1));
+        }
+    }
+
+}  // namespace
+
+
+
+
+
diff --git a/src/mongo/s/catalog/dist_lock_catalog_impl.cpp b/src/mongo/s/catalog/dist_lock_catalog_impl.cpp
index 9498c52da0d..57aab2dd2fc 100644
--- a/src/mongo/s/catalog/dist_lock_catalog_impl.cpp
+++ b/src/mongo/s/catalog/dist_lock_catalog_impl.cpp
@@ -41,6 +41,7 @@
 #include "mongo/db/lasterror.h"
 #include "mongo/db/query/find_and_modify_request.h"
 #include "mongo/rpc/get_status_from_command_result.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
 #include "mongo/s/type_lockpings.h"
 #include "mongo/s/type_locks.h"
 #include "mongo/s/write_ops/wc_error_detail.h"
@@ -115,11 +116,14 @@ namespace {
 
     /**
      * Extract the electionId from a command response.
+     *
+     * TODO: this needs to support OP_COMMAND metadata.
      */
     StatusWith<OID> extractElectionId(const BSONObj& responseObj) {
+
         BSONElement gleStatsElem;
         auto gleStatus = bsonExtractTypedField(responseObj,
-                                               kGLEStatsFieldName,
+                                               "$gleStats",
                                                Object,
                                                &gleStatsElem);
 
@@ -130,7 +134,7 @@ namespace {
         OID electionId;
 
         auto electionIdStatus = bsonExtractOIDField(gleStatsElem.Obj(),
-                                                    kGLEStatsElectionIdFieldName,
+                                                    "electionId",
                                                     &electionId);
 
         if (!electionIdStatus.isOK()) {
diff --git a/src/mongo/s/client/sharding_connection_hook.cpp b/src/mongo/s/client/sharding_connection_hook.cpp
index b6c334419a2..49e27684bfd 100644
--- a/src/mongo/s/client/sharding_connection_hook.cpp
+++ b/src/mongo/s/client/sharding_connection_hook.cpp
@@ -104,7 +104,11 @@ namespace {
             // For every DBClient created by mongos, add a hook that will capture the response from
             // commands we pass along from the client, so that we can target the correct node when
             // subsequent getLastError calls are made by mongos.
-            conn->setPostRunCommandHook(stdx::bind(&saveGLEStats, stdx::placeholders::_1, stdx::placeholders::_2));
+            conn->setReplyMetadataReader([](const BSONObj& metadataObj, StringData hostString)
+                                         -> Status {
+                saveGLEStats(metadataObj, hostString);
+                return Status::OK();
+            });
         }
 
         // For every DBClient created by mongos, add a hook that will append impersonated users
diff --git a/src/mongo/s/cluster_last_error_info.cpp b/src/mongo/s/cluster_last_error_info.cpp
index efa50d49101..30c79001be3 100644
--- a/src/mongo/s/cluster_last_error_info.cpp
+++ b/src/mongo/s/cluster_last_error_info.cpp
@@ -38,6 +38,7 @@
 #include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/lasterror.h"
 #include "mongo/db/stats/timer_stats.h"
+#include "mongo/rpc/metadata/sharding_metadata.h"
 #include "mongo/util/log.h"
 
 namespace mongo {
@@ -75,25 +76,39 @@ namespace mongo {
     static ServerStatusMetricField<TimerStats> displayGleLatency("getLastError.wtime",
                                                                  &gleWtimeStats);
 
-    void saveGLEStats(const BSONObj& result, const std::string& hostString) {
+    void saveGLEStats(const BSONObj& metadata, StringData hostString) {
         if (!haveClient()) {
+            // TODO: how can this happen?
             return;
         }
-        if (result[kGLEStatsFieldName].type() != Object) {
+
+        auto swShardingMetadata = rpc::ShardingMetadata::readFromMetadata(metadata);
+        if (swShardingMetadata.getStatus() == ErrorCodes::NoSuchKey) {
+            return;
+        }
+        else if (!swShardingMetadata.isOK()) {
+            warning() << "Got invalid sharding metadata " << swShardingMetadata.getStatus()
+                      << " metadata object was '" << metadata << "'";
             return;
         }
-        std::string errmsg;
-        ConnectionString shardConn = ConnectionString::parse(hostString, errmsg);
 
-        BSONElement subobj = result[kGLEStatsFieldName];
-        Timestamp lastOpTime = subobj[kGLEStatsLastOpTimeFieldName].timestamp();
-        OID electionId = subobj[kGLEStatsElectionIdFieldName].OID();
+        auto shardConn = ConnectionString::parse(hostString.toString());
+        // If we got the reply from this host, we expect that its 'hostString' must be valid.
+        if (!shardConn.isOK()) {
+            severe() << "got bad host string in saveGLEStats: " << hostString;
+        }
+        invariantOK(shardConn.getStatus());
+
+        auto shardingMetadata = std::move(swShardingMetadata.getValue());
+
         auto& clientInfo = cc();
-        LOG(4) << "saveGLEStats lastOpTime:" << lastOpTime 
-               << " electionId:" << electionId;
+        LOG(4) << "saveGLEStats lastOpTime:" << shardingMetadata.getLastOpTime()
+               << " electionId:" << shardingMetadata.getLastElectionId();
 
         ClusterLastErrorInfo::get(clientInfo).addHostOpTime(
-                shardConn, HostOpTime(lastOpTime, electionId));
+            shardConn.getValue(),
+            HostOpTime(shardingMetadata.getLastOpTime(),
+                       shardingMetadata.getLastElectionId()));
     }
 
 } // namespace mongo
diff --git a/src/mongo/s/cluster_last_error_info.h b/src/mongo/s/cluster_last_error_info.h
index e68724d78f2..a3110e3d40b 100644
--- a/src/mongo/s/cluster_last_error_info.h
+++ b/src/mongo/s/cluster_last_error_info.h
@@ -95,8 +95,8 @@ namespace mongo {
     };
 
     /**
-     * Looks for $gleStats in a command response, and fills in the ClusterLastErrorInfo for this
-     * thread's associated Client with the data, if found.
+     * Looks for $gleStats in a command's reply metadata, and fills in the ClusterLastErrorInfo
+     * for this thread's associated Client with the data, if found.
      *
      * This data will be used by subsequent GLE calls, to ensure we look for the correct
      * write on the correct PRIMARY.
@@ -104,6 +104,6 @@ namespace mongo {
      * conn: the std::string name of the hostAndPort where the command ran. This can be a replica
      *       set seed list.
      */
-    void saveGLEStats(const BSONObj& result, const std::string& conn);
+    void saveGLEStats(const BSONObj& metadataObj, StringData conn);
 
 }  // namespace mongo
diff --git a/src/mongo/s/d_state.cpp b/src/mongo/s/d_state.cpp
index 44c42890e46..927b6283517 100644
--- a/src/mongo/s/d_state.cpp
+++ b/src/mongo/s/d_state.cpp
@@ -1451,7 +1451,7 @@ namespace mongo {
     void usingAShardConnection( const string& addr ) {
     }
 
-    void saveGLEStats(const BSONObj& result, const std::string& conn) {
+    void saveGLEStats(const BSONObj& result, StringData hostString) {
         // Declared in cluster_last_error_info.h.
         //
         // This can be called in mongod, which is unfortunate.  To fix this,