SERVER-40078 Report lastCommitted wall clock time

6 files changed, 80 insertions, 30 deletions

diff --git a/src/mongo/rpc/metadata/oplog_query_metadata.cpp b/src/mongo/rpc/metadata/oplog_query_metadata.cpp
index 59d19207f20..4548928d1ca 100644
--- a/src/mongo/rpc/metadata/oplog_query_metadata.cpp
+++ b/src/mongo/rpc/metadata/oplog_query_metadata.cpp
@@ -39,12 +39,14 @@ namespace mongo {
 namespace rpc {
 
 using repl::OpTime;
+using repl::OpTimeAndWallTime;
 
 const char kOplogQueryMetadataFieldName[] = "$oplogQueryData";
 
 namespace {
 
 const char kLastOpCommittedFieldName[] = "lastOpCommitted";
+const char kLastCommittedWallFieldName[] = "lastCommittedWall";
 const char kLastOpAppliedFieldName[] = "lastOpApplied";
 const char kPrimaryIndexFieldName[] = "primaryIndex";
 const char kSyncSourceIndexFieldName[] = "syncSourceIndex";
@@ -54,7 +56,7 @@ const char kRBIDFieldName[] = "rbid";
 
 const int OplogQueryMetadata::kNoPrimary;
 
-OplogQueryMetadata::OplogQueryMetadata(OpTime lastOpCommitted,
+OplogQueryMetadata::OplogQueryMetadata(OpTimeAndWallTime lastOpCommitted,
                                        OpTime lastOpApplied,
                                        int rbid,
                                        int currentPrimaryIndex,
@@ -65,7 +67,8 @@ OplogQueryMetadata::OplogQueryMetadata(OpTime lastOpCommitted,
       _currentPrimaryIndex(currentPrimaryIndex),
       _currentSyncSourceIndex(currentSyncSourceIndex) {}
 
-StatusWith<OplogQueryMetadata> OplogQueryMetadata::readFromMetadata(const BSONObj& metadataObj) {
+StatusWith<OplogQueryMetadata> OplogQueryMetadata::readFromMetadata(const BSONObj& metadataObj,
+                                                                    bool requireWallTime) {
     BSONElement oqMetadataElement;
 
     Status status = bsonExtractTypedField(
@@ -89,11 +92,20 @@ StatusWith<OplogQueryMetadata> OplogQueryMetadata::readFromMetadata(const BSONOb
     if (!status.isOK())
         return status;
 
-    repl::OpTime lastOpCommitted;
-    status = bsonExtractOpTimeField(oqMetadataObj, kLastOpCommittedFieldName, &lastOpCommitted);
+    repl::OpTimeAndWallTime lastOpCommitted;
+    status =
+        bsonExtractOpTimeField(oqMetadataObj, kLastOpCommittedFieldName, &(lastOpCommitted.opTime));
     if (!status.isOK())
         return status;
 
+    BSONElement wallClockTimeElement;
+    status = bsonExtractTypedField(
+        oqMetadataObj, kLastCommittedWallFieldName, BSONType::Date, &wallClockTimeElement);
+    if (!status.isOK() && (status != ErrorCodes::NoSuchKey || requireWallTime))
+        return status;
+    if (status.isOK())
+        lastOpCommitted.wallTime = wallClockTimeElement.Date();
+
     repl::OpTime lastOpApplied;
     status = bsonExtractOpTimeField(oqMetadataObj, kLastOpAppliedFieldName, &lastOpApplied);
     if (!status.isOK())
@@ -104,7 +116,8 @@ StatusWith<OplogQueryMetadata> OplogQueryMetadata::readFromMetadata(const BSONOb
 
 Status OplogQueryMetadata::writeToMetadata(BSONObjBuilder* builder) const {
     BSONObjBuilder oqMetadataBuilder(builder->subobjStart(kOplogQueryMetadataFieldName));
-    _lastOpCommitted.append(&oqMetadataBuilder, kLastOpCommittedFieldName);
+    _lastOpCommitted.opTime.append(&oqMetadataBuilder, kLastOpCommittedFieldName);
+    oqMetadataBuilder.appendDate(kLastCommittedWallFieldName, _lastOpCommitted.wallTime);
     _lastOpApplied.append(&oqMetadataBuilder, kLastOpAppliedFieldName);
     oqMetadataBuilder.append(kRBIDFieldName, _rbid);
     oqMetadataBuilder.append(kPrimaryIndexFieldName, _currentPrimaryIndex);
diff --git a/src/mongo/rpc/metadata/oplog_query_metadata.h b/src/mongo/rpc/metadata/oplog_query_metadata.h
index d9a830bcf67..1f8cc2705be 100644
--- a/src/mongo/rpc/metadata/oplog_query_metadata.h
+++ b/src/mongo/rpc/metadata/oplog_query_metadata.h
@@ -53,7 +53,7 @@ public:
     static const int kNoPrimary = -1;
 
     OplogQueryMetadata() = default;
-    OplogQueryMetadata(repl::OpTime lastOpCommitted,
+    OplogQueryMetadata(repl::OpTimeAndWallTime lastOpCommitted,
                        repl::OpTime lastOpApplied,
                        int rbid,
                        int currentPrimaryIndex,
@@ -63,19 +63,23 @@ public:
      * format:
      * {
      *     lastOpCommitted: {ts: Timestamp(0, 0), term: 0},
+     *     lastCommittedWall: ISODate("2018-07-25T19:21:22.449Z")
      *     lastOpApplied: {ts: Timestamp(0, 0), term: 0},
      *     rbid: 0
      *     primaryIndex: 0,
      *     syncSourceIndex: 0
      * }
+     * requireWallTime is only false if FCV is less than 4.2 or the wall clock time is not read from
+     * this particular OplogQueryMetadata instance.
      */
-    static StatusWith<OplogQueryMetadata> readFromMetadata(const BSONObj& doc);
+    static StatusWith<OplogQueryMetadata> readFromMetadata(const BSONObj& doc,
+                                                           bool requireWallTime);
     Status writeToMetadata(BSONObjBuilder* builder) const;
 
     /**
      * Returns the OpTime of the most recently committed op of which the sender was aware.
      */
-    repl::OpTime getLastOpCommitted() const {
+    repl::OpTimeAndWallTime getLastOpCommitted() const {
         return _lastOpCommitted;
     }
 
@@ -115,7 +119,7 @@ public:
     std::string toString() const;
 
 private:
-    repl::OpTime _lastOpCommitted;
+    repl::OpTimeAndWallTime _lastOpCommitted;
     repl::OpTime _lastOpApplied;
     int _rbid = -1;
     int _currentPrimaryIndex = kNoPrimary;
diff --git a/src/mongo/rpc/metadata/oplog_query_metadata_test.cpp b/src/mongo/rpc/metadata/oplog_query_metadata_test.cpp
index fb73b359c17..859e8bbf10c 100644
--- a/src/mongo/rpc/metadata/oplog_query_metadata_test.cpp
+++ b/src/mongo/rpc/metadata/oplog_query_metadata_test.cpp
@@ -41,10 +41,11 @@ using repl::OpTime;
 
 TEST(ReplResponseMetadataTest, OplogQueryMetadataRoundtrip) {
     OpTime opTime1(Timestamp(1234, 100), 5);
+    Date_t committedWall = Date_t::min() + Seconds(opTime1.getSecs());
     OpTime opTime2(Timestamp(7777, 101), 6);
-    OplogQueryMetadata metadata(opTime1, opTime2, 6, 12, -1);
+    OplogQueryMetadata metadata({opTime1, committedWall}, opTime2, 6, 12, -1);
 
-    ASSERT_EQ(opTime1, metadata.getLastOpCommitted());
+    ASSERT_EQ(opTime1, metadata.getLastOpCommitted().opTime);
     ASSERT_EQ(opTime2, metadata.getLastOpApplied());
 
     BSONObjBuilder builder;
@@ -53,6 +54,8 @@ TEST(ReplResponseMetadataTest, OplogQueryMetadataRoundtrip) {
     BSONObj expectedObj(BSON(kOplogQueryMetadataFieldName << BSON(
                                  "lastOpCommitted"
                                  << BSON("ts" << opTime1.getTimestamp() << "t" << opTime1.getTerm())
+                                 << "lastCommittedWall"
+                                 << committedWall
                                  << "lastOpApplied"
                                  << BSON("ts" << opTime2.getTimestamp() << "t" << opTime2.getTerm())
                                  << "rbid"
@@ -65,12 +68,14 @@ TEST(ReplResponseMetadataTest, OplogQueryMetadataRoundtrip) {
     BSONObj serializedObj = builder.obj();
     ASSERT_BSONOBJ_EQ(expectedObj, serializedObj);
 
-    auto cloneStatus = OplogQueryMetadata::readFromMetadata(serializedObj);
+    auto cloneStatus =
+        OplogQueryMetadata::readFromMetadata(serializedObj, /*requireWallTime*/ true);
     ASSERT_OK(cloneStatus.getStatus());
 
     const auto& clonedMetadata = cloneStatus.getValue();
-    ASSERT_EQ(opTime1, clonedMetadata.getLastOpCommitted());
+    ASSERT_EQ(opTime1, clonedMetadata.getLastOpCommitted().opTime);
     ASSERT_EQ(opTime2, clonedMetadata.getLastOpApplied());
+    ASSERT_EQ(committedWall, clonedMetadata.getLastOpCommitted().wallTime);
     ASSERT_EQ(metadata.getRBID(), clonedMetadata.getRBID());
     ASSERT_EQ(metadata.getPrimaryIndex(), clonedMetadata.getPrimaryIndex());
     ASSERT_EQ(metadata.getSyncSourceIndex(), clonedMetadata.getSyncSourceIndex());
diff --git a/src/mongo/rpc/metadata/repl_set_metadata.cpp b/src/mongo/rpc/metadata/repl_set_metadata.cpp
index 4064bc498c6..12cd36df089 100644
--- a/src/mongo/rpc/metadata/repl_set_metadata.cpp
+++ b/src/mongo/rpc/metadata/repl_set_metadata.cpp
@@ -39,12 +39,14 @@ namespace mongo {
 namespace rpc {
 
 using repl::OpTime;
+using repl::OpTimeAndWallTime;
 
 const char kReplSetMetadataFieldName[] = "$replData";
 
 namespace {
 
 const char kLastOpCommittedFieldName[] = "lastOpCommitted";
+const char kLastCommittedWallFieldName[] = "lastCommittedWall";
 const char kLastOpVisibleFieldName[] = "lastOpVisible";
 const char kConfigVersionFieldName[] = "configVersion";
 const char kReplicaSetIdFieldName[] = "replicaSetId";
@@ -57,7 +59,7 @@ const char kTermFieldName[] = "term";
 const int ReplSetMetadata::kNoPrimary;
 
 ReplSetMetadata::ReplSetMetadata(long long term,
-                                 OpTime committedOpTime,
+                                 OpTimeAndWallTime committedOpTime,
                                  OpTime visibleOpTime,
                                  long long configVersion,
                                  OID id,
@@ -71,7 +73,8 @@ ReplSetMetadata::ReplSetMetadata(long long term,
       _currentPrimaryIndex(currentPrimaryIndex),
       _currentSyncSourceIndex(currentSyncSourceIndex) {}
 
-StatusWith<ReplSetMetadata> ReplSetMetadata::readFromMetadata(const BSONObj& metadataObj) {
+StatusWith<ReplSetMetadata> ReplSetMetadata::readFromMetadata(const BSONObj& metadataObj,
+                                                              bool requireWallTime) {
     BSONElement replMetadataElement;
 
     Status status = bsonExtractTypedField(
@@ -108,17 +111,32 @@ StatusWith<ReplSetMetadata> ReplSetMetadata::readFromMetadata(const BSONObj& met
     if (!status.isOK())
         return status;
 
-    repl::OpTime lastOpCommitted;
-    status = bsonExtractOpTimeField(replMetadataObj, kLastOpCommittedFieldName, &lastOpCommitted);
+    repl::OpTimeAndWallTime lastOpCommitted;
+    auto lastCommittedStatus = bsonExtractOpTimeField(
+        replMetadataObj, kLastOpCommittedFieldName, &(lastOpCommitted.opTime));
     // We check for NoSuchKey because these fields will be removed in SERVER-27668.
-    if (!status.isOK() && status != ErrorCodes::NoSuchKey)
-        return status;
+    if (!lastCommittedStatus.isOK() && lastCommittedStatus != ErrorCodes::NoSuchKey)
+        return lastCommittedStatus;
 
     repl::OpTime lastOpVisible;
     status = bsonExtractOpTimeField(replMetadataObj, kLastOpVisibleFieldName, &lastOpVisible);
     if (!status.isOK() && status != ErrorCodes::NoSuchKey)
         return status;
 
+    BSONElement wallClockTimeElement;
+    status = bsonExtractTypedField(
+        replMetadataObj, kLastCommittedWallFieldName, BSONType::Date, &wallClockTimeElement);
+
+    // Last committed OpTime is optional, so if last committed OpTime is missing, do not check for
+    // last committed wall clock time. Last committed wall clock time is also only required if
+    // FCV is 4.2.
+    if (!status.isOK() && lastCommittedStatus != ErrorCodes::NoSuchKey &&
+        (status != ErrorCodes::NoSuchKey || requireWallTime))
+        return status;
+    if (status.isOK()) {
+        lastOpCommitted.wallTime = wallClockTimeElement.Date();
+    }
+
     return ReplSetMetadata(
         term, lastOpCommitted, lastOpVisible, configVersion, id, primaryIndex, syncSourceIndex);
 }
@@ -126,7 +144,8 @@ StatusWith<ReplSetMetadata> ReplSetMetadata::readFromMetadata(const BSONObj& met
 Status ReplSetMetadata::writeToMetadata(BSONObjBuilder* builder) const {
     BSONObjBuilder replMetadataBuilder(builder->subobjStart(kReplSetMetadataFieldName));
     replMetadataBuilder.append(kTermFieldName, _currentTerm);
-    _lastOpCommitted.append(&replMetadataBuilder, kLastOpCommittedFieldName);
+    _lastOpCommitted.opTime.append(&replMetadataBuilder, kLastOpCommittedFieldName);
+    replMetadataBuilder.appendDate(kLastCommittedWallFieldName, _lastOpCommitted.wallTime);
     _lastOpVisible.append(&replMetadataBuilder, kLastOpVisibleFieldName);
     replMetadataBuilder.append(kConfigVersionFieldName, _configVersion);
     replMetadataBuilder.append(kReplicaSetIdFieldName, _replicaSetId);
diff --git a/src/mongo/rpc/metadata/repl_set_metadata.h b/src/mongo/rpc/metadata/repl_set_metadata.h
index 927b559ee82..7559120ac0c 100644
--- a/src/mongo/rpc/metadata/repl_set_metadata.h
+++ b/src/mongo/rpc/metadata/repl_set_metadata.h
@@ -54,7 +54,7 @@ public:
 
     ReplSetMetadata() = default;
     ReplSetMetadata(long long term,
-                    repl::OpTime committedOpTime,
+                    repl::OpTimeAndWallTime committedOpTime,
                     repl::OpTime visibleOpTime,
                     long long configVersion,
                     OID replicaSetId,
@@ -72,8 +72,10 @@ public:
      *     primaryIndex: 0,
      *     syncSourceIndex: 0
      * }
+     * requireWallTime is only false if FCV is less than 4.2 or the wall clock time is not read from
+     * this particular ReplSetMetadata instance.
      */
-    static StatusWith<ReplSetMetadata> readFromMetadata(const BSONObj& doc);
+    static StatusWith<ReplSetMetadata> readFromMetadata(const BSONObj& doc, bool requireWallTime);
     Status writeToMetadata(BSONObjBuilder* builder) const;
 
     /**
@@ -86,7 +88,7 @@ public:
     /**
      * Returns the OpTime of the most recently committed op of which the sender was aware.
      */
-    repl::OpTime getLastOpCommitted() const {
+    repl::OpTimeAndWallTime getLastOpCommitted() const {
         return _lastOpCommitted;
     }
 
@@ -140,7 +142,7 @@ public:
     std::string toString() const;
 
 private:
-    repl::OpTime _lastOpCommitted;
+    repl::OpTimeAndWallTime _lastOpCommitted;
     repl::OpTime _lastOpVisible;
     long long _currentTerm = -1;
     long long _configVersion = -1;
diff --git a/src/mongo/rpc/metadata/repl_set_metadata_test.cpp b/src/mongo/rpc/metadata/repl_set_metadata_test.cpp
index 1baa1000dd3..36e447889ec 100644
--- a/src/mongo/rpc/metadata/repl_set_metadata_test.cpp
+++ b/src/mongo/rpc/metadata/repl_set_metadata_test.cpp
@@ -38,17 +38,21 @@ namespace rpc {
 namespace {
 
 using repl::OpTime;
+using repl::OpTimeAndWallTime;
 
 TEST(ReplResponseMetadataTest, ReplicaSetIdNotSet) {
-    ASSERT_FALSE(ReplSetMetadata(3, OpTime(), OpTime(), 6, OID(), 12, -1).hasReplicaSetId());
+    ASSERT_FALSE(
+        ReplSetMetadata(3, OpTimeAndWallTime(), OpTime(), 6, OID(), 12, -1).hasReplicaSetId());
 }
 
 TEST(ReplResponseMetadataTest, Roundtrip) {
     OpTime opTime(Timestamp(1234, 100), 5);
     OpTime opTime2(Timestamp(7777, 100), 6);
-    ReplSetMetadata metadata(3, opTime, opTime2, 6, OID::gen(), 12, -1);
+    Date_t committedWallTime = Date_t::min() + Seconds(opTime.getSecs());
+    ReplSetMetadata metadata(3, {opTime, committedWallTime}, opTime2, 6, OID::gen(), 12, -1);
 
-    ASSERT_EQ(opTime, metadata.getLastOpCommitted());
+    ASSERT_EQ(opTime, metadata.getLastOpCommitted().opTime);
+    ASSERT_EQ(committedWallTime, metadata.getLastOpCommitted().wallTime);
     ASSERT_EQ(opTime2, metadata.getLastOpVisible());
     ASSERT_TRUE(metadata.hasReplicaSetId());
 
@@ -59,6 +63,8 @@ TEST(ReplResponseMetadataTest, Roundtrip) {
         BSON(kReplSetMetadataFieldName
              << BSON("term" << 3 << "lastOpCommitted"
                             << BSON("ts" << opTime.getTimestamp() << "t" << opTime.getTerm())
+                            << "lastCommittedWall"
+                            << committedWallTime
                             << "lastOpVisible"
                             << BSON("ts" << opTime2.getTimestamp() << "t" << opTime2.getTerm())
                             << "configVersion"
@@ -73,12 +79,13 @@ TEST(ReplResponseMetadataTest, Roundtrip) {
     BSONObj serializedObj = builder.obj();
     ASSERT_BSONOBJ_EQ(expectedObj, serializedObj);
 
-    auto cloneStatus = ReplSetMetadata::readFromMetadata(serializedObj);
+    auto cloneStatus = ReplSetMetadata::readFromMetadata(serializedObj, /*requireWallTime*/ true);
     ASSERT_OK(cloneStatus.getStatus());
 
     const auto& clonedMetadata = cloneStatus.getValue();
-    ASSERT_EQ(opTime, clonedMetadata.getLastOpCommitted());
+    ASSERT_EQ(opTime, clonedMetadata.getLastOpCommitted().opTime);
     ASSERT_EQ(opTime2, clonedMetadata.getLastOpVisible());
+    ASSERT_EQ(committedWallTime, clonedMetadata.getLastOpCommitted().wallTime);
     ASSERT_EQ(metadata.getConfigVersion(), clonedMetadata.getConfigVersion());
     ASSERT_EQ(metadata.getReplicaSetId(), clonedMetadata.getReplicaSetId());
 
@@ -94,7 +101,7 @@ TEST(ReplResponseMetadataTest, MetadataCanBeConstructedWhenMissingOplogQueryMeta
     BSONObj obj(BSON(kReplSetMetadataFieldName
                      << BSON("term" << 3 << "configVersion" << 6 << "replicaSetId" << id)));
 
-    auto status = ReplSetMetadata::readFromMetadata(obj);
+    auto status = ReplSetMetadata::readFromMetadata(obj, /*requireWallTime*/ true);
     ASSERT_OK(status.getStatus());
 
     const auto& metadata = status.getValue();