SERVER-41650 Use IDL to parse OpTime and wall fields

12 files changed, 156 insertions, 67 deletions

diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index d281ae99908..9d374e2135e 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -481,6 +481,7 @@ env.Library(
     ],
     LIBDEPS=[
         'optime',
+        'optime_and_wall_time_base',
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/logical_session_id',
         '$BUILD_DIR/mongo/idl/idl_parser',
@@ -488,6 +489,17 @@ env.Library(
 )
 
 env.Library(
+    target='optime_and_wall_time_base',
+    source=[
+        env.Idlc('optime_and_wall_time_base.idl')[0],
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/idl/idl_parser',
+    ],
+)
+
+env.Library(
     target='oplog_application_interface',
     source=[
         'oplog_applier.cpp',
@@ -991,6 +1003,7 @@ env.Library(target='optime',
                 'optime.cpp',
             ],
             LIBDEPS=[
+                'optime_and_wall_time_base',
                 '$BUILD_DIR/mongo/base',
                 '$BUILD_DIR/mongo/bson/util/bson_extract',
             ],
diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp
index 444435d4085..d3f03d1276c 100644
--- a/src/mongo/db/repl/initial_syncer.cpp
+++ b/src/mongo/db/repl/initial_syncer.cpp
@@ -159,16 +159,7 @@ StatusWith<OpTimeAndWallTime> parseOpTimeAndWallTime(const QueryResponseStatus&
                                              "no oplog entry found"};
     }
 
-    auto opTimeStatus = OpTime::parseFromOplogEntry(docs.front());
-    if (!opTimeStatus.getStatus().isOK()) {
-        return opTimeStatus.getStatus();
-    }
-    auto wallTimeStatus = OpTime::parseWallTimeFromOplogEntry(docs.front());
-    if (!wallTimeStatus.getStatus().isOK()) {
-        return wallTimeStatus.getStatus();
-    }
-    OpTimeAndWallTime result = {opTimeStatus.getValue(), wallTimeStatus.getValue()};
-    return result;
+    return OpTimeAndWallTime::parseOpTimeAndWallTimeFromOplogEntry(docs.front());
 }
 
 }  // namespace
diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp
index 9d95ee96821..9abdc1f9b7d 100644
--- a/src/mongo/db/repl/initial_syncer_test.cpp
+++ b/src/mongo/db/repl/initial_syncer_test.cpp
@@ -1429,7 +1429,10 @@ TEST_F(InitialSyncerTest,
     }
 
     initialSyncer->join();
-    ASSERT_EQUALS(ErrorCodes::NoSuchKey, _lastApplied);
+
+    // OpTimeAndWallTime now uses the IDL parser, so the status code returned is from
+    // IDLParserErrorContext
+    ASSERT_EQUALS(_lastApplied.getStatus().code(), 40414);
 }
 
 TEST_F(InitialSyncerTest,
diff --git a/src/mongo/db/repl/oplog_entry.h b/src/mongo/db/repl/oplog_entry.h
index 190de25a9cc..f298614b11b 100644
--- a/src/mongo/db/repl/oplog_entry.h
+++ b/src/mongo/db/repl/oplog_entry.h
@@ -121,6 +121,18 @@ public:
         getDurableReplOperation().setUpsert(std::move(value));
     }
 
+    void setTimestamp(Timestamp value) & {
+        getOpTimeAndWallTimeBase().setTimestamp(std::move(value));
+    }
+
+    void setTerm(boost::optional<std::int64_t> value) & {
+        getOpTimeAndWallTimeBase().setTerm(std::move(value));
+    }
+
+    void setWallClockTime(Date_t value) & {
+        getOpTimeAndWallTimeBase().setWallClockTime(std::move(value));
+    }
+
     /**
      * Sets the OpTime of the oplog entry.
      */
diff --git a/src/mongo/db/repl/oplog_entry.idl b/src/mongo/db/repl/oplog_entry.idl
index f2d1586914e..30d269f0120 100644
--- a/src/mongo/db/repl/oplog_entry.idl
+++ b/src/mongo/db/repl/oplog_entry.idl
@@ -34,6 +34,7 @@ global:
 imports:
     - "mongo/idl/basic_types.idl"
     - "mongo/db/logical_session_id.idl"
+    - "mongo/db/repl/optime_and_wall_time_base.idl"
     - "mongo/db/repl/replication_types.idl"
 
 enums:
@@ -90,16 +91,8 @@ structs:
         chained_structs:
             OperationSessionInfo : OperationSessionInfo
             DurableReplOperation: DurableReplOperation
+            OpTimeAndWallTimeBase: OpTimeAndWallTimeBase
         fields:
-            ts:
-                cpp_name: timestamp
-                type: timestamp
-                description: "The time when the oplog entry was created"
-            t:
-                cpp_name: term
-                type: long
-                optional: true # The term is optional for PV0 oplog entries.
-                description: "The term of the primary that created the oplog entry"
             h:
                 cpp_name: hash
                 type: long
@@ -118,10 +111,6 @@ structs:
                 type: objectid
                 optional: true
                 description: "An optional _id field for tests that manually insert oplog entries"
-            wall:
-                cpp_name: wallClockTime
-                type: date
-                description: "A wallclock time with MS resolution"
             stmtId:
                 cpp_name: statementId
                 type: StmtId
diff --git a/src/mongo/db/repl/oplog_entry_test.cpp b/src/mongo/db/repl/oplog_entry_test.cpp
index e1d4c614118..6b730867dcd 100644
--- a/src/mongo/db/repl/oplog_entry_test.cpp
+++ b/src/mongo/db/repl/oplog_entry_test.cpp
@@ -103,6 +103,37 @@ TEST(OplogEntryTest, Create) {
     ASSERT_EQ(entry.getOpTime(), entryOpTime);
 }
 
+TEST(OplogEntryTest, OpTimeAndWallTimeBaseNonStrictParsing) {
+    const BSONObj oplogEntryExtraField = BSON("ts" << Timestamp(0, 0) << "t" << 0LL << "op"
+                                                   << "c"
+                                                   << "ns" << nss.ns() << "wall" << Date_t() << "o"
+                                                   << BSON("_id" << 1) << "extraField" << 3);
+
+    // OpTimeAndWallTimeBase should be successfully created from an OplogEntry, even though it has
+    // extraneous fields.
+    UNIT_TEST_INTERNALS_IGNORE_UNUSED_RESULT_WARNINGS(OpTimeAndWallTimeBase::parse(
+        IDLParserErrorContext("OpTimeAndWallTimeBase"), oplogEntryExtraField));
+
+    // OplogEntryBase should still use strict parsing and throw an error when it has extraneous
+    // fields.
+    ASSERT_THROWS_CODE(
+        OplogEntryBase::parse(IDLParserErrorContext("OplogEntryBase"), oplogEntryExtraField),
+        AssertionException,
+        40415);
+
+    const BSONObj oplogEntryMissingTimestamp =
+        BSON("t" << 0LL << "op"
+                 << "c"
+                 << "ns" << nss.ns() << "wall" << Date_t() << "o" << BSON("_id" << 1));
+
+    // When an OplogEntryBase is created with a missing required field in a chained struct, it
+    // should throw an exception.
+    ASSERT_THROWS_CODE(
+        OplogEntryBase::parse(IDLParserErrorContext("OplogEntryBase"), oplogEntryMissingTimestamp),
+        AssertionException,
+        40414);
+}
+
 
 }  // namespace
 }  // namespace repl
diff --git a/src/mongo/db/repl/optime.cpp b/src/mongo/db/repl/optime.cpp
index 0c68be9e1a8..683203a4c88 100644
--- a/src/mongo/db/repl/optime.cpp
+++ b/src/mongo/db/repl/optime.cpp
@@ -37,6 +37,7 @@
 #include "mongo/bson/util/bson_extract.h"
 #include "mongo/db/repl/oplog.h"
 #include "mongo/db/repl/optime.h"
+#include "mongo/db/repl/optime_and_wall_time_base_gen.h"
 
 namespace mongo {
 namespace repl {
@@ -58,16 +59,6 @@ void OpTime::append(BSONObjBuilder* builder, const std::string& subObjName) cons
     opTimeBuilder.doneFast();
 }
 
-StatusWith<Date_t> OpTime::parseWallTimeFromOplogEntry(const BSONObj& obj) {
-    BSONElement wallClockTimeElement;
-    Status status = bsonExtractTypedField(
-        obj, OplogEntryBase::kWallClockTimeFieldName, BSONType::Date, &wallClockTimeElement);
-    if (!status.isOK())
-        return status;
-    auto wallClockTime = wallClockTimeElement.Date();
-    return wallClockTime;
-}
-
 StatusWith<OpTime> OpTime::parseFromOplogEntry(const BSONObj& obj) {
     Timestamp ts;
     Status status = bsonExtractTimestampField(obj, kTimestampFieldName, &ts);
@@ -123,6 +114,25 @@ BSONObj OpTime::asQuery() const {
     return builder.obj();
 }
 
+StatusWith<OpTimeAndWallTime> OpTimeAndWallTime::parseOpTimeAndWallTimeFromOplogEntry(
+    const BSONObj& bsonObject) {
+
+    try {
+        OpTimeAndWallTimeBase base = OpTimeAndWallTimeBase::parse(
+            IDLParserErrorContext("OpTimeAndWallTimeBase"), bsonObject);
+
+        long long term = OpTime::kUninitializedTerm;
+
+        if (base.getTerm()) {
+            term = base.getTerm().get();
+        }
+
+        return OpTimeAndWallTime(OpTime(base.getTimestamp(), term), base.getWallClockTime());
+    } catch (...) {
+        return exceptionToStatus();
+    }
+}
+
 }  // namespace repl
 
 BSONObjBuilder& operator<<(BSONObjBuilderValueStream& builder, const repl::OpTime& value) {
diff --git a/src/mongo/db/repl/optime.h b/src/mongo/db/repl/optime.h
index ed7c5df7e93..9d045da574e 100644
--- a/src/mongo/db/repl/optime.h
+++ b/src/mongo/db/repl/optime.h
@@ -31,6 +31,7 @@
 
 #include <tuple>
 
+#include "mongo/bson/bsonobj.h"
 #include "mongo/bson/timestamp.h"
 
 namespace mongo {
@@ -91,8 +92,6 @@ public:
     void append(BSONObjBuilder* builder, const std::string& subObjName) const;
     BSONObj toBSON() const;
 
-    static StatusWith<Date_t> parseWallTimeFromOplogEntry(const BSONObj& obj);
-
     static StatusWith<OpTime> parseFromOplogEntry(const BSONObj& obj);
 
     /**
@@ -160,9 +159,18 @@ private:
     long long _term = kInitialTerm;
 };
 
-struct OpTimeAndWallTime {
-    OpTime opTime;
+class OpTimeAndWallTime {
+public:
+    OpTime opTime = OpTime();
     Date_t wallTime = Date_t();
+
+    static StatusWith<OpTimeAndWallTime> parseOpTimeAndWallTimeFromOplogEntry(
+        const BSONObj& bsonObject);
+
+    OpTimeAndWallTime() {}
+
+    OpTimeAndWallTime(OpTime optime, Date_t wall) : opTime(optime), wallTime(wall) {}
+
     inline bool operator==(const OpTimeAndWallTime& rhs) const {
         return opTime == rhs.opTime && wallTime == rhs.wallTime;
     }
diff --git a/src/mongo/db/repl/optime_and_wall_time_base.idl b/src/mongo/db/repl/optime_and_wall_time_base.idl
new file mode 100644
index 00000000000..f9f1108fbab
--- /dev/null
+++ b/src/mongo/db/repl/optime_and_wall_time_base.idl
@@ -0,0 +1,55 @@
+# Copyright (C) 2019-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+
+# This IDL file specifies the fields for OpTimeAndWallTimeBase.
+
+global:
+    cpp_namespace: "mongo::repl"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+structs:
+    OpTimeAndWallTimeBase:
+        description: "A document that stores the time-related fields in an oplog entry. Should 
+                     never be used directly in server code. Instead, create an instance of 
+                     OpTimeAndWallTime."
+        strict: false
+        fields:
+            ts:
+                cpp_name: timestamp
+                type: timestamp
+                description: "The time when the oplog entry was created"
+            t:
+                cpp_name: term
+                type: long
+                optional: true # The term will no longer be optional, see SERVER-42258.
+                description: "The term of the primary that created the oplog entry"
+            wall:
+                cpp_name: wallClockTime
+                type: date
+                description: "A wallclock time with MS resolution"
\ No newline at end of file
diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h
index 419a5ad9989..81f9609eae5 100644
--- a/src/mongo/db/repl/replication_coordinator.h
+++ b/src/mongo/db/repl/replication_coordinator.h
@@ -72,7 +72,7 @@ namespace repl {
 class BackgroundSync;
 class IsMasterResponse;
 class OpTime;
-struct OpTimeAndWallTime;
+class OpTimeAndWallTime;
 class ReadConcernArgs;
 class ReplSetConfig;
 class ReplSetHeartbeatArgsV1;
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 05806b9b9db..5cf10287b04 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -651,32 +651,9 @@ StatusWith<OpTimeAndWallTime> ReplicationCoordinatorExternalStateImpl::loadLastO
                                                      << "Did not find any entries in "
                                                      << NamespaceString::kRsOplogNamespace.ns());
         }
-        BSONElement tsElement = oplogEntry[tsFieldName];
-        if (tsElement.eoo()) {
-            return StatusWith<OpTimeAndWallTime>(
-                ErrorCodes::NoSuchKey,
-                str::stream() << "Most recent entry in " << NamespaceString::kRsOplogNamespace.ns()
-                              << " missing \"" << tsFieldName << "\" field");
-        }
-        if (tsElement.type() != bsonTimestamp) {
-            return StatusWith<OpTimeAndWallTime>(
-                ErrorCodes::TypeMismatch,
-                str::stream() << "Expected type of \"" << tsFieldName << "\" in most recent "
-                              << NamespaceString::kRsOplogNamespace.ns()
-                              << " entry to have type Timestamp, but found "
-                              << typeName(tsElement.type()));
-        }
 
-        auto opTimeStatus = OpTime::parseFromOplogEntry(oplogEntry);
-        if (!opTimeStatus.isOK()) {
-            return opTimeStatus.getStatus();
-        }
-        auto wallTimeStatus = OpTime::parseWallTimeFromOplogEntry(oplogEntry);
-        if (!wallTimeStatus.isOK()) {
-            return wallTimeStatus.getStatus();
-        }
-        OpTimeAndWallTime parseResult = {opTimeStatus.getValue(), wallTimeStatus.getValue()};
-        return parseResult;
+        return OpTimeAndWallTime::parseOpTimeAndWallTimeFromOplogEntry(oplogEntry);
+
     } catch (const DBException& ex) {
         return StatusWith<OpTimeAndWallTime>(ex.toStatus());
     }
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index 4b1a4ff33e9..ba1739049fb 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -599,7 +599,7 @@ void ReplicationCoordinatorImpl::_finishLoadLocalConfig(
     // Do not check optime, if this node is an arbiter.
     bool isArbiter =
         myIndex.getValue() != -1 && localConfig.getMemberAt(myIndex.getValue()).isArbiter();
-    OpTimeAndWallTime lastOpTimeAndWallTime = {OpTime(), Date_t()};
+    OpTimeAndWallTime lastOpTimeAndWallTime = OpTimeAndWallTime();
     if (!isArbiter) {
         if (!lastOpTimeAndWallTimeStatus.isOK()) {
             warning() << "Failed to load timestamp and/or wall clock time of most recently applied "
@@ -1191,8 +1191,8 @@ void ReplicationCoordinatorImpl::_resetMyLastOpTimes(WithLock lk) {
     // Reset to uninitialized OpTime
     bool isRollbackAllowed = true;
     _setMyLastAppliedOpTimeAndWallTime(
-        lk, {OpTime(), Date_t()}, isRollbackAllowed, DataConsistency::Inconsistent);
-    _setMyLastDurableOpTimeAndWallTime(lk, {OpTime(), Date_t()}, isRollbackAllowed);
+        lk, OpTimeAndWallTime(), isRollbackAllowed, DataConsistency::Inconsistent);
+    _setMyLastDurableOpTimeAndWallTime(lk, OpTimeAndWallTime(), isRollbackAllowed);
     _stableOpTimeCandidates.clear();
 }