SERVER-13704 write commands handles ClockSkewException differently from legacy writes

22 files changed, 233 insertions, 212 deletions

diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index 801964498a0..d318d6467b3 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -1249,15 +1249,6 @@ var authCommandsLib = {
             ]
         },
         {
-            testname: "getoptime",
-            command: {getoptime: 1},
-            skipSharded: true,
-            testcases: [
-                { runOnDb: firstDbName, roles: roles_all, privileges: [ ] },
-                { runOnDb: secondDbName, roles: roles_all, privileges: [ ] }
-            ]
-        },
-        {
             testname: "getShardMap",
             command: {getShardMap: "x"},
             testcases: [
diff --git a/jstests/repl/master1.js b/jstests/repl/master1.js
index 9f021fc6a2a..93bfaf7862c 100644
--- a/jstests/repl/master1.js
+++ b/jstests/repl/master1.js
@@ -44,6 +44,10 @@ rt.stop( true );
 m = rt.start( true, null, true );
 assert.eq( op.ts.i, lastop().ts.i );
 am().save( {} );
-sleep( 3000 ); // make sure dies on its own before stop() called
 
-assert.eq( 47 /*EXIT_CLOCK_SKEW*/, rt.stop( true ) );
\ No newline at end of file
+// The above write should cause the server to terminate
+assert.throws(function() {
+    am().findOne();
+});
+
+assert.neq(0, rt.stop( true )); // fasserted
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 81a47c0978a..45b36857c03 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -222,6 +222,8 @@ env.Library('synchronization', [ 'util/concurrency/synchronization.cpp' ])
 
 env.Library('auth_helpers', ['client/auth_helpers.cpp'])
 
+env.Library('global_optime', ['db/global_optime.cpp'])
+
 env.Library('spin_lock', ["util/concurrency/spin_lock.cpp"])
 env.CppUnitTest('spin_lock_test', ['util/concurrency/spin_lock_test.cpp'],
                 LIBDEPS=['spin_lock', '$BUILD_DIR/third_party/shim_boost'])
@@ -892,6 +894,7 @@ env.Library("serveronly", serverOnlyFiles,
                      "defaultversion",
                      "geoparser",
                      "geoquery",
+                     "global_optime",
                      "index_key_validate",
                      "index_set",
                      'range_deleter',
diff --git a/src/mongo/bson/optime.cpp b/src/mongo/bson/optime.cpp
index 6cc01969366..f2f22fcdd91 100644
--- a/src/mongo/bson/optime.cpp
+++ b/src/mongo/bson/optime.cpp
@@ -20,86 +20,13 @@
 #include <ctime>
 
 #include "mongo/bson/inline_decls.h"
-#include "mongo/util/debug_util.h"
-#include "mongo/util/log.h"
-#include "mongo/util/startup_test.h"
 
 namespace mongo {
 
-    OpTime OpTime::last(0, 0);
-    boost::condition OpTime::notifier;
-    mongo::mutex OpTime::m("optime");
-
-    NOINLINE_DECL OpTime OpTime::skewed() {
-        bool toLog = false;
-        ONCE toLog = true;
-        RARELY toLog = true;
-        last.i++;
-        if ( last.i & 0x80000000 )
-            toLog = true;
-        if ( toLog ) {
-            log() << "clock skew detected  prev: " << last.secs << " now: " << (unsigned) time(0) 
-                  << std::endl;
-        }
-        if ( last.i & 0x80000000 ) {
-            log() << "error large clock skew detected, shutting down" << std::endl;
-            throw ClockSkewException();
-        }
-        return last;
-    }
-
-    /*static*/ OpTime OpTime::_now() {
-        OpTime result;
-        unsigned t = (unsigned) time(0);
-        if ( last.secs == t ) {
-            last.i++;
-            result = last;
-        }
-        else if ( t < last.secs ) {
-            result = skewed(); // separate function to keep out of the hot code path
-        }
-        else { 
-            last = OpTime(t, 1);
-            result = last;
-        }
-        notifier.notify_all();
-        return last;
-    }
-
-    OpTime OpTime::now(const mongo::mutex::scoped_lock&) {
-        return _now();
-    }
-
-    OpTime OpTime::getLast(const mongo::mutex::scoped_lock&) {
-        return last;
-    }
-
     OpTime OpTime::max() {
         unsigned int t = static_cast<unsigned int>(std::numeric_limits<int32_t>::max());
         unsigned int i = std::numeric_limits<uint32_t>::max();
         return OpTime(t, i);
     }
 
-    void OpTime::waitForDifferent(unsigned millis){
-        mutex::scoped_lock lk(m);
-        while (*this == last) {
-            if (!notifier.timed_wait(lk.boost(), boost::posix_time::milliseconds(millis)))
-                return; // timed out
-        }
-    }
-
-    struct TestOpTime : public StartupTest {
-        void run() {
-            OpTime t;
-            for ( int i = 0; i < 10; i++ ) {
-                OpTime s = OpTime::_now();
-                verify( s != t );
-                t = s;
-            }
-            OpTime q = t;
-            verify( q == t );
-            verify( !(q != t) );
-        }
-    } testoptime;
-
 }
diff --git a/src/mongo/bson/optime.h b/src/mongo/bson/optime.h
index 6d549a38824..79717e1b668 100644
--- a/src/mongo/bson/optime.h
+++ b/src/mongo/bson/optime.h
@@ -41,26 +41,14 @@ namespace mongo {
     class OpTime {
         unsigned i; // ordinal comes first so we can do a single 64 bit compare on little endian
         unsigned secs;
-        static OpTime last;
-        static OpTime skewed();
     public:
-        static void setLast(const Date_t &date) {
-            mutex::scoped_lock lk(m);
-            last = OpTime(date);
-            notifier.notify_all();
-        }
-        static void setLast(const OpTime &new_last) {
-            mutex::scoped_lock lk(m);
-            last = new_last;
-            notifier.notify_all();
-        }
-
         unsigned getSecs() const {
             return secs;
         }
         unsigned getInc() const {
             return i;
         }
+
         OpTime(Date_t date) {
             reinterpret_cast<unsigned long long&>(*this) = date.millis;
             dassert( (int)secs >= 0 );
@@ -83,21 +71,10 @@ namespace mongo {
             secs = 0;
             i = 0;
         }
-        // it isn't generally safe to not be locked for this. so use now(). some tests use this.
-        static OpTime _now();
-
-        static mongo::mutex m;
-
-        static OpTime now(const mongo::mutex::scoped_lock&);
-
-        static OpTime getLast(const mongo::mutex::scoped_lock&);
 
         // Maximum OpTime value.
         static OpTime max();
 
-        // Waits for global OpTime to be different from *this
-        void waitForDifferent(unsigned millis);
-
         /* We store OpTime's in the database as BSON Date datatype -- we needed some sort of
          64 bit "container" for these values.  While these are not really "Dates", that seems a
          better choice for now than say, Number, which is floating point.  Note the BinData type
@@ -152,8 +129,6 @@ namespace mongo {
         bool operator>=(const OpTime& r) const {
             return !(*this < r);
         }
-    private:
-        static boost::condition notifier;
     };
 #pragma pack()
 
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 6cc88baf434..9b54b14cb24 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -194,13 +194,7 @@ namespace mongo {
                 lastError.startRequest( m , le );
 
                 DbResponse dbresponse;
-                try {
-                    assembleResponse( m, dbresponse, port->remote() );
-                }
-                catch ( const ClockSkewException & ) {
-                    log() << "ClockSkewException - shutting down" << endl;
-                    exitCleanly( EXIT_CLOCK_SKEW );
-                }
+                assembleResponse( m, dbresponse, port->remote() );
 
                 if ( dbresponse.response ) {
                     port->reply(m, *dbresponse.response, dbresponse.responseTo);
diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp
index e570e4485de..f980c83e6df 100644
--- a/src/mongo/db/dbcommands.cpp
+++ b/src/mongo/db/dbcommands.cpp
@@ -382,25 +382,6 @@ namespace mongo {
         }
     } cmdProfile;
 
-    class CmdGetOpTime : public Command {
-    public:
-        virtual bool slaveOk() const {
-            return true;
-        }
-        virtual void help( stringstream& help ) const { help << "internal"; }
-        virtual bool isWriteCommandForConfigServer() const { return false; }
-        CmdGetOpTime() : Command("getoptime") { }
-        virtual void addRequiredPrivileges(const std::string& dbname,
-                                           const BSONObj& cmdObj,
-                                           std::vector<Privilege>* out) {} // No auth required
-
-        bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            mutex::scoped_lock lk(OpTime::m);
-            result.appendDate("optime", OpTime::now(lk).asDate());
-            return true;
-        }
-    } cmdgetoptime;
-
     class CmdDiagLogging : public Command {
     public:
         virtual bool slaveOk() const {
diff --git a/src/mongo/db/global_optime.cpp b/src/mongo/db/global_optime.cpp
new file mode 100644
index 00000000000..f7634822cb6
--- /dev/null
+++ b/src/mongo/db/global_optime.cpp
@@ -0,0 +1,78 @@
+/**
+ *    Copyright (C) 2014 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/global_optime.h"
+#include "mongo/util/concurrency/mutex.h"
+#include "mongo/util/log.h"
+
+namespace {
+    mongo::mutex globalOptimeMutex("globalOptime");
+    mongo::OpTime globalOpTime(0, 0);
+
+    bool skewed(const mongo::OpTime& val) {
+        if (val.getInc() & 0x80000000) {
+            mongo::warning() << "clock skew detected  prev: " << val.getSecs()
+                             << " now: " << (unsigned) time(0) << std::endl;
+            return true;
+        }
+
+        return false;
+    }
+}
+
+namespace mongo {
+    void setGlobalOptime(const OpTime& newTime) {
+        mutex::scoped_lock lk(globalOptimeMutex);
+        globalOpTime = newTime;
+    }
+
+    OpTime getLastSetOptime() {
+        mutex::scoped_lock lk(globalOptimeMutex);
+        return globalOpTime;
+    }
+
+    OpTime getNextGlobalOptime() {
+        mutex::scoped_lock lk(globalOptimeMutex);
+
+        const unsigned now = (unsigned) time(0);
+        const unsigned globalSecs = globalOpTime.getSecs();
+        if ( globalSecs == now ) {
+            globalOpTime = OpTime(globalSecs, globalOpTime.getInc() + 1);
+        }
+        else if ( now < globalSecs ) {
+            globalOpTime = OpTime(globalSecs, globalOpTime.getInc() + 1);
+            // separate function to keep out of the hot code path
+            fassert(17449, !skewed(globalOpTime));
+        }
+        else {
+            globalOpTime = OpTime(now, 1);
+        }
+
+        return globalOpTime;
+    }
+}
diff --git a/src/mongo/db/global_optime.h b/src/mongo/db/global_optime.h
new file mode 100644
index 00000000000..acb6e2d0ae6
--- /dev/null
+++ b/src/mongo/db/global_optime.h
@@ -0,0 +1,45 @@
+/**
+ *    Copyright (C) 2014 MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/bson/optime.h"
+
+namespace mongo {
+    void setGlobalOptime(const OpTime& newTime);
+
+    /**
+     * Returns the value of the global OpTime generated last time or set.
+     */
+    OpTime getLastSetOptime();
+
+    /**
+     * Generates a new and unique OpTime.
+     */
+    OpTime getNextGlobalOptime();
+}
diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp
index 46b75fd984d..4b47dc50396 100644
--- a/src/mongo/db/instance.cpp
+++ b/src/mongo/db/instance.cpp
@@ -56,6 +56,7 @@
 #include "mongo/db/storage/mmap_v1/dur_journal.h"
 #include "mongo/db/storage/mmap_v1/dur_recover.h"
 #include "mongo/db/storage/mmap_v1/dur_transaction.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/instance.h"
 #include "mongo/db/introspect.h"
 #include "mongo/db/json.h"
@@ -690,11 +691,10 @@ namespace mongo {
                     }
 
                     if (pass == 0) {
-                        mutex::scoped_lock lk(OpTime::m);
-                        last = OpTime::getLast(lk);
+                        last = getLastSetOptime();
                     }
                     else {
-                        last.waitForDifferent(1000/*ms*/);
+                        waitForOptimeChange(last, 1000/*ms*/);
                     }
                 }
 
diff --git a/src/mongo/db/ops/SConscript b/src/mongo/db/ops/SConscript
index 5ef3198bd1a..8748b5586e1 100644
--- a/src/mongo/db/ops/SConscript
+++ b/src/mongo/db/ops/SConscript
@@ -67,6 +67,7 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/expressions',
+        '$BUILD_DIR/mongo/global_optime',
         'update_common',
     ],
 )
diff --git a/src/mongo/db/ops/insert.cpp b/src/mongo/db/ops/insert.cpp
index 9d164437ed0..62c1fe739b6 100644
--- a/src/mongo/db/ops/insert.cpp
+++ b/src/mongo/db/ops/insert.cpp
@@ -29,6 +29,7 @@
  */
 
 #include "mongo/db/ops/insert.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/structure/catalog/namespace.h"
 #include "mongo/util/mongoutils/str.h"
 
@@ -122,8 +123,7 @@ namespace mongo {
                 // no-op
             }
             else if ( e.type() == Timestamp && e.timestampValue() == 0 ) {
-                mutex::scoped_lock lk(OpTime::m);
-                b.append( e.fieldName(), OpTime::now(lk) );
+                b.append( e.fieldName(), getNextGlobalOptime() );
             }
             else {
                 b.append( e );
diff --git a/src/mongo/db/ops/modifier_current_date.cpp b/src/mongo/db/ops/modifier_current_date.cpp
index 8e721def6db..e00f64631e6 100644
--- a/src/mongo/db/ops/modifier_current_date.cpp
+++ b/src/mongo/db/ops/modifier_current_date.cpp
@@ -30,6 +30,7 @@
 
 #include "mongo/base/error_codes.h"
 #include "mongo/bson/mutable/document.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/ops/field_checker.h"
 #include "mongo/db/ops/log_builder.h"
 #include "mongo/db/ops/path_support.h"
@@ -245,9 +246,7 @@ namespace mongo {
                 return s;
         }
         else {
-            mutex::scoped_lock lk(OpTime::m);
-            const OpTime timestamp = OpTime::now(lk);
-            Status s = elemToSet.setValueTimestamp(timestamp);
+            Status s = elemToSet.setValueTimestamp(getNextGlobalOptime());
             if (!s.isOK())
                 return s;
         }
diff --git a/src/mongo/db/ops/modifier_object_replace.cpp b/src/mongo/db/ops/modifier_object_replace.cpp
index 4748286b9d8..9c026d4f069 100644
--- a/src/mongo/db/ops/modifier_object_replace.cpp
+++ b/src/mongo/db/ops/modifier_object_replace.cpp
@@ -30,6 +30,7 @@
 
 #include "mongo/base/error_codes.h"
 #include "mongo/bson/mutable/document.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/ops/log_builder.h"
 #include "mongo/util/mongoutils/str.h"
 
@@ -40,7 +41,7 @@ namespace mongo {
     namespace {
         const char idFieldName[] = "_id";
 
-        void fixupTimestamps( const BSONObj& obj ) {
+        Status fixupTimestamps( const BSONObj& obj ) {
             BSONObjIterator i(obj);
             while (i.more()) {
                 BSONElement e = i.next();
@@ -52,11 +53,13 @@ namespace mongo {
                         *(reinterpret_cast<unsigned long long*>(
                               const_cast<char *>(e.value())));
                     if (timestamp == 0) {
-                        mutex::scoped_lock lk(OpTime::m);
-                        timestamp = OpTime::now(lk).asDate();
+                        OpTime ts(getNextGlobalOptime());
+                        timestamp = ts.asDate();
                     }
                 }
             }
+
+            return Status::OK();
         }
     }
 
@@ -99,9 +102,7 @@ namespace mongo {
         // We make a copy of the object here because the update driver does not guarantees, in
         // the case of object replacement, that the modExpr is going to outlive this mod.
         _val = modExpr.embeddedObject().getOwned();
-        fixupTimestamps(_val);
-
-        return Status::OK();
+        return fixupTimestamps(_val);
     }
 
     Status ModifierObjectReplace::prepare(mutablebson::Element root,
diff --git a/src/mongo/db/repl/consensus.cpp b/src/mongo/db/repl/consensus.cpp
index bd48303123c..719627aefff 100644
--- a/src/mongo/db/repl/consensus.cpp
+++ b/src/mongo/db/repl/consensus.cpp
@@ -28,6 +28,7 @@
 
 #include "mongo/db/repl/consensus.h"
 
+#include "mongo/db/global_optime.h"
 #include "mongo/db/repl/multicmd.h"
 #include "mongo/db/repl/replset_commands.h"
 
@@ -451,10 +452,9 @@ namespace mongo {
                     /* succeeded. */
                     LOG(1) << "replSet election succeeded, assuming primary role" << rsLog;
                     success = true;
-                    {
-                        mutex::scoped_lock lk(OpTime::m);
-                        setElectionTime(OpTime::now(lk));
-                    }
+
+                    setElectionTime(getNextGlobalOptime());
+
                     rs.assumePrimary();
                 }
             }
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index c45143687b7..5de7788a091 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/dbhash.h"
 #include "mongo/db/dbhelpers.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/index_builder.h"
 #include "mongo/db/instance.h"
 #include "mongo/db/namespace_string.h"
@@ -70,6 +71,17 @@ namespace mongo {
     static Collection* localOplogMainCollection = 0;
     static Collection* localOplogRSCollection = 0;
 
+    // Synchronizes the section where a new OpTime is generated and when it actually
+    // appears in the oplog.
+    static mongo::mutex newOpMutex("oplogNewOp");
+    static boost::condition newOptimeNotifier;
+
+    static void setNewOptime(const OpTime& newTime) {
+        mutex::scoped_lock lk(newOpMutex);
+        setGlobalOptime(newTime);
+        newOptimeNotifier.notify_all();
+    }
+
     void oplogCheckCloseDatabase( Database* db ) {
         verify( Lock::isW() );
         localDB = NULL;
@@ -142,7 +154,7 @@ namespace mongo {
             }
         }
 
-        OpTime::setLast( ts );
+        setNewOptime(ts);
     }
 
     /**
@@ -226,9 +238,11 @@ namespace mongo {
             return;
         }
 
-        mutex::scoped_lock lk2(OpTime::m);
+        mutex::scoped_lock lk2(newOpMutex);
+
+        OpTime ts(getNextGlobalOptime());
+        newOptimeNotifier.notify_all();
 
-        const OpTime ts = OpTime::now(lk2);
         long long hashNew;
         if( theReplSet ) {
             if (!theReplSet->box.getState().primary()) {
@@ -316,9 +330,11 @@ namespace mongo {
             return;
         }
 
-        mutex::scoped_lock lk2(OpTime::m);
+        mutex::scoped_lock lk2(newOpMutex);
+
+        OpTime ts(getNextGlobalOptime());
+        newOptimeNotifier.notify_all();
 
-        const OpTime ts = OpTime::now(lk2);
         Client::Context context("", 0);
 
         /* we jump through a bunch of hoops here to avoid copying the obj buffer twice --
@@ -444,11 +460,7 @@ namespace mongo {
 
             if( rs ) return;
 
-            DBDirectClient c;
-            BSONObj lastOp = c.findOne( ns, Query().sort(reverseNaturalObj) );
-            if ( !lastOp.isEmpty() ) {
-                OpTime::setLast( lastOp[ "ts" ].date() );
-            }
+            initOpTimeFromOplog(ns);
             return;
         }
 
@@ -737,4 +749,30 @@ namespace mongo {
                 !fieldB.eoo() ? &valueB : NULL );
         return failedUpdate;
     }
+
+    bool waitForOptimeChange(const OpTime& referenceTime, unsigned timeoutMillis) {
+        mutex::scoped_lock lk(newOpMutex);
+
+        while (referenceTime == getLastSetOptime()) {
+            if (!newOptimeNotifier.timed_wait(lk.boost(),
+                                              boost::posix_time::milliseconds(timeoutMillis)))
+                return false;
+        }
+
+        return true;
+
+    }
+
+    void initOpTimeFromOplog(const std::string& oplogNS) {
+        DBDirectClient c;
+        BSONObj lastOp = c.findOne(oplogNS,
+                                   Query().sort(reverseNaturalObj),
+                                   NULL,
+                                   QueryOption_SlaveOk);
+
+        if (!lastOp.isEmpty()) {
+            LOG(1) << "replSet setting last OpTime";
+            setNewOptime(lastOp[ "ts" ].date());
+        }
+    }
 }
diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h
index 119579ee3bf..2e4a1830b17 100644
--- a/src/mongo/db/repl/oplog.h
+++ b/src/mongo/db/repl/oplog.h
@@ -33,6 +33,7 @@ namespace mongo {
     class BSONObj;
     class Database;
     class TransactionExperiment;
+    class OpTime;
 
     // These functions redefine the function for logOp(),
     // for either master/slave or replica sets.
@@ -101,4 +102,15 @@ namespace mongo {
                                const BSONObj& op,
                                bool fromRepl = true,
                                bool convertUpdateToUpsert = false);
+
+    /**
+     * Waits until the given timeout for the OpTime from the oplog to change.
+     * Returns true if the OpTime changed occured before the timeout.
+     */
+    bool waitForOptimeChange(const OpTime& referenceTime, unsigned timeoutMillis);
+
+    /**
+     * Initializes the global OpTime with the value from the timestamp of the last oplog entry.
+     */
+    void initOpTimeFromOplog(const std::string& oplogNS);
 }
diff --git a/src/mongo/db/repl/rs.cpp b/src/mongo/db/repl/rs.cpp
index 3ba266724c0..94a6ab8979c 100644
--- a/src/mongo/db/repl/rs.cpp
+++ b/src/mongo/db/repl/rs.cpp
@@ -145,13 +145,7 @@ namespace {
         LOG(1) << "replSet waiting for global write lock";
         Lock::GlobalWrite lk;
 
-        // Make sure that new OpTimes are higher than existing ones even with clock skew
-        DBDirectClient c;
-        BSONObj lastOp = c.findOne( "local.oplog.rs", Query().sort(reverseNaturalObj), NULL, QueryOption_SlaveOk );
-        if ( !lastOp.isEmpty() ) {
-            LOG(1) << "replSet setting last OpTime";
-            OpTime::setLast( lastOp[ "ts" ].date() );
-        }
+        initOpTimeFromOplog("local.oplog.rs");
 
         // Generate new election unique id
         elect.setElectionId(OID::gen());
diff --git a/src/mongo/dbtests/querytests.cpp b/src/mongo/dbtests/querytests.cpp
index 7190885849d..4ce36bbe88f 100644
--- a/src/mongo/dbtests/querytests.cpp
+++ b/src/mongo/dbtests/querytests.cpp
@@ -33,6 +33,7 @@
 #include "mongo/client/dbclientcursor.h"
 #include "mongo/db/clientcursor.h"
 #include "mongo/db/dbhelpers.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/instance.h"
 #include "mongo/db/json.h"
 #include "mongo/db/kill_current_op.h"
@@ -590,9 +591,10 @@ namespace QueryTests {
                                 BSON( "create" << "querytests.OplogReplaySlaveReadTill" <<
                                      "capped" << true << "size" << 8192 ),
                                 info );
-            Date_t one = OpTime::_now().asDate();
-            Date_t two = OpTime::_now().asDate();
-            Date_t three = OpTime::_now().asDate();
+
+            Date_t one = getNextGlobalOptime().asDate();
+            Date_t two = getNextGlobalOptime().asDate();
+            Date_t three = getNextGlobalOptime().asDate();
             insert( ns, BSON( "ts" << one ) );
             insert( ns, BSON( "ts" << two ) );
             insert( ns, BSON( "ts" << three ) );
diff --git a/src/mongo/dbtests/replsettests.cpp b/src/mongo/dbtests/replsettests.cpp
index c17e1e98427..573be60d3d7 100644
--- a/src/mongo/dbtests/replsettests.cpp
+++ b/src/mongo/dbtests/replsettests.cpp
@@ -32,6 +32,7 @@
 #include "mongo/pch.h"
 
 #include "mongo/db/db.h"
+#include "mongo/db/global_optime.h"
 #include "mongo/db/index_builder.h"
 #include "mongo/db/instance.h"
 #include "mongo/db/json.h"
@@ -242,13 +243,7 @@ namespace ReplSetTests {
     class TestInitApplyOp : public Base {
     public:
         void run() {
-
-            OpTime o;
-
-            {
-                mongo::mutex::scoped_lock lk2(OpTime::m);
-                o = OpTime::now(lk2);
-            }
+            OpTime o(getNextGlobalOptime());
 
             BSONObjBuilder b;
             b.append("ns","dummy");
@@ -289,7 +284,7 @@ namespace ReplSetTests {
     class TestInitApplyOp2 : public Base {
     public:
         void run() {
-            OpTime o = OpTime::_now();
+            OpTime o(getNextGlobalOptime());
 
             BSONObjBuilder b;
             b.appendTimestamp("ts", o.asLL());
@@ -338,10 +333,9 @@ namespace ReplSetTests {
 
         BSONObj updateFail() {
             BSONObjBuilder b;
-            {
-                mongo::mutex::scoped_lock lk2(OpTime::m);
-                b.appendTimestamp("ts", OpTime::now(lk2).asLL());
-            }
+            OpTime ts(getNextGlobalOptime());
+
+            b.appendTimestamp("ts", ts.asLL());
             b.append("op", "u");
             b.append("o", BSON("$set" << BSON("x" << 456)));
             b.append("o2", BSON("_id" << 123 << "x" << 123));
@@ -385,10 +379,9 @@ namespace ReplSetTests {
     class CappedUpdate : public CappedInitialSync {
         void updateSucceed() {
             BSONObjBuilder b;
-            {
-                mongo::mutex::scoped_lock lk2(OpTime::m);
-                b.appendTimestamp("ts", OpTime::now(lk2).asLL());
-            }
+            OpTime ts(getNextGlobalOptime());
+
+            b.appendTimestamp("ts", ts.asLL());
             b.append("op", "u");
             b.append("o", BSON("$set" << BSON("x" << 789)));
             b.append("o2", BSON("x" << 456));
@@ -433,10 +426,9 @@ namespace ReplSetTests {
     class CappedInsert : public CappedInitialSync {
         void insertSucceed() {
             BSONObjBuilder b;
-            {
-                mongo::mutex::scoped_lock lk2(OpTime::m);
-                b.appendTimestamp("ts", OpTime::now(lk2).asLL());
-            }
+            OpTime ts(getNextGlobalOptime());
+
+            b.appendTimestamp("ts", ts.asLL());
             b.append("op", "i");
             b.append("o", BSON("_id" << 123 << "x" << 456));
             b.append("ns", cappedNs());
@@ -462,11 +454,7 @@ namespace ReplSetTests {
 
         void addOp(const string& op, BSONObj o, BSONObj* o2 = NULL, const char* coll = NULL,
                    int version = 0) {
-            OpTime ts;
-            {
-                Lock::GlobalWrite lk;
-                ts = OpTime::_now();
-            }
+            OpTime ts(getNextGlobalOptime());
 
             BSONObjBuilder b;
             b.appendTimestamp("ts", ts.asLL());
diff --git a/src/mongo/s/grid.cpp b/src/mongo/s/grid.cpp
index e37b3244a44..868a74124b6 100644
--- a/src/mongo/s/grid.cpp
+++ b/src/mongo/s/grid.cpp
@@ -589,18 +589,6 @@ namespace mongo {
         return false;
     }
 
-    unsigned long long Grid::getNextOpTime() const {
-        ScopedDbConnection conn(configServer.getPrimary().getConnString(), 30);
-
-        BSONObj result;
-        massert( 10421,
-                 "getoptime failed",
-                 conn->simpleCommand( "admin" , &result , "getoptime" ) );
-        conn.done();
-
-        return result["optime"]._numberLong();
-    }
-
     bool Grid::_isSpecialLocalDB( const string& dbName ) {
         return ( dbName == "local" ) || ( dbName == "admin" ) || ( dbName == "config" );
     }
diff --git a/src/mongo/util/exit_code.h b/src/mongo/util/exit_code.h
index ac6dec938e7..5362a3a07b1 100644
--- a/src/mongo/util/exit_code.h
+++ b/src/mongo/util/exit_code.h
@@ -35,7 +35,7 @@ namespace mongo {
         EXIT_OOM_MALLOC = 42 ,
         EXIT_OOM_REALLOC = 43 ,
         EXIT_FS = 45 ,
-        EXIT_CLOCK_SKEW = 47 ,
+        EXIT_CLOCK_SKEW = 47 , // OpTime clock skew, deprecated
         EXIT_NET_ERROR = 48 ,
         EXIT_WINDOWS_SERVICE_STOP = 49 ,
         EXIT_POSSIBLE_CORRUPTION = 60 , // this means we detected a possible corruption situation, like a buf overflow