SERVER-6825 convert updates to upserts on oplog application

During initial sync, you must apply updates as updates and upserts as upserts,
since it is possible that an in-place update will move a document backwards and
cause the cloner to miss it.  We need to detect this situation on the secondary.

However, during normal replication, we need all updates to be upserts.
Consider the situation where the oplog is replayed (possibly due to a server
crash).  If an update and subsequent delete are played and then replyed,
the reply will hit an error on the update because the document will not exist.
Converting the update to an upsert will allow oplog application to proceed;
while it may result in an incomplete document, this document will be deleted
soon afterward.

4 files changed, 9 insertions, 7 deletions

diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp
index fb6c302e722..d7f87074c5b 100644
--- a/src/mongo/db/oplog.cpp
+++ b/src/mongo/db/oplog.cpp
@@ -720,7 +720,7 @@ namespace mongo {
     /** @param fromRepl false if from ApplyOpsCmd
         @return true if was and update should have happened and the document DNE.  see replset initial sync code.
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl ) {
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl, bool convertUpdateToUpsert) {
         LOG(6) << "applying op: " << op << endl;
         bool failedUpdate = false;
 
@@ -789,7 +789,7 @@ namespace mongo {
 
             OpDebug debug;
             BSONObj updateCriteria = op.getObjectField("o2");
-            bool upsert = fields[3].booleanSafe();
+            bool upsert = fields[3].booleanSafe() || convertUpdateToUpsert;
             UpdateResult ur = updateObjects(ns, o, updateCriteria, upsert, /*multi*/ false,
                                             /*logop*/ false , debug, /*fromMigrate*/ false,
                                             QueryPlanSelectionPolicy::idElseNatural() );
diff --git a/src/mongo/db/oplog.h b/src/mongo/db/oplog.h
index 9e38bb6baec..283927a9061 100644
--- a/src/mongo/db/oplog.h
+++ b/src/mongo/db/oplog.h
@@ -156,5 +156,5 @@ namespace mongo {
      * @param fromRepl really from replication or for testing/internal/command/etc...
      * Returns if the op was an update that could not be applied (true on failure)
      */
-    bool applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
+    bool applyOperation_inlock(const BSONObj& op, bool fromRepl = true, bool convertUpdateToUpsert = false);
 }
diff --git a/src/mongo/db/repl/rs_sync.cpp b/src/mongo/db/repl/rs_sync.cpp
index 8a67ab92a00..f7fdaf55e96 100644
--- a/src/mongo/db/repl/rs_sync.cpp
+++ b/src/mongo/db/repl/rs_sync.cpp
@@ -50,7 +50,7 @@ namespace replset {
     /* apply the log op that is in param o
        @return bool success (true) or failure (false)
     */
-    bool SyncTail::syncApply(const BSONObj &op) {
+    bool SyncTail::syncApply(const BSONObj &op, bool convertUpdateToUpsert) {
         const char *ns = op.getStringField("ns");
         verify(ns);
 
@@ -79,7 +79,9 @@ namespace replset {
 
         Client::Context ctx(ns, dbpath, false);
         ctx.getClient()->curop()->reset();
-        bool ok = !applyOperation_inlock(op);
+        // For non-initial-sync, we convert updates to upserts
+        // to suppress errors when replaying oplog entries.
+        bool ok = !applyOperation_inlock(op, true, convertUpdateToUpsert);
         getDur().commitIfNeeded();
 
         return ok;
@@ -111,7 +113,7 @@ namespace replset {
              it != ops.end();
              ++it) {
             try {
-                fassert(16359, st->syncApply(*it));
+                fassert(16359, st->syncApply(*it, true));
             } catch (DBException& e) {
                 error() << "writer worker caught exception: " << e.what() 
                         << " on: " << it->toString() << endl;
diff --git a/src/mongo/db/repl/rs_sync.h b/src/mongo/db/repl/rs_sync.h
index 4b97bcf79ec..f47538d9742 100644
--- a/src/mongo/db/repl/rs_sync.h
+++ b/src/mongo/db/repl/rs_sync.h
@@ -37,7 +37,7 @@ namespace replset {
     public:
         SyncTail(BackgroundSyncInterface *q);
         virtual ~SyncTail();
-        virtual bool syncApply(const BSONObj &o);
+        virtual bool syncApply(const BSONObj &o, bool convertUpdateToUpsert = false);
         void oplogApplication();
         bool peek(BSONObj* obj);