SERVER-24927 Ignore NamespaceNotFound errors during initial sync apply ops, because they are transient

3 files changed, 53 insertions, 18 deletions

diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp
index 3a87828e76b..6c4d88132eb 100644
--- a/src/mongo/db/repl/sync_tail.cpp
+++ b/src/mongo/db/repl/sync_tail.cpp
@@ -1019,32 +1019,39 @@ void multiInitialSyncApply(const std::vector<BSONObj>& ops, SyncTail* st) {
     initializeWriterThread();
 
     OperationContextImpl txn;
-    txn.setReplicatedWrites(false);
-    DisableDocumentValidation validationDisabler(&txn);
+    Status status = multiInitialSyncApply_noAbort(&txn, ops, st);
+    fassertNoTrace(15915, status);
+}
+
+Status multiInitialSyncApply_noAbort(OperationContext* txn,
+                                     const std::vector<BSONObj>& ops,
+                                     SyncTail* st) {
+    txn->setReplicatedWrites(false);
+    DisableDocumentValidation validationDisabler(txn);
 
     // allow us to get through the magic barrier
-    txn.lockState()->setIsBatchWriter(true);
+    txn->lockState()->setIsBatchWriter(true);
 
     // This function is only called in initial sync, as its name suggests.
     bool inSteadyStateReplication = false;
 
     for (std::vector<BSONObj>::const_iterator it = ops.begin(); it != ops.end(); ++it) {
         try {
-            const Status s = SyncTail::syncApply(&txn, *it, inSteadyStateReplication);
+            const Status s = SyncTail::syncApply(txn, *it, inSteadyStateReplication);
             if (!s.isOK()) {
                 // Don't retry on commands.
                 SyncTail::OplogEntry entry(*it);
                 if (entry.opType[0] == 'c') {
                     error() << "Error applying command (" << it->toString() << "): " << s;
-                    fassertFailedNoTrace(40353);
+                    return s;
                 }
 
                 // We might need to fetch the missing docs from the sync source.
-                if (st->shouldRetry(&txn, *it)) {
-                    const Status s2 = SyncTail::syncApply(&txn, *it, inSteadyStateReplication);
+                if (st->shouldRetry(txn, *it)) {
+                    const Status s2 = SyncTail::syncApply(txn, *it, inSteadyStateReplication);
                     if (!s2.isOK()) {
                         severe() << "Error applying operation (" << it->toString() << "): " << s2;
-                        fassertFailedNoTrace(15915);
+                        return s2;
                     }
                 }
 
@@ -1053,16 +1060,24 @@ void multiInitialSyncApply(const std::vector<BSONObj>& ops, SyncTail* st) {
                 // subsequently got deleted and no longer exists on the Sync Target at all
             }
         } catch (const DBException& e) {
+            // SERVER-24927 If we have a NamespaceNotFound exception, then this document will be
+            // dropped before initial sync ends anyways and we should ignore it.
+            SyncTail::OplogEntry entry(*it);
+            if (e.getCode() == ErrorCodes::NamespaceNotFound &&
+                isCrudOpType(entry.opType.rawData())) {
+                continue;
+            }
+
             severe() << "writer worker caught exception: " << causedBy(e)
                      << " on: " << it->toString();
 
             if (inShutdown()) {
-                return;
+                return Status::OK();
             }
-
-            fassertFailedNoTrace(16361);
+            return e.toStatus();
         }
     }
+    return Status::OK();
 }
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/sync_tail.h b/src/mongo/db/repl/sync_tail.h
index 7e5cf2ec2a7..bfb290c2ca2 100644
--- a/src/mongo/db/repl/sync_tail.h
+++ b/src/mongo/db/repl/sync_tail.h
@@ -220,6 +220,9 @@ private:
 // These free functions are used by the thread pool workers to write ops to the db.
 void multiSyncApply(const std::vector<BSONObj>& ops, SyncTail* st);
 void multiInitialSyncApply(const std::vector<BSONObj>& ops, SyncTail* st);
+Status multiInitialSyncApply_noAbort(OperationContext* txn,
+                                     const std::vector<BSONObj>& ops,
+                                     SyncTail* st);
 
 }  // namespace repl
 }  // namespace mongo
diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp
index ad4f9b95c35..0d0250e4814 100644
--- a/src/mongo/db/repl/sync_tail_test.cpp
+++ b/src/mongo/db/repl/sync_tail_test.cpp
@@ -45,6 +45,7 @@
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/repl/bgsync.h"
 #include "mongo/db/repl/operation_context_repl_mock.h"
+#include "mongo/db/repl/oplog_interface_local.h"
 #include "mongo/db/repl/replication_coordinator_global.h"
 #include "mongo/db/repl/replication_coordinator_mock.h"
 #include "mongo/db/repl/sync_tail.h"
@@ -429,13 +430,9 @@ Status IdempotencyTest::runOp(const BSONObj& op) {
 }
 
 Status IdempotencyTest::runOps(std::initializer_list<BSONObj> ops) {
-    for (auto& op : ops) {
-        Status status = SyncTail::syncApply(_txn.get(), op, false);
-        if (!status.isOK()) {
-            return status;
-        }
-    }
-    return Status::OK();
+    std::vector<BSONObj> opsVector(ops);
+    SyncTail syncTail(nullptr, SyncTail::MultiSyncApplyFunc());
+    return multiInitialSyncApply_noAbort(_txn.get(), opsVector, &syncTail);
 }
 
 BSONObj IdempotencyTest::createCollection() {
@@ -544,4 +541,24 @@ TEST_F(IdempotencyTest, ResyncOnRenameCollection) {
     ASSERT_EQUALS(runOp(op), ErrorCodes::OplogOperationUnsupported);
 }
 
+TEST_F(IdempotencyTest, MultiInitialSyncApplySkipsDocumentOnNamespaceNotFound) {
+    BSONObj emptyDoc;
+    NamespaceString nss("test", "foo");
+    NamespaceString badNss("test", "bad");
+    auto doc1 = BSON("_id" << 1);
+    auto doc2 = BSON("_id" << 2);
+    auto doc3 = BSON("_id" << 3);
+    auto op0 = makeCreateCollectionOplogEntry(nextOpTime(), nss);
+    auto op1 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc1);
+    auto op2 = makeInsertDocumentOplogEntry(nextOpTime(), badNss, doc2);
+    auto op3 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc3);
+    runOps({op0, op1, op2, op3});
+
+    OplogInterfaceLocal collectionReader(_txn.get(), nss.ns());
+    auto iter = collectionReader.makeIterator();
+    ASSERT_EQUALS(doc3, unittest::assertGet(iter->next()).first);
+    ASSERT_EQUALS(doc1, unittest::assertGet(iter->next()).first);
+    ASSERT_EQUALS(ErrorCodes::NoSuchKey, iter->next().getStatus());
+}
+
 }  // namespace