SERVER-18594 Fix RS Catalog drop coll unit test

4 files changed, 198 insertions, 16 deletions

diff --git a/src/mongo/s/catalog/catalog_manager.cpp b/src/mongo/s/catalog/catalog_manager.cpp
index 9e93c59c162..666459bc95e 100644
--- a/src/mongo/s/catalog/catalog_manager.cpp
+++ b/src/mongo/s/catalog/catalog_manager.cpp
@@ -44,8 +44,8 @@
 #include "mongo/db/write_concern_options.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/s/catalog/dist_lock_manager.h"
-#include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_chunk.h"
+#include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_database.h"
 #include "mongo/s/catalog/type_shard.h"
 #include "mongo/s/client/shard.h"
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_drop_coll_test.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set_drop_coll_test.cpp
index 19f28ea9f80..ad0b5419e00 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_drop_coll_test.cpp
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_drop_coll_test.cpp
@@ -197,6 +197,8 @@ TEST_F(DropColl2ShardTest, Basic) {
 
     expectChangeLogInsert(
         configHost(), testClient(), network()->now(), "dropCollection", dropNS().ns(), BSONObj());
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, NSNotFound) {
@@ -241,6 +243,8 @@ TEST_F(DropColl2ShardTest, NSNotFound) {
 
     expectChangeLogInsert(
         configHost(), testClient(), network()->now(), "dropCollection", dropNS().ns(), BSONObj());
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, ConfigTargeterError) {
@@ -251,6 +255,8 @@ TEST_F(DropColl2ShardTest, ConfigTargeterError) {
         ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, DistLockBusy) {
@@ -270,6 +276,10 @@ TEST_F(DropColl2ShardTest, DistLockBusy) {
                           "dropCollection.start",
                           dropNS().ns(),
                           BSONObj());
+
+    expectGetShards({shard1(), shard2()});
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, FirstShardTargeterError) {
@@ -292,12 +302,14 @@ TEST_F(DropColl2ShardTest, FirstShardTargeterError) {
                           BSONObj());
 
     expectGetShards({shard1(), shard2()});
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, FirstShardDropError) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -311,9 +323,12 @@ TEST_F(DropColl2ShardTest, FirstShardDropError) {
 
     expectGetShards({shard1(), shard2()});
 
-    onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "drop bad network"};
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so drop command will fail.
+        return BSON("ok" << 1);
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, FirstShardDropCmdError) {
@@ -333,9 +348,14 @@ TEST_F(DropColl2ShardTest, FirstShardDropCmdError) {
 
     expectGetShards({shard1(), shard2()});
 
+    // drop command will be sent to all shards even if we get a not ok response from one shard.
     onCommand([](const RemoteCommandRequest& request) {
         return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized);
     });
+
+    expectDrop(shard2());
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, SecondShardTargeterError) {
@@ -360,12 +380,14 @@ TEST_F(DropColl2ShardTest, SecondShardTargeterError) {
     expectGetShards({shard1(), shard2()});
 
     expectDrop(shard1());
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, SecondShardDropError) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -381,9 +403,12 @@ TEST_F(DropColl2ShardTest, SecondShardDropError) {
 
     expectDrop(shard1());
 
-    onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "drop bad network"};
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so drop command will fail.
+        return BSON("ok" << 1);
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, SecondShardDropCmdError) {
@@ -408,6 +433,8 @@ TEST_F(DropColl2ShardTest, SecondShardDropCmdError) {
     onCommand([](const RemoteCommandRequest& request) {
         return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized);
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, CleanupChunkError) {
@@ -434,12 +461,74 @@ TEST_F(DropColl2ShardTest, CleanupChunkError) {
         return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized << "errmsg"
                          << "bad delete");
     });
+
+    future.timed_get(kFutureTimeout);
+}
+
+TEST_F(DropColl2ShardTest, SSVCmdErrorOnShard1) {
+    auto future = launchAsync([this] {
+        auto status = catalogManager()->dropCollection(operationContext(), dropNS());
+        ASSERT_EQ(ErrorCodes::Unauthorized, status.code());
+        ASSERT_FALSE(status.reason().empty());
+    });
+
+    expectChangeLogCreate(configHost(), BSON("ok" << 1));
+    expectChangeLogInsert(configHost(),
+                          testClient(),
+                          network()->now(),
+                          "dropCollection.start",
+                          dropNS().ns(),
+                          BSONObj());
+
+    expectGetShards({shard1(), shard2()});
+
+    expectDrop(shard1());
+    expectDrop(shard2());
+
+    expectRemoveChunks();
+
+    onCommand([](const RemoteCommandRequest& request) {
+        return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized << "errmsg"
+                         << "bad");
+    });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, SSVErrorOnShard1) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
+        ASSERT_FALSE(status.reason().empty());
+    });
+
+    expectChangeLogCreate(configHost(), BSON("ok" << 1));
+    expectChangeLogInsert(configHost(),
+                          testClient(),
+                          network()->now(),
+                          "dropCollection.start",
+                          dropNS().ns(),
+                          BSONObj());
+
+    expectGetShards({shard1(), shard2()});
+
+    expectDrop(shard1());
+    expectDrop(shard2());
+
+    expectRemoveChunks();
+
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so ssv command will fail.
+        return BSON("ok" << 1);
+    });
+
+    future.timed_get(kFutureTimeout);
+}
+
+TEST_F(DropColl2ShardTest, UnsetCmdErrorOnShard1) {
+    auto future = launchAsync([this] {
+        auto status = catalogManager()->dropCollection(operationContext(), dropNS());
+        ASSERT_EQ(ErrorCodes::Unauthorized, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -458,15 +547,20 @@ TEST_F(DropColl2ShardTest, SSVErrorOnShard1) {
 
     expectRemoveChunks();
 
+    expectSetShardVersionZero(shard1());
+
     onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "bad test network"};
+        return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized << "errmsg"
+                         << "bad");
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, UnsetErrorOnShard1) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -487,15 +581,51 @@ TEST_F(DropColl2ShardTest, UnsetErrorOnShard1) {
 
     expectSetShardVersionZero(shard1());
 
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so ssv command will fail.
+        return BSON("ok" << 1);
+    });
+
+    future.timed_get(kFutureTimeout);
+}
+
+TEST_F(DropColl2ShardTest, SSVCmdErrorOnShard2) {
+    auto future = launchAsync([this] {
+        auto status = catalogManager()->dropCollection(operationContext(), dropNS());
+        ASSERT_EQ(ErrorCodes::Unauthorized, status.code());
+        ASSERT_FALSE(status.reason().empty());
+    });
+
+    expectChangeLogCreate(configHost(), BSON("ok" << 1));
+    expectChangeLogInsert(configHost(),
+                          testClient(),
+                          network()->now(),
+                          "dropCollection.start",
+                          dropNS().ns(),
+                          BSONObj());
+
+    expectGetShards({shard1(), shard2()});
+
+    expectDrop(shard1());
+    expectDrop(shard2());
+
+    expectRemoveChunks();
+
+    expectSetShardVersionZero(shard1());
+    expectUnsetSharding(shard1());
+
     onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "bad test network"};
+        return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized << "errmsg"
+                         << "bad");
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, SSVErrorOnShard2) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -517,15 +647,53 @@ TEST_F(DropColl2ShardTest, SSVErrorOnShard2) {
     expectSetShardVersionZero(shard1());
     expectUnsetSharding(shard1());
 
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so ssv command will fail.
+        return BSON("ok" << 1);
+    });
+
+    future.timed_get(kFutureTimeout);
+}
+
+TEST_F(DropColl2ShardTest, UnsetCmdErrorOnShard2) {
+    auto future = launchAsync([this] {
+        auto status = catalogManager()->dropCollection(operationContext(), dropNS());
+        ASSERT_EQ(ErrorCodes::Unauthorized, status.code());
+        ASSERT_FALSE(status.reason().empty());
+    });
+
+    expectChangeLogCreate(configHost(), BSON("ok" << 1));
+    expectChangeLogInsert(configHost(),
+                          testClient(),
+                          network()->now(),
+                          "dropCollection.start",
+                          dropNS().ns(),
+                          BSONObj());
+
+    expectGetShards({shard1(), shard2()});
+
+    expectDrop(shard1());
+    expectDrop(shard2());
+
+    expectRemoveChunks();
+
+    expectSetShardVersionZero(shard1());
+    expectUnsetSharding(shard1());
+
+    expectSetShardVersionZero(shard2());
+
     onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "bad test network"};
+        return BSON("ok" << 0 << "code" << ErrorCodes::Unauthorized << "errmsg"
+                         << "bad");
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 TEST_F(DropColl2ShardTest, UnsetErrorOnShard2) {
     auto future = launchAsync([this] {
         auto status = catalogManager()->dropCollection(operationContext(), dropNS());
-        ASSERT_EQ(ErrorCodes::HostUnreachable, status.code());
+        ASSERT_EQ(ErrorCodes::CallbackCanceled, status.code());
         ASSERT_FALSE(status.reason().empty());
     });
 
@@ -549,9 +717,12 @@ TEST_F(DropColl2ShardTest, UnsetErrorOnShard2) {
 
     expectSetShardVersionZero(shard2());
 
-    onCommand([](const RemoteCommandRequest& request) {
-        return Status{ErrorCodes::HostUnreachable, "bad test network"};
+    onCommand([this](const RemoteCommandRequest& request) {
+        shutdownExecutor();  // shutdown executor so unset command will fail.
+        return BSON("ok" << 1);
     });
+
+    future.timed_get(kFutureTimeout);
 }
 
 }  // unnamed namespace
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
index 6d7c1b933f1..d906566aaab 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.cpp
@@ -85,6 +85,7 @@ void CatalogManagerReplSetTestFixture::setUp() {
         stdx::make_unique<repl::ReplicationExecutor>(network.release(), nullptr, 0));
 
     _networkTestEnv = stdx::make_unique<NetworkTestEnv>(executor.get(), _mockNetwork);
+    _executor = executor.get();
 
     std::unique_ptr<CatalogManagerReplicaSet> cm(stdx::make_unique<CatalogManagerReplicaSet>());
 
@@ -117,6 +118,12 @@ void CatalogManagerReplSetTestFixture::tearDown() {
     _service.reset();
 }
 
+void CatalogManagerReplSetTestFixture::shutdownExecutor() {
+    if (_executor) {
+        _executor->shutdown();
+    }
+}
+
 CatalogManagerReplicaSet* CatalogManagerReplSetTestFixture::catalogManager() const {
     auto cm = dynamic_cast<CatalogManagerReplicaSet*>(grid.catalogManager());
     invariant(cm);
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
index c50bd7ad76a..15f69603ad5 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set_test_fixture.h
@@ -50,6 +50,7 @@ class StatusWith;
 
 namespace executor {
 class NetworkInterfaceMock;
+class TaskExecutor;
 }  // namespace executor
 
 /**
@@ -138,6 +139,8 @@ protected:
 
     void tearDown() override;
 
+    void shutdownExecutor();
+
 private:
     std::unique_ptr<ServiceContext> _service;
     ServiceContext::UniqueClient _client;
@@ -148,6 +151,7 @@ private:
     RemoteCommandTargeterMock* _configTargeter;
 
     executor::NetworkInterfaceMock* _mockNetwork;
+    executor::TaskExecutor* _executor;
     std::unique_ptr<executor::NetworkTestEnv> _networkTestEnv;
 };