SERVER-29459 Add 'includePendingDrops' option to listCollections command

3 files changed, 139 insertions, 5 deletions

diff --git a/jstests/core/list_collections1.js b/jstests/core/list_collections1.js
index 2aeb9ec9fbd..2997d250362 100644
--- a/jstests/core/list_collections1.js
+++ b/jstests/core/list_collections1.js
@@ -292,4 +292,22 @@
     assert.throws(function() {
         cursor.hasNext();
     });
+
+    //
+    // Test parsing of the 'includePendingDrops' flag. If included, its argument must be of
+    // 'boolean' type. Functional testing of the 'includePendingDrops' flag is done in
+    // "jstests/replsets".
+    //
+
+    // Bad argument types.
+    assert.commandFailedWithCode(mydb.runCommand("listCollections", {includePendingDrops: {}}),
+                                 ErrorCodes.TypeMismatch);
+    assert.commandFailedWithCode(mydb.runCommand("listCollections", {includePendingDrops: "s"}),
+                                 ErrorCodes.TypeMismatch);
+
+    // Valid argument types.
+    assert.commandWorked(mydb.runCommand("listCollections", {includePendingDrops: 1}));
+    assert.commandWorked(mydb.runCommand("listCollections", {includePendingDrops: true}));
+    assert.commandWorked(mydb.runCommand("listCollections", {includePendingDrops: false}));
+
 }());
diff --git a/jstests/replsets/drop_collections_two_phase.js b/jstests/replsets/drop_collections_two_phase.js
new file mode 100644
index 00000000000..00c4d6cc176
--- /dev/null
+++ b/jstests/replsets/drop_collections_two_phase.js
@@ -0,0 +1,101 @@
+/**
+ * Test to ensure that two phase drop behavior for collections on replica sets works properly.
+ *
+ * Uses a 2 node replica set to verify both phases of a 2-phase collection drop: the 'Prepare' and
+ * 'Commit' phase. Executing a 'drop' collection command should put that collection into the
+ * 'Prepare' phase. The 'Commit' phase (physically dropping the collection) of a drop operation with
+ * optime T should only be executed when C >= T, where C is the majority commit point of the replica
+ * set.
+ */
+
+(function() {
+    "use strict";
+
+    // List all collections in a given database. Use 'args' as the command arguments.
+    function listCollections(database, args) {
+        var args = args || {};
+        var failMsg = "'listCollections' command failed";
+        var res = assert.commandWorked(database.runCommand("listCollections", args), failMsg);
+        return res.cursor.firstBatch;
+    }
+
+    // Set a fail point on a specified node.
+    function setFailPoint(node, failpoint, mode) {
+        assert.commandWorked(node.adminCommand({configureFailPoint: failpoint, mode: mode}));
+    }
+
+    var testName = "drop_collections_two_phase";
+    var replTest = new ReplSetTest({name: testName, nodes: 2});
+
+    replTest.startSet();
+    replTest.initiate();
+    replTest.awaitReplication();
+
+    var primary = replTest.getPrimary();
+    var secondary = replTest.getSecondary();
+
+    var primaryDB = primary.getDB(testName);
+    var collToDrop = "collectionToDrop";
+    var collections, collection;
+
+    // Create the collection that will be dropped and let it replicate.
+    primaryDB.createCollection(collToDrop);
+    replTest.awaitReplication();
+
+    // Pause application on secondary so that commit point doesn't advance, meaning that a dropped
+    // collection on the primary will remain in 'drop-pending' state.
+    jsTestLog("Pausing oplog application on the secondary node.");
+    setFailPoint(secondary, "rsSyncApplyStop", "alwaysOn");
+
+    // Make sure the collection was created.
+    collections = listCollections(primaryDB);
+    collection = collections.find(c => c.name === collToDrop);
+    assert(collection, "Collection '" + collToDrop + "' wasn't created properly");
+    assert.eq(collToDrop, collection.name);
+
+    // Drop the collection on the primary.
+    jsTestLog("Dropping collection '" + collToDrop + "' on primary node.");
+    assert.commandWorked(primaryDB.runCommand({drop: collToDrop}, {writeConcern: {w: 1}}));
+
+    // Make sure the collection is now in 'drop-pending' state. The collection name should be of the
+    // format "system.drop.<optime>.<collectionName>", where 'optime' is the optime of the
+    // collection drop operation, encoded as a string, and 'collectionName' is the original
+    // collection name.
+    var pendingDropRegex = new RegExp("system\.drop\..*\." + collToDrop + "$");
+
+    collections = listCollections(primaryDB, {includePendingDrops: true});
+    collection = collections.find(c => pendingDropRegex.test(c.name));
+    assert(collection,
+           "Collection was not found in the 'system.drop' namespace. Full collection list: " +
+               tojson(collections));
+
+    // Make sure the collection doesn't appear in the normal collection list. Also check that
+    // the default 'listCollections' behavior excludes drop-pending collections.
+    collections = listCollections(primaryDB, {includePendingDrops: false});
+    assert.eq(undefined, collections.find(c => c.name === collToDrop));
+
+    collections = listCollections(primaryDB);
+    assert.eq(undefined, collections.find(c => c.name === collToDrop));
+
+    // Let the secondary apply the collection drop operation, so that the replica set commit point
+    // will advance, and the 'Commit' phase of the collection drop will complete on the primary.
+    jsTestLog("Restarting oplog application on the secondary node.");
+    setFailPoint(secondary, "rsSyncApplyStop", "off");
+
+    jsTestLog("Waiting for collection drop operation to replicate to all nodes.");
+    replTest.awaitReplication();
+
+    // Make sure the collection has been fully dropped. It should not appear as
+    // a normal collection or under the 'system.drop' namespace any longer. Physical collection
+    // drops may happen asynchronously, any time after the drop operation is committed, so we wait
+    // to make sure the collection is eventually dropped.
+    assert.soonNoExcept(function() {
+        var collections = listCollections(primaryDB, {includePendingDrops: true});
+        assert.eq(undefined, collections.find(c => c.name === collToDrop));
+        assert.eq(undefined, collections.find(c => pendingDropRegex.test(c.name)));
+        return true;
+    });
+
+    replTest.stopSet();
+
+}());
diff --git a/src/mongo/db/commands/list_collections.cpp b/src/mongo/db/commands/list_collections.cpp
index bbf9e37e18d..cf38981e14f 100644
--- a/src/mongo/db/commands/list_collections.cpp
+++ b/src/mongo/db/commands/list_collections.cpp
@@ -34,6 +34,7 @@
 #include "mongo/bson/bsonmisc.h"
 #include "mongo/bson/bsonobj.h"
 #include "mongo/bson/simple_bsonobj_comparator.h"
+#include "mongo/bson/util/bson_extract.h"
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/catalog/collection.h"
 #include "mongo/db/catalog/collection_catalog_entry.h"
@@ -147,7 +148,9 @@ BSONObj buildViewBson(const ViewDefinition& view) {
     return b.obj();
 }
 
-BSONObj buildCollectionBson(OperationContext* opCtx, const Collection* collection) {
+BSONObj buildCollectionBson(OperationContext* opCtx,
+                            const Collection* collection,
+                            bool includePendingDrops) {
 
     if (!collection) {
         return {};
@@ -161,8 +164,8 @@ BSONObj buildCollectionBson(OperationContext* opCtx, const Collection* collectio
 
     // Drop-pending collections are replicated collections that have been marked for deletion.
     // These collections are considered dropped and should not be returned in the results for this
-    // command.
-    if (nss.isDropPendingNamespace()) {
+    // command, unless specified explicitly by the 'includePendingDrops' command argument.
+    if (nss.isDropPendingNamespace() && !includePendingDrops) {
         return {};
     }
 
@@ -232,6 +235,8 @@ public:
              string& errmsg,
              BSONObjBuilder& result) {
         unique_ptr<MatchExpression> matcher;
+
+        // Check for 'filter' argument.
         BSONElement filterElt = jsobj["filter"];
         if (!filterElt.eoo()) {
             if (filterElt.type() != mongo::Object) {
@@ -256,6 +261,16 @@ public:
             return appendCommandStatus(result, parseCursorStatus);
         }
 
+        // Check for 'includePendingDrops' flag. The default is to not include drop-pending
+        // collections.
+        bool includePendingDrops;
+        Status status = bsonExtractBooleanFieldWithDefault(
+            jsobj, "includePendingDrops", false, &includePendingDrops);
+
+        if (!status.isOK()) {
+            return appendCommandStatus(result, status);
+        }
+
         AutoGetDb autoDb(opCtx, dbname, MODE_S);
 
         Database* db = autoDb.getDb();
@@ -268,14 +283,14 @@ public:
                 for (auto&& collName : *collNames) {
                     auto nss = NamespaceString(db->name(), collName);
                     Collection* collection = db->getCollection(opCtx, nss);
-                    BSONObj collBson = buildCollectionBson(opCtx, collection);
+                    BSONObj collBson = buildCollectionBson(opCtx, collection, includePendingDrops);
                     if (!collBson.isEmpty()) {
                         _addWorkingSetMember(opCtx, collBson, matcher.get(), ws.get(), root.get());
                     }
                 }
             } else {
                 for (auto&& collection : *db) {
-                    BSONObj collBson = buildCollectionBson(opCtx, collection);
+                    BSONObj collBson = buildCollectionBson(opCtx, collection, includePendingDrops);
                     if (!collBson.isEmpty()) {
                         _addWorkingSetMember(opCtx, collBson, matcher.get(), ws.get(), root.get());
                     }