SERVER-24506 Cloner support for views

11 files changed, 160 insertions, 29 deletions

diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp
index 92eb63a29b8..989deeffc7f 100644
--- a/src/mongo/client/dbclient.cpp
+++ b/src/mongo/client/dbclient.cpp
@@ -583,15 +583,6 @@ list<string> DBClientWithCommands::getDatabaseNames() {
     return names;
 }
 
-list<string> DBClientWithCommands::getCollectionNames(const string& db) {
-    list<BSONObj> infos = getCollectionInfos(db);
-    list<string> names;
-    for (list<BSONObj>::iterator it = infos.begin(); it != infos.end(); ++it) {
-        names.push_back(db + "." + (*it)["name"].valuestr());
-    }
-    return names;
-}
-
 list<BSONObj> DBClientWithCommands::getCollectionInfos(const string& db, const BSONObj& filter) {
     list<BSONObj> infos;
 
diff --git a/src/mongo/client/dbclientinterface.h b/src/mongo/client/dbclientinterface.h
index 23387aa932a..880f94aa4e4 100644
--- a/src/mongo/client/dbclientinterface.h
+++ b/src/mongo/client/dbclientinterface.h
@@ -692,12 +692,6 @@ public:
     std::list<std::string> getDatabaseNames();
 
     /**
-     * Get a list of all the current collections in db.
-     * Returns fully qualified names.
-     */
-    std::list<std::string> getCollectionNames(const std::string& db);
-
-    /**
      * { name : "<short collection name>",
      *   options : { }
      * }
diff --git a/src/mongo/db/cloner.cpp b/src/mongo/db/cloner.cpp
index d43b145d776..a9e9d763871 100644
--- a/src/mongo/db/cloner.cpp
+++ b/src/mongo/db/cloner.cpp
@@ -47,6 +47,7 @@
 #include "mongo/db/catalog/index_create.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/copydb.h"
+#include "mongo/db/commands/list_collections_filter.h"
 #include "mongo/db/commands/rename_collection.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/dbdirectclient.h"
@@ -568,7 +569,8 @@ Status Cloner::copyDb(OperationContext* txn,
         // getCollectionInfos may make a remote call, which may block indefinitely, so release
         // the global lock that we are entering with.
         Lock::TempRelease tempRelease(txn->lockState());
-        std::list<BSONObj> initialCollections = _conn->getCollectionInfos(opts.fromDB);
+        std::list<BSONObj> initialCollections = _conn->getCollectionInfos(
+            opts.fromDB, ListCollectionsFilter::makeTypeCollectionFilter());
         auto status = filterCollectionsForClone(opts, initialCollections);
         if (!status.isOK()) {
             return status.getStatus();
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 2f53565f93c..556f8e43878 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -170,6 +170,16 @@ env.Library(
     ],
 )
 
+env.Library(
+    target="list_collections_filter",
+    source=[
+        'list_collections_filter.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+    ],
+)
+
 env.CppUnitTest(
     target="index_filter_commands_test",
     source=[
diff --git a/src/mongo/db/commands/list_collections_filter.cpp b/src/mongo/db/commands/list_collections_filter.cpp
new file mode 100644
index 00000000000..6b04c0a286d
--- /dev/null
+++ b/src/mongo/db/commands/list_collections_filter.cpp
@@ -0,0 +1,57 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands/list_collections_filter.h"
+
+#include "mongo/db/jsobj.h"
+
+namespace mongo {
+
+// TODO SERVER-25493: Remove $exists clause once MongoDB versions <= 3.2 are no longer supported.
+BSONObj ListCollectionsFilter::makeTypeCollectionFilter() {
+    return BSON("$or" << BSON_ARRAY(BSON("type"
+                                         << "collection")
+                                    << BSON("type" << BSON("$exists" << false))));
+}
+
+BSONObj ListCollectionsFilter::makeTypeViewFilter() {
+    return BSON("type"
+                << "view");
+}
+
+BSONObj ListCollectionsFilter::addTypeCollectionFilter(const BSONObj& filter) {
+    return BSON("$and" << BSON_ARRAY(filter << makeTypeCollectionFilter()));
+}
+
+BSONObj ListCollectionsFilter::addTypeViewFilter(const BSONObj& filter) {
+    return BSON("$and" << BSON_ARRAY(filter << makeTypeViewFilter()));
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/commands/list_collections_filter.h b/src/mongo/db/commands/list_collections_filter.h
new file mode 100644
index 00000000000..1b29e10e7a9
--- /dev/null
+++ b/src/mongo/db/commands/list_collections_filter.h
@@ -0,0 +1,65 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/disallow_copying.h"
+#include "mongo/bson/bsonobj.h"
+
+namespace mongo {
+
+class ListCollectionsFilter {
+    MONGO_DISALLOW_COPYING(ListCollectionsFilter);
+
+public:
+    /**
+     * Returns an object that can be passed to the listCollections command as a filter to limit
+     * results to collection namespaces.
+     */
+    static BSONObj makeTypeCollectionFilter();
+
+    /**
+     * Returns an object that can be passed to the listCollections command as a filter to limit
+     * results to view namespaces.
+     */
+    static BSONObj makeTypeViewFilter();
+
+    /**
+     * Injects into an existing listCollections filter a clause to limit results to collection
+     * namespaces.
+     */
+    static BSONObj addTypeCollectionFilter(const BSONObj& filter);
+
+    /**
+     * Injects into an existing listCollections filter a clause to limit results to view
+     * namespaces.
+     */
+    static BSONObj addTypeViewFilter(const BSONObj& filter);
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index 3fd9ec9a787..dae02917a07 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -775,6 +775,7 @@ env.Library(
     ],
     LIBDEPS=[
         'collection_cloner',
+        '$BUILD_DIR/mongo/db/commands/list_collections_filter',
         '$BUILD_DIR/mongo/executor/task_executor_interface',
     ],
 )
@@ -1072,6 +1073,7 @@ env.Library(
         '$BUILD_DIR/mongo/client/clientdriver',
         '$BUILD_DIR/mongo/db/auth/authcore',
         '$BUILD_DIR/mongo/db/commands',
+        '$BUILD_DIR/mongo/db/commands/list_collections_filter',
         '$BUILD_DIR/mongo/rpc/client_metadata',
         '$BUILD_DIR/mongo/db/concurrency/lock_manager',
         '$BUILD_DIR/mongo/db/curop',
diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp
index 35ba3254681..8888b958fba 100644
--- a/src/mongo/db/repl/database_cloner.cpp
+++ b/src/mongo/db/repl/database_cloner.cpp
@@ -38,6 +38,7 @@
 
 #include "mongo/client/remote_command_retry_scheduler.h"
 #include "mongo/db/catalog/collection_options.h"
+#include "mongo/db/commands/list_collections_filter.h"
 #include "mongo/db/repl/storage_interface.h"
 #include "mongo/rpc/metadata/server_selection_metadata.h"
 #include "mongo/stdx/functional.h"
@@ -94,7 +95,10 @@ DatabaseCloner::DatabaseCloner(executor::TaskExecutor* executor,
       _dbWorkThreadPool(dbWorkThreadPool),
       _source(source),
       _dbname(dbname),
-      _listCollectionsFilter(listCollectionsFilter),
+      _listCollectionsFilter(
+          listCollectionsFilter.isEmpty()
+              ? ListCollectionsFilter::makeTypeCollectionFilter()
+              : ListCollectionsFilter::addTypeCollectionFilter(listCollectionsFilter)),
       _listCollectionsPredicate(listCollectionsPred ? listCollectionsPred : acceptAllPred),
       _storageInterface(si),
       _collectionWork(collWork),
@@ -128,7 +132,7 @@ DatabaseCloner::~DatabaseCloner() {
     DESTRUCTOR_GUARD(shutdown(); join(););
 }
 
-const std::vector<BSONObj>& DatabaseCloner::getCollectionInfos() const {
+const std::vector<BSONObj>& DatabaseCloner::getCollectionInfos_forTest() const {
     LockGuard lk(_mutex);
     return _collectionInfos;
 }
diff --git a/src/mongo/db/repl/database_cloner.h b/src/mongo/db/repl/database_cloner.h
index bb6ecf13c77..c5f3996b8c2 100644
--- a/src/mongo/db/repl/database_cloner.h
+++ b/src/mongo/db/repl/database_cloner.h
@@ -106,6 +106,8 @@ public:
      * 'onCompletion' will be called exactly once.
      *
      * Takes ownership of the passed StorageInterface object.
+     *
+     * 'listCollectionsFilter' will be extended to include collections only, filtering out views.
      */
     DatabaseCloner(executor::TaskExecutor* executor,
                    OldThreadPool* dbWorkThreadPool,
@@ -120,11 +122,9 @@ public:
     virtual ~DatabaseCloner();
 
     /**
-     * Returns collection info objects read from listCollections result.
-     * This will return an empty vector until we have processed the last
-     * batch of results from listCollections.
+     * Returns collection info objects read from listCollections result and will not include views.
      */
-    const std::vector<BSONObj>& getCollectionInfos() const;
+    const std::vector<BSONObj>& getCollectionInfos_forTest() const;
 
     std::string getDiagnosticString() const override;
 
diff --git a/src/mongo/db/repl/database_cloner_test.cpp b/src/mongo/db/repl/database_cloner_test.cpp
index 77516249d51..27f345e5443 100644
--- a/src/mongo/db/repl/database_cloner_test.cpp
+++ b/src/mongo/db/repl/database_cloner_test.cpp
@@ -32,6 +32,7 @@
 #include <memory>
 #include <utility>
 
+#include "mongo/db/commands/list_collections_filter.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/repl/base_cloner_test_fixture.h"
 #include "mongo/db/repl/database_cloner.h"
@@ -197,7 +198,10 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithoutFilter) {
     ASSERT_EQUALS(nss.db().toString(), noiRequest.dbname);
     ASSERT_EQUALS("listCollections", std::string(noiRequest.cmdObj.firstElementFieldName()));
     ASSERT_EQUALS(1, noiRequest.cmdObj.firstElement().numberInt());
-    ASSERT_FALSE(noiRequest.cmdObj.hasField("filter"));
+    ASSERT_TRUE(noiRequest.cmdObj.hasField("filter"));
+    BSONElement filterElement = noiRequest.cmdObj.getField("filter");
+    ASSERT_TRUE(filterElement.isABSONObj());
+    ASSERT_EQUALS(ListCollectionsFilter::makeTypeCollectionFilter(), filterElement.Obj());
     ASSERT_FALSE(net->hasReadyRequests());
     ASSERT_TRUE(_databaseCloner->isActive());
 }
@@ -230,7 +234,8 @@ TEST_F(DatabaseClonerTest, FirstRemoteCommandWithFilter) {
     ASSERT_EQUALS(1, noiRequest.cmdObj.firstElement().numberInt());
     BSONElement filterElement = noiRequest.cmdObj.getField("filter");
     ASSERT_TRUE(filterElement.isABSONObj());
-    ASSERT_EQUALS(listCollectionsFilter, filterElement.Obj());
+    ASSERT_EQUALS(ListCollectionsFilter::addTypeCollectionFilter(listCollectionsFilter),
+                  filterElement.Obj());
     ASSERT_FALSE(net->hasReadyRequests());
     ASSERT_TRUE(_databaseCloner->isActive());
 }
@@ -313,7 +318,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsPredicate) {
     ASSERT_EQUALS(getDetectableErrorStatus(), getStatus());
     ASSERT_TRUE(_databaseCloner->isActive());
 
-    const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos();
+    const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos_forTest();
     ASSERT_EQUALS(2U, collectionInfos.size());
     ASSERT_EQUALS(sourceInfos[0], collectionInfos[0]);
     ASSERT_EQUALS(sourceInfos[2], collectionInfos[1]);
@@ -339,7 +344,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) {
     ASSERT_TRUE(_databaseCloner->isActive());
 
     {
-        const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos();
+        const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos_forTest();
         ASSERT_EQUALS(1U, collectionInfos.size());
         ASSERT_EQUALS(sourceInfos[0], collectionInfos[0]);
     }
@@ -354,7 +359,7 @@ TEST_F(DatabaseClonerTest, ListCollectionsMultipleBatches) {
     ASSERT_TRUE(_databaseCloner->isActive());
 
     {
-        const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos();
+        const std::vector<BSONObj>& collectionInfos = _databaseCloner->getCollectionInfos_forTest();
         ASSERT_EQUALS(2U, collectionInfos.size());
         ASSERT_EQUALS(sourceInfos[0], collectionInfos[0]);
         ASSERT_EQUALS(sourceInfos[1], collectionInfos[1]);
diff --git a/src/mongo/db/repl/rs_initialsync.cpp b/src/mongo/db/repl/rs_initialsync.cpp
index ea66da6ba7b..77e10cde892 100644
--- a/src/mongo/db/repl/rs_initialsync.cpp
+++ b/src/mongo/db/repl/rs_initialsync.cpp
@@ -42,6 +42,7 @@
 #include "mongo/db/catalog/document_validation.h"
 #include "mongo/db/client.h"
 #include "mongo/db/cloner.h"
+#include "mongo/db/commands/list_collections_filter.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/dbhelpers.h"
@@ -332,8 +333,8 @@ Status _initialSync(BackgroundSync* bgsync) {
         CloneOptions options;
         options.fromDB = db;
         log() << "fetching and creating collections for " << db;
-        std::list<BSONObj> initialCollections =
-            r.conn()->getCollectionInfos(options.fromDB);  // may uassert
+        std::list<BSONObj> initialCollections = r.conn()->getCollectionInfos(
+            options.fromDB, ListCollectionsFilter::makeTypeCollectionFilter());  // may uassert
         auto fetchStatus = cloner.filterCollectionsForClone(options, initialCollections);
         if (!fetchStatus.isOK()) {
             return fetchStatus.getStatus();