SERVER-29725 mongoe application

Embedded MongoDB in a standalone application. To be able to connect with the shell and perform testing on embedded.

10 files changed, 353 insertions, 71 deletions

diff --git a/src/mongo/client/embedded/SConscript b/src/mongo/client/embedded/SConscript
index 1130e37cdc6..3025372251d 100644
--- a/src/mongo/client/embedded/SConscript
+++ b/src/mongo/client/embedded/SConscript
@@ -109,8 +109,10 @@ env.Library(
         '$BUILD_DIR/mongo/db/commands/mongod_fcv',
         '$BUILD_DIR/mongo/db/op_observer_d',
         '$BUILD_DIR/mongo/db/repair_database_and_check_version',
+        '$BUILD_DIR/mongo/db/repl/repl_set_status_commands',
         '$BUILD_DIR/mongo/db/repl/storage_interface_impl',
         '$BUILD_DIR/mongo/db/storage/mobile/storage_mobile',
+        '$BUILD_DIR/mongo/db/storage/storage_init_d',
         '$BUILD_DIR/mongo/util/version_impl',
         'embedded_commands',
         'embedded_options',
@@ -171,6 +173,32 @@ capiTest = capiTestEnv.Program(
 env.RegisterUnitTest(capiTest[0])
 env.Alias('embedded_capi', capiTest)
 
+mongoe_env = env.Clone()
+mongoe_env.InjectThirdPartyIncludePaths(libraries=['yaml'])
+mongoe = mongoe_env.Program(
+    target='mongoe',
+    source=[
+        'mongoe_main.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/db/commands/shell_protocol',
+        '$BUILD_DIR/mongo/db/mongod_options',
+        '$BUILD_DIR/mongo/db/server_options',
+        '$BUILD_DIR/mongo/transport/transport_layer_manager',
+        '$BUILD_DIR/mongo/util/signal_handlers',
+        'embedded',
+    ],
+    INSTALL_ALIAS=[
+        'mobile-test',
+    ],
+)
+
+hygienic = get_option('install-mode') == 'hygienic'
+
+if not hygienic:
+    env.Install('#/', mongoe)
+
+
 if not env['MONGO_HAVE_LIBMONGOC']:
     Return()
 
diff --git a/src/mongo/client/embedded/embedded.cpp b/src/mongo/client/embedded/embedded.cpp
index 328217bd157..9f99b33d253 100644
--- a/src/mongo/client/embedded/embedded.cpp
+++ b/src/mongo/client/embedded/embedded.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/global_settings.h"
 #include "mongo/db/index_rebuilder.h"
 #include "mongo/db/kill_sessions_local.h"
+#include "mongo/db/logical_clock.h"
 #include "mongo/db/op_observer_impl.h"
 #include "mongo/db/op_observer_registry.h"
 #include "mongo/db/repair_database_and_check_version.h"
@@ -104,6 +105,9 @@ GlobalInitializerRegisterer replicationManagerInitializer(
         auto serviceContext = context->serviceContext();
         repl::StorageInterface::set(serviceContext, std::make_unique<repl::StorageInterfaceImpl>());
 
+        auto logicalClock = stdx::make_unique<LogicalClock>(serviceContext);
+        LogicalClock::set(serviceContext, std::move(logicalClock));
+
         auto replCoord = std::make_unique<ReplicationCoordinatorEmbedded>(serviceContext);
         repl::ReplicationCoordinator::set(serviceContext, std::move(replCoord));
         repl::setOplogCollectionName(serviceContext);
@@ -113,6 +117,7 @@ GlobalInitializerRegisterer replicationManagerInitializer(
         auto serviceContext = context->serviceContext();
 
         repl::ReplicationCoordinator::set(serviceContext, nullptr);
+        LogicalClock::set(serviceContext, nullptr);
         repl::StorageInterface::set(serviceContext, nullptr);
 
         return Status::OK();
diff --git a/src/mongo/client/embedded/mongoe_main.cpp b/src/mongo/client/embedded/mongoe_main.cpp
new file mode 100644
index 00000000000..3a7bcd9ec38
--- /dev/null
+++ b/src/mongo/client/embedded/mongoe_main.cpp
@@ -0,0 +1,136 @@
+/**
+*    Copyright (C) 2018 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects
+*    for all of the code used other than as permitted herein. If you modify
+*    file(s) with this exception, you may extend this exception to your
+*    version of the file(s), but you are not obligated to do so. If you do not
+*    wish to do so, delete this exception statement from your version. If you
+*    delete this exception statement from all source files in the program,
+*    then also delete it in the license file.
+*/
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/base/init.h"
+#include "mongo/client/embedded/embedded.h"
+#include "mongo/db/mongod_options.h"
+#include "mongo/db/service_context.h"
+#include "mongo/transport/transport_layer.h"
+#include "mongo/transport/transport_layer_manager.h"
+#include "mongo/util/exit.h"
+#include "mongo/util/log.h"
+#include "mongo/util/options_parser/options_parser.h"
+#include "mongo/util/options_parser/startup_options.h"
+#include "mongo/util/signal_handlers.h"
+#include "mongo/util/text.h"
+
+#include <yaml-cpp/yaml.h>
+
+namespace mongo {
+namespace {
+
+MONGO_INITIALIZER_WITH_PREREQUISITES(SignalProcessingStartup, ("ThreadNameInitializer"))
+(InitializerContext*) {
+    // Make sure we call this as soon as possible but before any other threads are started. Before
+    // embedded::initialize is too early and after is too late. So instead we hook in during the
+    // global initialization at the right place.
+    startSignalProcessingThread();
+    return Status::OK();
+}
+
+int mongoeMain(int argc, char* argv[], char** envp) {
+    ServiceContext* serviceContext = nullptr;
+
+    registerShutdownTask([&]() {
+        if (!serviceContext)
+            return;
+
+        if (auto tl = serviceContext->getTransportLayer()) {
+            log(logger::LogComponent::kNetwork) << "shutdown: going to close listening sockets...";
+            tl->shutdown();
+        }
+
+        embedded::shutdown(serviceContext);
+    });
+
+    setupSignalHandlers();
+
+    log() << "MongoDB embedded standalone application, for testing purposes only";
+
+    try {
+        optionenvironment::OptionSection startupOptions("Options");
+        uassertStatusOK(addMongodOptions(&startupOptions));
+
+        // Manually run the code that's equivalent to the MONGO_INITIALIZERs for mongod. We can't do
+        // this in initializers because embedded uses a different options format. However as long as
+        // we store the options in the same place it will be valid for embedded too. Adding all
+        // options mongod we don't have to maintain a separate set for this executable, some will be
+        // unused but that's fine as this is just an executable for testing purposes anyway.
+        std::vector<std::string> args;
+        std::map<std::string, std::string> env;
+
+        args.reserve(argc);
+        std::copy(argv, argv + argc, std::back_inserter(args));
+
+        optionenvironment::OptionsParser parser;
+        uassertStatusOK(
+            parser.run(startupOptions, args, env, &optionenvironment::startupOptionsParsed));
+        uassertStatusOK(storeMongodOptions(optionenvironment::startupOptionsParsed));
+
+        // Add embedded specific options that's not available in mongod here.
+        YAML::Emitter yaml;
+        serviceContext = embedded::initialize(yaml.c_str());
+
+        auto tl =
+            transport::TransportLayerManager::createWithConfig(&serverGlobalParams, serviceContext);
+        uassertStatusOK(tl->setup());
+
+        serviceContext->setTransportLayer(std::move(tl));
+
+        uassertStatusOK(serviceContext->getServiceExecutor()->start());
+        uassertStatusOK(serviceContext->getTransportLayer()->start());
+    } catch (const std::exception& ex) {
+        error() << ex.what();
+        return EXIT_BADOPTIONS;
+    }
+
+    return waitForShutdown();
+}
+
+}  // namespace
+}  // namespace mongo
+
+#if defined(_WIN32)
+// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters
+// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters.  The
+// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent
+// and makes them available through the argv() and envp() members.  This enables mongoDbMain()
+// to process UTF-8 encoded arguments and environment variables without regard to platform.
+int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) {
+    mongo::WindowsCommandLine wcl(argc, argvW, envpW);
+    return mongo::mongoeMain(argc, wcl.argv(), wcl.envp());
+}
+#else
+int main(int argc, char* argv[], char** envp) {
+    return mongo::mongoeMain(argc, argv, envp);
+}
+#endif
diff --git a/src/mongo/client/embedded/replication_coordinator_embedded.cpp b/src/mongo/client/embedded/replication_coordinator_embedded.cpp
index e0dfc14ce22..85d272a6bb7 100644
--- a/src/mongo/client/embedded/replication_coordinator_embedded.cpp
+++ b/src/mongo/client/embedded/replication_coordinator_embedded.cpp
@@ -346,7 +346,7 @@ Status ReplicationCoordinatorEmbedded::checkIfWriteConcernCanBeSatisfied(
 }
 
 Status ReplicationCoordinatorEmbedded::checkReplEnabledForCommand(BSONObjBuilder*) {
-    UASSERT_NOT_IMPLEMENTED;
+    return Status(ErrorCodes::NoReplicationEnabled, "no replication on embedded");
 }
 
 HostAndPort ReplicationCoordinatorEmbedded::chooseNewSyncSource(const OpTime&) {
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 2d7a24f47ca..1dce82f94de 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -183,6 +183,7 @@ env.Library(
 env.Library(
     target="standalone",
     source=[
+        "collection_to_capped.cpp",
         "count_cmd.cpp",
         "create_indexes.cpp",
         "current_op.cpp",
@@ -229,6 +230,22 @@ env.Library(
     ],
 )
 
+# Commands required by the shell to connect
+env.Library(
+    target="shell_protocol",
+    source=[
+        "test_commands.cpp",
+        "whats_my_uri_cmd.cpp",
+    ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/catalog/catalog_helpers',
+        '$BUILD_DIR/mongo/db/catalog/catalog_impl',
+        '$BUILD_DIR/mongo/db/commands',
+        'test_commands_enabled',
+    ],
+)
+
+
 # Commands that should only be present in mongod
 env.Library(
     target="mongod",
@@ -236,7 +253,6 @@ env.Library(
         "apply_ops_cmd.cpp",
         "clone.cpp",
         "clone_collection.cpp",
-        "collection_to_capped.cpp",
         "compact.cpp",
         "copydb.cpp",
         "copydb_start_commands.cpp",
@@ -260,7 +276,6 @@ env.Library(
         "set_feature_compatibility_version_command.cpp",
         "shutdown_d.cpp",
         "snapshot_management.cpp",
-        "test_commands.cpp",
         "top_command.cpp",
         "touch.cpp",
         "txn_cmds.cpp",
@@ -289,8 +304,9 @@ env.Library(
         'mongod_fcv',
         'mongod_fsync',
         'profile_common',
-        'standalone',
         'servers',
+        'shell_protocol',
+        'standalone',
         'test_commands_enabled',
     ],
 )
diff --git a/src/mongo/db/commands/dbcommands_d.cpp b/src/mongo/db/commands/dbcommands_d.cpp
index 44de9bbc2c4..a0b4be7ebd1 100644
--- a/src/mongo/db/commands/dbcommands_d.cpp
+++ b/src/mongo/db/commands/dbcommands_d.cpp
@@ -309,31 +309,6 @@ public:
 
 } cmdFileMD5;
 
-/* Returns client's uri */
-class CmdWhatsMyUri : public BasicCommand {
-public:
-    CmdWhatsMyUri() : BasicCommand("whatsmyuri") {}
-    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
-        return AllowedOnSecondary::kAlways;
-    }
-    virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
-        return false;
-    }
-    std::string help() const override {
-        return "{whatsmyuri:1}";
-    }
-    virtual void addRequiredPrivileges(const std::string& dbname,
-                                       const BSONObj& cmdObj,
-                                       std::vector<Privilege>* out) const {}  // No auth required
-    virtual bool run(OperationContext* opCtx,
-                     const string& dbname,
-                     const BSONObj& cmdObj,
-                     BSONObjBuilder& result) {
-        result << "you" << opCtx->getClient()->clientAddress(true /*includePort*/);
-        return true;
-    }
-} cmdWhatsMyUri;
-
 class AvailableQueryOptions : public BasicCommand {
 public:
     AvailableQueryOptions() : BasicCommand("availableQueryOptions", "availablequeryoptions") {}
diff --git a/src/mongo/db/commands/whats_my_uri_cmd.cpp b/src/mongo/db/commands/whats_my_uri_cmd.cpp
new file mode 100644
index 00000000000..1827c12b318
--- /dev/null
+++ b/src/mongo/db/commands/whats_my_uri_cmd.cpp
@@ -0,0 +1,68 @@
+/**
+ *    Copyright (C) 2012-2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands.h"
+
+namespace mongo {
+namespace {
+
+/* Returns client's uri */
+class CmdWhatsMyUri : public BasicCommand {
+public:
+    CmdWhatsMyUri() : BasicCommand("whatsmyuri") {}
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kAlways;
+    }
+
+    virtual bool supportsWriteConcern(const BSONObj& cmd) const override {
+        return false;
+    }
+
+    std::string help() const override {
+        return "{whatsmyuri:1}";
+    }
+
+    void addRequiredPrivileges(const std::string& dbname,
+                               const BSONObj& cmdObj,
+                               std::vector<Privilege>* out) const override {}  // No auth required
+
+    bool run(OperationContext* opCtx,
+             const std::string& dbname,
+             const BSONObj& cmdObj,
+             BSONObjBuilder& result) override {
+        result << "you" << opCtx->getClient()->clientAddress(true /*includePort*/);
+        return true;
+    }
+
+} cmdWhatsMyUri;
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index af3d35d6490..c5662a4964e 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -1038,10 +1038,24 @@ env.CppUnitTest('isself_test',
                 )
 
 env.Library(
+    target='repl_set_status_commands',
+    source=[
+        'repl_set_command.cpp',
+        'repl_set_get_status_cmd.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/auth/authcore',
+        '$BUILD_DIR/mongo/db/commands',
+        '$BUILD_DIR/mongo/db/lasterror',
+        '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
+    ],
+)
+
+env.Library(
     target='repl_set_commands',
     source=[
         'repl_set_commands.cpp',
-        'repl_set_command.cpp',
         'repl_set_request_votes.cpp',
     ],
     LIBDEPS=[
@@ -1054,6 +1068,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
         '$BUILD_DIR/mongo/db/storage/storage_options',
         'drop_pending_collection_reaper',
+        'repl_set_status_commands',
         'repl_settings',
         'replica_set_messages',
         'replication_process',
diff --git a/src/mongo/db/repl/repl_set_commands.cpp b/src/mongo/db/repl/repl_set_commands.cpp
index 8b6cc7dd2e2..d96486aa7d5 100644
--- a/src/mongo/db/repl/repl_set_commands.cpp
+++ b/src/mongo/db/repl/repl_set_commands.cpp
@@ -176,47 +176,6 @@ public:
     }
 } cmdReplSetRBID;
 
-class CmdReplSetGetStatus : public ReplSetCommand {
-public:
-    std::string help() const override {
-        return "Report status of a replica set from the POV of this server\n"
-               "{ replSetGetStatus : 1 }\n"
-               "http://dochub.mongodb.org/core/replicasetcommands";
-    }
-    CmdReplSetGetStatus() : ReplSetCommand("replSetGetStatus") {}
-    virtual bool run(OperationContext* opCtx,
-                     const string&,
-                     const BSONObj& cmdObj,
-                     BSONObjBuilder& result) {
-        if (cmdObj["forShell"].trueValue())
-            LastError::get(opCtx->getClient()).disable();
-
-        Status status = ReplicationCoordinator::get(opCtx)->checkReplEnabledForCommand(&result);
-        if (!status.isOK())
-            return CommandHelpers::appendCommandStatus(result, status);
-
-        bool includeInitialSync = false;
-        Status initialSyncStatus =
-            bsonExtractBooleanFieldWithDefault(cmdObj, "initialSync", false, &includeInitialSync);
-        if (!initialSyncStatus.isOK()) {
-            return CommandHelpers::appendCommandStatus(result, initialSyncStatus);
-        }
-
-        auto responseStyle = ReplicationCoordinator::ReplSetGetStatusResponseStyle::kBasic;
-        if (includeInitialSync) {
-            responseStyle = ReplicationCoordinator::ReplSetGetStatusResponseStyle::kInitialSync;
-        }
-        status =
-            ReplicationCoordinator::get(opCtx)->processReplSetGetStatus(&result, responseStyle);
-        return CommandHelpers::appendCommandStatus(result, status);
-    }
-
-private:
-    ActionSet getAuthActionSet() const override {
-        return ActionSet{ActionType::replSetGetStatus};
-    }
-} cmdReplSetGetStatus;
-
 class CmdReplSetGetConfig : public ReplSetCommand {
 public:
     std::string help() const override {
diff --git a/src/mongo/db/repl/repl_set_get_status_cmd.cpp b/src/mongo/db/repl/repl_set_get_status_cmd.cpp
new file mode 100644
index 00000000000..fb6e7fd7006
--- /dev/null
+++ b/src/mongo/db/repl/repl_set_get_status_cmd.cpp
@@ -0,0 +1,80 @@
+/**
+*    Copyright (C) 2018 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects for
+*    all of the code used other than as permitted herein. If you modify file(s)
+*    with this exception, you may extend this exception to your version of the
+*    file(s), but you are not obligated to do so. If you do not wish to do so,
+*    delete this exception statement from your version. If you delete this
+*    exception statement from all source files in the program, then also delete
+*    it in the license file.
+*/
+
+#include "mongo/bson/util/bson_extract.h"
+#include "mongo/db/lasterror.h"
+#include "mongo/db/repl/repl_set_command.h"
+
+namespace mongo {
+namespace repl {
+
+class CmdReplSetGetStatus : public ReplSetCommand {
+public:
+    std::string help() const override {
+        return "Report status of a replica set from the POV of this server\n"
+               "{ replSetGetStatus : 1 }\n"
+               "http://dochub.mongodb.org/core/replicasetcommands";
+    }
+
+    CmdReplSetGetStatus() : ReplSetCommand("replSetGetStatus") {}
+
+    bool run(OperationContext* opCtx,
+             const std::string&,
+             const BSONObj& cmdObj,
+             BSONObjBuilder& result) override {
+        if (cmdObj["forShell"].trueValue())
+            LastError::get(opCtx->getClient()).disable();
+
+        Status status = ReplicationCoordinator::get(opCtx)->checkReplEnabledForCommand(&result);
+        if (!status.isOK())
+            return CommandHelpers::appendCommandStatus(result, status);
+
+        bool includeInitialSync = false;
+        Status initialSyncStatus =
+            bsonExtractBooleanFieldWithDefault(cmdObj, "initialSync", false, &includeInitialSync);
+        if (!initialSyncStatus.isOK()) {
+            return CommandHelpers::appendCommandStatus(result, initialSyncStatus);
+        }
+
+        auto responseStyle = ReplicationCoordinator::ReplSetGetStatusResponseStyle::kBasic;
+        if (includeInitialSync) {
+            responseStyle = ReplicationCoordinator::ReplSetGetStatusResponseStyle::kInitialSync;
+        }
+        status =
+            ReplicationCoordinator::get(opCtx)->processReplSetGetStatus(&result, responseStyle);
+        return CommandHelpers::appendCommandStatus(result, status);
+    }
+
+private:
+    ActionSet getAuthActionSet() const override {
+        return ActionSet{ActionType::replSetGetStatus};
+    }
+} cmdReplSetGetStatus;
+
+}  // namespace repl
+}  // namespace mongo