SERVER-37823 Server Side Traffic Capture

Adds support for special commands which dump wire protocol traffic to
disk.

11 files changed, 332 insertions, 5 deletions

diff --git a/jstests/auth/commands_builtin_roles.js b/jstests/auth/commands_builtin_roles.js
index f309435d6d6..32674cd8a41 100644
--- a/jstests/auth/commands_builtin_roles.js
+++ b/jstests/auth/commands_builtin_roles.js
@@ -143,7 +143,13 @@ function checkForNonExistentRoles() {
     }
 }
 
-var opts = {auth: "", enableExperimentalStorageDetailsCmd: ""};
+const dbPath = MongoRunner.toRealDir("$dataDir/commands_built_in_roles/");
+mkdir(dbPath);
+var opts = {
+    auth: "",
+    enableExperimentalStorageDetailsCmd: "",
+    setParameter: "trafficRecordingDirectory=" + dbPath
+};
 var impls = {createUsers: createUsers, runOneTest: runOneTest};
 
 checkForNonExistentRoles();
@@ -159,7 +165,11 @@ conn = new ShardingTest({
     shards: 2,
     mongos: 1,
     keyFile: "jstests/libs/key1",
-    other: {shardOptions: opts, shardAsReplicaSet: false}
+    other: {
+        shardOptions: opts,
+        shardAsReplicaSet: false,
+        mongosOptions: {setParameter: "trafficRecordingDirectory=" + dbPath}
+    }
 });
 authCommandsLib.runTests(conn, impls);
 conn.stop();
diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index 6899d0646aa..99b6d9a6924 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -5972,7 +5972,29 @@ var authCommandsLib = {
                       {killCursors: "$cmd.aggregate", cursors: [response.cursor.id]}));
               }
           }
-        }
+        },
+        {
+          testname: "startRecordingTraffic",
+          command: {startRecordingTraffic: 1, filename: "notARealPath"},
+          testcases: [
+              {runOnDb: adminDbName, roles: roles_hostManager},
+          ],
+          teardown: (db, response) => {
+              if (response.ok) {
+                  assert.commandWorked(db.runCommand({stopRecordingTraffic: 1}));
+              }
+          }
+        },
+        {
+          testname: "stopRecordingTraffic",
+          command: {stopRecordingTraffic: 1},
+          testcases: [
+              {runOnDb: adminDbName, roles: roles_hostManager},
+          ],
+          setup: function(db) {
+              db.runCommand({startRecordingTraffic: 1, filename: "notARealPath"});
+          }
+        },
     ],
 
     /************* SHARED TEST LOGIC ****************/
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index c5bafd7671b..21bc90b7138 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -514,7 +514,9 @@
             expectFailure: true,
         },
         stageDebug: {skip: isAnInternalCommand},
+        startRecordingTraffic: {skip: isUnrelated},
         startSession: {skip: isAnInternalCommand},
+        stopRecordingTraffic: {skip: isUnrelated},
         top: {skip: "tested in views/views_stats.js"},
         touch: {
             command: {touch: "view", data: true},
diff --git a/jstests/noPassthrough/traffic_reading.js b/jstests/noPassthrough/traffic_reading.js
new file mode 100644
index 00000000000..18642f03be4
--- /dev/null
+++ b/jstests/noPassthrough/traffic_reading.js
@@ -0,0 +1,81 @@
+// tests for the traffic_recording commands.
+(function() {
+    // Variables for this test
+    const recordingDir = MongoRunner.toRealDir("$dataDir/traffic_recording/");
+    const recordingFile = "recording.txt";
+    const recordingFilePath = MongoRunner.toRealDir(recordingDir + "/" + recordingFile);
+    const replayFilePath = MongoRunner.toRealDir(recordingDir + "/replay.txt");
+
+    // Create the recording directory if it does not already exist
+    mkdir(recordingDir);
+
+    // Create the options and run mongod
+    var opts = {auth: "", setParameter: "trafficRecordingDirectory=" + recordingDir};
+    m = MongoRunner.runMongod(opts);
+
+    // Get the port of the host
+    var serverPort = m.port;
+
+    // Create necessary users
+    adminDB = m.getDB("admin");
+    const testDB = m.getDB("test");
+    const coll = testDB.getCollection("foo");
+    adminDB.createUser({user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
+    adminDB.auth("admin", "pass");
+
+    // Start recording traffic
+    assert.commandWorked(
+        adminDB.runCommand({'startRecordingTraffic': 1, 'filename': 'recording.txt'}));
+
+    // Run a few commands
+    assert.commandWorked(testDB.runCommand({"serverStatus": 1}));
+    assert.commandWorked(coll.insert({"name": "foo biz bar"}));
+    assert.eq("foo biz bar", coll.findOne().name);
+    assert.commandWorked(coll.insert({"name": "foo bar"}));
+    assert.eq("foo bar", coll.findOne({"name": "foo bar"}).name);
+    assert.commandWorked(coll.deleteOne({}));
+    assert.eq(1, coll.aggregate().toArray().length);
+    assert.commandWorked(coll.update({}, {}));
+
+    // Stop recording traffic
+    assert.commandWorked(testDB.runCommand({'stopRecordingTraffic': 1}));
+
+    // Shutdown Mongod
+    MongoRunner.stopMongod(m, null, {user: 'admin', pwd: 'password'});
+
+    // Counters
+    var numRequest = 0;
+    var numResponse = 0;
+    var opTypes = {};
+
+    // Pass filepath to traffic_reader helper method to get recorded info in BSON
+    var res = convertTrafficRecordingToBSON(recordingFilePath);
+
+    // Iterate through the results and assert the above commands are properly recorded
+    res.forEach((obj) => {
+        assert.eq(obj["rawop"]["header"]["opcode"], 2013);
+        assert.eq(obj["seenconnectionnum"], 1);
+        var responseTo = obj["rawop"]["header"]["responseto"];
+        if (responseTo == 0) {
+            assert.eq(obj["destendpoint"], serverPort.toString());
+            numRequest++;
+        } else {
+            assert.eq(obj["srcendpoint"], serverPort.toString());
+            numResponse++;
+        }
+        opTypes[obj["opType"]] = (opTypes[obj["opType"]] || 0) + 1;
+    });
+
+    // Assert there is a response for every request
+    assert.eq(numResponse, numRequest);
+
+    // Assert the opTypes were correct
+    assert.eq(opTypes['isMaster'], opTypes["ismaster"]);
+    assert.eq(opTypes['find'], 2);
+    assert.eq(opTypes['insert'], 2);
+    assert.eq(opTypes['delete'], 1);
+    assert.eq(opTypes['update'], 1);
+    assert.eq(opTypes['aggregate'], 1);
+    assert.eq(opTypes['stopRecordingTraffic'], 1);
+
+})();
diff --git a/jstests/noPassthrough/traffic_reading_legacy.js b/jstests/noPassthrough/traffic_reading_legacy.js
new file mode 100644
index 00000000000..9224edf926a
--- /dev/null
+++ b/jstests/noPassthrough/traffic_reading_legacy.js
@@ -0,0 +1,72 @@
+// tests for the traffic_recording commands.
+(function() {
+    var baseName = "jstests_traffic_recording";
+
+    // Variables for this test
+    const recordingDir = MongoRunner.toRealDir("$dataDir/traffic_recording/");
+    const recordingFile = "recording.txt";
+    const recordingFilePath = MongoRunner.toRealDir(recordingDir + "/" + recordingFile);
+
+    // Create the recording directory if it does not already exist
+    mkdir(recordingDir);
+
+    // Create the options and run mongod
+    var opts = {auth: "", setParameter: "trafficRecordingDirectory=" + recordingDir};
+    m = MongoRunner.runMongod(opts);
+
+    // Get the port of the host
+    var serverPort = m.port;
+
+    // Set the readMode and writeMode to legacy
+    m.forceReadMode("legacy");
+    m.forceWriteMode("legacy");
+
+    // Create necessary users
+    adminDB = m.getDB("admin");
+    const testDB = m.getDB("test");
+    const coll = testDB.getCollection("foo");
+    adminDB.createUser({user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
+    adminDB.auth("admin", "pass");
+
+    // Start recording traffic
+    assert.commandWorked(
+        adminDB.runCommand({'startRecordingTraffic': 1, 'filename': 'recording.txt'}));
+
+    // Run a few commands
+    testDB.runCommand({"serverStatus": 1});
+    coll.insert({"name": "foo biz bar"});
+    coll.findOne();
+    coll.insert({"name": "foo bar"});
+    coll.findOne({"name": "foo bar"});
+    coll.deleteOne({});
+
+    // Stop recording traffic
+    assert.commandWorked(testDB.runCommand({'stopRecordingTraffic': 1}));
+
+    // Shutdown Mongod
+    MongoRunner.stopMongod(m, null, {user: 'admin', pwd: 'password'});
+
+    // Counters
+    var opCodes = {};
+
+    // Pass filepath to traffic_reader helper method to get recorded info in BSON
+    var res = convertTrafficRecordingToBSON(recordingFilePath);
+
+    // Iterate through the results and assert the above commands are properly recorded
+    res.forEach((obj) => {
+        opCodes[obj["rawop"]["header"]["opcode"]] =
+            (opCodes[obj["rawop"]["header"]["opcode"]] || 0) + 1;
+        assert.eq(obj["seenconnectionnum"], 1);
+        var responseTo = obj["rawop"]["header"]["responseto"];
+        if (responseTo == 0) {
+            assert.eq(obj["destendpoint"], serverPort.toString());
+        } else {
+            assert.eq(obj["srcendpoint"], serverPort.toString());
+        }
+    });
+
+    // ensure legacy operations worked properly
+    assert.eq(opCodes[2002], 2);
+    assert.eq(opCodes[2006], 1);
+
+})();
diff --git a/jstests/noPassthrough/traffic_recording.js b/jstests/noPassthrough/traffic_recording.js
new file mode 100644
index 00000000000..03828809a81
--- /dev/null
+++ b/jstests/noPassthrough/traffic_recording.js
@@ -0,0 +1,126 @@
+// tests for the traffic_recording commands.
+(function() {
+    function getDB(client) {
+        let db = client.getDB("admin");
+        db.auth("admin", "pass");
+
+        return db;
+    }
+
+    function runTest(client, restartCommand) {
+        let db = getDB(client);
+
+        let res = db.runCommand({'startRecordingTraffic': 1, 'filename': 'notARealPath'});
+        assert.eq(res.ok, false);
+        assert.eq(res["errmsg"], "Traffic recording directory not set");
+
+        const path = MongoRunner.toRealDir("$dataDir/traffic_recording/");
+        mkdir(path);
+
+        if (!jsTest.isMongos(client)) {
+            setJsTestOption("enableTestCommands", 0);
+            client = restartCommand({
+                trafficRecordingDirectory: path,
+                AlwaysRecordTraffic: "notARealPath",
+                enableTestCommands: 0,
+            });
+            setJsTestOption("enableTestCommands", 1);
+            assert.eq(null, client, "AlwaysRecordTraffic and not enableTestCommands should fail");
+        }
+
+        client = restartCommand({
+            trafficRecordingDirectory: path,
+            AlwaysRecordTraffic: "notARealPath",
+            enableTestCommands: 1
+        });
+        assert.neq(null, client, "AlwaysRecordTraffic and with enableTestCommands should suceed");
+        db = getDB(client);
+
+        assert(db.runCommand({"serverStatus": 1}).trafficRecording.running);
+
+        client = restartCommand({trafficRecordingDirectory: path});
+        db = getDB(client);
+
+        res = db.runCommand({'startRecordingTraffic': 1, 'filename': 'notARealPath'});
+        assert.eq(res.ok, true);
+
+        // Running the command again should fail
+        res = db.runCommand({'startRecordingTraffic': 1, 'filename': 'notARealPath'});
+        assert.eq(res.ok, false);
+        assert.eq(res["errmsg"], "Traffic recording already active");
+
+        // Running the serverStatus command should return the relevant information
+        res = db.runCommand({"serverStatus": 1});
+        assert("trafficRecording" in res);
+        let trafficStats = res["trafficRecording"];
+        assert.eq(trafficStats["running"], true);
+
+        // Assert that the current file size is growing
+        res = db.runCommand({"serverStatus": 1});
+        assert("trafficRecording" in res);
+        let trafficStats2 = res["trafficRecording"];
+        assert.eq(trafficStats2["running"], true);
+        assert(trafficStats2["currentFileSize"] >= trafficStats["currentFileSize"]);
+
+        // Running the stopRecordingTraffic command should succeed
+        res = db.runCommand({'stopRecordingTraffic': 1});
+        assert.eq(res.ok, true);
+
+        // Running the stopRecordingTraffic command again should fail
+        res = db.runCommand({'stopRecordingTraffic': 1});
+        assert.eq(res.ok, false);
+        assert.eq(res["errmsg"], "Traffic recording not active");
+
+        // Running the serverStatus command should return running is false
+        res = db.runCommand({"serverStatus": 1});
+        assert("trafficRecording" in res);
+        trafficStats = res["trafficRecording"];
+        assert.eq(trafficStats["running"], false);
+
+        return client;
+    }
+
+    {
+        let m = MongoRunner.runMongod({auth: ""});
+
+        let db = m.getDB("admin");
+
+        db.createUser({user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
+        db.auth("admin", "pass");
+
+        m = runTest(m, function(setParams) {
+            if (m) {
+                MongoRunner.stopMongod(m, null, {user: 'admin', pwd: 'pass'});
+            }
+            m = MongoRunner.runMongod({auth: "", setParameter: setParams});
+
+            if (m) {
+                m.getDB("admin").createUser(
+                    {user: "admin", pwd: "pass", roles: jsTest.adminUserRoles});
+            }
+
+            return m;
+        });
+
+        MongoRunner.stopMongod(m, null, {user: 'admin', pwd: 'pass'});
+    }
+
+    {
+        let shardTest = new ShardingTest({
+            config: 1,
+            mongos: 1,
+            shards: 0,
+        });
+
+        runTest(shardTest.s, function(setParams) {
+            shardTest.restartMongos(0, {
+                restart: true,
+                setParameter: setParams,
+            });
+
+            return shardTest.s;
+        });
+
+        shardTest.stop();
+    }
+})();
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index 9dff71e3862..0c680da3df3 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -412,7 +412,9 @@
         shutdown: {skip: "does not forward command to primary shard"},
         split: {skip: "does not forward command to primary shard"},
         splitVector: {skip: "does not forward command to primary shard"},
+        startRecordingTraffic: {skip: "executes locally on mongos (not sent to any remote node)"},
         startSession: {skip: "executes locally on mongos (not sent to any remote node)"},
+        stopRecordingTraffic: {skip: "executes locally on mongos (not sent to any remote node)"},
         update: {
             skipProfilerCheck: true,
             sendsDbVersion: false,
diff --git a/jstests/sharding/libs/last_stable_mongos_commands.js b/jstests/sharding/libs/last_stable_mongos_commands.js
index fdd1bc0fa4a..e9470a1794a 100644
--- a/jstests/sharding/libs/last_stable_mongos_commands.js
+++ b/jstests/sharding/libs/last_stable_mongos_commands.js
@@ -15,5 +15,11 @@ const commandsRemovedFromMongosIn42 = [
 // These commands were added in mongos 4.2, so will not appear in the listCommands output of a 4.0
 // mongos. We will allow these commands to have a test defined without always existing on the mongos
 // being used.
-const commandsAddedToMongosIn42 =
-    ['abortTransaction', 'commitTransaction', 'dropConnections', 'setIndexCommitQuorum'];
+const commandsAddedToMongosIn42 = [
+    'abortTransaction',
+    'commitTransaction',
+    'dropConnections',
+    'setIndexCommitQuorum',
+    'startRecordingTraffic',
+    'stopRecordingTraffic',
+];
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index e54deea9f30..814b79b6308 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -289,7 +289,9 @@
         splitChunk: {skip: "primary only"},
         splitVector: {skip: "primary only"},
         stageDebug: {skip: "primary only"},
+        startRecordingTraffic: {skip: "does not return user data"},
         startSession: {skip: "does not return user data"},
+        stopRecordingTraffic: {skip: "does not return user data"},
         top: {skip: "does not return user data"},
         touch: {skip: "does not return user data"},
         unsetSharding: {skip: "does not return user data"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index 2941787dc71..7f7e8fff3f1 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -324,7 +324,9 @@
         splitChunk: {skip: "primary only"},
         splitVector: {skip: "primary only"},
         stageDebug: {skip: "primary only"},
+        startRecordingTraffic: {skip: "does not return user data"},
         startSession: {skip: "does not return user data"},
+        stopRecordingTraffic: {skip: "does not return user data"},
         top: {skip: "does not return user data"},
         touch: {skip: "does not return user data"},
         unsetSharding: {skip: "does not return user data"},
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index e3be4c90f7c..4ce841f18f9 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -294,7 +294,9 @@
         splitChunk: {skip: "primary only"},
         splitVector: {skip: "primary only"},
         stageDebug: {skip: "primary only"},
+        startRecordingTraffic: {skip: "does not return user data"},
         startSession: {skip: "does not return user data"},
+        stopRecordingTraffic: {skip: "does not return user data"},
         top: {skip: "does not return user data"},
         touch: {skip: "does not return user data"},
         unsetSharding: {skip: "does not return user data"},