SERVER-34226 Registration Integration Tests

7 files changed, 530 insertions, 0 deletions

diff --git a/jstests/free_mon/free_mon_http_down.js b/jstests/free_mon/free_mon_http_down.js
new file mode 100644
index 00000000000..aff229614c4
--- /dev/null
+++ b/jstests/free_mon/free_mon_http_down.js
@@ -0,0 +1,26 @@
+// Validate registration retries if the web server is down.
+//
+load("jstests/free_mon/libs/free_mon.js");
+
+(function() {
+    'use strict';
+
+    let mock_web = new FreeMonWebServer(FAULT_FAIL_REGISTER);
+
+    mock_web.start();
+
+    let options = {
+        setParameter: "cloudFreeMonitoringEndpointURL=" + mock_web.getURL(),
+        enableFreeMonitoring: "on",
+        verbose: 1,
+    };
+
+    const conn = MongoRunner.runMongod(options);
+    assert.neq(null, conn, 'mongod was unable to start up');
+
+    mock_web.waitRegisters(3);
+
+    MongoRunner.stopMongod(conn);
+
+    mock_web.stop();
+})();
diff --git a/jstests/free_mon/free_mon_http_validate.js b/jstests/free_mon/free_mon_http_validate.js
new file mode 100644
index 00000000000..aff5ad5e8ec
--- /dev/null
+++ b/jstests/free_mon/free_mon_http_validate.js
@@ -0,0 +1,35 @@
+// Ensure free monitoring gives up if registration fails
+//
+load("jstests/free_mon/libs/free_mon.js");
+
+(function() {
+    'use strict';
+
+    let mock_web = new FreeMonWebServer(FAULT_INVALID_REGISTER);
+
+    mock_web.start();
+
+    let options = {
+        setParameter: "cloudFreeMonitoringEndpointURL=" + mock_web.getURL(),
+        enableFreeMonitoring: "on",
+        verbose: 1,
+    };
+
+    const conn = MongoRunner.runMongod(options);
+    assert.neq(null, conn, 'mongod was unable to start up');
+
+    mock_web.waitRegisters(1);
+
+    // Sleep for some more time in case free monitoring would still try to register
+    sleep(20 * 1000);
+
+    // Ensure it only tried to register once since we gave it a bad response.
+    const stats = mock_web.queryStats();
+    print(tojson(stats));
+
+    assert.eq(stats.registers, 1);
+
+    MongoRunner.stopMongod(conn);
+
+    mock_web.stop();
+})();
diff --git a/jstests/free_mon/free_mon_register.js b/jstests/free_mon/free_mon_register.js
new file mode 100644
index 00000000000..362cc74cb04
--- /dev/null
+++ b/jstests/free_mon/free_mon_register.js
@@ -0,0 +1,40 @@
+// Validate registration works
+//
+load("jstests/free_mon/libs/free_mon.js");
+
+(function() {
+    'use strict';
+
+    let mock_web = new FreeMonWebServer();
+
+    mock_web.start();
+
+    let options = {
+        setParameter: "cloudFreeMonitoringEndpointURL=" + mock_web.getURL(),
+        enableFreeMonitoring: "on",
+        verbose: 1,
+    };
+
+    const conn = MongoRunner.runMongod(options);
+    assert.neq(null, conn, 'mongod was unable to start up');
+
+    WaitForRegistration(conn);
+
+    const stats = mock_web.queryStats();
+    print(tojson(stats));
+
+    assert.eq(stats.registers, 1);
+
+    const last_register = mock_web.query("last_register");
+    print(tojson(last_register));
+
+    assert.eq(last_register.version, 1);
+    assert.eq(last_register.payload.buildInfo.bits, 64);
+    assert.eq(last_register.payload.buildInfo.ok, 1);
+    assert.eq(last_register.payload.storageEngine.readOnly, false);
+    assert.eq(last_register.payload.isMaster.ok, 1);
+
+    MongoRunner.stopMongod(conn);
+
+    mock_web.stop();
+})();
diff --git a/jstests/free_mon/libs/free_mon.js b/jstests/free_mon/libs/free_mon.js
new file mode 100644
index 00000000000..5e507ce3641
--- /dev/null
+++ b/jstests/free_mon/libs/free_mon.js
@@ -0,0 +1,157 @@
+
+/**
+ * Control the Free Monitoring Mock Webserver.
+ */
+
+// These faults must match the list of faults in mock_http_server.py, see the
+// SUPPORTED_FAULT_TYPES list in mock_http_server.py
+const FAULT_FAIL_REGISTER = "fail_register";
+const FAULT_INVALID_REGISTER = "invalid_register";
+
+class FreeMonWebServer {
+    /**
+    * Create a new webserver.
+    *
+    * @param {string} fault_type
+    */
+    constructor(fault_type) {
+        this.python = "/opt/mongodbtoolchain/v2/bin/python3";
+        this.fault_type = fault_type;
+
+        if (_isWindows()) {
+            const paths = ["c:\\python36\\python.exe", "c:\\python\\python36\\python.exe"];
+            for (let p of paths) {
+                if (fileExists(p)) {
+                    this.python = p;
+                }
+            }
+        }
+
+        print("Using python interpreter: " + this.python);
+        this.web_server_py = "jstests/free_mon/libs/mock_http_server.py";
+        this.control_py = "jstests/free_mon/libs/mock_http_control.py";
+
+        this.pid = undefined;
+        this.port = -1;
+    }
+
+    /**
+     * Get the Port.
+     *
+     * @return {number} port number of http server
+     */
+    getPort() {
+        return port;
+    }
+
+    /**
+     * Get the URL.
+     *
+     * @return {string} url of http server
+     */
+    getURL() {
+        return "http://localhost:" + this.port;
+    }
+
+    /**
+     *  Start the Mock HTTP Server.
+     */
+    start() {
+        this.port = allocatePort();
+        print("Mock Web server is listening on port: " + this.port);
+
+        let args = [this.python, "-u", this.web_server_py, "--port=" + this.port];
+        if (this.fault_type) {
+            args.push("--fault=" + this.fault_type);
+        }
+
+        this.pid = _startMongoProgram({args: args});
+
+        assert(checkProgram(this.pid));
+
+        // Wait for the web server to start
+        assert.soon(function() {
+            return rawMongoProgramOutput().search("Mock Web Server Listening") !== -1;
+        });
+
+        print("Mock HTTP Server sucessfully started.");
+    }
+
+    /**
+     *  Stop the Mock HTTP Server.
+     */
+    stop() {
+        stopMongoProgramByPid(this.pid);
+    }
+
+    /**
+     * Query the HTTP server.
+     *
+     * @param {string} query type
+     *
+     * @return {object} Object representation of JSON from the server.
+     */
+    query(query) {
+        const out_file = "out_" + this.port + ".txt";
+        const python_command = this.python + " -u " + this.control_py + " --port=" + this.port +
+            " --query=" + query + " > " + out_file;
+
+        let ret = 0;
+        if (_isWindows()) {
+            ret = runProgram('cmd.exe', '/c', python_command);
+        } else {
+            ret = runProgram('/bin/sh', '-c', python_command);
+        }
+
+        assert.eq(ret, 0);
+
+        const result = cat(out_file);
+
+        try {
+            return JSON.parse(result);
+        } catch (e) {
+            jsTestLog("Failed to parse: " + result + "\n" + result);
+            throw e;
+        }
+    }
+
+    /**
+     * Query the stats page for the HTTP server.
+     *
+     * @return {object} Object representation of JSON from the server.
+     */
+    queryStats() {
+        return this.query("stats");
+    }
+
+    /**
+     * Wait for N register calls to be received by web server.
+     *
+     * @throws assert.soon() exception
+     */
+    waitRegisters(count) {
+        const qs = this.queryStats.bind(this);
+        // Wait for registration to occur
+        assert.soon(function() {
+            const stats = qs();
+            print("QS : " + tojson(stats));
+            return stats.registers == count;
+        }, "Failed to web server register", 60 * 1000);
+    }
+}
+
+/**
+ * Wait for registration information to be populated in the database.
+ *
+ * @param {object} conn
+ */
+function WaitForRegistration(conn) {
+    const admin = conn.getDB("admin");
+
+    // Wait for registration to occur
+    assert.soon(function() {
+        const docs = admin.system.version.find({_id: "free_monitoring"});
+        const da = docs.toArray();
+        return da.length != 0;
+    }, "Failed to register", 60 * 1000);
+}
diff --git a/jstests/free_mon/libs/mock_http_common.py b/jstests/free_mon/libs/mock_http_common.py
new file mode 100644
index 00000000000..7840894062c
--- /dev/null
+++ b/jstests/free_mon/libs/mock_http_common.py
@@ -0,0 +1,19 @@
+"""Common code for mock free monitoring http endpoint."""
+import json
+
+URL_PATH_STATS = "/stats"
+URL_PATH_LAST_REGISTER = "/last_register"
+URL_PATH_LAST_METRICS = "/last_metrics"
+
+class Stats:
+    """Stats class shared between client and server."""
+
+    def __init__(self):
+        self.register_calls = 0
+        self.metrics_calls = 0
+
+    def __repr__(self):
+        return json.dumps({
+            'metrics': self.metrics_calls,
+            'registers': self.register_calls,
+        })
diff --git a/jstests/free_mon/libs/mock_http_control.py b/jstests/free_mon/libs/mock_http_control.py
new file mode 100644
index 00000000000..fc72021c143
--- /dev/null
+++ b/jstests/free_mon/libs/mock_http_control.py
@@ -0,0 +1,47 @@
+#! /usr/bin/env python3
+"""
+Python script to interact with mock free monitoring HTTP server.
+"""
+
+import argparse
+import json
+import logging
+import sys
+import urllib.request
+
+import mock_http_common
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(description='MongoDB Mock Free Monitoring Endpoint.')
+
+    parser.add_argument('-p', '--port', type=int, default=8000, help="Port to listen on")
+
+    parser.add_argument('-v', '--verbose', action='count', help="Enable verbose tracing")
+
+    parser.add_argument('--query', type=str, help="Query endpoint <name>")
+
+    args = parser.parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    url_str = "http://localhost:" + str(args.port)
+    if args.query == "stats":
+        url_str += mock_http_common.URL_PATH_STATS
+    elif args.query == "last_register":
+        url_str += mock_http_common.URL_PATH_LAST_REGISTER
+    elif args.query == "last_metrics":
+        url_str += mock_http_common.URL_PATH_LAST_METRICS
+    else:
+        print("Unknown query type")
+        sys.exit(1)
+
+    with urllib.request.urlopen(url_str) as f:
+        print(f.read().decode('utf-8'))
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+
+    main()
diff --git a/jstests/free_mon/libs/mock_http_server.py b/jstests/free_mon/libs/mock_http_server.py
new file mode 100644
index 00000000000..cafdf989550
--- /dev/null
+++ b/jstests/free_mon/libs/mock_http_server.py
@@ -0,0 +1,206 @@
+#! /usr/bin/env python3
+"""Mock Free Monitoring Endpoint."""
+
+import argparse
+import collections
+import http.server
+import json
+import logging
+import socketserver
+import sys
+import urllib.parse
+
+import bson
+from bson.codec_options import CodecOptions
+from bson.json_util import dumps
+import mock_http_common
+
+# Pass this data out of band instead of storing it in FreeMonHandler since the
+# BaseHTTPRequestHandler does not call the methods as object methods but as class methods. This
+# means there is not self.
+stats = mock_http_common.Stats()
+last_metrics = None
+last_register = None
+
+fault_type = None
+
+"""Fault which causes the server to return an HTTP failure on register."""
+FAULT_FAIL_REGISTER = "fail_register"
+
+"""Fault which causes the server to return a response with a document with a bad version."""
+FAULT_INVALID_REGISTER = "invalid_register"
+
+# List of supported fault types
+SUPPORTED_FAULT_TYPES = [
+    FAULT_FAIL_REGISTER,
+    FAULT_INVALID_REGISTER
+]
+
+# Supported POST URL types
+URL_POST_REGISTER = '/register'
+URL_POST_METRICS = '/metrics'
+
+
+class FreeMonHandler(http.server.BaseHTTPRequestHandler):
+    """
+    Handle requests from Free Monitoring and test commands
+    """
+
+    def do_GET(self):
+        """Serve a Test GET request."""
+        parts = urllib.parse.urlsplit(self.path)
+        path = parts[2]
+
+        if path == mock_http_common.URL_PATH_STATS:
+            self._do_stats()
+        elif path == mock_http_common.URL_PATH_LAST_REGISTER:
+            self._do_last_register()
+        elif path == mock_http_common.URL_PATH_LAST_METRICS:
+            self._do_last_metrics()
+        else:
+            self.send_response(http.HTTPStatus.NOT_FOUND)
+            self.end_headers()
+            self.wfile.write("Unknown URL".encode())
+
+    def do_POST(self):
+        """Serve a Free Monitoring POST request."""
+        parts = urllib.parse.urlsplit(self.path)
+        path = parts[2]
+
+        if path == URL_POST_REGISTER:
+            self._do_registration()
+        elif path == URL_POST_METRICS:
+            self._do_metrics()
+        else:
+            self.send_response(http.HTTPStatus.NOT_FOUND)
+            self.end_headers()
+            self.wfile.write("Unknown URL".encode())
+
+    def _send_header(self):
+        self.send_response(http.HTTPStatus.OK)
+        self.send_header("content-type", "application/octet-stream")
+        self.end_headers()
+
+    def _do_registration(self):
+        global stats
+        global last_register
+        clen = int(self.headers.get('content-length'))
+
+        stats.register_calls += 1
+
+        raw_input = self.rfile.read(clen)
+        decoded_doc = bson.BSON.decode(raw_input)
+        last_register = dumps(decoded_doc)
+
+        if fault_type == FAULT_FAIL_REGISTER:
+            self.send_response(http.HTTPStatus.INTERNAL_SERVER_ERROR)
+            self.send_header("content-type", "application/octet-stream")
+            self.end_headers()
+            return
+
+        if fault_type == FAULT_INVALID_REGISTER:
+            data = bson.BSON.encode({
+                'version': bson.int64.Int64(42),
+                'haltMetricsUploading': False,
+                'id': 'mock123',
+                'informationalURL': 'http://www.example.com/123',
+                'message': 'Welcome to the Mock Free Monitoring Endpoint',
+                'reportingInterval': bson.int64.Int64(1),
+            })
+        else:
+            data = bson.BSON.encode({
+                'version': bson.int64.Int64(1),
+                'haltMetricsUploading': False,
+                'id': 'mock123',
+                'informationalURL': 'http://www.example.com/123',
+                'message': 'Welcome to the Mock Free Monitoring Endpoint',
+                'reportingInterval': bson.int64.Int64(1),
+            })
+
+        self._send_header()
+
+        self.wfile.write(data)
+
+    def _do_metrics(self):
+        global stats
+        global last_metrics
+        clen = int(self.headers.get('content-length'))
+
+        stats.metrics_calls += 1
+
+        raw_input = self.rfile.read(clen)
+        decoded_doc = bson.BSON.decode(raw_input)
+        last_metrics = dumps(decoded_doc)
+
+        data = bson.BSON.encode({
+            'version': bson.int64.Int64(1),
+            'haltMetricsUploading': False,
+            'permanentlyDelete': False,
+            'id': 'mock123',
+            'reportingInterval': bson.int64.Int64(1),
+            'message': 'Thanks for all the metrics',
+        })
+
+        # TODO: test what if header is sent first?
+        self._send_header()
+
+        self.wfile.write(data)
+
+    def _do_stats(self):
+        self._send_header()
+
+        self.wfile.write(str(stats).encode('utf-8'))
+
+    def _do_last_register(self):
+        self._send_header()
+
+        self.wfile.write(str(last_register).encode('utf-8'))
+
+    def _do_last_metrics(self):
+        self._send_header()
+
+        self.wfile.write(str(last_metrics).encode('utf-8'))
+
+
+def run(port, server_class=http.server.HTTPServer, handler_class=FreeMonHandler):
+    """Run web server."""
+    server_address = ('', port)
+
+    http.server.HTTPServer.protocol_version = "HTTP/1.1"
+
+    httpd = server_class(server_address, handler_class)
+
+    print("Mock Web Server Listening on %s" % (str(server_address)))
+
+    httpd.serve_forever()
+
+
+def main():
+    """Main Method."""
+    global fault_type
+
+    parser = argparse.ArgumentParser(description='MongoDB Mock Free Monitoring Endpoint.')
+
+    parser.add_argument('-p', '--port', type=int, default=8000, help="Port to listen on")
+
+    parser.add_argument('-v', '--verbose', action='count', help="Enable verbose tracing")
+
+    parser.add_argument('--fault', type=str, help="Type of fault to inject")
+
+    args = parser.parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    if args.fault:
+        if args.fault not in SUPPORTED_FAULT_TYPES:
+            print("Unsupported fault type %s, supports types are %s" % (args.fault, SUPPORTED_FAULT_TYPES))
+            sys.exit(1)
+
+        fault_type = args.fault
+
+    run(args.port)
+
+
+if __name__ == '__main__':
+
+    main()