SERVER-36077 Create new resmoke.py test suite for running mongoebench on an Android device

(cherry picked from commit 4d21b46427c3ff65ed4fc2ae463054d7d86ad564)
(cherry picked from commit 64359931e3478ecbac756b2fcef25187e8fe04ce)

Also changed the referenced Evergreen project from mongodb-mongo-master
to mongodb-mongo-v4.0.

3 files changed, 270 insertions, 9 deletions

diff --git a/buildscripts/mobile/README-Android.md b/buildscripts/mobile/README-Android.md
index ec04bdd7e8e..ed9cc744eb9 100644
--- a/buildscripts/mobile/README-Android.md
+++ b/buildscripts/mobile/README-Android.md
@@ -13,6 +13,16 @@ The ADB profiler is a custom script which provides
   * CPU statistics - cpu.json
 `python buildscripts/mobile/adb_monitor.py`
 
+
+## Enable USB Debugging on Android Device
+Enabling USB debugging can differ by device, see https://developer.android.com/studio/debug/dev-options
+1. Enable USB debugging via ADB, example
+  * Select Settings/About phone(or tablet)
+  * Select Build number 7 times, to enable Developer Options
+  * Select Settings/Developer Options/USB Debugging
+1. Connect the Android device to the computer via USB cable
+1. Select "Aways allow from this computer" and OK, when the prompt "Allow USB debugging?" appears on the device
+
 ## Run the ADB Profiler Wirelessly
 1. Ensure the local computer and Android device are on the same network
 1. Connect the Android device to the computer via USB cable
@@ -21,7 +31,7 @@ The ADB profiler is a custom script which provides
    * `adb tcpip 5555`
 1. Disconnect the USB cable
 1. Identify the Android's IP address
-   * Settings/About phone/Status
+   * Settings/About phone(or tablet)/Status
    * `adb_ip=<ip_address>`, i.e., adb_ip=10.4.123.244
 1. Connect wirelessly to the Android device
    * `adb connect $adb_ip`
diff --git a/buildscripts/mobile/adb_monitor.py b/buildscripts/mobile/adb_monitor.py
index 0542b5e9428..f9922e6cec4 100644
--- a/buildscripts/mobile/adb_monitor.py
+++ b/buildscripts/mobile/adb_monitor.py
@@ -6,6 +6,7 @@ import logging
 import optparse
 import os
 import pipes
+import re
 import shlex
 import sys
 import tempfile
@@ -52,24 +53,61 @@ class Adb(object):
         self._tempfile = None
 
     @staticmethod
-    def _adb_cmd(adb_command, output_file=None, append_file=False, output_string=False):
+    def adb_cmd(adb_command, output_file=None, append_file=False, output_string=False):
         """Run an adb command and return result."""
         cmd = runcommand.RunCommand("adb {}".format(adb_command), output_file, append_file)
-        if output_string:
+        if output_string or not output_file:
             return cmd.execute_with_output()
         return cmd.execute_save_output()
 
+    @staticmethod
+    def shell(adb_shell_command):
+        """Run an adb shell command and return output_string.
+
+        Raise an exception if the exit status is non-zero.
+
+        Since the adb shell command does not return an exit status. We simulate it by
+        saving the exit code in the output and then stripping if off.
+
+        See https://stackoverflow.com/questions/9379400/adb-error-codes
+        """
+        cmd_prefix = "set -o errexit; function _exit_ { echo __EXIT__:$?; } ; trap _exit_ EXIT ;"
+        cmd = runcommand.RunCommand("adb shell {} {}".format(cmd_prefix, adb_shell_command))
+        cmd_output = cmd.execute_with_output()
+        if "__EXIT__" in cmd_output:
+            exit_code = int(cmd_output.split()[-1].split(":")[1])
+            cmd_output_stripped = re.split("__EXIT__.*\n", cmd_output)[0]
+            if exit_code:
+                raise RuntimeError("{}: {}".format(exit_code, cmd_output_stripped))
+            return cmd_output_stripped
+        return cmd_output
+
     def devices(self):
         """Return the available ADB devices and the uptime."""
-        return self._adb_cmd("devices -l", output_string=True)
+        return self.adb_cmd("devices -l", output_string=True)
 
     def device_available(self):
         """Return the the uptime of the connected device."""
         # If the device is not available this will throw an exception.
-        return self._adb_cmd("shell uptime", output_string=True)
+        return self.adb_cmd("shell uptime", output_string=True)
+
+    def push(self, files, remote_dir, sync=False):
+        """Push a list of files over adb to remote_dir."""
+        # We can specify files as a single file name or a list of files.
+        if isinstance(files, list):
+            files = " ".join(files)
+        sync_opt = "--sync " if sync else ""
+        return self.adb_cmd("push {}{} {}".format(sync_opt, files, remote_dir), output_string=True)
+
+    def pull(self, files, local_dir):
+        """Pull a list of remote files over adb to local_dir."""
+        # We can specify files as a single file name or a list of files.
+        if isinstance(files, list):
+            files = " ".join(files)
+        return self.adb_cmd("pull {} {}".format(files, local_dir), output_string=True)
 
     def _battery_cmd(self, option, output_file=None, append_file=False):
-        self._adb_cmd("shell dumpsys batterystats {}".format(option), output_file, append_file)
+        self.adb_cmd("shell dumpsys batterystats {}".format(option), output_file, append_file)
 
     def battery(self, reset=False, output_file=None, append_file=False):
         """Collect the battery stats and save to the output_file."""
@@ -79,7 +117,7 @@ class Adb(object):
 
     def memory(self, output_file=None, append_file=False):
         """Collect the memory stats and save to the output_file."""
-        self._adb_cmd("shell dumpsys meminfo -c -d", output_file, append_file)
+        self.adb_cmd("shell dumpsys meminfo -c -d", output_file, append_file)
 
     def systrace_start(self, output_file=None):
         """Start the systrace.py script to collect CPU usage."""
@@ -232,7 +270,7 @@ class AdbSampleBasedResourceMonitor(AdbResourceMonitor):
             self, output_file, should_stop, adb_cmd, num_samples, sample_interval_ms):
         """Initialize AdbSampleBasedResourceMonitor."""
         AdbResourceMonitor.__init__(self, output_file, should_stop)
-        self._adb_cmd = adb_cmd
+        self.adb_cmd = adb_cmd
         self._num_samples = num_samples
         self._sample_interval_ms = sample_interval_ms
 
@@ -258,7 +296,7 @@ class AdbSampleBasedResourceMonitor(AdbResourceMonitor):
         """Collect sample."""
         LOGGER.debug("%s: Collecting sample %d of %d", self._output_file, collected_samples,
                      self._num_samples)
-        self._adb_cmd(output_file=self._output_file, append_file=True)
+        self.adb_cmd(output_file=self._output_file, append_file=True)
 
 
 class AdbContinuousResourceMonitor(AdbResourceMonitor):
diff --git a/buildscripts/mobile/benchrun_embedded_setup_android.py b/buildscripts/mobile/benchrun_embedded_setup_android.py
new file mode 100644
index 00000000000..cbda82ac7ec
--- /dev/null
+++ b/buildscripts/mobile/benchrun_embedded_setup_android.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+"""Setup an Android device to run the benchrun_embedded test suite."""
+
+from __future__ import print_function
+
+import glob
+import logging
+import optparse
+import os
+import posixpath
+import shutil
+import sys
+import tarfile
+import tempfile
+import time
+import urllib
+
+# pylint: disable=wrong-import-position
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from buildscripts.mobile import adb_monitor
+
+# Initialize the global logger.
+logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO)
+logging.Formatter.converter = time.gmtime
+LOGGER = logging.getLogger(__name__)
+
+
+def download_and_untar(url, root_dir):
+    """Download url and untar into root_dir."""
+    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".tgz").name
+    LOGGER.info("Downloading %s", url)
+    urllib.urlretrieve(url, temp_file)
+    with tarfile.open(temp_file, "r:gz") as tar:
+        tar.extractall(root_dir)
+    os.remove(temp_file)
+
+
+def push_directory_contents(adb, local_dir, remote_dir):
+    """Push contents of local_dir to remote_dir via adb."""
+    # Push the contents of temp_dir.
+    paths = glob.glob(os.path.join(local_dir, "*"))
+    paths.sort()
+    paths_msg = paths
+    if isinstance(paths, list):
+        paths_msg = [os.path.basename(path) for path in paths]
+        paths_msg = "{}{}".format(paths_msg[:5], "" if len(paths) <= 5 else " ...")
+    LOGGER.info("Pushing %s to %s", paths_msg, remote_dir)
+    adb.push(paths, remote_dir)
+
+
+def download_and_push(adb, url, remote_dir, local_dir=None):
+    """Download url and push directory to remote_dir via adb.
+
+    If local_dir is defined, then save the unzipped tar there.
+    """
+    temp_dir = tempfile.mkdtemp()
+    download_and_untar(url, temp_dir)
+    push_directory_contents(adb, temp_dir, remote_dir)
+    if local_dir:
+        if os.path.exists(local_dir):
+            LOGGER.info("Removing local path %s", local_dir)
+            shutil.rmtree(local_dir)
+        LOGGER.info("Saving local copy to %s", local_dir)
+        shutil.move(temp_dir, local_dir)
+    else:
+        shutil.rmtree(temp_dir)
+
+
+def create_empty_remote_dirs(adb, dirs):
+    """Create empty remote directories via adb."""
+    # We can specify dirs as a single directory name or as list.
+    if isinstance(dirs, str):
+        dirs = [dirs]
+    # Keep directories in order, so we do not delete a root level later.
+    dirs.sort()
+    for remote_dir in dirs:
+        LOGGER.info("Creating remote directory %s", remote_dir)
+        adb.shell(
+            "if [ -d {remote_dir} ]; then rm -fr {remote_dir}; fi; mkdir -p {remote_dir}".format(
+                remote_dir=remote_dir))
+
+
+def move_sdk_files(adb, sdk_root_dir):
+    """Move all the files in bin and lib into sdk_root_dir."""
+    LOGGER.info("Moving SDK bin & lib files to %s", sdk_root_dir)
+    adb_command = "lib_dir=$(find {} -name 'lib')".format(sdk_root_dir)
+    adb_command = "{}; bin_dir=$(find {} -name 'bin')".format(adb_command, sdk_root_dir)
+    adb_command = "{}; mv $lib_dir/* $bin_dir/* {}".format(adb_command, sdk_root_dir)
+    adb.shell(adb_command)
+
+
+def main():
+    """Execute Main program."""
+
+    benchrun_root = "/data/local/tmp/benchrun_embedded"
+
+    parser = optparse.OptionParser()
+    program_options = optparse.OptionGroup(parser, "Program Options")
+    device_options = optparse.OptionGroup(parser, "Device Options")
+    sdk_options = optparse.OptionGroup(parser, "Embedded Test SDK Options")
+    json_options = optparse.OptionGroup(parser, "JSON benchrun file Options")
+
+    program_options.add_option("--adbBinary", dest="adb_binary",
+                               help="The path for adb. Defaults to '%default', which is in $PATH.",
+                               default="adb")
+
+    device_options.add_option(
+        "--rootRemoteDir", dest="embedded_root_dir",
+        help="The remote root directory to store the files. Defaults to '%default'.",
+        default=benchrun_root)
+
+    device_options.add_option(
+        "--dbDir", dest="db_dir",
+        help=("The remote dbpath directory used by mongoebench."
+              " Will be created if it does not exist. Defaults to '%default'."),
+        default=posixpath.join(benchrun_root, "db"))
+
+    device_options.add_option(
+        "--resultsDir", dest="results_dir",
+        help=("The remote directory to store the mongoebench results."
+              " Will be created if it does not exist. Defaults to '%default'."),
+        default=posixpath.join(benchrun_root, "results"))
+
+    device_options.add_option(
+        "--sdkRemoteDir", dest="sdk_remote_dir",
+        help="The remote directory to store the embedded SDK files. Defaults to '%default'.",
+        default=posixpath.join(benchrun_root, "sdk"))
+
+    device_options.add_option("--benchrunJsonRemoteDir", dest="json_remote_dir",
+                              help="The remote directory to store the benchrun JSON files."
+                              " Defaults to '%default'.", default=posixpath.join(
+                                  benchrun_root, "testcases"))
+
+    sdk_url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-v4.0/embedded-sdk-test/embedded-sdk-android-arm64-latest.tgz"
+    sdk_options.add_option(
+        "--sdkUrl", dest="sdk_url",
+        help=("The embedded SDK test URL. This tarball must contain mongoebench and"
+              " any required shared object (.so) libraries. Defaults to '%default'."),
+        default=sdk_url)
+
+    sdk_options.add_option("--sdkLocalDir", dest="sdk_local_dir",
+                           help="The local directory of embedded SDK files to be copied."
+                           "If specified, overrides --sdkUrl.", default=None)
+
+    sdk_options.add_option(
+        "--sdkSaveLocalDir", dest="sdk_save_local_dir",
+        help=("The local directory to save the downloaded embedded SDK as an unzipped tarball."
+              " Only used if the embedded SDK tarball is downloaded. Note - this will delete"
+              " the existing directory."), default=None)
+
+    json_url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-v4.0/benchrun_embedded/benchrun_json_files.tgz"
+    json_options.add_option(
+        "--benchrunJsonUrl", dest="json_url",
+        help=("The benchrun JSON files URL. This tarball must contain all the JSON"
+              " files to be used in the benchrun embedded test."
+              " Defaults to '%default'."), default=json_url)
+
+    json_options.add_option("--benchrunJsonLocalDir", dest="json_local_dir",
+                            help="The local directory of benchrun JSON files to be copied."
+                            "If specified, overrides --benchrunJsonUrl.", default=None)
+
+    json_options.add_option(
+        "--benchrunJsonSaveLocalDir", dest="json_save_local_dir",
+        help=("The local directory to save the downloaded benchrun JSON as an unzipped tarball."
+              " Only used if the benchrun JSON files tarball is downloaded. Note - this will"
+              " delete the existing directory.  Defaults to '%default'."), default=os.path.join(
+                  "benchrun_embedded", "testcases"))
+
+    json_options.add_option(
+        "--noBenchrunJsonSaveLocal", action="store_true", dest="no_json_save_local_dir",
+        help=("Disable saving downloaded benchrun JSON as an unzipped tarball."), default=False)
+
+    parser.add_option_group(program_options)
+    parser.add_option_group(device_options)
+    parser.add_option_group(sdk_options)
+    parser.add_option_group(json_options)
+    options, _ = parser.parse_args()
+
+    if options.no_json_save_local_dir:
+        options.json_save_local_dir = None
+
+    adb = adb_monitor.Adb(options.adb_binary)
+    adb.device_available()
+    LOGGER.info("Detected devices by adb:\n%s%s", adb.devices(), adb.device_available())
+
+    # Create/empty remote directories.
+    create_empty_remote_dirs(adb, [
+        options.embedded_root_dir, options.db_dir, options.results_dir, options.sdk_remote_dir,
+        options.json_remote_dir
+    ])
+
+    # Download, untar and push Embedded SDK Tests & Benchrun JSON files.
+    # Unfortunately gunzip may not exist on the Android device, so we cannot use this remote command:
+    #   curl URL | tar -xzv -C LOCAL_DIR
+
+    if options.sdk_local_dir:
+        push_directory_contents(adb, options.sdk_local_dir, options.sdk_remote_dir)
+    else:
+        download_and_push(adb, options.sdk_url, options.sdk_remote_dir, options.sdk_save_local_dir)
+    move_sdk_files(adb, options.sdk_remote_dir)
+
+    if options.json_local_dir:
+        push_directory_contents(adb, options.json_local_dir, options.json_remote_dir)
+    else:
+        download_and_push(adb, options.json_url, options.json_remote_dir,
+                          options.json_save_local_dir)
+
+
+if __name__ == "__main__":
+    main()