diff options
author | Andrew Morrow <andrew.morrow@10gen.com> | 2019-12-09 19:22:42 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-12-09 19:22:42 +0000 |
commit | be05c9ea62d6f01895545628992f1899426b88c4 (patch) | |
tree | 232741ad863ff71b4b09b70a4b692c6010c5ff26 | |
parent | 4f00d6e3d7e54a2d131b8216d4b4ef9bfd7a7c6b (diff) | |
download | mongo-be05c9ea62d6f01895545628992f1899426b88c4.tar.gz |
SERVER-44546 Remove mobile variants and embedded benchmarks
60 files changed, 9 insertions, 4424 deletions
diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py index 563f769ac85..cd636585e22 100755 --- a/buildscripts/bypass_compile_and_fetch_binaries.py +++ b/buildscripts/bypass_compile_and_fetch_binaries.py @@ -343,8 +343,6 @@ def fetch_artifacts(evg_api, build_id, revision): # Need to extract certain files from the pre-existing artifacts.tgz. extract_files = [ executable_name("mongobridge"), - executable_name("mongoebench"), - executable_name("mongoed"), executable_name("mongotmock"), executable_name("wt"), ] diff --git a/buildscripts/mobile/MongoProfiler.tracetemplate b/buildscripts/mobile/MongoProfiler.tracetemplate Binary files differdeleted file mode 100644 index d2334527b59..00000000000 --- a/buildscripts/mobile/MongoProfiler.tracetemplate +++ /dev/null diff --git a/buildscripts/mobile/README-Android.md b/buildscripts/mobile/README-Android.md deleted file mode 100644 index ed9cc744eb9..00000000000 --- a/buildscripts/mobile/README-Android.md +++ /dev/null @@ -1,40 +0,0 @@ -# Profiling an Application on Android -How to profile a test application for Android - -## Set up Your Local Environment -1. Download ADB SDK -1. Add the SDK platform-tools to your PATH - * export PATH="$PATH:$HOME/Library/Android/sdk/platform-tools" - -## Run the ADB profiler -The ADB profiler is a custom script which provides - * Battery statistics - battery.csv - * Memory statistics - memory.csv - * CPU statistics - cpu.json -`python buildscripts/mobile/adb_monitor.py` - - -## Enable USB Debugging on Android Device -Enabling USB debugging can differ by device, see https://developer.android.com/studio/debug/dev-options -1. Enable USB debugging via ADB, example - * Select Settings/About phone(or tablet) - * Select Build number 7 times, to enable Developer Options - * Select Settings/Developer Options/USB Debugging -1. Connect the Android device to the computer via USB cable -1. Select "Aways allow from this computer" and OK, when the prompt "Allow USB debugging?" appears on the device - -## Run the ADB Profiler Wirelessly -1. Ensure the local computer and Android device are on the same network -1. Connect the Android device to the computer via USB cable -1. Set the Android device's ADB listening port - * `adb devices` - * `adb tcpip 5555` -1. Disconnect the USB cable -1. Identify the Android's IP address - * Settings/About phone(or tablet)/Status - * `adb_ip=<ip_address>`, i.e., adb_ip=10.4.123.244 -1. Connect wirelessly to the Android device - * `adb connect $adb_ip` -1. Ensure you can connect to the Android device - * `adb shell uptime` -1. Run the ADB profiler as detailed above diff --git a/buildscripts/mobile/README-iOS.md b/buildscripts/mobile/README-iOS.md deleted file mode 100644 index 6451ff731f1..00000000000 --- a/buildscripts/mobile/README-iOS.md +++ /dev/null @@ -1,47 +0,0 @@ -# Profiling an Application on iOS -How to profile a test application for iOS - -## Set up Your Mac OSX Environment -1. Download Xcode - -## Set up Xcode -1. Enable the Developer app on the iOS device - * Settings/General/Device Management: Select the Developer app and Trust - -## MongoProfiler -The `MongoProfiler` is a custom profiling template which includes - * Energy Log - this must be imported from the iOS device and should be captured untethered - Captures battery level - * Activity Monitor - this can be captured via USB cable or wirelessly - Captures CPU load - * Virtual Memory Trace - this can be captured via USB cable or wirelessly - Captures memory usage - -## Run Profiler on the Mac -1. Connect the iOS device to the Mac via USB cable -1. Start Instruments on Mac - * Can be started from within Xcode or from the Spotlight Search -1. Connect to the iOS device and select an app to profile -1. Open a Custom profiling template and select `MongoProfiler` -1. Start recording -1. Start the application on the iOS device -1. Stop recording in Instruments - -## Run Profiler Wirelessly on the Mac -1. Ensure the Mac and iOS device are on the same WiFi network -1. Connect the iOS device to the Mac via USB cable -1. Pair the devices in Xcode -See https://help.apple.com/xcode/mac/9.0/index.html?localePath=en.lproj#/devbc48d1bad - * Select Windows/Devices & Simulators - * Select the Devices tab, select the iOS device and select `Connect via network` -1. Disconnect the USB cable -1. Follow the instructions from step 2 in `Run profiler on the Mac` - -## Measuring Energy Usage on the iOS device -Unfortunately Instruments cannot measure Energy usage by recording from the Mac, but only from the iOS device. -1. Select Settings/Developer/Logging on the iOS device -1. Enable Energy -1. Select `Start Recording` (note this can be done before recording within Instruments for the other probes) -1. After the test is completed, select `Stop Recording` -1. From within Instruments on the Mac (note this can be done using the `MongoProfiler` so all probes are captured together) - * Select File, `Import Logged Data From Device` diff --git a/buildscripts/mobile/__init__.py b/buildscripts/mobile/__init__.py deleted file mode 100644 index 4b7a2bb941b..00000000000 --- a/buildscripts/mobile/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Empty.""" diff --git a/buildscripts/mobile/adb_monitor.py b/buildscripts/mobile/adb_monitor.py deleted file mode 100644 index c6ff6139b7e..00000000000 --- a/buildscripts/mobile/adb_monitor.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -"""ADB utilities to collect adb samples from a locally connected Android device.""" - -import argparse -import distutils.spawn # pylint: disable=no-name-in-module -import logging -import os -import pipes -import re -import shlex -import sys -import tempfile -import threading -import time -import warnings - -# pylint: disable=wrong-import-position -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -from buildscripts.util import fileops -from buildscripts.util import runcommand - -# Default program options. -DEFAULT_ADB_BINARY = "adb" -DEFAULT_BATTERY_FILE = "battery.csv" -DEFAULT_CPU_FILE = "cpu.json" -DEFAULT_NUM_SAMPLES = 0 -DEFAULT_LOG_LEVEL = "info" -DEFAULT_MEMORY_FILE = "memory.csv" -DEFAULT_PYTHON27 = "python2" -DEFAULT_SAMPLE_INTERVAL_MS = 500 - -LOG_LEVELS = ["debug", "error", "info", "warning"] - -# Initialize the global logger. -LOGGER = logging.getLogger(__name__) - - -def parse_command_line(): - """Parse command line options. - - :return: Argparser object. - """ - parser = argparse.ArgumentParser() - - program_options = parser.add_argument_group("Program Options") - battery_options = parser.add_argument_group("Battery Options") - memory_options = parser.add_argument_group("Memory Options") - systrace_options = parser.add_argument_group("Systrace Options") - - program_options.add_argument( - "--adbBinary", dest="adb_binary", - help="The path for adb. Defaults to '%(default)s', which is in $PATH.", - default=DEFAULT_ADB_BINARY) - - program_options.add_argument( - "--python27", dest="python27", - help="The path for python2.7, required by systrace. Defaults to '%(default)s', which is in" - " $PATH.", default=DEFAULT_PYTHON27) - - program_options.add_argument( - "--samples", dest="num_samples", - help="Number of samples to collect, 0 indicates infinite. [Default: %(default)d]", type=int, - default=DEFAULT_NUM_SAMPLES) - - program_options.add_argument( - "--collectionTime", dest="collection_time_secs", - help="Time in seconds to collect samples, if specifed overrides '--samples'.", type=int, - default=None) - - program_options.add_argument( - "--sampleIntervalMs", dest="sample_interval_ms", - help="Time in milliseconds between collecting a sample. [Default: %(default)d]", type=int, - default=DEFAULT_SAMPLE_INTERVAL_MS) - - program_options.add_argument( - "--logLevel", dest="log_level", choices=LOG_LEVELS, - help=f"The log level. Accepted values are: {LOG_LEVELS}. [default: '%(default)s'].", - default=DEFAULT_LOG_LEVEL) - - battery_options.add_argument( - "--batteryFile", dest="battery_file", - help="The destination file for battery stats (CSV format). [Default: %(default)s].", - default=DEFAULT_BATTERY_FILE) - - battery_options.add_argument("--noBattery", dest="battery_file", - help="Disable collection of battery samples.", - action="store_const", const=None) - - memory_options.add_argument( - "--memoryFile", dest="memory_file", - help="The destination file for memory stats (CSV format). [Default: %(default)s].", - default=DEFAULT_MEMORY_FILE) - - memory_options.add_argument("--noMemory", dest="memory_file", - help="Disable collection of memory samples.", action="store_const", - const=None) - - systrace_options.add_argument( - "--cpuFile", dest="cpu_file", - help="The destination file for CPU stats (JSON format). [Default: %(default)s].", - default=DEFAULT_CPU_FILE) - - systrace_options.add_argument("--noCpu", dest="cpu_file", - help="Disable collection of CPU samples.", action="store_const", - const=None) - - return parser - - -def create_files_mtime(files): - """Create dict of file names and it's modified time. - - param files: List file names. - return: Dict of file names with value of the file's modified time. - """ - return {file_name: fileops.getmtime(file_name) for file_name in files if file_name} - - -def find_executable(binary_file): - """Find if binary_file exists in $PATH. Raise exception if it cannot be found. - - param binary_file: Name of binary to find. - return: Full path of binary_file. - """ - binary_path = distutils.spawn.find_executable(binary_file) - if not binary_path: - raise EnvironmentError(f"Executable '{binary_file}' does not exist or is not in the PATH.") - return binary_path - - -class Adb(object): - """Class to abstract calls to adb.""" - - def __init__(self, adb_binary=DEFAULT_ADB_BINARY, logger=LOGGER, python27=DEFAULT_PYTHON27): - """Initialize the Adb object.""" - self._cmd = None - self._tempfile = None - self.logger = logger - self.python27 = find_executable(python27) - adb_path = find_executable(adb_binary) - - # We support specifying a path the adb binary to use; however, systrace.py only - # knows how to find it using the PATH environment variable. It is possible that - # 'adb_binary' is an absolute path specified by the user, so we add its parent - # directory to the PATH manually. - adb_dir = os.path.dirname(adb_path) - if adb_dir: - os.environ["PATH"] = "{}{}{}".format(os.environ["PATH"], os.path.pathsep, adb_dir) - - # systrace.py should be in <adb_dir>/systrace/systrace.py - self.systrace_script = os.path.join(adb_dir, "systrace", "systrace.py") - if not os.path.isfile(self.systrace_script): - raise EnvironmentError("Script '{}' cannot be found.".format(self.systrace_script)) - - @staticmethod - def adb_cmd(adb_command, output_file=None, append_file=False, output_string=False): - """Run an adb command and return result.""" - cmd = runcommand.RunCommand("adb {}".format(adb_command), output_file, append_file) - if output_string or not output_file: - return cmd.execute_with_output() - return cmd.execute_save_output() - - @staticmethod - def shell(adb_shell_command): - """Run an adb shell command and return output_string. - - Raise an exception if the exit status is non-zero. - - Since the adb shell command does not return an exit status. We simulate it by - saving the exit code in the output and then stripping if off. - - See https://stackoverflow.com/questions/9379400/adb-error-codes - """ - cmd_prefix = "set -o errexit; function _exit_ { echo __EXIT__:$?; } ; trap _exit_ EXIT ;" - cmd = runcommand.RunCommand("adb shell {} {}".format(cmd_prefix, adb_shell_command)) - cmd_output = cmd.execute_with_output() - if "__EXIT__" in cmd_output: - exit_code = int(cmd_output.split()[-1].split(":")[1]) - cmd_output_stripped = re.split("__EXIT__.*\n", cmd_output)[0] - if exit_code: - raise RuntimeError("{}: {}".format(exit_code, cmd_output_stripped)) - return cmd_output_stripped - return cmd_output - - def devices(self): - """Return the available ADB devices and the uptime.""" - return self.adb_cmd("devices -l", output_string=True) - - def device_available(self): - """Return the the uptime of the connected device.""" - # If the device is not available this will throw an exception. - return self.adb_cmd("shell uptime", output_string=True) - - def push(self, files, remote_dir, sync=False): - """Push a list of files over adb to remote_dir.""" - # We can specify files as a single file name or a list of files. - if isinstance(files, list): - files = " ".join(files) - sync_opt = "--sync " if sync else "" - return self.adb_cmd("push {}{} {}".format(sync_opt, files, remote_dir), output_string=True) - - def pull(self, files, local_dir): - """Pull a list of remote files over adb to local_dir.""" - # We can specify files as a single file name or a list of files. - if isinstance(files, list): - files = " ".join(files) - return self.adb_cmd("pull {} {}".format(files, local_dir), output_string=True) - - def _battery_cmd(self, option, output_file=None, append_file=False): - self.adb_cmd("shell dumpsys batterystats {}".format(option), output_file, append_file) - - def battery(self, output_file, append_file=False, reset=False): - """Collect the battery stats and save to the output_file.""" - if reset: - self._battery_cmd("--reset") - self._battery_cmd("--checkin", output_file, append_file) - - def memory(self, output_file, append_file=False): - """Collect the memory stats and save to the output_file.""" - self.adb_cmd("shell dumpsys meminfo -c -d", output_file, append_file) - - def systrace_start(self, output_file=None): - """Start the systrace.py script to collect CPU usage.""" - self._tempfile = tempfile.NamedTemporaryFile(delete=False).name - self._cmd = runcommand.RunCommand(output_file=self._tempfile, propagate_signals=False) - # systrace.py currently only supports python 2.7. - self._cmd.add_file(self.python27) - self._cmd.add_file(self.systrace_script) - self._cmd.add("--json") - self._cmd.add("-o") - self._cmd.add_file(output_file) - self._cmd.add("dalvik sched freq idle load") - self._cmd.start_process() - - def systrace_stop(self, output_file=None): - """Stop the systrace.py script.""" - self._cmd.send_to_process(b"bye") - with open(self._tempfile) as fh: - buff = fh.read() - os.remove(self._tempfile) - self.logger.debug("systrace_stop: %s", buff) - if "Wrote trace" not in buff: - self.logger.error("CPU file not saved: %s", buff) - if output_file and os.path.isfile(output_file): - os.remove(output_file) - - -class AdbControl(object): # pylint: disable=too-many-instance-attributes - """Class to controls calls to adb.""" - - _JOIN_TIMEOUT = 24 * 60 * 60 # 24 hours (a long time to have the monitor run for) - - def __init__( # pylint: disable=too-many-arguments - self, adb, logger=LOGGER, battery_file=None, memory_file=None, cpu_file=None, - append_file=False, num_samples=DEFAULT_NUM_SAMPLES, collection_time_secs=None, - sample_interval_ms=DEFAULT_SAMPLE_INTERVAL_MS): - """Initialize AdbControl object.""" - - self.adb = adb - - self.logger = logger - - output_files = [fn for fn in [battery_file, memory_file, cpu_file] if fn] - if not output_files: - raise ValueError("There are no collection sample files selected.") - self.battery_file = battery_file - self.memory_file = memory_file - self.cpu_file = cpu_file - - # The AdbResourceMonitor will always append results to the specified file. - # If append_file is specified in this init, then if there's an existing file - # we do not overwrite it. - for output_file in output_files: - if not append_file: - fileops.create_empty(output_file) - - # collection_time_secs overrides num_samples - self.num_samples = num_samples if not collection_time_secs else 0 - self.collection_time_secs = collection_time_secs - self.sample_interval_ms = sample_interval_ms - - self._should_stop = threading.Event() - self._should_stop.clear() - self._sample_based_threads = [] - self._all_threads = [] - - def start(self): - """Start adb sample collection.""" - if self.cpu_file: - monitor = AdbContinuousResourceMonitor(self.cpu_file, self._should_stop, - self.adb.systrace_start, self.adb.systrace_stop) - self._all_threads.append(monitor) - monitor.start() - - if self.battery_file: - monitor = AdbSampleBasedResourceMonitor(self.battery_file, self._should_stop, - self.adb.battery, self.num_samples, - self.sample_interval_ms) - self._sample_based_threads.append(monitor) - self._all_threads.append(monitor) - monitor.start() - - if self.memory_file: - monitor = AdbSampleBasedResourceMonitor(self.memory_file, self._should_stop, - self.adb.memory, self.num_samples, - self.sample_interval_ms) - self._sample_based_threads.append(monitor) - self._all_threads.append(monitor) - monitor.start() - - def stop(self): - """Stop adb sample collection.""" - self._should_stop.set() - self.wait() - - def wait(self): - """Wait for all sample collections to complete.""" - try: - # We either wait for the specified amount of time or for the sample-based monitors - # to have collected the specified number of samples. - if self.collection_time_secs: - self._should_stop.wait(self.collection_time_secs) - else: - for thread in self._sample_based_threads: - # We must specify a timeout to threading.Thread.join() to ensure that the - # wait is interruptible. The main thread would otherwise never be able to - # receive a KeyboardInterrupt. - thread.join(self._JOIN_TIMEOUT) - except KeyboardInterrupt: - # The user has interrupted the script, so we signal to all of the monitor threads - # that they should exit as quickly as they can. - pass - finally: - self._should_stop.set() - # Wait for all of the monitor threads to exit, by specifying a timeout to - # threading.Thread.join() in case the user tries to interrupt the script again. - for thread in self._all_threads: - thread.join(self._JOIN_TIMEOUT) - - self.logger.info("Collections stopped.") - - # If any of the monitor threads encountered an error, then reraise the exception in the - # main thread. - for thread in self._all_threads: - if thread.exception is not None: - raise thread.exception - - -class AdbResourceMonitor(threading.Thread): - """Thread to collect information about a specific resource using adb.""" - - def __init__(self, output_file, should_stop, logger=LOGGER): - """Initialize the AdbResourceMonitor object.""" - threading.Thread.__init__(self, name="AdbResourceMonitor {}".format(output_file)) - self._output_file = output_file - self._should_stop = should_stop - self.logger = logger - self.exception = None - - def run(self): - """Collect adb samples.""" - try: - self._do_monitoring() - except Exception as err: # pylint: disable=broad-except - self.logger.error("%s: Encountered an error: %s", self._output_file, err) - self.exception = err - self._should_stop.set() - - -class AdbSampleBasedResourceMonitor(AdbResourceMonitor): - """Subclass for ADB sample based monitor.""" - - def __init__( # pylint: disable=too-many-arguments - self, output_file, should_stop, adb_cmd, num_samples, sample_interval_ms): - """Initialize AdbSampleBasedResourceMonitor.""" - AdbResourceMonitor.__init__(self, output_file, should_stop) - self.adb_cmd = adb_cmd - self._num_samples = num_samples - self._sample_interval_ms = sample_interval_ms - - def _do_monitoring(self): - """Monitor function.""" - collected_samples = 0 - now = time.time() - - while not self._should_stop.is_set(): - if self._num_samples > 0 and collected_samples >= self._num_samples: - break - if collected_samples > 0: - self.logger.debug("%s: Sleeping %d ms.", self._output_file, - self._sample_interval_ms) - self._should_stop.wait(self._sample_interval_ms / 1000.0) - collected_samples += 1 - self._take_sample(collected_samples) - - total_time_ms = (time.time() - now) * 1000 - self.logger.info("%s: Stopping monitoring, %d samples collected in %d ms.", - self._output_file, collected_samples, total_time_ms) - - def _take_sample(self, collected_samples): - """Collect sample.""" - self.logger.debug("%s: Collecting sample %d of %d", self._output_file, collected_samples, - self._num_samples) - self.adb_cmd(output_file=self._output_file, append_file=True) - - -class AdbContinuousResourceMonitor(AdbResourceMonitor): - """Subclass for ADB continuous sample based monitoring.""" - - def __init__(self, output_file, should_stop, adb_start_cmd, adb_stop_cmd): - """Initialize AdbContinuousResourceMonitor.""" - AdbResourceMonitor.__init__(self, output_file, should_stop) - self._adb_start_cmd = adb_start_cmd - self._adb_stop_cmd = adb_stop_cmd - - def _do_monitoring(self): - """Monitor function.""" - self.logger.debug("%s: Starting monitoring.", self._output_file) - now = time.time() - self._adb_start_cmd(output_file=self._output_file) - self._should_stop.wait() - total_time_ms = (time.time() - now) * 1000 - self.logger.info("%s: Stopping monitoring after %d ms.", self._output_file, total_time_ms) - self._adb_stop_cmd(output_file=self._output_file) - - -def monitor_device(adb_control, files_mtime): - """Run monitoring on device and collect results. - - param adb_control: AdbControl object. - param files_mtime: Dict of files with their modified time. - """ - adb_control.start() - try: - adb_control.wait() - finally: - files_saved = [ - path for path in files_mtime - if fileops.getmtime(path) > files_mtime[path] and not fileops.is_empty(path) - ] - LOGGER.info("Files saved: %s", files_saved) - - -def main(): - """Execute Main program.""" - - logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO) - logging.Formatter.converter = time.gmtime - - parser = parse_command_line() - options = parser.parse_args() - - files_mtime = create_files_mtime([options.battery_file, options.memory_file, options.cpu_file]) - - if not files_mtime: - parser.error("Must specify one ouptut file") - - LOGGER.setLevel(options.log_level.upper()) - LOGGER.info( - "This program can be cleanly terminated by issuing the following command:" - "\n\t\t'kill -INT %d'", os.getpid()) - - adb = Adb(adb_binary=options.adb_binary, python27=options.python27) - LOGGER.info("Detected devices by adb:\n%s%s", adb.devices(), adb.device_available()) - - adb_control = AdbControl(adb=adb, battery_file=options.battery_file, - memory_file=options.memory_file, cpu_file=options.cpu_file, - num_samples=options.num_samples, - collection_time_secs=options.collection_time_secs, - sample_interval_ms=options.sample_interval_ms) - - monitor_device(adb_control, files_mtime) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/mobile/benchrun_embedded_setup_android.py b/buildscripts/mobile/benchrun_embedded_setup_android.py deleted file mode 100644 index 8cbba5cbb13..00000000000 --- a/buildscripts/mobile/benchrun_embedded_setup_android.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 -"""Setup an Android device to run the benchrun_embedded test suite.""" - -import glob -import logging -import optparse -import os -import posixpath -import shutil -import sys -import tarfile -import tempfile -import time -import urllib.error -import urllib.parse -import urllib.request - -# pylint: disable=wrong-import-position -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -from buildscripts.mobile import adb_monitor - -# Initialize the global logger. -LOGGER = logging.getLogger(__name__) - - -def download_and_untar(url, root_dir): - """Download url and untar into root_dir.""" - temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".tgz").name - LOGGER.info("Downloading %s", url) - urllib.request.urlretrieve(url, temp_file) - with tarfile.open(temp_file, "r:gz") as tar: - tar.extractall(root_dir) - os.remove(temp_file) - - -def push_directory_contents(adb, local_dir, remote_dir): - """Push contents of local_dir to remote_dir via adb.""" - # Push the contents of temp_dir. - paths = glob.glob(os.path.join(local_dir, "*")) - paths.sort() - paths_msg = paths - if isinstance(paths, list): - paths_msg = [os.path.basename(path) for path in paths] - paths_msg = "{}{}".format(paths_msg[:5], "" if len(paths) <= 5 else " ...") - LOGGER.info("Pushing %s to %s", paths_msg, remote_dir) - adb.push(paths, remote_dir) - - -def download_and_push(adb, url, remote_dir, local_dir=None): - """Download url and push directory to remote_dir via adb. - - If local_dir is defined, then save the unzipped tar there. - """ - temp_dir = tempfile.mkdtemp() - download_and_untar(url, temp_dir) - push_directory_contents(adb, temp_dir, remote_dir) - if local_dir: - if os.path.exists(local_dir): - LOGGER.info("Removing local path %s", local_dir) - shutil.rmtree(local_dir) - LOGGER.info("Saving local copy to %s", local_dir) - shutil.move(temp_dir, local_dir) - else: - shutil.rmtree(temp_dir) - - -def create_empty_remote_dirs(adb, dirs): - """Create empty remote directories via adb.""" - # We can specify dirs as a single directory name or as list. - if isinstance(dirs, str): - dirs = [dirs] - # Keep directories in order, so we do not delete a root level later. - dirs.sort() - for remote_dir in dirs: - LOGGER.info("Creating remote directory %s", remote_dir) - adb.shell( - "if [ -d {remote_dir} ]; then rm -fr {remote_dir}; fi; mkdir -p {remote_dir}".format( - remote_dir=remote_dir)) - - -def move_sdk_files(adb, sdk_root_dir): - """Move all the files in bin and lib into sdk_root_dir.""" - LOGGER.info("Moving SDK bin & lib files to %s", sdk_root_dir) - adb_command = "lib_dir=$(find {} -name 'lib')".format(sdk_root_dir) - adb_command = "{}; bin_dir=$(find {} -name 'bin')".format(adb_command, sdk_root_dir) - adb_command = "{}; mv $lib_dir/* $bin_dir/* {}".format(adb_command, sdk_root_dir) - adb.shell(adb_command) - - -def main(): - """Execute Main program.""" - - logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO) - logging.Formatter.converter = time.gmtime - - benchrun_root = "/data/local/tmp/benchrun_embedded" - - parser = optparse.OptionParser() - program_options = optparse.OptionGroup(parser, "Program Options") - device_options = optparse.OptionGroup(parser, "Device Options") - sdk_options = optparse.OptionGroup(parser, "Embedded Test SDK Options") - json_options = optparse.OptionGroup(parser, "JSON benchrun file Options") - - program_options.add_option("--adbBinary", dest="adb_binary", - help="The path for adb. Defaults to '%default', which is in $PATH.", - default="adb") - - device_options.add_option( - "--rootRemoteDir", dest="embedded_root_dir", - help="The remote root directory to store the files. Defaults to '%default'.", - default=benchrun_root) - - device_options.add_option( - "--dbDir", dest="db_dir", - help=("The remote dbpath directory used by mongoebench." - " Will be created if it does not exist. Defaults to '%default'."), - default=posixpath.join(benchrun_root, "db")) - - device_options.add_option( - "--resultsDir", dest="results_dir", - help=("The remote directory to store the mongoebench results." - " Will be created if it does not exist. Defaults to '%default'."), - default=posixpath.join(benchrun_root, "results")) - - device_options.add_option( - "--sdkRemoteDir", dest="sdk_remote_dir", - help="The remote directory to store the embedded SDK files. Defaults to '%default'.", - default=posixpath.join(benchrun_root, "sdk")) - - device_options.add_option( - "--benchrunJsonRemoteDir", dest="json_remote_dir", - help="The remote directory to store the benchrun JSON files." - " Defaults to '%default'.", default=posixpath.join(benchrun_root, "testcases")) - - sdk_url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-master/embedded-sdk-test/embedded-sdk-android-arm64-latest.tgz" - sdk_options.add_option( - "--sdkUrl", dest="sdk_url", - help=("The embedded SDK test URL. This tarball must contain mongoebench and" - " any required shared object (.so) libraries. Defaults to '%default'."), - default=sdk_url) - - sdk_options.add_option( - "--sdkLocalDir", dest="sdk_local_dir", - help="The local directory of embedded SDK files to be copied." - "If specified, overrides --sdkUrl.", default=None) - - sdk_options.add_option( - "--sdkSaveLocalDir", dest="sdk_save_local_dir", - help=("The local directory to save the downloaded embedded SDK as an unzipped tarball." - " Only used if the embedded SDK tarball is downloaded. Note - this will delete" - " the existing directory."), default=None) - - json_url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-master/benchrun_embedded/benchrun_json_files.tgz" - json_options.add_option( - "--benchrunJsonUrl", dest="json_url", - help=("The benchrun JSON files URL. This tarball must contain all the JSON" - " files to be used in the benchrun embedded test." - " Defaults to '%default'."), default=json_url) - - json_options.add_option( - "--benchrunJsonLocalDir", dest="json_local_dir", - help="The local directory of benchrun JSON files to be copied." - "If specified, overrides --benchrunJsonUrl.", default=None) - - json_options.add_option( - "--benchrunJsonSaveLocalDir", dest="json_save_local_dir", - help=("The local directory to save the downloaded benchrun JSON as an unzipped tarball." - " Only used if the benchrun JSON files tarball is downloaded. Note - this will" - " delete the existing directory. Defaults to '%default'."), default=os.path.join( - "benchrun_embedded", "testcases")) - - json_options.add_option( - "--noBenchrunJsonSaveLocal", action="store_true", dest="no_json_save_local_dir", - help=("Disable saving downloaded benchrun JSON as an unzipped tarball."), default=False) - - parser.add_option_group(program_options) - parser.add_option_group(device_options) - parser.add_option_group(sdk_options) - parser.add_option_group(json_options) - options, _ = parser.parse_args() - - if options.no_json_save_local_dir: - options.json_save_local_dir = None - - adb = adb_monitor.Adb(options.adb_binary) - adb.device_available() - LOGGER.info("Detected devices by adb:\n%s%s", adb.devices(), adb.device_available()) - - # Create/empty remote directories. - create_empty_remote_dirs(adb, [ - options.embedded_root_dir, options.db_dir, options.results_dir, options.sdk_remote_dir, - options.json_remote_dir - ]) - - # Download, untar and push Embedded SDK Tests & Benchrun JSON files. - # Unfortunately gunzip may not exist on the Android device, so we cannot use this remote command: - # curl URL | tar -xzv -C LOCAL_DIR - - if options.sdk_local_dir: - push_directory_contents(adb, options.sdk_local_dir, options.sdk_remote_dir) - else: - download_and_push(adb, options.sdk_url, options.sdk_remote_dir, options.sdk_save_local_dir) - move_sdk_files(adb, options.sdk_remote_dir) - - if options.json_local_dir: - push_directory_contents(adb, options.json_local_dir, options.json_remote_dir) - else: - download_and_push(adb, options.json_url, options.json_remote_dir, - options.json_save_local_dir) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml deleted file mode 100644 index 6db291dcde1..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/aggregation*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml deleted file mode 100644 index c44747998c6..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/commands*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml deleted file mode 100644 index 55350189472..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/insert*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml deleted file mode 100644 index 56896e5b6e1..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml +++ /dev/null @@ -1,19 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/*.json - exclude_files: - - benchrun_embedded/testcases/**/aggregation*.json - - benchrun_embedded/testcases/**/commands*.json - - benchrun_embedded/testcases/**/insert*.json - - benchrun_embedded/testcases/**/mixed*.json - - benchrun_embedded/testcases/**/multi*.json - - benchrun_embedded/testcases/**/queries*.json - - benchrun_embedded/testcases/**/remove*.json - - benchrun_embedded/testcases/**/update*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml deleted file mode 100644 index cd0deb1d09f..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml +++ /dev/null @@ -1,11 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/mixed*.json - - benchrun_embedded/testcases/**/multi*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml deleted file mode 100644 index 9ba8ef7db43..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/queries*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml deleted file mode 100644 index f625a2e874b..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/remove*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml deleted file mode 100644 index a9ad8734a1b..00000000000 --- a/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: benchrun_embedded_test - -selector: - roots: - - benchrun_embedded/testcases/**/update*.json - -executor: - hooks: - - class: CollectEmbeddedResources - - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 33f00f48d19..be04f1baad4 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -24,7 +24,6 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # We default to search for executables in the current working directory or in /data/multiversion # which are both part of the PATH. DEFAULT_DBTEST_EXECUTABLE = os.path.join(os.curdir, "dbtest") -DEFAULT_MONGOEBENCH_EXECUTABLE = "mongoebench" DEFAULT_MONGO_EXECUTABLE = "mongo" DEFAULT_MONGOD_EXECUTABLE = "mongod" DEFAULT_MONGOS_EXECUTABLE = "mongos" @@ -50,9 +49,6 @@ DEFAULTS = { "archive_limit_mb": 5000, "archive_limit_tests": 10, "base_port": 20000, - "benchrun_device": "Desktop", - "benchrun_embedded_root": "/data/local/tmp/benchrun_embedded", - "benchrun_report_root": "benchrun_embedded/results", "buildlogger_url": "https://logkeeper.mongodb.org", "continue_on_failure": False, "dbpath_prefix": None, @@ -67,7 +63,6 @@ DEFAULTS = { "mongo_executable": None, "mongod_executable": None, "mongod_set_parameters": None, - "mongoebench_executable": None, "mongos_executable": None, "mongos_set_parameters": None, "no_journal": False, @@ -313,9 +308,6 @@ MONGOD_EXECUTABLE = None # The --setParameter options passed to mongod. MONGOD_SET_PARAMETERS = None -# The path to the mongoebench executable used by resmoke.py. -MONGOEBENCH_EXECUTABLE = None - # The path to the mongos executable used by resmoke.py. MONGOS_EXECUTABLE = None @@ -437,11 +429,6 @@ BENCHMARK_LIST_TESTS = None BENCHMARK_MIN_TIME = None BENCHMARK_REPETITIONS = None -# Embedded Benchrun Test options. -BENCHRUN_DEVICE = None -BENCHRUN_EMBEDDED_ROOT = None -BENCHRUN_REPORT_ROOT = None - ## # Internally used configuration options that aren't exposed to the user ## @@ -465,7 +452,7 @@ DEFAULT_LIBFUZZER_TEST_LIST = "build/libfuzzer_tests.txt" # therefore might not be available when creating a test membership map. EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE, - DEFAULT_MONGOEBENCH_EXECUTABLE, DEFAULT_LIBFUZZER_TEST_LIST) + DEFAULT_LIBFUZZER_TEST_LIST) # Where to look for logging and suite configuration files CONFIG_DIR = None diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index f6ed3417e82..dd9139dd561 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -134,11 +134,6 @@ def _make_parser(): # pylint: disable=too-many-statements " started by resmoke.py. The argument is specified as bracketed YAML -" " i.e. JSON with support for single quoted and unquoted keys.")) - parser.add_option( - "--mongoebench", dest="mongoebench_executable", metavar="PATH", - help=("The path to the mongoebench (benchrun embedded) executable for" - " resmoke.py to use.")) - parser.add_option("--mongos", dest="mongos_executable", metavar="PATH", help="The path to the mongos executable for resmoke.py to use.") @@ -394,22 +389,10 @@ def _make_parser(): # pylint: disable=too-many-statements benchmark_options.add_option("--benchmarkRepetitions", type="int", dest="benchmark_repetitions", metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help) - benchrun_devices = ["Android", "Desktop"] - benchmark_options.add_option( - "--benchrunDevice", dest="benchrun_device", metavar="DEVICE", type="choice", action="store", - choices=benchrun_devices, help=("The device to run the benchrun test on, choose from {}." - " Defaults to DEVICE='%default'.".format(benchrun_devices))) - - benchmark_options.add_option("--benchrunReportRoot", dest="benchrun_report_root", - metavar="PATH", help="The root path for benchrun test report.") - - benchmark_options.add_option("--benchrunEmbeddedRoot", dest="benchrun_embedded_root", - metavar="PATH", - help="The root path on the mobile device, for a benchrun test.") + parser.set_defaults(dry_run="off", find_suites=False, list_suites=False, logger_file="console", + shuffle="auto", stagger_jobs="off", suite_files="with_server", + majority_read_concern="on") - parser.set_defaults(benchrun_device="Desktop", dry_run="off", find_suites=False, - list_suites=False, logger_file="console", shuffle="auto", - stagger_jobs="off", suite_files="with_server", majority_read_concern="on") return parser @@ -592,8 +575,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb") _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests") _config.BASE_PORT = int(config.pop("base_port")) - _config.BENCHRUN_DEVICE = config.pop("benchrun_device") - _config.BENCHRUN_EMBEDDED_ROOT = config.pop("benchrun_embedded_root") _config.BUILDLOGGER_URL = config.pop("buildlogger_url") _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix")) _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable")) @@ -616,7 +597,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable")) _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters") - _config.MONGOEBENCH_EXECUTABLE = _expand_user(config.pop("mongoebench_executable")) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters") _config.NO_JOURNAL = config.pop("no_journal") @@ -667,7 +647,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many if benchmark_min_time is not None: _config.BENCHMARK_MIN_TIME = datetime.timedelta(seconds=benchmark_min_time) _config.BENCHMARK_REPETITIONS = config.pop("benchmark_repetitions") - _config.BENCHRUN_REPORT_ROOT = config.pop("benchrun_report_root") # Config Dir options. _config.CONFIG_DIR = config.pop("config_dir") diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 497266c1711..faa60039e48 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -693,7 +693,6 @@ _SELECTOR_REGISTRY = { "cpp_integration_test": (_CppTestSelectorConfig, _CppTestSelector), "cpp_unit_test": (_CppTestSelectorConfig, _CppTestSelector), "benchmark_test": (_CppTestSelectorConfig, _CppTestSelector), - "benchrun_embedded_test": (_FileBasedSelectorConfig, _Selector), "sdam_json_test": (_FileBasedSelectorConfig, _Selector), "db_test": (_DbTestSelectorConfig, _DbTestSelector), "fsm_workload_test": (_JSTestSelectorConfig, _JSTestSelector), diff --git a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py deleted file mode 100644 index f39399d6dc8..00000000000 --- a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Module for generating and collecting embedded resource results.""" - -import os - -from buildscripts.mobile import adb_monitor -from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib.testing.hooks import interface - - -class CollectEmbeddedResources(interface.Hook): # pylint: disable=too-many-instance-attributes - """CollectEmbeddedResources class. - - CollectEmbeddedResources starts and stops the resource monitoring for each test. - """ - - DESCRIPTION = "Embedded resources" - - def __init__(self, hook_logger, fixture, sample_interval_ms=500, threads=1): - """Initialize CollectEmbeddedResources.""" - interface.Hook.__init__(self, hook_logger, fixture, CollectEmbeddedResources.DESCRIPTION) - self.hook_logger = hook_logger - self.adb = None - self.adb_control = None - if _config.BENCHRUN_DEVICE == "Android": - self.report_root = _config.BENCHRUN_REPORT_ROOT - self.sample_interval_ms = sample_interval_ms - self.threads = threads - self.battery_file = "battery.csv" - self.cpu_file = "cpu.json" - self.memory_file = "memory.csv" - self.adb = adb_monitor.Adb(logger=hook_logger) - - def before_test(self, test, test_report): - """Start ADB monitoring.""" - if self.adb: - battery_file = self._report_path(test, "battery.csv") - cpu_file = self._report_path(test, "cpu.json") - memory_file = self._report_path(test, "memory.csv") - self.adb_control = adb_monitor.AdbControl( - self.adb, logger=self.hook_logger, battery_file=battery_file, cpu_file=cpu_file, - memory_file=memory_file, sample_interval_ms=self.sample_interval_ms) - self.hook_logger.info("Starting ADB monitoring for test %s", test.short_name()) - self.hook_logger.info("ADB resource files: %s %s %s", battery_file, cpu_file, - memory_file) - self.adb_control.start() - - def after_test(self, test, test_report): - """Stop ADB monitoring.""" - if self.adb_control: - self.hook_logger.info("Stopping ADB monitoring for test %s", test.short_name()) - self.adb_control.stop() - - def _report_path(self, test, report_name): - """Return the report path. Reports are stored in <report_root>/<testname>/thread<num>/.""" - return os.path.join(self.report_root, test.short_name(), "thread{}".format(self.threads), - report_name) diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py deleted file mode 100644 index 62f442dba32..00000000000 --- a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Module for generating the test results file fed into the perf plugin.""" - -import collections -import datetime -import glob -import json -import os -import re - -from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib.testing.hooks import combine_benchmark_results as cbr - - -class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults): - """CombineBenchrunEmbeddedResults class. - - The CombineBenchrunEmbeddedResults hook combines test results from - individual benchmark embedded files to a single file. This is useful for - generating the json file to feed into the Evergreen performance - visualization plugin. - """ - - DESCRIPTION = "Combine JSON results from embedded benchrun" - - def __init__(self, hook_logger, fixture): - """Initialize CombineBenchrunEmbeddedResults.""" - cbr.CombineBenchmarkResults.__init__(self, hook_logger, fixture) - self.report_root = _config.BENCHRUN_REPORT_ROOT - - def before_test(self, test, test_report): - """Remove any existing mongoebench reports for this test.""" - for bm_report in self._test_result_files(test): - os.remove(bm_report) - - def after_test(self, test, test_report): - """Update test report.""" - for bm_report in self._test_result_files(test): - test_name = test.short_name() - thread_count = self._parse_report_name(bm_report) - with open(bm_report, "r") as report_file: - report_dict = json.load(report_file) - if test_name not in self.benchmark_reports: - self.benchmark_reports[test_name] = _BenchrunEmbeddedThreadsReport() - self.benchmark_reports[test_name].add_report(thread_count, report_dict) - - def before_suite(self, test_report): - """Set suite start time.""" - self.create_time = datetime.datetime.now() - # Remove any existing perf reports. - if self.report_file and os.path.isfile(self.report_file): - os.remove(self.report_file) - - def _generate_perf_plugin_report(self): - """Format the data to look like a perf plugin report.""" - perf_report = { - "start": self._strftime(self.create_time), - "end": self._strftime(self.end_time), - "errors": [], # There are no errors if we have gotten this far. - "results": [] - } - - for name, report in list(self.benchmark_reports.items()): - test_report = {"name": name, "results": report.generate_perf_plugin_dict()} - - perf_report["results"].append(test_report) - - return perf_report - - def _test_result_files(self, test): - """Return a list of existing test result files based on the test.short_name().""" - return glob.glob( - os.path.join(self.report_root, test.short_name(), "**", "mongoebench[.]*[.]json")) - - def _parse_report_name(self, report_path): - """Parse mongoebench report path and return thread_count. - - The format of the mongoebench report file name is defined in - ../testing/testcases/benchrun_embedded_test.py - as self.report_root/<test_name>/thread<num threads>/mongoebench.<iteration num>.json - """ - _, report_subpath = report_path.split(self.report_root + os.sep) - _, thread_name, _ = report_subpath.split(os.sep) - return re.findall(r"\d+", thread_name)[0] - - -class _BenchrunEmbeddedThreadsReport(object): - """_BenchrunEmbeddedThreadsReport class. - - Class representation of a report for all thread levels of a single - benchmark test. Each report is designed to correspond to one graph - in the Evergreen perf plugin. - - A raw mongoebench report looks like the following: - { - "note" : "values per second", - "errCount" : { "$numberLong" : "0" }, - "trapped" : "error: not implemented", - "insertLatencyAverageMicros" : 389.4926654182272, - "totalOps" : { "$numberLong" : "12816" }, - "totalOps/s" : 2563.095938304905, - "findOne" : 0, - "insert" : 2563.095938304905, - "delete" : 0, - "update" : 0, - "query" : 0, - "command" : 0, - "findOnes" : { "$numberLong" : "0" }, - "inserts" : { "$numberLong" : "12816" }, - "deletes" : { "$numberLong" : "0" }, - "updates" : { "$numberLong" : "0" }, - "queries" : { "$numberLong" : "0" }, - "commands" : { "$numberLong" : "0" } - } - """ - - def __init__(self): - # list of benchmark runs for each thread. - self.thread_benchmark_map = collections.defaultdict(list) - - def add_report(self, thread_count, report): - """Add to report.""" - self.thread_benchmark_map[thread_count].append(report) - - def generate_perf_plugin_dict(self): - """Generate perf plugin data points of the following format. - - "1": { - "error_values": [ - 0, - 0, - 0 - ], - "ops_per_sec": 9552.108279243452, - "ops_per_sec_values": [ - 9574.812658450564, - 9522.642340821469, - 9536.252775275878 - ] - }, - """ - - res = {} - for thread_count, reports in list(self.thread_benchmark_map.items()): - thread_report = {"error_values": [], "ops_per_sec_values": []} - - for report in reports: - thread_report["error_values"].append(report["errCount"]["$numberLong"]) - thread_report["ops_per_sec_values"].append(report["totalOps/s"]) - thread_report["ops_per_sec"] = sum(thread_report["ops_per_sec_values"]) / len(reports) - - res[thread_count] = thread_report - - return res diff --git a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py deleted file mode 100644 index d4666c94074..00000000000 --- a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py +++ /dev/null @@ -1,147 +0,0 @@ -"""The unittest.TestCase for tests using benchrun embedded (mongoebench).""" - -import os -import posixpath - -from buildscripts.mobile import adb_monitor -from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib import core -from buildscripts.resmokelib import parser -from buildscripts.resmokelib import utils -from buildscripts.resmokelib.testing.testcases import interface - - -class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes - interface.ProcessTestCase): - """A Benchrun embedded test to execute.""" - - REGISTERED_NAME = "benchrun_embedded_test" - - def __init__(self, logger, mongoebench_config_file, program_options=None): - """Initialize the BenchrunEmbeddedTestCase with the executable to run.""" - - interface.ProcessTestCase.__init__(self, logger, "Benchmark embedded test", - mongoebench_config_file) - parser.validate_benchmark_options() - - self.benchrun_config_file = mongoebench_config_file - - # Command line options override the YAML configuration. - self.benchrun_executable = utils.default_if_none(_config.MONGOEBENCH_EXECUTABLE, - _config.DEFAULT_MONGOEBENCH_EXECUTABLE) - self.benchrun_repetitions = utils.default_if_none(_config.BENCHMARK_REPETITIONS, - _config.DEFAULT_BENCHMARK_REPETITIONS) - self.suite_benchrun_options = program_options - self.benchrun_threads = 1 - if program_options and "threads" in program_options: - self.benchrun_threads = program_options["threads"] - self.report_root = _config.BENCHRUN_REPORT_ROOT - self.benchrun_options = {} - - # Set the dbpath. - dbpath = utils.default_if_none(_config.DBPATH_PREFIX, _config.DEFAULT_DBPATH_PREFIX) - self.dbpath = os.path.join(dbpath, "mongoebench") - - self.android_device = _config.BENCHRUN_DEVICE == "Android" - # If Android device, then the test runs via adb shell. - if self.android_device: - self.adb = adb_monitor.Adb() - self.android_benchrun_root = _config.BENCHRUN_EMBEDDED_ROOT - self.device_report_root = posixpath.join(self.android_benchrun_root, "results") - self.dbpath = posixpath.join(self.android_benchrun_root, "db") - self.benchrun_config_file = posixpath.join(self.android_benchrun_root, "testcases", - os.path.basename(self.benchrun_config_file)) - ld_library_path = "LD_LIBRARY_PATH={}".format( - posixpath.join(self.android_benchrun_root, "sdk")) - mongoebench = posixpath.join(self.android_benchrun_root, "sdk", "mongoebench") - self.benchrun_executable = "adb shell {} {}".format(ld_library_path, mongoebench) - - def configure(self, fixture, *args, **kwargs): - """Configure BenchrunEmbeddedTestCase.""" - interface.ProcessTestCase.configure(self, fixture, *args, **kwargs) - - # 1. Set the default benchmark options. - benchrun_options = {"time": _config.DEFAULT_BENCHMARK_MIN_TIME.total_seconds()} - - # 2. Override Benchmark options with options set through `program_options` in the suite - # configuration. - suite_benchrun_options = utils.default_if_none(self.suite_benchrun_options, {}) - benchrun_options.update(suite_benchrun_options) - - # 3. Override Benchmark options with options set through resmoke's command line. - resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME} - - for key, value in list(resmoke_benchrun_options.items()): - if value is not None: - # 4. sanitize options before passing them to Benchmark's command line. - if key == "time": - value = value.total_seconds() - benchrun_options[key] = value - - self.benchrun_options = benchrun_options - - # Create the test report directory. - utils.rmtree(self._report_dir(), ignore_errors=True) - try: - os.makedirs(self._report_dir()) - except os.error: - # Directory already exists. - pass - - # Create the dbpath. - if self.android_device: - self.adb.shell("rm -fr {}".format(self.dbpath)) - self.adb.shell("mkdir {}".format(self.dbpath)) - else: - utils.rmtree(self.dbpath, ignore_errors=True) - try: - os.makedirs(self.dbpath) - except os.error: - # Directory already exists. - pass - - def run_test(self): - """Run the test for specified number of iterations.""" - for iter_num in range(self.benchrun_repetitions): - # Set the output file for each iteration. - local_report_path = self._report_path(iter_num) - device_report_path = self._device_report_path(iter_num) - self.benchrun_options["output"] = device_report_path - interface.ProcessTestCase.run_test(self) - self._move_report(device_report_path, local_report_path) - - def _move_report(self, remote_path, local_path): - """Move report from device to local directory.""" - if self.android_device: - # Pull test result from the Android device and then delete it from the device. - self.logger.info("Moving report %s from device to local %s ...", remote_path, - local_path) - self.adb.pull(remote_path, local_path) - self.adb.shell("rm {}".format(remote_path)) - - def _device_report_path(self, iter_num): - """Return the device report path.""" - if self.android_device: - # The mongoebench report is generated on the remote device. - return posixpath.join(self.device_report_root, self._report_name(iter_num)) - return self._report_path(iter_num) - - def _report_path(self, iter_num): - """Return the local report path.""" - return os.path.join(self._report_dir(), self._report_name(iter_num)) - - def _report_dir(self): - """Return the report directory. Reports are stored in <report_root>/<testname>/<thread>.""" - return os.path.join(self.report_root, self.short_name(), - "thread{}".format(self.benchrun_threads)) - - @staticmethod - def _report_name(iter_num): - """Return the constructed report name of the form mongoebench.<iteration num>.json.""" - return "mongoebench.{}.json".format(iter_num) - - def _make_process(self): - # The 'commands' argument for core.programs.generic_program must be a list. - commands = self.benchrun_executable.split() - commands.append(self.benchrun_config_file) - return core.programs.generic_program(self.logger, commands, **self.benchrun_options) diff --git a/buildscripts/tests/mobile/__init__.py b/buildscripts/tests/mobile/__init__.py deleted file mode 100644 index 4b7a2bb941b..00000000000 --- a/buildscripts/tests/mobile/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Empty.""" diff --git a/buildscripts/tests/mobile/test_adb_monitor.py b/buildscripts/tests/mobile/test_adb_monitor.py deleted file mode 100644 index c7d1ba55852..00000000000 --- a/buildscripts/tests/mobile/test_adb_monitor.py +++ /dev/null @@ -1,876 +0,0 @@ -""" Unit tests for adb_monitor. """ -# pylint: disable=protected-access,no-self-use,missing-docstring,too-many-public-methods - -import os -import unittest - -from unittest.mock import MagicMock, mock_open, patch - -import buildscripts.mobile.adb_monitor as adb_monitor - -ADB_MONITOR = "buildscripts.mobile.adb_monitor" - - -def ns(module): - """Get the namespace.""" - # pylint: disable=invalid-name - return f"{ADB_MONITOR}.{module}" - - -class TestParseCommandLine(unittest.TestCase): - def test_parse_command_line(self): - options = adb_monitor.parse_command_line().parse_args([]) - self.assertEqual(options.adb_binary, adb_monitor.DEFAULT_ADB_BINARY) - self.assertEqual(options.python27, adb_monitor.DEFAULT_PYTHON27) - self.assertEqual(options.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - self.assertEqual(options.log_level, adb_monitor.DEFAULT_LOG_LEVEL) - self.assertEqual(options.battery_file, adb_monitor.DEFAULT_BATTERY_FILE) - self.assertEqual(options.memory_file, adb_monitor.DEFAULT_MEMORY_FILE) - self.assertEqual(options.cpu_file, adb_monitor.DEFAULT_CPU_FILE) - self.assertEqual(options.num_samples, adb_monitor.DEFAULT_NUM_SAMPLES) - self.assertEqual(options.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - self.assertIsNone(options.collection_time_secs) - - def test_parse_command_line_partial_args(self): - adb_binary = "myadb" - python27 = "mypython2" - collection_time_secs = 35 - battery_file = "mybattery" - arg_list = [ - "--adbBinary", adb_binary, "--python27", python27, "--collectionTime", - str(collection_time_secs), "--batteryFile", battery_file, "--noMemory" - ] - options = adb_monitor.parse_command_line().parse_args(arg_list) - self.assertEqual(options.adb_binary, adb_binary) - self.assertEqual(options.python27, python27) - self.assertEqual(options.collection_time_secs, collection_time_secs) - self.assertEqual(options.battery_file, battery_file) - - self.assertEqual(options.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - self.assertEqual(options.log_level, adb_monitor.DEFAULT_LOG_LEVEL) - self.assertIsNone(options.memory_file) - self.assertEqual(options.cpu_file, adb_monitor.DEFAULT_CPU_FILE) - self.assertEqual(options.num_samples, adb_monitor.DEFAULT_NUM_SAMPLES) - self.assertEqual(options.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - - def test_parse_command_line_no_files(self): - arg_list = ["--noBattery", "--noCpu", "--noMemory"] - options = adb_monitor.parse_command_line().parse_args(arg_list) - self.assertIsNone(options.battery_file) - self.assertIsNone(options.cpu_file) - self.assertIsNone(options.memory_file) - - def test_parse_command_line_no_battery_first(self): - battery_file = "mybattery" - arg_list = ["--noBattery", "--batteryFile", battery_file] - options = adb_monitor.parse_command_line().parse_args(arg_list) - self.assertEqual(options.battery_file, battery_file) - - def test_parse_command_line_no_battery_second(self): - battery_file = "mybattery" - arg_list = ["--batteryFile", battery_file, "--noBattery"] - options = adb_monitor.parse_command_line().parse_args(arg_list) - self.assertIsNone(options.battery_file) - - -class TestMonitorDevice(unittest.TestCase): - @patch(ns("fileops"), return_value=False) - def test_monitor_device(self, mock_fileops): - """Basic test monitor device.""" - files_mtime = {"file1": 0, "file2": 0} - mock_fileops.getmtime.return_value = 10 - mock_adb_control = MagicMock() - adb_monitor.monitor_device(mock_adb_control, files_mtime) - mock_adb_control.start.assert_called_once() - mock_adb_control.wait.assert_called_once() - self.assertEqual(mock_fileops.getmtime.call_count, len(files_mtime)) - self.assertEqual(mock_fileops.is_empty.call_count, len(files_mtime)) - - @patch(ns("fileops"), return_value=True) - def test_monitor_device_empty_file(self, mock_fileops): - files_mtime = {"file1": 0, "file2": 0} - mock_fileops.getmtime.return_value = 10 - mock_adb_control = MagicMock() - adb_monitor.monitor_device(mock_adb_control, files_mtime) - mock_adb_control.start.assert_called_once() - mock_adb_control.wait.assert_called_once() - self.assertEqual(mock_fileops.getmtime.call_count, len(files_mtime)) - self.assertEqual(mock_fileops.is_empty.call_count, len(files_mtime)) - - @patch(ns("fileops"), return_value=False) - def test_monitor_device_earlier_mtime(self, mock_fileops): - files_mtime = {"file1": 10, "file2": 10} - mock_fileops.getmtime.return_value = 6 - mock_adb_control = MagicMock() - adb_monitor.monitor_device(mock_adb_control, files_mtime) - mock_adb_control.start.assert_called_once() - mock_adb_control.wait.assert_called_once() - self.assertEqual(mock_fileops.getmtime.call_count, len(files_mtime)) - mock_fileops.is_empty.assert_not_called() - - @patch(ns("fileops"), return_value=False) - def test_monitor_device_no_files(self, mock_fileops): - files_mtime = {} - mock_fileops.getmtime.return_value = 10 - mock_adb_control = MagicMock() - adb_monitor.monitor_device(mock_adb_control, files_mtime) - mock_adb_control.start.assert_called_once() - mock_adb_control.wait.assert_called_once() - mock_fileops.getmtime.assert_not_called() - mock_fileops.is_empty.assert_not_called() - - -class TestOutputFilesMtime(unittest.TestCase): - @patch(ns("fileops.getmtime")) - def test_output_files_mtime(self, mock_getmtime): - mtime = 11 - mock_getmtime.return_value = mtime - files = ["file1", "file2"] - m_files = adb_monitor.create_files_mtime(files) - self.assertEqual(len(m_files), len(files)) - for fn in files: - self.assertEqual(m_files[fn], mtime) - - @patch(ns("fileops.getmtime")) - def test_output_files_mtime_no_files(self, mock_getmtime): - # pylint: disable=unused-argument - m_files = adb_monitor.create_files_mtime([]) - self.assertEqual(len(m_files), 0) - - -class TestFindExecutable(unittest.TestCase): - def test_find_executable(self): - my_binary = "mybinary" - with patch(ns("distutils.spawn.find_executable"), side_effect=lambda x: x): - adb_binary = adb_monitor.find_executable(my_binary) - self.assertEqual(adb_binary, my_binary) - - def test_find_executable_not_found(self): - with patch(ns("distutils.spawn.find_executable"), side_effect=lambda x: x),\ - self.assertRaises(EnvironmentError): - adb_monitor.find_executable(None) - - -class TestAdb(unittest.TestCase): - @patch(ns("os.path.isfile"), return_value=True) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test___init__(self, mock_find_executable, mock_isfile): - # pylint: disable=unused-argument - os_path = os.environ["PATH"] - adb = adb_monitor.Adb() - self.assertTrue(adb.systrace_script.startswith(os.path.join("systrace", "systrace.py"))) - self.assertEqual(adb.logger, adb_monitor.LOGGER) - self.assertEqual(adb.python27, adb_monitor.DEFAULT_PYTHON27) - self.assertEqual(adb.logger, adb_monitor.LOGGER) - self.assertEqual(os.environ["PATH"], os_path) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test___init__adb_binary(self, mock_find_executable, mock_os): - # pylint: disable=unused-argument - adb_dir = os.path.join("/root", "adb_dir") - adb_path = os.path.join(adb_dir, "adb") - mock_os.environ = {"PATH": os.environ["PATH"]} - mock_os.path.pathsep = os.path.pathsep - mock_os.path.dirname = lambda x: x - adb = adb_monitor.Adb(adb_binary=adb_path) - self.assertTrue(adb.systrace_script.startswith(adb_dir)) - self.assertIn(os.path.pathsep + adb_dir, mock_os.environ["PATH"]) - - @patch(ns("os.environ")) - @patch(ns("os.path.isfile"), return_value=True) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test___init__python27_binary(self, mock_find_executable, mock_isfile, mock_os_environ): - # pylint: disable=unused-argument - python_dir = os.path.join("/root", "python27_dir") - python27_path = os.path.join(python_dir, "python2") - adb = adb_monitor.Adb(python27=python27_path) - self.assertEqual(adb.python27, python27_path) - - @patch(ns("os.environ")) - @patch(ns("os.path.isfile"), return_value=False) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test___init__bad_systrace(self, mock_find_executable, mock_isfile, mock_os_environ): - # pylint: disable=unused-argument - with self.assertRaises(EnvironmentError): - adb_monitor.Adb() - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_adb_cmd_output(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - adb_result = adb_monitor.Adb.adb_cmd("mycmd") - self.assertEqual(adb_result, mock_runcmd.RunCommand().execute_with_output()) - self.assertNotEqual(adb_result, mock_runcmd.RunCommand().execute_save_output()) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_adb_cmd_output_string(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - adb_result = adb_monitor.Adb.adb_cmd("mycmd", output_string=True) - self.assertEqual(adb_result, mock_runcmd.RunCommand().execute_with_output()) - self.assertNotEqual(adb_result, mock_runcmd.RunCommand().execute_save_output()) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_adb_cmd_save_output(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - adb_result = adb_monitor.Adb.adb_cmd("mycmd", output_file="myfile") - self.assertEqual(adb_result, mock_runcmd.RunCommand().execute_save_output()) - self.assertNotEqual(adb_result, mock_runcmd.RunCommand().execute_with_output()) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_adb_cmd_all_params(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - adb_result = adb_monitor.Adb.adb_cmd("mycmd", output_file="myfile", append_file=True, - output_string=True) - self.assertNotEqual(adb_result, mock_runcmd.RunCommand().execute_save_output()) - self.assertEqual(adb_result, mock_runcmd.RunCommand().execute_with_output()) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_shell(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - cmd_output = adb_monitor.Adb.shell("mycmd") - self.assertEqual(cmd_output, mock_runcmd.RunCommand().execute_with_output()) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_shell_stripped(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - output = "output from shell" - mock_runcmd.RunCommand().execute_with_output.return_value = output + "__EXIT__:0\n" - cmd_output = adb_monitor.Adb.shell("mycmd") - self.assertEqual(cmd_output, output) - - @patch(ns("os")) - @patch(ns("runcommand")) - def test_shell_error(self, mock_runcmd, mock_os): - # pylint: disable=unused-argument - output = "output from shell" - mock_runcmd.RunCommand().execute_with_output.return_value = output + "__EXIT__:1\n" - with self.assertRaises(RuntimeError): - adb_monitor.Adb.shell("mycmd") - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_devices(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.devices() - mock_runcmd.RunCommand.assert_called_once_with("adb devices -l", unittest.mock.ANY, - unittest.mock.ANY) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_device_available(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.device_available() - mock_runcmd.RunCommand.assert_called_once_with("adb shell uptime", unittest.mock.ANY, - unittest.mock.ANY) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_push(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - files = "myfile" - remote_dir = "/remotedir" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.push(files, remote_dir) - push_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - self.assertIn(files, push_cmd) - self.assertIn(remote_dir, push_cmd) - self.assertNotIn("--sync", push_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_push_list(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - files = ["myfile", "file2"] - remote_dir = "/remotedir" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.push(files, remote_dir) - push_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - self.assertIn(" ".join(files), push_cmd) - self.assertIn(remote_dir, push_cmd) - self.assertNotIn("--sync", push_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_push_sync(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - files = ["myfile", "file2"] - remote_dir = "/remotedir" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.push(files, remote_dir, sync=True) - push_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - for file_name in files: - self.assertIn(file_name, push_cmd) - self.assertIn(remote_dir, push_cmd) - self.assertIn("--sync", push_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_pull(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - files = "myfile" - local_dir = "/localdir" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.pull(files, local_dir) - pull_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - self.assertIn(files, pull_cmd) - self.assertIn(local_dir, pull_cmd) - self.assertNotIn("--sync", pull_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_pull_files(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - files = ["myfile", "file2"] - local_dir = "/localdir" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.pull(files, local_dir) - pull_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - for file_name in files: - self.assertIn(file_name, pull_cmd) - self.assertIn(local_dir, pull_cmd) - self.assertNotIn("--sync", pull_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test__battery_cmd(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - option = "myopt" - battery_cmd = "shell dumpsys batterystats " + option - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb._battery_cmd(option) - mock_runcmd.RunCommand.assert_called_once_with("adb " + battery_cmd, unittest.mock.ANY, - unittest.mock.ANY) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test__battery_cmd_save_output(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - option = "myopt" - battery_cmd = "shell dumpsys batterystats " + option - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb._battery_cmd(option, "myfile") - mock_runcmd.RunCommand.assert_called_once_with("adb " + battery_cmd, unittest.mock.ANY, - unittest.mock.ANY) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_battery(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.battery("myfile") - battery_cmd = mock_runcmd.RunCommand.call_args_list[0][0][0] - self.assertIn("--checkin", battery_cmd) - self.assertNotIn("--reset", battery_cmd) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_battery_reset(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.battery("myfile", reset=True, append_file=True) - battery_cmd1 = mock_runcmd.RunCommand.call_args_list[0][0][0] - battery_cmd2 = mock_runcmd.RunCommand.call_args_list[1][0][0] - self.assertIn("--reset", battery_cmd1) - self.assertIn("--checkin", battery_cmd2) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_memory(self, mock_runcmd, mock_find_executable, mock_os): - # pylint: disable=unused-argument - memory_cmd = "shell dumpsys meminfo -c -d" - mock_os.path.dirname.return_value = "adb_dir" - adb = adb_monitor.Adb() - adb.memory("myfile") - mock_runcmd.RunCommand.assert_called_once_with("adb " + memory_cmd, unittest.mock.ANY, - unittest.mock.ANY) - - @patch(ns("tempfile.NamedTemporaryFile")) - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_systrace_start(self, mock_runcmd, mock_find_executable, mock_os, mock_tempfile): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - systrace_script = "systrace.py" - mock_os.path.join.return_value = systrace_script - adb = adb_monitor.Adb() - adb.systrace_start() - mock_runcmd.RunCommand.assert_called_once_with(output_file=mock_tempfile().name, - propagate_signals=False) - self.assertEqual(adb.systrace_script, systrace_script) - self.assertEqual(mock_runcmd.RunCommand().add_file.call_count, 3) - mock_runcmd.RunCommand().add_file.assert_any_call(adb_monitor.DEFAULT_PYTHON27) - mock_runcmd.RunCommand().add_file.assert_any_call(systrace_script) - self.assertEqual(mock_runcmd.RunCommand().add.call_count, 3) - mock_runcmd.RunCommand().add.assert_any_call("--json") - mock_runcmd.RunCommand().add.assert_any_call("-o") - mock_runcmd.RunCommand().add.assert_any_call("dalvik sched freq idle load") - mock_runcmd.RunCommand().start_process.assert_called_once() - - @patch(ns("tempfile.NamedTemporaryFile")) - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - @patch(ns("runcommand")) - def test_systrace_start_output_file(self, mock_runcmd, mock_find_executable, mock_os, - mock_tempfile): - # pylint: disable=unused-argument - output_file = "myfile" - mock_os.path.dirname.return_value = "adb_dir" - systrace_script = "systrace.py" - mock_os.path.join.return_value = systrace_script - adb = adb_monitor.Adb() - adb.systrace_start(output_file) - mock_runcmd.RunCommand.assert_called_once_with(output_file=mock_tempfile().name, - propagate_signals=False) - self.assertEqual(adb.systrace_script, systrace_script) - self.assertEqual(mock_runcmd.RunCommand().add_file.call_count, 3) - mock_runcmd.RunCommand().add_file.assert_any_call(adb_monitor.DEFAULT_PYTHON27) - mock_runcmd.RunCommand().add_file.assert_any_call(systrace_script) - mock_runcmd.RunCommand().add_file.assert_any_call(output_file) - self.assertEqual(mock_runcmd.RunCommand().add.call_count, 3) - mock_runcmd.RunCommand().add.assert_any_call("--json") - mock_runcmd.RunCommand().add.assert_any_call("-o") - mock_runcmd.RunCommand().add.assert_any_call("dalvik sched freq idle load") - mock_runcmd.RunCommand().start_process.assert_called_once() - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test_systrace_stop(self, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - systrace_script = "systrace.py" - systrace_output = "Systrace: Wrote trace" - mock_os.path.join.return_value = systrace_script - adb = adb_monitor.Adb() - adb._cmd = MagicMock() - with patch(ADB_MONITOR + ".open", mock_open(read_data=systrace_output)): - adb.systrace_stop() - adb._cmd.send_to_process.assert_called_once_with(b"bye") - mock_os.remove.assert_called_once_with(adb._tempfile) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test_systrace_stop_output_file(self, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - systrace_script = "systrace.py" - systrace_output = "Systrace: Wrote trace" - output_file = "myfile" - mock_os.path.join.return_value = systrace_script - adb = adb_monitor.Adb() - adb._cmd = MagicMock() - with patch(ADB_MONITOR + ".open", mock_open(read_data=systrace_output)): - adb.systrace_stop(output_file=output_file) - adb._cmd.send_to_process.assert_called_once_with(b"bye") - mock_os.remove.assert_called_once_with(adb._tempfile) - - @patch(ns("os")) - @patch(ns("find_executable"), side_effect=lambda x: x) - def test_systrace_stop_no_trace(self, mock_find_executable, mock_os): - # pylint: disable=unused-argument - mock_os.path.dirname.return_value = "adb_dir" - systrace_script = "systrace.py" - systrace_output = "Systrace: did not Write trace" - output_file = "myfile" - mock_os.path.join.return_value = systrace_script - adb = adb_monitor.Adb() - adb._cmd = MagicMock() - with patch(ADB_MONITOR + ".open", mock_open(read_data=systrace_output)): - adb.systrace_stop(output_file=output_file) - adb._cmd.send_to_process.assert_called_once_with(b"bye") - self.assertEqual(mock_os.remove.call_count, 2) - mock_os.remove.assert_any_call(adb._tempfile) - mock_os.remove.assert_any_call(output_file) - - -class TestAdbControl(unittest.TestCase): - def test___init___no_files(self): - mock_adb = MagicMock() - with self.assertRaises(ValueError): - adb_monitor.AdbControl(mock_adb) - - @patch(ns("fileops.create_empty")) - def test___init___all_files(self, mock_create_empty): - mock_adb = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - adb_control = adb_monitor.AdbControl(mock_adb, battery_file=battery_file, - memory_file=memory_file, cpu_file=cpu_file) - self.assertEqual(adb_control.adb, mock_adb) - self.assertEqual(adb_control.battery_file, battery_file) - self.assertEqual(adb_control.memory_file, memory_file) - self.assertEqual(adb_control.cpu_file, cpu_file) - self.assertEqual(adb_control.num_samples, 0) - self.assertIsNone(adb_control.collection_time_secs) - self.assertEqual(adb_control.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - self.assertEqual(mock_create_empty.call_count, 3) - mock_create_empty.assert_any_call(battery_file) - mock_create_empty.assert_any_call(memory_file) - mock_create_empty.assert_any_call(cpu_file) - - @patch(ns("fileops.create_empty")) - def test___init___battery_file(self, mock_create_empty): - mock_adb = MagicMock() - battery_file = "mybattery" - adb_control = adb_monitor.AdbControl(mock_adb, battery_file=battery_file) - self.assertEqual(adb_control.adb, mock_adb) - self.assertEqual(adb_control.battery_file, battery_file) - self.assertIsNone(adb_control.memory_file) - self.assertIsNone(adb_control.cpu_file) - self.assertEqual(adb_control.num_samples, 0) - self.assertIsNone(adb_control.collection_time_secs) - self.assertEqual(adb_control.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - mock_create_empty.assert_called_once_with(battery_file) - - @patch(ns("fileops.create_empty")) - def test___init___all_params(self, mock_create_empty): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - num_samples = 5 - collection_time_secs = 10 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - cpu_file=cpu_file, append_file=True, num_samples=num_samples, collection_time_secs=10, - sample_interval_ms=sample_interval_ms) - self.assertEqual(adb_control.adb, mock_adb) - self.assertEqual(adb_control.battery_file, battery_file) - self.assertEqual(adb_control.memory_file, memory_file) - self.assertEqual(adb_control.cpu_file, cpu_file) - self.assertEqual(adb_control.num_samples, 0) - self.assertEqual(adb_control.collection_time_secs, collection_time_secs) - self.assertEqual(adb_control.sample_interval_ms, sample_interval_ms) - mock_create_empty.assert_not_called() - - @patch(ns("fileops.create_empty")) - def test___init___num_samples(self, mock_create_empty): - # pylint: disable=unused-argument - mock_adb = MagicMock() - battery_file = "mybattery" - num_samples = 5 - adb_control = adb_monitor.AdbControl(mock_adb, battery_file=battery_file, - num_samples=num_samples) - self.assertEqual(adb_control.adb, mock_adb) - self.assertEqual(adb_control.battery_file, battery_file) - self.assertIsNone(adb_control.memory_file) - self.assertIsNone(adb_control.cpu_file) - self.assertEqual(adb_control.num_samples, num_samples) - self.assertIsNone(adb_control.collection_time_secs) - self.assertEqual(adb_control.sample_interval_ms, adb_monitor.DEFAULT_SAMPLE_INTERVAL_MS) - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_start_all(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - num_samples = 5 - collection_time_secs = 10 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - cpu_file=cpu_file, append_file=True, num_samples=num_samples, - collection_time_secs=collection_time_secs, sample_interval_ms=sample_interval_ms) - adb_control.start() - self.assertEqual(len(adb_control._all_threads), 3) - self.assertEqual(len(adb_control._sample_based_threads), 2) - mock_continuous_monitor.assert_called_once_with( - cpu_file, adb_control._should_stop, mock_adb.systrace_start, mock_adb.systrace_stop) - mock_continuous_monitor().start.assert_called_once() - self.assertEqual(mock_sample_monitor.call_count, 2) - mock_sample_monitor.assert_any_call(battery_file, adb_control._should_stop, - mock_adb.battery, 0, sample_interval_ms) - mock_sample_monitor.assert_any_call(memory_file, adb_control._should_stop, mock_adb.memory, - 0, sample_interval_ms) - self.assertEqual(mock_sample_monitor().start.call_count, 2) - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_start_cpu(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - cpu_file = "mycpu" - collection_time_secs = 10 - adb_control = adb_monitor.AdbControl(mock_adb, logger=mock_logger, cpu_file=cpu_file, - append_file=True, - collection_time_secs=collection_time_secs) - adb_control.start() - self.assertEqual(len(adb_control._all_threads), 1) - self.assertEqual(len(adb_control._sample_based_threads), 0) - mock_continuous_monitor.assert_called_once_with( - cpu_file, adb_control._should_stop, mock_adb.systrace_start, mock_adb.systrace_stop) - mock_continuous_monitor().start.assert_called_once() - mock_sample_monitor.assert_not_called() - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_start_battery(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - num_samples = 5 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, append_file=True, - num_samples=num_samples, sample_interval_ms=sample_interval_ms) - adb_control.start() - self.assertEqual(len(adb_control._all_threads), 1) - self.assertEqual(len(adb_control._sample_based_threads), 1) - mock_continuous_monitor.assert_not_called() - mock_sample_monitor.assert_called_once_with(battery_file, adb_control._should_stop, - mock_adb.battery, num_samples, - sample_interval_ms) - mock_sample_monitor().start.assert_called_once() - - def test_stop(self): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - num_samples = 5 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, append_file=True, - num_samples=num_samples, sample_interval_ms=sample_interval_ms) - adb_control._should_stop = MagicMock() - adb_control.wait = MagicMock() - adb_control.stop() - adb_control._should_stop.set.assert_called_once() - adb_control.wait.assert_called_once() - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_wait(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - num_samples = 5 - collection_time_secs = 10 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - cpu_file=cpu_file, append_file=True, num_samples=num_samples, collection_time_secs=10, - sample_interval_ms=sample_interval_ms) - mock_sample_monitor().exception = None - mock_continuous_monitor().exception = None - adb_control._should_stop = MagicMock() - adb_control.start() - adb_control.wait() - adb_control._should_stop.wait.assert_called_once_with(collection_time_secs) - adb_control._should_stop.set.assert_called_once() - mock_continuous_monitor().join.assert_called_once_with(adb_control._JOIN_TIMEOUT) - self.assertEqual(mock_sample_monitor().join.call_count, 2) - mock_sample_monitor().join.assert_called_with(adb_control._JOIN_TIMEOUT) - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_wait_keyboard_interrupt(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - num_samples = 5 - collection_time_secs = 10 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - cpu_file=cpu_file, append_file=True, num_samples=num_samples, collection_time_secs=10, - sample_interval_ms=sample_interval_ms) - mock_sample_monitor().exception = None - mock_continuous_monitor().exception = None - adb_control._should_stop = MagicMock() - adb_control._should_stop.wait.side_effect = KeyboardInterrupt() - adb_control.start() - adb_control.wait() - adb_control._should_stop.wait.assert_called_once_with(collection_time_secs) - adb_control._should_stop.set.assert_called_once() - mock_continuous_monitor().join.assert_called_once_with(adb_control._JOIN_TIMEOUT) - self.assertEqual(mock_sample_monitor().join.call_count, 2) - mock_sample_monitor().join.assert_called_with(adb_control._JOIN_TIMEOUT) - - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_wait_no_collection_time(self, mock_sample_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - num_samples = 5 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - append_file=True, num_samples=num_samples, sample_interval_ms=sample_interval_ms) - mock_sample_monitor().exception = None - adb_control._should_stop = MagicMock() - adb_control.start() - adb_control.wait() - adb_control._should_stop.wait.assert_not_called() - adb_control._should_stop.set.assert_called_once() - self.assertEqual(mock_sample_monitor().join.call_count, 4) - mock_sample_monitor().join.assert_called_with(adb_control._JOIN_TIMEOUT) - - @patch(ns("AdbContinuousResourceMonitor")) - @patch(ns("AdbSampleBasedResourceMonitor")) - def test_wait_thread_exception(self, mock_sample_monitor, mock_continuous_monitor): - mock_adb = MagicMock() - mock_logger = MagicMock() - battery_file = "mybattery" - memory_file = "mymemory" - cpu_file = "mycpu" - num_samples = 5 - collection_time_secs = 10 - sample_interval_ms = 25 - adb_control = adb_monitor.AdbControl( - mock_adb, logger=mock_logger, battery_file=battery_file, memory_file=memory_file, - cpu_file=cpu_file, append_file=True, num_samples=num_samples, collection_time_secs=10, - sample_interval_ms=sample_interval_ms) - mock_sample_monitor().exception = RuntimeError("my run time exception") - mock_continuous_monitor().exception = None - adb_control._should_stop = MagicMock() - adb_control.start() - with self.assertRaises(RuntimeError): - adb_control.wait() - adb_control._should_stop.wait.assert_called_once_with(collection_time_secs) - adb_control._should_stop.set.assert_called_once() - mock_continuous_monitor().join.assert_called_once_with(adb_control._JOIN_TIMEOUT) - self.assertEqual(mock_sample_monitor().join.call_count, 2) - mock_sample_monitor().join.assert_called_with(adb_control._JOIN_TIMEOUT) - - -class TestAdbResourceMonitor(unittest.TestCase): - @patch(ns("threading")) - def test_run(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - should_stop = MagicMock() - arm = adb_monitor.AdbResourceMonitor(output_file, should_stop) - arm._do_monitoring = MagicMock() - arm.run() - self.assertEqual(arm._output_file, output_file) - arm._do_monitoring.assert_called_once() - self.assertIsNone(arm.exception) - should_stop.set.assert_not_called() - - @patch(ns("threading")) - def test_run_exception(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - should_stop = MagicMock() - exception = RuntimeError() - arm = adb_monitor.AdbResourceMonitor(output_file, should_stop) - arm._do_monitoring = MagicMock(side_effect=exception) - arm.run() - self.assertEqual(arm._output_file, output_file) - arm._do_monitoring.assert_called_once() - self.assertEqual(arm.exception, exception) - should_stop.set.assert_called_once() - - -class TestAdbSampleBasedResourceMonitor(unittest.TestCase): - @patch(ns("threading")) - def test__do_monitoring(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - mock_should_stop = MagicMock() - mock_should_stop.is_set.return_value = False - mock_adb_cmd = MagicMock() - num_samples = 5 - sample_interval_ms = 50 - arm = adb_monitor.AdbSampleBasedResourceMonitor(output_file, mock_should_stop, mock_adb_cmd, - num_samples, sample_interval_ms) - arm._take_sample = MagicMock() - arm._do_monitoring() - self.assertEqual(mock_should_stop.is_set.call_count, num_samples + 1) - self.assertEqual(arm._take_sample.call_count, num_samples) - self.assertEqual(mock_should_stop.wait.call_count, num_samples - 1) - mock_should_stop.wait.assert_called_with(sample_interval_ms / 1000.0) - - def _is_set(self): - self.num_collected += 1 - return self.num_collected == self.num_samples - - @patch(ns("threading")) - def test__do_monitoring_no_samples(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - mock_should_stop = MagicMock() - self.num_collected = 0 # pylint: disable=attribute-defined-outside-init - self.num_samples = 5 # pylint: disable=attribute-defined-outside-init - mock_should_stop.is_set = self._is_set - mock_adb_cmd = MagicMock() - num_samples = 0 - sample_interval_ms = 50 - arm = adb_monitor.AdbSampleBasedResourceMonitor(output_file, mock_should_stop, mock_adb_cmd, - num_samples, sample_interval_ms) - arm._take_sample = MagicMock() - arm._do_monitoring() - self.assertEqual(arm._take_sample.call_count, self.num_samples - 1) - self.assertEqual(mock_should_stop.wait.call_count, self.num_samples - 2) - - @patch(ns("threading")) - def test__take_sample(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - mock_should_stop = MagicMock() - self.num_collected = 0 # pylint: disable=attribute-defined-outside-init - self.num_samples = 5 # pylint: disable=attribute-defined-outside-init - mock_should_stop.is_set = self._is_set - mock_adb_cmd = MagicMock() - arm = adb_monitor.AdbSampleBasedResourceMonitor(output_file, mock_should_stop, mock_adb_cmd, - 10, 20) - arm._take_sample(2) - mock_adb_cmd.assert_called_once_with(output_file=output_file, append_file=True) - - -class TestAdbContinuousResourceMonitor(unittest.TestCase): - @patch(ns("threading")) - def test__do_monitoring(self, mock_threading): - # pylint: disable=unused-argument - output_file = "myfile" - mock_should_stop = MagicMock() - mock_adb_start_cmd = MagicMock() - mock_adb_stop_cmd = MagicMock() - acrm = adb_monitor.AdbContinuousResourceMonitor(output_file, mock_should_stop, - mock_adb_start_cmd, mock_adb_stop_cmd) - acrm._do_monitoring() - mock_should_stop.wait.assert_called_once() - mock_adb_start_cmd.assert_called_with(output_file=output_file) - mock_adb_stop_cmd.assert_called_with(output_file=output_file) diff --git a/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py b/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py deleted file mode 100755 index b069e79146b..00000000000 --- a/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -"""Unit tests for the resmokelib.testing.hooks.combine_benchrun_embedded_results module.""" - -import datetime -import os -import unittest - -import mock - -import buildscripts.resmokelib.testing.hooks.combine_benchrun_embedded_results as cber - -# pylint: disable=missing-docstring,protected-access,attribute-defined-outside-init - -_BM_REPORT_INSERT_1 = { - "note": "values per second", "errCount": {"$numberLong": "0"}, - "trapped": "error: not implemented", "insertLatencyAverageMicros": 389.4926654182272, - "totalOps": {"$numberLong": "12816"}, "totalOps/s": 2563.095938304905, "findOne": 0, - "insert": 2563.095938304905, "delete": 0, "update": 0, "query": 0, "command": 0, "findOnes": { - "$numberLong": "0" - }, "inserts": {"$numberLong": "12816"}, "deletes": {"$numberLong": "0"}, "updates": { - "$numberLong": "0" - }, "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} -} - -_BM_REPORT_INSERT_2 = { - "note": "values per second", "errCount": {"$numberLong": "0"}, "trapped": - "error: not implemented", "insertLatencyAverageMicros": 2563.095938304905, "totalOps": { - "$numberLong": "7404" - }, "totalOps/s": 2409.05, "findOne": 0, "insert": 2409.05, "delete": 0, "update": 0, - "query": 0, "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": {"$numberLong": "7404"}, - "deletes": {"$numberLong": "0"}, "updates": {"$numberLong": "0"}, - "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} -} - -_BM_REPORT_DELETE = { - "note": "values per second", "errCount": {"$numberLong": "0"}, - "trapped": "error: not implemented", "insertLatencyAverageMicros": "1234.56", "totalOps": { - "$numberLong": "2345" - }, "totalOps/s": 1234.56, "findOne": 0, "insert": 0, "delete": 1234.56, "update": 0, "query": - 0, "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": {"$numberLong": "0"}, - "deletes": {"$numberLong": "2345"}, "updates": {"$numberLong": "0"}, - "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} -} - -_BM_REPORT_UPDATE = { - "note": "values per second", "errCount": {"$numberLong": "0"}, - "trapped": "error: not implemented", "insertLatencyAverageMicros": 654.321, "totalOps": { - "$numberLong": "4521" - }, "totalOps/s": 4521.00, "findOne": 0, "insert": 0, "delete": 0, "update": 4521.00, "query": - 0, "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": {"$numberLong": "0"}, - "deletes": {"$numberLong": "0"}, "updates": {"$numberLong": "4521"}, - "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} -} - -_BM_REPORT_MULTI = { - "note": "values per second", "errCount": {"$numberLong": "0"}, "trapped": - "error: not implemented", "insertLatencyAverageMicros": 111.111, - "totalOps": {"$numberLong": "11532"}, "totalOps/s": 5766.00, "findOne": 0, "insert": 2490.00, - "delete": 0, "update": 9042.00, "query": 0, "command": 0, "findOnes": {"$numberLong": "0"}, - "inserts": {"$numberLong": "2490.00"}, "deletes": {"$numberLong": "0"}, "updates": { - "$numberLong": "9042" - }, "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} -} - -_BM_ALL_REPORTS = [ - _BM_REPORT_INSERT_1, _BM_REPORT_INSERT_2, _BM_REPORT_DELETE, _BM_REPORT_UPDATE, _BM_REPORT_MULTI -] - -# 12/31/2999 @ 11:59pm (UTC) -_START_TIME = 32503679999 - -# 01/01/3000 @ 12:00am (UTC) -_END_TIME = 32503680000 - - -class CombineBenchrunEmbeddedResultsFixture(unittest.TestCase): - - # Mock the hook's parent class because we're testing only functionality of this hook and - # not anything related to or inherit from the parent class. - @mock.patch("buildscripts.resmokelib.testing.hooks.interface.Hook", autospec=True) - def setUp(self, MockHook): # pylint: disable=arguments-differ,unused-argument - self.cber_hook = cber.CombineBenchrunEmbeddedResults(None, None) - self.cber_hook.create_time = datetime.datetime.utcfromtimestamp(_START_TIME) - self.cber_hook.end_time = datetime.datetime.utcfromtimestamp(_END_TIME) - - -class TestCombineBenchmarkResults(CombineBenchrunEmbeddedResultsFixture): - def _setup_reports(self, reports, test_name, num_threads): - self.total_ops_per_sec = 0 - self.num_tests = len(reports) - self.cber_hook.benchmark_reports[test_name] = cber._BenchrunEmbeddedThreadsReport() - for rep in reports: - self.cber_hook.benchmark_reports[test_name].add_report(num_threads, rep) - self.total_ops_per_sec += rep["totalOps/s"] - self.ops_per_sec = self.total_ops_per_sec / self.num_tests - self.report = self.cber_hook._generate_perf_plugin_report() - - def test_generate_one_report(self): - test_name = "test_cber1" - num_threads = "2" - self._setup_reports([_BM_REPORT_MULTI], test_name, num_threads) - report_0 = self.report["results"][0] - self.assertEqual(report_0["name"], test_name) - self.assertEqual(report_0["results"][str(num_threads)]["ops_per_sec"], self.ops_per_sec) - - def test_generate_all_reports(self): - test_name = "test_cber2" - thread_num = "1" - self._setup_reports(_BM_ALL_REPORTS, test_name, thread_num) - self.assertEqual(len(list(self.report.keys())), 4) - report_0 = self.report["results"][0] - self.assertEqual(report_0["name"], test_name) - self.assertEqual(report_0["results"][thread_num]["ops_per_sec"], self.ops_per_sec) - self.assertEqual(self.report["start"], "2999-12-31T23:59:59Z") - self.assertEqual(self.report["end"], "3000-01-01T00:00:00Z") - - def test_parse_report_name(self): - self.cber_hook.report_root = os.path.join("benchrun_embedded", "results") - test_name = "test1" - thread_num = 3 - file_name = os.path.join(self.cber_hook.report_root, test_name, - "thread{}".format(thread_num), "mongoebench.0.json") - report_threads = self.cber_hook._parse_report_name(file_name) - self.assertEqual(thread_num, int(report_threads)) - - -class TestBenchrunEmbeddedThreadsReport(CombineBenchrunEmbeddedResultsFixture): - def test_generate_single_thread_perf_plugin_dict(self): - thread_report = cber._BenchrunEmbeddedThreadsReport() - thread_num = "1" - thread_report.add_report(thread_num, _BM_REPORT_INSERT_1) - perf_report = thread_report.generate_perf_plugin_dict() - self.assertEqual(len(list(perf_report.keys())), 1) - self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) - self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) - - thread_report.add_report(thread_num, _BM_REPORT_INSERT_2) - perf_report = thread_report.generate_perf_plugin_dict() - self.assertEqual(len(list(perf_report.keys())), 1) - ops_per_sec = (_BM_REPORT_INSERT_1["totalOps/s"] + _BM_REPORT_INSERT_2["totalOps/s"]) / 2 - self.assertEqual(perf_report[thread_num]["ops_per_sec"], ops_per_sec) - self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 2) - - def test_generate_multi_thread_perf_plugin_dict(self): - thread_report = cber._BenchrunEmbeddedThreadsReport() - thread_num = "1" - thread_report.add_report(thread_num, _BM_REPORT_INSERT_1) - perf_report = thread_report.generate_perf_plugin_dict() - self.assertEqual(len(list(perf_report.keys())), 1) - self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) - self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) - - thread_num = "2" - thread_report.add_report(thread_num, _BM_REPORT_INSERT_2) - perf_report = thread_report.generate_perf_plugin_dict() - self.assertEqual(len(list(perf_report.keys())), 2) - self.assertEqual(perf_report["1"]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) - self.assertEqual(len(perf_report["1"]["ops_per_sec_values"]), 1) - self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_2["totalOps/s"]) - self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index db27ac5281c..929cead27c3 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -71,18 +71,6 @@ variables: resmoke_jobs_max: 1 - func: "send benchmark results" -- &benchrun_embedded_template - name: benchrun_embedded_template - depends_on: - - generate_benchrun_embedded_files - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --help - resmoke_jobs_max: 1 - - func: "send benchmark results" - - &jepsen_config_vars jepsen_key_time_limit: --key-time-limit 15 jepsen_protocol_version: --protocol-version 1 @@ -191,19 +179,6 @@ variables: mongod_options: --mongodUsablePorts ${standard_port} ${secret_port} --dbPath=${db_path} --logPath=${log_path} mongod_extra_options: --mongodOptions=\"--setParameter enableTestCommands=1 --setParameter logComponentVerbosity='{storage:{recovery:2}}'\" -- &benchrun_embedded - name: benchrun_embedded - execution_tasks: - - benchrun_embedded_aggregation - - benchrun_embedded_commands - - benchrun_embedded_insert - - benchrun_embedded_misc - - benchrun_embedded_mixed_and_multi - - benchrun_embedded_queries - - benchrun_embedded_remove - - benchrun_embedded_update - - generate_benchrun_embedded_files - - &libfuzzertests name: libfuzzertests! execution_tasks: @@ -593,15 +568,6 @@ functions: bucket: mciuploads extract_to: src - "fetch benchrun embedded files": &fetch_benchrun_embedded_files - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${project}/${build_variant}/${revision}/benchrun_json_files.tgz - bucket: mciuploads - extract_to: src/benchrun_embedded/testcases - "fetch corpus": &fetch_corpus command: s3.get params: @@ -1328,17 +1294,6 @@ functions: - *set_up_credentials - *fetch_benchmarks - "do benchmark embedded setup": - - command: manifest.load - - *git_get_project - - *set_task_expansion_macros - - *set_up_virtualenv - - *upload_pip_requirements - - *fetch_artifacts - - *get_buildnumber - - *set_up_credentials - - *fetch_benchrun_embedded_files - "do multiversion setup": &do_multiversion_setup command: shell.exec params: @@ -3796,8 +3751,6 @@ tasks: - "pytests/**" - "./test*" - "./mongobridge*" - - "./mongoebench*" - - "./mongoed*" - "./mongotmock*" - "./wt*" - "buildscripts/**" @@ -4647,154 +4600,6 @@ tasks: resmoke_jobs_max: 1 - func: "send benchmark results" -- <<: *task_template - name: generate_benchrun_embedded_files - tags: ["benchrun"] - commands: - - command: manifest.load - - func: "git get project" - - func: "fetch binaries" - - func: "extract binaries" - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - - ${activate_virtualenv} - - mkdir -p /data/db - rm -rf /data/db/* - - ./mongod --fork --logpath=/data/db/mongod.log - PATH="$(pwd):$PATH" ./src/third_party/scripts/mongo-perf_get_sources.sh - ./mongo --eval 'db.getSiblingDB("admin").shutdownServer()' - - command: archive.targz_pack - params: - target: "src/benchrun_json_files.tgz" - source_dir: "${workdir}/src/benchrun_embedded/testcases" - include: - - "**.json" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/benchrun_json_files.tgz - remote_file: ${project}/${build_variant}/${revision}/benchrun_json_files.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: benchrun embedded JSON config files - -- <<: *task_template - name: upload_benchrun_embedded_files - tags: ["benchrun"] - depends_on: - - generate_benchrun_embedded_files - patchable: false - commands: - - command: s3Copy.copy - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - s3_copy_files: - - source: - path: ${project}/${build_variant}/${revision}/benchrun_json_files.tgz - bucket: mciuploads - destination: - path: ${project}/benchrun_embedded/benchrun_json_files.tgz - bucket: mciuploads - build_variants: [rhel-62-64-bit-mobile] - -- <<: *benchrun_embedded_template - name: benchrun_embedded_aggregation - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_aggregation - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_commands - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_commands - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_insert - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_insert - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_misc - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_misc - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_mixed_and_multi - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_mixed_and_multi - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_queries - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_queries - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_remove - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_remove - resmoke_jobs_max: 1 - - func: "send benchmark results" - -- <<: *benchrun_embedded_template - name: benchrun_embedded_update - tags: ["benchrun", "perf"] - commands: - - func: "do benchmark embedded setup" - - func: "run tests" - vars: - resmoke_args: --suites=benchrun_embedded_update - resmoke_jobs_max: 1 - - func: "send benchmark results" - - <<: *run_jepsen_template name: jepsen_register_findAndModify tags: ["jepsen"] @@ -6076,15 +5881,6 @@ tasks: resmoke_args: --suites=core --storageEngine=wiredTiger - <<: *task_template - name: jsCore_mobile - tags: ["jscore"] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_args: --suites=core --mongod=./mongoed --excludeWithAnyTags=requires_scripting,requires_auth,requires_sharding,does_not_support_stepdowns,requires_background_index,requires_ttl_index,incompatible_with_embedded,incompatible_with_embedded_todo_investigate,requires_replication,requires_capped,requires_profiling,requires_javascript,requires_fsync,requires_compact,requires_snapshot_read - -- <<: *task_template name: jsCore_ese tags: ["jscore", "encrypt"] depends_on: @@ -11740,161 +11536,6 @@ buildvariants: - name: .updatefuzzer - name: secondary_reads_passthrough_gen -- name: rhel-62-64-bit-mobile - display_name: RHEL 6.2 (mobile) - run_on: - - rhel62-small - batchtime: 10080 # 7 days - expansions: &rhel-62-64-bit-mobile-expansions - # Transactions are only supported on replica sets, and replication is not supported by the - # mobile storage engine. - test_flags: >- - --storageEngine=mobile - --excludeWithAnyTags=requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact,requires_document_locking,requires_snapshot_read,requires_db_locking,uses_ttl - --excludeWithAnyTags=SERVER-32869,SERVER-40561 - compile_flags: >- - -j$(grep -c ^processor /proc/cpuinfo) - --mobile-se=on - --ssl=off - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - MONGO_DISTMOD=rhel62 - scons_cache_scope: shared - build_mongoreplay: true - additional_targets: install-embedded-test - display_tasks: - - *benchrun_embedded - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel62-large - - name: aggregation - - name: aggregation_auth - - name: aggregation_facet_unwind_passthrough - - name: auth_gen - - name: .benchrun !.perf - - name: .benchrun .perf - distros: - - centos6-perf - - name: .misc_js !.non_mobile - - name: concurrency - - name: disk_mobile - - name: failpoints_auth - - name: integration_tests_standalone - distros: - - rhel62-large - - name: .jscore .common !.txns !.sharding - - name: jsCore_minimum_batch_size - - name: jsCore_mobile - - name: jsCore_op_query - - name: jsonSchema - - name: jstestfuzz_gen - - name: jstestfuzz_concurrent_gen - - name: powercycle_mobile - - name: session_jscore_passthrough - -- name: ubuntu1604-arm64-mobile - display_name: "Ubuntu 16.04 arm64 (mobile)" - run_on: - - ubuntu1604-arm64-large - batchtime: 10080 # 7 days - expansions: - test_flags: >- - --storageEngine=mobile - --excludeWithAnyTags=requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact,requires_document_locking,requires_snapshot_read,requires_db_locking,uses_ttl - --excludeWithAnyTags=SERVER-32869,SERVER-40561 - compile_flags: >- - -j$(grep -c ^processor /proc/cpuinfo) - --mobile-se=on - --ssl=off - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - CCFLAGS="-march=armv8-a+crc -mtune=generic" - MONGO_DISTMOD=ubuntu1604 - resmoke_jobs_max: 8 # Avoid starting too many mongod's on ARM test servers - build_mongoreplay: true - additional_targets: install-embedded-test - tasks: - - name: compile_all_run_unittests_TG - - name: aggregation - - name: aggregation_auth - - name: aggregation_facet_unwind_passthrough - - name: auth_gen - - name: .misc_js !.non_mobile - - name: concurrency - - name: disk_mobile - - name: failpoints_auth - - name: integration_tests_standalone - - name: .jscore .common !.txns !.sharding - - name: jsCore_minimum_batch_size - - name: jsCore_mobile - - name: jsCore_op_query - - name: jsonSchema - - name: jstestfuzz_gen - - name: jstestfuzz_concurrent_gen - - name: session_jscore_passthrough - -- name: rhel-62-64-bit-required-mobile - display_name: "! RHEL 6.2 (mobile)" - batchtime: 60 # 1 hour - run_on: - - rhel62-small - expansions: - <<: *rhel-62-64-bit-mobile-expansions - burn_in_tests_build_variant: rhel-62-64-bit-mobile - tasks: - # This build variant exists specifically to test that newly added or modified JavaScript tests are - # correctly tagged with "requires_replication" and "requires_sharding" prior to the changes being - # pushed. It uses the task list from the rhel-62-64-bit-mobile build variant to - # determine the resmoke.py YAML suite configurations to run the tests under. Do not add more tasks - # to this list. - - name: compile_TG - requires: - - name: burn_in_tests_gen - distros: - - rhel62-large - - name: burn_in_tests_gen - - name: jsCore_mobile - -- name: macos-mobile - display_name: macOS (mobile) - run_on: - - macos-1014 - batchtime: 10080 # 7 days - expansions: - # Transactions are only supported on replica sets, and replication is not supported by the - # mobile storage engine. - test_flags: >- - --storageEngine=mobile - --excludeWithAnyTags=requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact,requires_document_locking,requires_snapshot_read,requires_db_locking,uses_ttl - --excludeWithAnyTags=SERVER-32869,SERVER-40561 - compile_env: DEVELOPER_DIR=/Applications/Xcode10.2.app - compile_flags: >- - -j$(sysctl -n hw.logicalcpu) - --libc++ - --mobile-se=on - --ssl=off - CPPPATH=/opt/mongodbtoolchain/v3/include - --variables-files=etc/scons/xcode_macosx.vars - resmoke_jobs_max: 4 - build_mongoreplay: true - additional_targets: install-embedded-test - tasks: - - name: compile_all_run_unittests_TG - - name: aggregation - - name: aggregation_facet_unwind_passthrough - - name: .auth !.multiversion !.gle !.audit !.replica_sets !.sharding - - name: .misc_js !.non_mobile - - name: concurrency - - name: disk_mobile - - name: integration_tests_standalone - - name: .jscore .common !.txns !.sharding - - name: jsCore_minimum_batch_size - - name: jsCore_mobile - - name: jsCore_op_query - - name: jsonSchema - - name: jstestfuzz_gen - - name: jstestfuzz_concurrent_gen - - name: session_jscore_passthrough - - name: linux-64-ephemeralForTest display_name: Linux (ephemeralForTest) run_on: diff --git a/jstests/core_standalone/write_concern.js b/jstests/core_standalone/write_concern.js index c02da6bb6f8..aa5a706ea71 100644 --- a/jstests/core_standalone/write_concern.js +++ b/jstests/core_standalone/write_concern.js @@ -1,4 +1,4 @@ -// This test verifies writeConcern behavior on a standalone mongod or embedded mongoed. +// This test verifies writeConcern behavior on a standalone mongod. (function() { 'use strict'; @@ -13,4 +13,4 @@ assert.commandWorked(col.insert({_id: "majority"}, {writeConcern: {w: "majority" // writeConcern: 2 should not work on standalone assert.writeError(col.insert({_id: 2}, {writeConcern: {w: 2}}), "expected writeConcern: 2 to fail"); -})();
\ No newline at end of file +})(); diff --git a/jstests/libs/mongoebench.js b/jstests/libs/mongoebench.js deleted file mode 100644 index a0d6f1b512d..00000000000 --- a/jstests/libs/mongoebench.js +++ /dev/null @@ -1,48 +0,0 @@ -"use strict"; - -var {runMongoeBench} = (function() { - /** - * Spawns a mongoebench process with the specified options. - * - * If a plain JavaScript object is specified as the 'config' parameter, then it is serialized to - * a file as a JSON string which is then specified as the config file for the mongoebench - * process. - */ - function runMongoeBench(config, options = {}) { - const args = ["mongoebench"]; - - if (typeof config === "object") { - const filename = MongoRunner.dataPath + "mongoebench_config.json"; - writeFile(filename, tojson(config)); - args.push(filename); - } else if (typeof config === "string") { - args.push(config); - } else { - throw new Error("'config' parameter must be a string or an object"); - } - - if (!options.hasOwnProperty("dbpath")) { - options.dbpath = MongoRunner.dataDir; - } - - for (let key of Object.keys(options)) { - const value = options[key]; - if (value === null || value === undefined) { - throw new Error( - "Value '" + value + "' for '" + key + - "' option is ambiguous; specify {flag: ''} to add --flag command line" + - " options'"); - } - - args.push("--" + key); - if (value !== "") { - args.push(value.toString()); - } - } - - const exitCode = _runMongoProgram(...args); - assert.eq(0, exitCode, "encountered an error in mongoebench"); - } - - return {runMongoeBench}; -})(); diff --git a/jstests/noPassthrough/mongoebench_test.js b/jstests/noPassthrough/mongoebench_test.js deleted file mode 100644 index cb531963e2b..00000000000 --- a/jstests/noPassthrough/mongoebench_test.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Tests for the mongoebench executable. - */ -(function() { -"use strict"; - -load("jstests/libs/mongoebench.js"); // for runMongoeBench - -if (jsTest.options().storageEngine !== "mobile") { - print("Skipping test because storage engine isn't mobile"); - return; -} - -const dbpath = MongoRunner.dataPath + "mongoebench_test"; -resetDbpath(dbpath); - -// Test that the operations in the "pre" section of the configuration are run exactly once. -runMongoeBench( // Force clang-format to break this line. - { - pre: [{ - op: "insert", - ns: "test.mongoebench_test", - doc: {pre: {"#SEQ_INT": {seq_id: 0, start: 0, step: 1, unique: true}}} - }], - ops: [{ - op: "update", - ns: "test.mongoebench_test", - update: {$inc: {ops: 1}}, - multi: true, - }] - }, - {dbpath}); - -const output = cat(dbpath + "/perf.json"); -const stats = assert.doesNotThrow( - JSON.parse, [output], "failed to parse output file as strict JSON: " + output); -assert.eq({$numberLong: "0"}, - stats.errCount, - () => "stats file reports errors but exit code was zero: " + tojson(stats)); -assert(stats.hasOwnProperty("totalOps/s"), - () => "stats file doesn't report ops per second: " + tojson(stats)); - -const conn = MongoRunner.runMongod({dbpath, noCleanData: true}); -assert.neq(null, conn, "failed to start mongod after running mongoebench"); - -const db = conn.getDB("test"); -const count = db.mongoebench_test.find().itcount(); -assert.eq(1, count, "ops in 'pre' section ran more than once or didn't run at all"); - -MongoRunner.stopMongod(conn); -})(); diff --git a/src/mongo/db/ftdc/ftdc_system_stats_android.cpp b/src/mongo/db/ftdc/ftdc_system_stats_android.cpp deleted file mode 100644 index 10f3468c918..00000000000 --- a/src/mongo/db/ftdc/ftdc_system_stats_android.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/db/ftdc/ftdc_system_stats.h" - -#include <memory> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/base/string_data.h" -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/db/ftdc/collector.h" -#include "mongo/db/ftdc/controller.h" -#include "mongo/util/processinfo.h" -#include "mongo/util/procparser.h" - -namespace mongo { - -namespace { - -static const std::vector<StringData> kCpuKeys{ - "btime"_sd, "cpu"_sd, "ctxt"_sd, "processes"_sd, "procs_blocked"_sd, "procs_running"_sd}; - -static const std::vector<StringData> kMemKeys{ - "MemTotal"_sd, - "MemFree"_sd, - "Cached"_sd, - "Dirty"_sd, - "Buffers"_sd, - "SwapTotal"_sd, - "SwapCached"_sd, - "SwapFree"_sd, - "Active"_sd, - "Inactive"_sd, - "Active(anon)"_sd, - "Inactive(anon)"_sd, - "Active(file)"_sd, - "Inactive(file)"_sd, -}; - -/** - * Collect metrics from the Linux /proc file system. - */ -class LinuxSystemMetricsCollector final : public SystemMetricsCollector { -public: - LinuxSystemMetricsCollector() : _disks(procparser::findPhysicalDisks("/sys/block"_sd)) { - for (const auto& disk : _disks) { - _disksStringData.emplace_back(disk); - } - } - - void collect(OperationContext* opCtx, BSONObjBuilder& builder) override { - { - BSONObjBuilder subObjBuilder(builder.subobjStart("cpu"_sd)); - - // Include the number of cpus to simplify client calculations - ProcessInfo p; - subObjBuilder.append("num_cpus", p.getNumCores()); - - processStatusErrors( - procparser::parseProcStatFile("/proc/stat"_sd, kCpuKeys, &subObjBuilder), - &subObjBuilder); - subObjBuilder.doneFast(); - } - - { - BSONObjBuilder subObjBuilder(builder.subobjStart("memory"_sd)); - processStatusErrors( - procparser::parseProcMemInfoFile("/proc/meminfo"_sd, kMemKeys, &subObjBuilder), - &subObjBuilder); - subObjBuilder.doneFast(); - } - - // Skip the disks section if we could not find any disks. - // This can happen when we do not have permission to /sys/block for instance. - if (!_disksStringData.empty()) { - BSONObjBuilder subObjBuilder(builder.subobjStart("disks"_sd)); - processStatusErrors(procparser::parseProcDiskStatsFile( - "/proc/diskstats"_sd, _disksStringData, &subObjBuilder), - &subObjBuilder); - subObjBuilder.doneFast(); - } - } - -private: - // List of physical disks to collect stats from as string from findPhysicalDisks. - std::vector<std::string> _disks; - - // List of physical disks to collect stats from as StringData to pass to parseProcDiskStatsFile. - std::vector<StringData> _disksStringData; -}; - -} // namespace - -void installSystemMetricsCollector(FTDCController* controller) { - controller->addPeriodicCollector(std::make_unique<LinuxSystemMetricsCollector>()); -} - -} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_iOS-sim.cpp b/src/mongo/db/ftdc/ftdc_system_stats_iOS-sim.cpp deleted file mode 100644 index a25484a0158..00000000000 --- a/src/mongo/db/ftdc/ftdc_system_stats_iOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/ftdc/ftdc_system_stats_osx.cpp" diff --git a/src/mongo/db/ftdc/ftdc_system_stats_iOS.cpp b/src/mongo/db/ftdc/ftdc_system_stats_iOS.cpp deleted file mode 100644 index a25484a0158..00000000000 --- a/src/mongo/db/ftdc/ftdc_system_stats_iOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/ftdc/ftdc_system_stats_osx.cpp" diff --git a/src/mongo/db/ftdc/ftdc_system_stats_tvOS-sim.cpp b/src/mongo/db/ftdc/ftdc_system_stats_tvOS-sim.cpp deleted file mode 100644 index a25484a0158..00000000000 --- a/src/mongo/db/ftdc/ftdc_system_stats_tvOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/ftdc/ftdc_system_stats_osx.cpp" diff --git a/src/mongo/db/ftdc/ftdc_system_stats_tvOS.cpp b/src/mongo/db/ftdc/ftdc_system_stats_tvOS.cpp deleted file mode 100644 index a25484a0158..00000000000 --- a/src/mongo/db/ftdc/ftdc_system_stats_tvOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/ftdc/ftdc_system_stats_osx.cpp" diff --git a/src/mongo/embedded/SConscript b/src/mongo/embedded/SConscript index 49baf7939e9..c5a9ed4da18 100644 --- a/src/mongo/embedded/SConscript +++ b/src/mongo/embedded/SConscript @@ -137,35 +137,3 @@ env.Library( '$BUILD_DIR/mongo/util/options_parser/options_parser', ], ) - -if get_option('link-model') != 'dynamic-sdk': - mongoed = yamlEnv.Program( - target='mongoed', - source=[ - 'mongoed_main.cpp', - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/db/commands/shell_protocol', - '$BUILD_DIR/mongo/db/mongod_options', - '$BUILD_DIR/mongo/db/repl/repl_set_status_commands', - '$BUILD_DIR/mongo/db/server_options', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger' if get_option('wiredtiger') == 'on' else [], - '$BUILD_DIR/mongo/embedded/embedded', - '$BUILD_DIR/mongo/embedded/embedded_integration_helpers', - '$BUILD_DIR/mongo/transport/service_entry_point', - '$BUILD_DIR/mongo/transport/transport_layer_manager', - '$BUILD_DIR/mongo/util/signal_handlers', - ], - AIB_COMPONENT='embedded-test', - AIB_COMPONENTS_EXTRA=[ - 'tests', - ] - ) - - env.Alias('all', mongoed) # This ensures it compiles and links, but doesn't copy it anywhere. - - hygienic = get_option('install-mode') == 'hygienic' - install_mongoed = env.Install('#/', mongoed) - if hygienic: - # TODO: remove when archive.tgz is built by hygienic - env.Alias("install-embedded-test", install_mongoed[0]) diff --git a/src/mongo/embedded/mongo_embedded/java/embedded-android/AndroidManifest.xml b/src/mongo/embedded/mongo_embedded/java/embedded-android/AndroidManifest.xml deleted file mode 100644 index 8d3ffd30527..00000000000 --- a/src/mongo/embedded/mongo_embedded/java/embedded-android/AndroidManifest.xml +++ /dev/null @@ -1,2 +0,0 @@ -<manifest xmlns:android="http://schemas.android.com/apk/res/android" - package="com.mongodb.embedded" /> diff --git a/src/mongo/embedded/mongo_embedded/java/embedded-android/build.gradle b/src/mongo/embedded/mongo_embedded/java/embedded-android/build.gradle deleted file mode 100644 index 9c81b4e01c5..00000000000 --- a/src/mongo/embedded/mongo_embedded/java/embedded-android/build.gradle +++ /dev/null @@ -1,95 +0,0 @@ -apply plugin: 'com.android.library' -apply plugin: 'optional-base' -apply plugin: 'maven' -apply plugin: 'signing' - -android { - compileSdkVersion 28 - - defaultConfig { - minSdkVersion 21 - targetSdkVersion 28 - } - - buildTypes { - release { - minifyEnabled false - } - } - - sourceSets { - main { - manifest.srcFile 'AndroidManifest.xml' - java.srcDir '../src' - jniLibs.srcDirs '../jniLibs' - } - } -} - -task sourcesJar(type: Jar) { - from android.sourceSets.main.java.srcDirs - classifier = 'sources' -} - -dependencies { - api "net.java.dev.jna:jna:$jnaVersion@aar" - api 'org.slf4j:slf4j-api:1.7.6', optional -} - -artifacts { - archives sourcesJar -} - -signing { - sign configurations.archives -} - -uploadArchives { - repositories { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - - pom.project { - name "${libraryName}-android" - packaging 'aar' - description "${pomDescription} - Android" - url pomUrl - - scm { - connection "scm:${pomScmConnection}" - developerConnection "scm:${pomScmConnection}" - url pomScmUrl - } - - licenses { - license { - name pomLicenseName - url pomLicenseUrl - } - } - - developers { - developer { - id 'mongodb' - name 'MongoDB Inc.' - email 'build@mongodb.com' - } - } - } - } - } -} - -def deployer = uploadArchives.repositories.mavenDeployer -deployer.pom*.whenConfigured { pom -> - pom.dependencies.find { dep -> dep.groupId == 'org.slf4j' }.optional = true -} - diff --git a/src/mongo/embedded/mongoed_main.cpp b/src/mongo/embedded/mongoed_main.cpp deleted file mode 100644 index 0c31a224374..00000000000 --- a/src/mongo/embedded/mongoed_main.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault - -#include "mongo/platform/basic.h" - -#include "mongo/base/init.h" -#include "mongo/db/mongod_options.h" -#include "mongo/db/mongod_options_general_gen.h" -#include "mongo/db/mongod_options_replication_gen.h" -#include "mongo/db/server_options_general_gen.h" -#include "mongo/db/service_context.h" -#include "mongo/embedded/embedded.h" -#include "mongo/embedded/embedded_options.h" -#include "mongo/embedded/embedded_options_helpers.h" -#include "mongo/embedded/service_entry_point_embedded.h" -#include "mongo/transport/service_entry_point_impl.h" -#include "mongo/transport/transport_layer.h" -#include "mongo/transport/transport_layer_manager.h" -#include "mongo/util/exit.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/options_parser.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/signal_handlers.h" -#include "mongo/util/text.h" - -#include <yaml-cpp/yaml.h> - -namespace mongo { -namespace { - -class ServiceEntryPointMongoe : public ServiceEntryPointImpl { -public: - explicit ServiceEntryPointMongoe(ServiceContext* svcCtx) - : ServiceEntryPointImpl(svcCtx), - _sepEmbedded(std::make_unique<ServiceEntryPointEmbedded>()) {} - - DbResponse handleRequest(OperationContext* opCtx, const Message& request) final { - return _sepEmbedded->handleRequest(opCtx, request); - } - -private: - std::unique_ptr<ServiceEntryPointEmbedded> _sepEmbedded; -}; - -MONGO_INITIALIZER_WITH_PREREQUISITES(SignalProcessingStartup, ("ThreadNameInitializer")) -(InitializerContext*) { - // Make sure we call this as soon as possible but before any other threads are started. Before - // embedded::initialize is too early and after is too late. So instead we hook in during the - // global initialization at the right place. - startSignalProcessingThread(); - return Status::OK(); -} - -int mongoedMain(int argc, char* argv[], char** envp) { - ServiceContext* serviceContext = nullptr; - - registerShutdownTask([&]() { - if (!serviceContext) - return; - - if (auto tl = serviceContext->getTransportLayer()) { - log(logger::LogComponent::kNetwork) << "shutdown: going to close listening sockets..."; - tl->shutdown(); - } - - if (auto sep = serviceContext->getServiceEntryPoint()) { - if (sep->shutdown(Seconds(10))) { - embedded::shutdown(serviceContext); - } else { - log(logger::LogComponent::kNetwork) << "Failed to shutdown service entry point " - "within timelimit, skipping embedded " - "shutdown."; - } - } - }); - - setupSignalHandlers(); - - log() << "MongoDB embedded standalone application, for testing purposes only"; - - try { - optionenvironment::OptionSection startupOptions("Options"); - // Adding all options mongod we don't have to maintain a separate set for this executable, - // some will be unused but that's fine as this is just an executable for testing purposes - // anyway. - uassertStatusOK(addGeneralServerOptionDefinitions(&startupOptions)); - uassertStatusOK(addMongodGeneralOptions(&startupOptions)); - uassertStatusOK(addMongodReplicationOptions(&startupOptions)); - uassertStatusOK(embedded::addOptions(&startupOptions)); - uassertStatusOK( - embedded_integration_helpers::parseCommandLineOptions(argc, argv, startupOptions)); - - serviceContext = embedded::initialize(""); - - // storeMongodOptions() triggers cmdline censoring, which must happen after initializers. - uassertStatusOK(storeMongodOptions(optionenvironment::startupOptionsParsed)); - - // Override the ServiceEntryPoint with one that can support transport layers. - serviceContext->setServiceEntryPoint( - std::make_unique<ServiceEntryPointMongoe>(serviceContext)); - - auto tl = - transport::TransportLayerManager::createWithConfig(&serverGlobalParams, serviceContext); - uassertStatusOK(tl->setup()); - - serviceContext->setTransportLayer(std::move(tl)); - - uassertStatusOK(serviceContext->getServiceExecutor()->start()); - uassertStatusOK(serviceContext->getTransportLayer()->start()); - } catch (const std::exception& ex) { - error() << ex.what(); - return EXIT_BADOPTIONS; - } - - return waitForShutdown(); -} - -} // namespace -} // namespace mongo - -#if defined(_WIN32) -// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters -// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The -// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent -// and makes them available through the argv() and envp() members. This enables mongoDbMain() -// to process UTF-8 encoded arguments and environment variables without regard to platform. -int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { - mongo::WindowsCommandLine wcl(argc, argvW, envpW); - return mongo::mongoedMain(argc, wcl.argv(), wcl.envp()); -} -#else -int main(int argc, char* argv[], char** envp) { - return mongo::mongoedMain(argc, argv, envp); -} -#endif diff --git a/src/mongo/platform/stack_locator_android.cpp b/src/mongo/platform/stack_locator_android.cpp deleted file mode 100644 index 0570a338ed2..00000000000 --- a/src/mongo/platform/stack_locator_android.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "stack_locator_pthread_getattr_np.cpp" diff --git a/src/mongo/platform/stack_locator_iOS-sim.cpp b/src/mongo/platform/stack_locator_iOS-sim.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_iOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/platform/stack_locator_iOS.cpp b/src/mongo/platform/stack_locator_iOS.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_iOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/platform/stack_locator_tvOS-sim.cpp b/src/mongo/platform/stack_locator_tvOS-sim.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_tvOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/platform/stack_locator_tvOS.cpp b/src/mongo/platform/stack_locator_tvOS.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_tvOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/platform/stack_locator_watchOS-sim.cpp b/src/mongo/platform/stack_locator_watchOS-sim.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_watchOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/platform/stack_locator_watchOS.cpp b/src/mongo/platform/stack_locator_watchOS.cpp deleted file mode 100644 index a99bbd567d4..00000000000 --- a/src/mongo/platform/stack_locator_watchOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/stack_locator_osx.cpp" diff --git a/src/mongo/shell/bench.cpp b/src/mongo/shell/bench.cpp index 9e7dfe3d3a7..765120a8f94 100644 --- a/src/mongo/shell/bench.cpp +++ b/src/mongo/shell/bench.cpp @@ -836,9 +836,9 @@ BenchRunWorker::BenchRunWorker(size_t id, BenchRunWorker::~BenchRunWorker() { try { - // We explicitly call join() on the started thread to ensure that any thread-local variables - // (e.g. 'currentClient' when running through mongoebench) have been destructed before - // returning from BenchRunWorker's destructor. + // We explicitly call join() on the started thread to ensure + // that any thread-local variables have been destructed + // before returning from BenchRunWorker's destructor. _thread.join(); } catch (...) { severe() << "caught exception in destructor: " << exceptionToStatus(); diff --git a/src/mongo/tools/SConscript b/src/mongo/tools/SConscript index 10acd257809..6e3e8719c85 100644 --- a/src/mongo/tools/SConscript +++ b/src/mongo/tools/SConscript @@ -32,34 +32,5 @@ mongobridge = env.Program( AIB_COMPONENT='tools', ) -mongoebench = yamlEnv.Program( - target='mongoebench', - source=[ - 'mongoebench_main.cpp', - 'mongoebench_options.cpp', - 'mongoebench_options_init.cpp', - env.Idlc('mongoebench_options.idl')[0], - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/db/dbdirectclient', - '$BUILD_DIR/mongo/db/storage/storage_options', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger' if get_option('wiredtiger') == 'on' else [], - '$BUILD_DIR/mongo/embedded/embedded', - '$BUILD_DIR/mongo/embedded/embedded_integration_helpers', - '$BUILD_DIR/mongo/shell/benchrun', - '$BUILD_DIR/mongo/util/signal_handlers', - ], - AIB_COMPONENT="embedded-test", - AIB_COMPONENTS_EXTRA=[ - "tests", - "benchmarks", - ], -) - -# TODO: remove this when hygienic is driving all tarball creation -install_mongoebench = env.Install("#/", mongoebench) -env.Alias("install-embedded-test", [install_mongoebench[0]]) install_mongobridge = env.Install("#/", mongobridge) env.Alias("install-tools", [install_mongobridge[0]]) - -env.Alias('all', mongoebench) # This ensures it compiles and links, but doesn't copy it anywhere. diff --git a/src/mongo/tools/mongoebench_main.cpp b/src/mongo/tools/mongoebench_main.cpp deleted file mode 100644 index 74f81264289..00000000000 --- a/src/mongo/tools/mongoebench_main.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault - -#include "mongo/platform/basic.h" - -#include <boost/filesystem.hpp> - -#include "mongo/base/init.h" -#include "mongo/base/shim.h" -#include "mongo/db/dbdirectclient.h" -#include "mongo/db/service_context.h" -#include "mongo/embedded/embedded.h" -#include "mongo/embedded/embedded_options.h" -#include "mongo/embedded/embedded_options_helpers.h" -#include "mongo/scripting/bson_template_evaluator.h" -#include "mongo/shell/bench.h" -#include "mongo/tools/mongoebench_options.h" -#include "mongo/tools/mongoebench_options_gen.h" -#include "mongo/util/exit.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/signal_handlers.h" -#include "mongo/util/text.h" - -namespace mongo { -namespace { - -/** - * DBDirectClientWithOwnOpCtx is a version of DBDirectClient that owns its own OperationContext. - * - * Since benchRun originally existed only in the mongo shell and was used with only - * DBClientConnections, there isn't a part of the BenchRunConfig or BenchRunWorker interfaces that - * is aware of having an OperationContext. In particular, the mongo shell lacks a ServiceContext and - * therefore also lacks the rest of the Client and OperationContext hierarchy. We shove an - * OperationContext onto the DBDirectClient to work around this limitation for mongoebench to work. - */ -class DBDirectClientWithOwnOpCtx : public DBDirectClient { -public: - DBDirectClientWithOwnOpCtx() - : DBDirectClient(nullptr), _threadClient(getGlobalServiceContext()) { - _opCtx = cc().makeOperationContext(); - setOpCtx(_opCtx.get()); - } - -private: - ThreadClient _threadClient; - ServiceContext::UniqueOperationContext _opCtx; -}; - -MONGO_INITIALIZER_WITH_PREREQUISITES(SignalProcessingStartup, ("ThreadNameInitializer")) -(InitializerContext*) { - // Make sure we call this as soon as possible but before any other threads are started. Before - // embedded::initialize is too early and after is too late. So instead we hook in during the - // global initialization at the right place. - startSignalProcessingThread(); - return Status::OK(); -} - -int mongoeBenchMain(int argc, char* argv[], char** envp) { - ServiceContext* serviceContext = nullptr; - - registerShutdownTask([&]() { - if (serviceContext) { - embedded::shutdown(serviceContext); - } - }); - - setupSignalHandlers(); - - log() << "MongoDB embedded benchRun application, for testing purposes only"; - - try { - optionenvironment::OptionSection startupOptions("Options"); - uassertStatusOK(embedded::addOptions(&startupOptions)); - uassertStatusOK(addMongoeBenchOptions(&startupOptions)); - uassertStatusOK( - embedded_integration_helpers::parseCommandLineOptions(argc, argv, startupOptions)); - serviceContext = embedded::initialize(nullptr); - } catch (const std::exception& ex) { - error() << ex.what(); - return EXIT_BADOPTIONS; - } - - // If a "pre" section was present in the benchRun config file, then we run its operations once - // before running the operations from the "ops" section. - if (mongoeBenchGlobalParams.preConfig) { - auto conn = mongoeBenchGlobalParams.preConfig->createConnection(); - boost::optional<LogicalSessionIdToClient> lsid; - - PseudoRandom rng(mongoeBenchGlobalParams.preConfig->randomSeed); - BsonTemplateEvaluator bsonTemplateEvaluator(mongoeBenchGlobalParams.preConfig->randomSeed); - BenchRunStats stats; - BenchRunOp::State state(&rng, &bsonTemplateEvaluator, &stats); - - for (auto&& op : mongoeBenchGlobalParams.preConfig->ops) { - op.executeOnce(conn.get(), lsid, *mongoeBenchGlobalParams.preConfig, &state); - } - } - - // If an "ops" section was present in the benchRun config file, then we repeatedly run its - // operations across the configured number of threads for the configured number of seconds. - if (mongoeBenchGlobalParams.opsConfig) { - const double seconds = mongoeBenchGlobalParams.opsConfig->seconds; - auto runner = std::make_unique<BenchRunner>(mongoeBenchGlobalParams.opsConfig.release()); - runner->start(); - - sleepmillis(static_cast<long long>(seconds * 1000)); - - BSONObj stats = BenchRunner::finish(runner.release()); - log() << "writing stats to " << mongoeBenchGlobalParams.outputFile.string() << ": " - << stats; - - boost::filesystem::ofstream outfile(mongoeBenchGlobalParams.outputFile); - outfile << stats.jsonString() << '\n'; - } - - shutdown(EXIT_CLEAN); -} - -std::unique_ptr<DBClientBase> benchRunConfigCreateConnectionImplProvider(const BenchRunConfig&) { - return std::unique_ptr<DBClientBase>(new DBDirectClientWithOwnOpCtx()); -} - -auto benchRunConfigCreateConnectionImplRegistration = MONGO_WEAK_FUNCTION_REGISTRATION( - BenchRunConfig::createConnectionImpl, benchRunConfigCreateConnectionImplProvider); -} // namespace - -} // namespace mongo - -#if defined(_WIN32) -// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters as -// main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The -// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent and -// makes them available through the argv() and envp() members. This enables mongoeBenchMain() to -// process UTF-8 encoded arguments and environment variables without regard to platform. -int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { - mongo::WindowsCommandLine wcl(argc, argvW, envpW); - return mongo::mongoeBenchMain(argc, wcl.argv(), wcl.envp()); -} -#else -int main(int argc, char* argv[], char** envp) { - return mongo::mongoeBenchMain(argc, argv, envp); -} -#endif diff --git a/src/mongo/tools/mongoebench_options.cpp b/src/mongo/tools/mongoebench_options.cpp deleted file mode 100644 index f2982ee028a..00000000000 --- a/src/mongo/tools/mongoebench_options.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/tools/mongoebench_options.h" - -#include <algorithm> -#include <fstream> -#include <iostream> -#include <iterator> - -#include "mongo/base/status.h" -#include "mongo/db/storage/storage_options.h" -#include "mongo/platform/random.h" -#include "mongo/shell/bench.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/str.h" - -namespace mongo { - -MongoeBenchGlobalParams mongoeBenchGlobalParams; - -void printMongoeBenchHelp(std::ostream* out) { - *out << "Usage: mongoebench <config file> [options]" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; -} - -bool handlePreValidationMongoeBenchOptions(const moe::Environment& params) { - if (params.count("help")) { - printMongoeBenchHelp(&std::cout); - return false; - } - return true; -} - -namespace { - -BSONObj getBsonFromJsonFile(const std::string& filename) { - std::ifstream infile(filename.c_str()); - std::string data((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>()); - return fromjson(data); -} - -boost::filesystem::path kDefaultOutputFile("perf.json"); - -} // namespace - -Status storeMongoeBenchOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - if (!params.count("benchRunConfigFile")) { - return {ErrorCodes::BadValue, "No benchRun config file was specified"}; - } - - BSONObj config = getBsonFromJsonFile(params["benchRunConfigFile"].as<std::string>()); - for (auto&& elem : config) { - const auto fieldName = elem.fieldNameStringData(); - if (fieldName == "pre") { - mongoeBenchGlobalParams.preConfig.reset( - BenchRunConfig::createFromBson(elem.wrap("ops"))); - } else if (fieldName == "ops") { - mongoeBenchGlobalParams.opsConfig.reset(BenchRunConfig::createFromBson(elem.wrap())); - } else { - return {ErrorCodes::BadValue, - str::stream() << "Unrecognized key in benchRun config file: " << fieldName}; - } - } - - int64_t seed = params.count("seed") ? static_cast<int64_t>(params["seed"].as<long>()) - : SecureRandom().nextInt64(); - - if (mongoeBenchGlobalParams.preConfig) { - mongoeBenchGlobalParams.preConfig->randomSeed = seed; - } - - if (mongoeBenchGlobalParams.opsConfig) { - mongoeBenchGlobalParams.opsConfig->randomSeed = seed; - - if (params.count("threads")) { - mongoeBenchGlobalParams.opsConfig->parallel = params["threads"].as<unsigned>(); - } - - if (params.count("time")) { - mongoeBenchGlobalParams.opsConfig->seconds = params["time"].as<double>(); - } - } - - if (params.count("output")) { - mongoeBenchGlobalParams.outputFile = - boost::filesystem::path(params["output"].as<std::string>()); - } else { - boost::filesystem::path dbpath(storageGlobalParams.dbpath); - mongoeBenchGlobalParams.outputFile = dbpath / kDefaultOutputFile; - } - - mongoeBenchGlobalParams.outputFile = mongoeBenchGlobalParams.outputFile.lexically_normal(); - auto parentPath = mongoeBenchGlobalParams.outputFile.parent_path(); - if (!parentPath.empty() && !boost::filesystem::exists(parentPath)) { - return {ErrorCodes::NonExistentPath, - str::stream() << "Directory containing output file must already exist, but " - << parentPath.string() << " wasn't found"}; - } - - return Status::OK(); -} - -} // namespace mongo diff --git a/src/mongo/tools/mongoebench_options.h b/src/mongo/tools/mongoebench_options.h deleted file mode 100644 index fabd676d8b2..00000000000 --- a/src/mongo/tools/mongoebench_options.h +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <boost/filesystem.hpp> -#include <cstdint> -#include <iosfwd> -#include <memory> -#include <string> -#include <vector> - -#include "mongo/base/status.h" - -namespace mongo { - -class BenchRunConfig; - -namespace optionenvironment { - -class OptionSection; -class Environment; - -} // namespace optionenvironment - -namespace moe = mongo::optionenvironment; - -struct MongoeBenchGlobalParams { - std::unique_ptr<BenchRunConfig> preConfig; - std::unique_ptr<BenchRunConfig> opsConfig; - boost::filesystem::path outputFile; -}; - -extern MongoeBenchGlobalParams mongoeBenchGlobalParams; - -Status addMongoeBenchOptions(moe::OptionSection* options); - -void printMongoeBenchHelp(std::ostream* out); - -/** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ -bool handlePreValidationMongoeBenchOptions(const moe::Environment& params); - -Status storeMongoeBenchOptions(const moe::Environment& params, - const std::vector<std::string>& args); - -} // namespace mongo diff --git a/src/mongo/tools/mongoebench_options.idl b/src/mongo/tools/mongoebench_options.idl deleted file mode 100644 index b86795f8790..00000000000 --- a/src/mongo/tools/mongoebench_options.idl +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2019-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# <http://www.mongodb.com/licensing/server-side-public-license>. -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. -# - -global: - cpp_namespace: "mongo" - configs: - source: [ cli, ini ] - initializer: - register: addMongoeBenchOptions - -configs: - help: - description: 'Show this usage information' - arg_vartype: Switch - - benchRunConfigFile: - description: 'Config file for benchRun' - arg_vartype: String - hidden: true - positional: 1 - - seed: - description: 'Random seed to use' - arg_vartype: Long - - threads: - description: 'Number of benchRun worker threads' - single_name: t - arg_vartype: Unsigned - default: { expr: '1U' } - - time: - description: 'Seconds to run benchRun for' - single_name: s - arg_vartype: Double - default: 1.0 - - output: - description: 'Output file for benchRun stats (defaults to <dbPath>/perf.json)' - single_name: o - arg_vartype: String diff --git a/src/mongo/tools/mongoebench_options_init.cpp b/src/mongo/tools/mongoebench_options_init.cpp deleted file mode 100644 index 4ed8f4d75c5..00000000000 --- a/src/mongo/tools/mongoebench_options_init.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/tools/mongoebench_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { -namespace { - -MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoeBenchOptions)(InitializerContext* context) { - return addMongoeBenchOptions(&moe::startupOptions); -} - -GlobalInitializerRegisterer mongoeBenchOptionsStore( - "MongoeBenchOptions_Store", - [](InitializerContext* context) { - if (!handlePreValidationMongoeBenchOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - return storeMongoeBenchOptions(moe::startupOptionsParsed, context->args()); - }, - DeinitializerFunction(nullptr), - {"BeginStartupOptionStorage", "EmbeddedOptions_Store"}, - {"EndStartupOptionStorage"}); - -} // namespace -} // namespace mongo diff --git a/src/mongo/util/dns_query_android-impl.h b/src/mongo/util/dns_query_android-impl.h deleted file mode 100644 index 3b2860424e6..00000000000 --- a/src/mongo/util/dns_query_android-impl.h +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#ifndef MONGO_UTIL_DNS_QUERY_PLATFORM_INCLUDE_WHITELIST -#error Do not include the DNS Query platform implementation headers. Please use "mongo/util/dns_query.h" instead. -#endif - -#include <stdexcept> -#include <string> -#include <vector> - -#include "mongo/util/assert_util.h" - -namespace mongo { -namespace dns { -namespace { - -enum class DNSQueryClass { kInternet }; - -enum class DNSQueryType { kSRV, kTXT, kAddress }; - -[[noreturn]] void throwNotSupported() { - uasserted(ErrorCodes::InternalErrorNotSupported, "srv_nsearch not supported on android"); -} - -class ResourceRecord { -public: - explicit ResourceRecord() = default; - - std::vector<std::string> txtEntry() const { - throwNotSupported(); - } - - std::string addressEntry() const { - throwNotSupported(); - } - - SRVHostEntry srvHostEntry() const { - throwNotSupported(); - } -}; - -using DNSResponse = std::vector<ResourceRecord>; - -class DNSQueryState { -public: - DNSResponse lookup(const std::string&, const DNSQueryClass, const DNSQueryType) { - throwNotSupported(); - } - - DNSQueryState() { - throwNotSupported(); - } -}; -} // namespace -} // namespace dns -} // namespace mongo diff --git a/src/mongo/util/processinfo_android.cpp b/src/mongo/util/processinfo_android.cpp deleted file mode 100644 index 0adf5ae2a2d..00000000000 --- a/src/mongo/util/processinfo_android.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_linux.cpp" diff --git a/src/mongo/util/processinfo_iOS-sim.cpp b/src/mongo/util/processinfo_iOS-sim.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_iOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/mongo/util/processinfo_iOS.cpp b/src/mongo/util/processinfo_iOS.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_iOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/mongo/util/processinfo_tvOS-sim.cpp b/src/mongo/util/processinfo_tvOS-sim.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_tvOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/mongo/util/processinfo_tvOS.cpp b/src/mongo/util/processinfo_tvOS.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_tvOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/mongo/util/processinfo_watchOS-sim.cpp b/src/mongo/util/processinfo_watchOS-sim.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_watchOS-sim.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/mongo/util/processinfo_watchOS.cpp b/src/mongo/util/processinfo_watchOS.cpp deleted file mode 100644 index 9c6f9a6d1f3..00000000000 --- a/src/mongo/util/processinfo_watchOS.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/util/processinfo_osx.cpp" diff --git a/src/third_party/scripts/mongo-perf_get_sources.sh b/src/third_party/scripts/mongo-perf_get_sources.sh deleted file mode 100755 index b1f5cc60115..00000000000 --- a/src/third_party/scripts/mongo-perf_get_sources.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# This script downloads the mongodb/mongo-perf repository and generates mongoebench-compatible JSON -# config files equivalent to the JavaScript test cases. -# -# Turn on strict error checking, like perl use 'strict'. -set -xeuo pipefail -IFS=$'\n\t' - -if [ "$#" -ne 0 ]; then - echo "This script does not take any arguments" - exit 1 -fi - -# We add the current working directory to the PATH because that is where the mongo shell may have -# been installed. The benchrun.py script is going to run the mongo shell binary that's on the PATH. -PATH=$PATH:$(pwd) - -# The mongo shell processes spawned by the benchrun.py script will attempt to connect to a mongod, -# so we just error out if we find that one isn't already running. -mongo --eval 'db.adminCommand({ping: 1})' && rc=$? || rc=$? -if [ "$rc" -ne 0 ]; then - echo "This script requires a mongod to be running on port 27017" - exit 2 -fi - -NAME=mongo-perf -SRC_ROOT=$(mktemp -d /tmp/$NAME.XXXXXX) -trap "rm -rf $SRC_ROOT" EXIT -DEST_DIR=$(git rev-parse --show-toplevel)/benchrun_embedded/testcases - -git clone --branch=master https://github.com/mongodb/mongo-perf.git $SRC_ROOT - -pushd $SRC_ROOT - -# We pin to a particular commit of the mongodb/mongo-perf repository to make it clear what version -# of the JavaScript test cases we are running. -git checkout 7070ac74dd35fde2f59af01b155191382357ed1d - -# We use Python to get the number of CPUs in a platform-agnostic way. -NUM_CPUS=$(python -c 'import multiprocessing; print(multiprocessing.cpu_count())') - -# Generating the JSON config files sequentially takes ~6 minutes due to how certain JavaScript test -# cases build up very large documents only to eventually realize upon trying to serialize them out -# to a file that they are too big. We use `xargs -P` to speed up generating the JSON config files. -# -# We don't generate JSON config files for tests that are tagged with "capped" or "where" because -# they aren't supported by embedded. -# -# We generate JSON config files for tests that are tagged with "aggregation_identityview" or -# "query_identityview" while using --readCmd=true because the find command is necessary to read from -# a view. We use --readCmd=false for all other tests to match what etc/perf.yml does. -find testcases -type f -print0 | xargs -0 -I% -n1 -P$NUM_CPUS \ - python2 benchrun.py --testfiles % \ - --threads 1 \ - --excludeFilter capped \ - --excludeFilter where \ - --generateMongoeBenchConfigFiles mongoebench/ \ - --readCmd false \ - --writeCmd true \ - --excludeFilter aggregation_identityview \ - --excludeFilter query_identityview - -find testcases -type f -print0 | xargs -0 -I% -n1 -P$NUM_CPUS \ - python2 benchrun.py --testfiles % \ - --threads 1 \ - --excludeFilter capped \ - --excludeFilter where \ - --generateMongoeBenchConfigFiles mongoebench/ \ - --readCmd true \ - --writeCmd true \ - --includeFilter aggregation_identityview query_identityview - -popd - -test -d $DEST_DIR && rm -r $DEST_DIR -mkdir -p $(dirname $DEST_DIR) - -mv $SRC_ROOT/mongoebench $DEST_DIR |