diff options
author | Richard Samuels <richard.samuels@mongodb.com> | 2020-06-18 14:13:39 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-07-13 21:19:45 +0000 |
commit | b62fbbebc64f1ab105db49a1cc410af6d878add1 (patch) | |
tree | 72942c749626b0afb48e9fb941dc4db361e4b63f | |
parent | b93e662c43eac608d4bcdaf19957082614377a60 (diff) | |
download | mongo-b62fbbebc64f1ab105db49a1cc410af6d878add1.tar.gz |
SERVER-47591 Add build variant that runs some tests with live record
-rw-r--r-- | buildscripts/resmokelib/core/process.py | 7 | ||||
-rw-r--r-- | buildscripts/resmokelib/undodb/__init__.py | 49 | ||||
-rw-r--r-- | etc/evergreen.yml | 75 |
3 files changed, 83 insertions, 48 deletions
diff --git a/buildscripts/resmokelib/core/process.py b/buildscripts/resmokelib/core/process.py index 0cadf16d703..2c458e4320f 100644 --- a/buildscripts/resmokelib/core/process.py +++ b/buildscripts/resmokelib/core/process.py @@ -11,6 +11,7 @@ import os.path import subprocess import sys import threading +from datetime import datetime from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import errors @@ -122,9 +123,13 @@ class Process(object): if _config.UNDO_RECORDER_PATH is not None and ("mongod" in self.args[0] or "mongos" in self.args[0]): + now_str = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + recorder_output_file = "{logger}-{process}-{pid}-{t}.undo".format( + logger=self.logger.name.replace('/', '-'), + process=os.path.basename(self.args[0]), pid=self.pid, t=now_str) recorder_args = [ _config.UNDO_RECORDER_PATH, "--thread-fuzzing", "-p", - str(self.pid) + str(self.pid), "-o", recorder_output_file ] self._recorder = subprocess.Popen(recorder_args, bufsize=buffer_size, env=self.env, creationflags=creation_flags) diff --git a/buildscripts/resmokelib/undodb/__init__.py b/buildscripts/resmokelib/undodb/__init__.py index 8ab078779ea..881871c8584 100644 --- a/buildscripts/resmokelib/undodb/__init__.py +++ b/buildscripts/resmokelib/undodb/__init__.py @@ -6,54 +6,11 @@ _HELP = """ Info on how to install undodb. """ -_MESSAGE = """ -Instructions for setting up and using UndoDB's reversible debugger +_MESSAGE = """Instructions for setting up and using UndoDB's reversible debugger -UndoDB is only supported on linux platforms. It will not work on macOS or Windows. - -1. Download the tarball from Google Drive. - - https://drive.google.com/open?id=18dx1hHRrPgc27TtvvCe9rLXSYx_rKjzq - - You must be logged into your MongoDB/10gen google account to access this link. - This file has MongoDB's private undodb key-server parameters baked into it, - so do not share this file outside of the company. - - If you're using an Evergreen virtual workstation, use the following command - to copy the tarball over. - - evergreen host rsync --host <HOST_ID> -l /path/to/local/undodb-*.tgz -r /home/ubuntu/undo.tgz - -2. Untar and install: - - tar xzf undodb-*.tgz - cd undodb-* - sudo make install - -There is good README help in the undodb directory if you have questions. - -3. To use UndoDB, you first need to make a recording by running a test suite - in resmoke.py with the recorder - - ./buildscripts/resmoke.py run --recordWith live-record [your other resmoke args] - - This will generate one recording per invocation of each mongod and mongos process. - The recordings are stored in the current working directory. - - Once you have the recording, you're ready to use `udb` to start debugging: - - udb --undodb-gdb-exe /opt/mongodbtoolchain/gdb/bin/gdb path/to/recording.undo - - There is a quick reference guide of UndoDB commands at: - https://undo.io/media/uploads/files/A5_UndoDB_quick_reference_guide_June_2019.pdf - - Please also refer to Will Schultz's intro talk for a getting started primer: - - https://mongodb.zoom.com/rec/share/5eBrDqHJ7k5If6uX9Fn7Wo0sGKT6T6a8gydK-_QOxBkaEyZzwv6Yf4tjTB4cS0f1 - - If you have any questions, suggestions or run into hiccups, please reach out - to #server-tig. +Setup and usage instructions have can be found at: + https://wiki.corp.mongodb.com/display/KERNEL/UndoDB+Usage """ _COMMAND = "undodb" diff --git a/etc/evergreen.yml b/etc/evergreen.yml index bf050587d26..cc8d08ac676 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1467,7 +1467,8 @@ functions: ${san_symbolizer} \ ${snmp_config_path} \ ${resmoke_wrapper} \ - $python buildscripts/resmoke.py run \ + $python buildscripts/resmoke.py run \ + ${record_with} \ ${resmoke_args} \ $extra_args \ ${test_flags} \ @@ -1489,6 +1490,15 @@ functions: resmoke_exit_code=$? set -o errexit + if [[ -n "${record_with}" ]]; then + recording_size=$(du -ch *.undo | grep total) + echo "UndoDB produced recordings that were $recording_size (uncompressed) on disk" + if [[ $resmoke_exit_code = 0 ]]; then + echo "Resmoke exited successfully. UndoDB recordings will not be saved." + rm *.undo + fi + fi + # 74 is exit code for IOError on POSIX systems, which is raised when the machine is # shutting down. # @@ -3266,6 +3276,31 @@ functions: - *tar_system_resource_information - *archive_system_resource_information + "tar UndoDB recordings": &tar_undodb_recordings + command: archive.targz_pack + params: + target: undodb-recordings.tgz + source_dir: src + include: + - "./*.undo" + + "archive UndoDB recordings": &archive_undodb_recordings + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: undodb-recordings.tgz + remote_file: ${project}/${build_variant}/${revision}/undo/undodb-recordings-${task_id}-execution-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/gzip + display_name: UndoDB Recordings - Execution ${execution} + optional: true + + "save UndoDB recordings": + - *tar_undodb_recordings + - *archive_undodb_recordings + ### Attach report & artifacts ### "attach scons logs": - command: s3.put @@ -3348,6 +3383,7 @@ post: - func: "save hang analyzer debugger files" - func: "save disk statistics" - func: "save system resource information" + - func: "save UndoDB recordings" - func: "print OOM messages" - func: "umount shared scons directory" - func: "cleanup FUSE watchdog" @@ -12332,3 +12368,40 @@ buildvariants: distros: - rhel62-large - name: validate_commit_message + +- name: live-record + display_name: "~ RHEL 6.2 Shared Library (with UndoDB live-record)" + batchtime: 1440 # 1 day + stepback: true + modules: + - enterprise + run_on: + - rhel62-small + expansions: + compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic + multiversion_platform: rhel62 + multiversion_edition: enterprise + has_packages: false + scons_cache_scope: shared + scons_cache_mode: all + target_resmoke_time: 10 + max_sub_suites: 100 + large_distro_name: rhel62-large + num_scons_link_jobs_available: 0.99 + record_with: --recordWith /opt/undodb5/bin/live-record + tasks: + - name: compile_without_package_TG + distros: + - rhel62-large + - name: .jscore .common + - name: jsCore_minimum_batch_size + - name: jsCore_op_query + - name: replica_sets_multi_stmt_txn_jscore_passthrough + - name: replica_sets_large_txns_format_jscore_passthrough + - name: sharded_collections_causally_consistent_jscore_txns_passthrough + - name: sharded_jscore_op_query_txns + - name: sharded_jscore_txns + - name: sharded_jscore_txns_sharded_collections + - name: sharding_jscore_passthrough + - name: sharding_jscore_op_query_passthrough + - name: sharded_collections_causally_consistent_jscore_txns_passthrough |