diff options
-rw-r--r-- | .pylintrc | 2 | ||||
-rwxr-xr-x | buildscripts/mongosymb.py | 72 | ||||
-rw-r--r-- | buildscripts/resmokelib/parser.py | 2 | ||||
-rw-r--r-- | buildscripts/resmokelib/setup_multiversion/download.py | 9 | ||||
-rw-r--r-- | buildscripts/resmokelib/setup_multiversion/evergreen_conn.py | 150 | ||||
-rw-r--r-- | buildscripts/resmokelib/setup_multiversion/setup_multiversion.py | 130 | ||||
-rw-r--r-- | buildscripts/resmokelib/symbolizer/__init__.py | 235 | ||||
-rw-r--r-- | buildscripts/resmokelib/utils/evergreen_conn.py | 233 | ||||
-rw-r--r-- | buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py | 50 | ||||
-rw-r--r-- | buildscripts/tests/resmokelib/utils/test_evergreen_conn.py (renamed from buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py) | 4 |
10 files changed, 639 insertions, 248 deletions
diff --git a/.pylintrc b/.pylintrc index 3935170435f..75ae207605d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -31,7 +31,7 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$ # R0801 - duplicate-code - See PM-1380 # E0611 - no-name-in-module -disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,no-else-return,redefined-variable-type,too-few-public-methods,unused-import,useless-object-inheritance,deprecated-module,unnecessary-pass,duplicate-code,no-else-raise,deprecated-method,exec-used,no-name-in-module,raise-missing-from, unnecessary-comprehension,super-with-arguments,consider-using-sys-exit,import-outside-toplevel,no-else-continue,no-else-break +disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,no-else-return,redefined-variable-type,too-few-public-methods,unused-import,useless-object-inheritance,deprecated-module,unnecessary-pass,duplicate-code,no-else-raise,deprecated-method,exec-used,no-name-in-module,raise-missing-from, unnecessary-comprehension,super-with-arguments,consider-using-sys-exit,import-outside-toplevel,no-else-continue,no-else-break,too-many-arguments,too-many-locals [IMPORTS] known-third-party=boto3,botocore,psutil,yaml,xmlrunner diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 44cfd56008d..a8544cdf928 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -61,21 +61,17 @@ def parse_input(trace_doc, dbg_path_resolver): def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format, - **_kwargs): + **kwargs): """Return a list of symbolized stack frames from a trace_doc in MongoDB stack dump format.""" + # Keep frames in kwargs to avoid changing the function signature. + frames = kwargs.get("frames") + if frames is None: + frames = preprocess_frames(dbg_path_resolver, trace_doc, input_format) + if not symbolizer_path: symbolizer_path = os.environ.get("MONGOSYMB_SYMBOLIZER_PATH", "llvm-symbolizer") - if input_format == "classic": - frames = parse_input(trace_doc, dbg_path_resolver) - elif input_format == "thin": - frames = trace_doc["backtrace"] - for frame in frames: - frame["path"] = dbg_path_resolver.get_dbg_file(frame) - else: - raise ValueError('Unknown input format "{}"'.format(input_format)) - symbolizer_args = [symbolizer_path] for dh in dsym_hint: symbolizer_args.append("-dsym-hint={}".format(dh)) @@ -123,16 +119,33 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i return frames +def preprocess_frames(dbg_path_resolver, trace_doc, input_format): + """Process the paths in frame objects.""" + if input_format == "classic": + frames = parse_input(trace_doc, dbg_path_resolver) + elif input_format == "thin": + frames = trace_doc["backtrace"] + for frame in frames: + frame["path"] = dbg_path_resolver.get_dbg_file(frame) + else: + raise ValueError('Unknown input format "{}"'.format(input_format)) + return frames + + class PathDbgFileResolver(object): """PathDbgFileResolver class.""" def __init__(self, bin_path_guess): """Initialize PathDbgFileResolver.""" self._bin_path_guess = os.path.realpath(bin_path_guess) + self.mci_build_dir = None def get_dbg_file(self, soinfo): """Return dbg file name.""" path = soinfo.get("path", "") + # TODO: make identifying mongo shared library directory more robust + if self.mci_build_dir is None and path.startswith("/data/mci/"): + self.mci_build_dir = path.split("/src/", maxsplit=1)[0] return path if path else self._bin_path_guess @@ -143,6 +156,7 @@ class S3BuildidDbgFileResolver(object): """Initialize S3BuildidDbgFileResolver.""" self._cache_dir = cache_dir self._s3_bucket = s3_bucket + self.mci_build_dir = None def get_dbg_file(self, soinfo): """Return dbg file name.""" @@ -182,32 +196,41 @@ def classic_output(frames, outfile, **kwargs): # pylint: disable=unused-argumen outfile.write(" {path:s}!!!\n".format(**symbinfo)) -def make_argument_parser(**kwargs): +def make_argument_parser(parser=None, **kwargs): """Make and return an argparse.""" - parser = argparse.ArgumentParser(**kwargs) + if parser is None: + parser = argparse.ArgumentParser(**kwargs) + parser.add_argument('--dsym-hint', default=[], action='append') parser.add_argument('--symbolizer-path', default='') parser.add_argument('--input-format', choices=['classic', 'thin'], default='classic') parser.add_argument('--output-format', choices=['classic', 'json'], default='classic', help='"json" shows some extra information') parser.add_argument('--debug-file-resolver', choices=['path', 's3'], default='path') + parser.add_argument('--src-dir-to-move', action="store", type=str, default=None, + help="Specify a src dir to move to /data/mci/{original_buildid}/src") + s3_group = parser.add_argument_group( "s3 options", description='Options used with \'--debug-file-resolver s3\'') s3_group.add_argument('--s3-cache-dir') s3_group.add_argument('--s3-bucket') - parser.add_argument('path_to_executable') + + # Look for symbols in the cwd by default. + parser.add_argument('path_to_executable', nargs="?") return parser -def main(): +def main(options): """Execute Main program.""" - options = make_argument_parser(description=__doc__).parse_args() - # Skip over everything before the first '{' since it is likely to be log line prefixes. # Additionally, using raw_decode() to ignore extra data after the closing '}' to allow maximal # sloppiness in copy-pasting input. trace_doc = sys.stdin.read() + + if not trace_doc or not trace_doc.strip(): + print("Please provide the backtrace through stdin for symbolization;" + "e.g. `your/symbolization/command < /file/with/stacktrace`") trace_doc = trace_doc[trace_doc.find('{'):] trace_doc = json.JSONDecoder().raw_decode(trace_doc)[0] @@ -242,10 +265,23 @@ def main(): elif options.debug_file_resolver == 's3': resolver = S3BuildidDbgFileResolver(options.s3_cache_dir, options.s3_bucket) - frames = symbolize_frames(trace_doc, resolver, **vars(options)) + frames = preprocess_frames(resolver, trace_doc, options.input_format) + + if options.src_dir_to_move and resolver.mci_build_dir is not None: + try: + os.makedirs(resolver.mci_build_dir) + os.symlink( + os.path.join(os.getcwd(), options.src_dir_to_move), + os.path.join(resolver.mci_build_dir, 'src')) + except FileExistsError: + pass + + frames = symbolize_frames(frames=frames, trace_doc=trace_doc, dbg_path_resolver=resolver, + **vars(options)) output_fn(frames, sys.stdout, indent=2) if __name__ == '__main__': - main() + symbolizer_options = make_argument_parser(description=__doc__).parse_args() + main(symbolizer_options) sys.exit(0) diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index ddc4e056fca..cbbd4fb971b 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -8,6 +8,7 @@ from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin from buildscripts.resmokelib.setup_multiversion import SetupMultiversionPlugin +from buildscripts.resmokelib.symbolizer import SymbolizerPlugin from buildscripts.resmokelib.undodb import UndoDbPlugin _PLUGINS = [ @@ -16,6 +17,7 @@ _PLUGINS = [ UndoDbPlugin(), SetupMultiversionPlugin(), PowercyclePlugin(), + SymbolizerPlugin(), ] diff --git a/buildscripts/resmokelib/setup_multiversion/download.py b/buildscripts/resmokelib/setup_multiversion/download.py index d2243329423..e2a16effecd 100644 --- a/buildscripts/resmokelib/setup_multiversion/download.py +++ b/buildscripts/resmokelib/setup_multiversion/download.py @@ -26,7 +26,7 @@ def download_from_s3(url): """Download file from S3 bucket by a given URL.""" if not url: - raise DownloadError("Download URL not found.") + raise DownloadError("Download URL not found") LOGGER.info("Downloading.", url=url) filename = os.path.join(tempfile.gettempdir(), url.split('/')[-1].split('?')[0]) @@ -91,11 +91,8 @@ def extract_archive(archive_file, install_dir): try: os.makedirs(install_dir) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(install_dir): - pass - else: - raise + except FileExistsError: + pass _rsync_move_dir(temp_dir, install_dir) shutil.rmtree(temp_dir) diff --git a/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py b/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py deleted file mode 100644 index 5f4eec6c4c2..00000000000 --- a/buildscripts/resmokelib/setup_multiversion/evergreen_conn.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Helper functions to interact with evergreen.""" -import os - -import structlog -from requests import HTTPError - -from evergreen import RetryingEvergreenApi - -EVERGREEN_HOST = "https://evergreen.mongodb.com" -EVERGREEN_CONFIG_LOCATIONS = ( - # Common for machines in Evergreen - os.path.join(os.getcwd(), ".evergreen.yml"), - # Common for local machines - os.path.expanduser(os.path.join("~", ".evergreen.yml")), -) - -GENERIC_EDITION = "base" -GENERIC_PLATFORM = "linux_x86_64" -GENERIC_ARCHITECTURE = "x86_64" - -LOGGER = structlog.getLogger(__name__) - - -class EvergreenConnError(Exception): - """Errors in evergreen_conn.py.""" - - pass - - -def get_evergreen_api(evergreen_config): - """Return evergreen API.""" - config_to_pass = evergreen_config - if not config_to_pass: - # Pickup the first config file found in common locations. - for file in EVERGREEN_CONFIG_LOCATIONS: - if os.path.isfile(file): - config_to_pass = file - break - try: - evg_api = RetryingEvergreenApi.get_api(config_file=config_to_pass) - except Exception as ex: - LOGGER.error("Most likely something is wrong with evergreen config file.", - config_file=config_to_pass) - raise ex - else: - return evg_api - - -def get_buildvariant_name(config, edition, platform, architecture, major_minor_version): - """Return Evergreen buildvariant name.""" - - buildvariant_name = "" - evergreen_buildvariants = config.evergreen_buildvariants - - for buildvariant in evergreen_buildvariants: - if (buildvariant.edition == edition and buildvariant.platform == platform - and buildvariant.architecture == architecture): - versions = buildvariant.versions - if major_minor_version in versions: - buildvariant_name = buildvariant.name - break - elif not versions: - buildvariant_name = buildvariant.name - - return buildvariant_name - - -def get_generic_buildvariant_name(config, major_minor_version): - """Return Evergreen buildvariant name for generic platform.""" - - LOGGER.info("Falling back to generic architecture.", edition=GENERIC_EDITION, - platform=GENERIC_PLATFORM, architecture=GENERIC_ARCHITECTURE) - - generic_buildvariant_name = get_buildvariant_name( - config=config, edition=GENERIC_EDITION, platform=GENERIC_PLATFORM, - architecture=GENERIC_ARCHITECTURE, major_minor_version=major_minor_version) - - if not generic_buildvariant_name: - raise EvergreenConnError("Generic architecture buildvariant not found.") - - return generic_buildvariant_name - - -def get_evergreen_project_and_version(config, evg_api, commit_hash): - """Return evergreen project and version by commit hash.""" - - for evg_project in config.evergreen_projects: - try: - version_id = evg_project.replace("-", "_") + "_" + commit_hash - evg_version = evg_api.version_by_id(version_id) - except HTTPError: - continue - else: - LOGGER.debug("Found evergreen version.", - evergreen_version=f"{EVERGREEN_HOST}/version/{evg_version.version_id}") - return evg_project, evg_version - - raise EvergreenConnError(f"Evergreen version for commit hash {commit_hash} not found.") - - -def get_evergreen_versions(evg_api, evg_project): - """Return the list of evergreen versions by evergreen project name.""" - return evg_api.versions_by_project(evg_project) - - -def get_compile_artifact_urls(evg_api, evg_version, buildvariant_name): - """Return compile urls from buildvariant in Evergreen version.""" - compile_artifact_urls = {} - - try: - build_id = evg_version.build_variants_map[buildvariant_name] - except KeyError: - raise EvergreenConnError(f"Buildvariant {buildvariant_name} not found.") - else: - evg_build = evg_api.build_by_id(build_id) - LOGGER.debug("Found evergreen build.", evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") - evg_tasks = evg_build.get_tasks() - compile_task = None - push_task = None - - for evg_task in evg_tasks: - # Only set the compile task if there isn't one already, otherwise - # newer tasks like "archive_dist_test_debug" take precedence. - if evg_task.display_name in ("compile", "archive_dist_test") and compile_task is None: - compile_task = evg_task - elif evg_task.display_name == "push": - push_task = evg_task - if compile_task and push_task: - break - - if compile_task and push_task and compile_task.status == push_task.status == "success": - LOGGER.info("Found successful evergreen tasks.", - compile_task=f"{EVERGREEN_HOST}/task/{compile_task.task_id}", - push_task=f"{EVERGREEN_HOST}/task/{push_task.task_id}") - evg_artifacts = compile_task.artifacts - for artifact in evg_artifacts: - compile_artifact_urls[artifact.name] = artifact.url - - # Tack on the project id for generating a friendly decompressed name for the artifacts. - compile_artifact_urls["project_id"] = compile_task.project_id - - elif compile_task and push_task: - LOGGER.warning("Found evergreen tasks, but they are not both successful.", - compile_task=f"{EVERGREEN_HOST}/task/{compile_task.task_id}", - push_task=f"{EVERGREEN_HOST}/task/{push_task.task_id}") - else: - LOGGER.error("There are no `compile` and/or 'push' tasks in the evergreen build.", - evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") - - return compile_artifact_urls diff --git a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py index 94d84656dfb..2e24cbd40eb 100644 --- a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py +++ b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py @@ -15,7 +15,8 @@ import structlog import yaml from buildscripts.resmokelib.plugin import PluginInterface, Subcommand -from buildscripts.resmokelib.setup_multiversion import config, download, evergreen_conn, github_conn +from buildscripts.resmokelib.setup_multiversion import config, download, github_conn +from buildscripts.resmokelib.utils import evergreen_conn SUBCOMMAND = "setup-multiversion" @@ -43,27 +44,30 @@ class SetupMultiversion(Subcommand): """Main class for the setup multiversion subcommand.""" # pylint: disable=too-many-instance-attributes - def __init__(self, options): + def __init__(self, download_options, install_dir="", link_dir="", platform=None, edition=None, + architecture=None, use_latest=None, versions=None, evergreen_config=None, + github_oauth_token=None, debug=None, ignore_failed_push=False): """Initialize.""" - setup_logging(options.debug) + setup_logging(debug) cwd = os.getcwd() - self.install_dir = os.path.join(cwd, options.install_dir) - self.link_dir = os.path.join(cwd, options.link_dir) + self.install_dir = os.path.join(cwd, install_dir) + self.link_dir = os.path.join(cwd, link_dir) - self.edition = options.edition.lower() if options.edition else None - self.platform = options.platform.lower() if options.platform else None - self.architecture = options.architecture.lower() if options.architecture else None - self.use_latest = options.use_latest - self.versions = options.versions + self.edition = edition.lower() if edition else None + self.platform = platform.lower() if platform else None + self.architecture = architecture.lower() if architecture else None + self.use_latest = use_latest + self.versions = versions + self.ignore_failed_push = ignore_failed_push - self.download_binaries = options.download_binaries - self.download_symbols = options.download_symbols - self.download_artifacts = options.download_artifacts + self.download_binaries = download_options.download_binaries + self.download_symbols = download_options.download_symbols + self.download_artifacts = download_options.download_artifacts - self.evg_api = evergreen_conn.get_evergreen_api(options.evergreen_config) + self.evg_api = evergreen_conn.get_evergreen_api(evergreen_config) # In evergreen github oauth token is stored as `token ******`, so we remove the leading part - self.github_oauth_token = options.github_oauth_token.replace( - "token ", "") if options.github_oauth_token else None + self.github_oauth_token = github_oauth_token.replace("token ", + "") if github_oauth_token else None with open(config.SETUP_MULTIVERSION_CONFIG) as file_handle: raw_yaml = yaml.safe_load(file_handle) self.config = config.SetupMultiversionConfig(raw_yaml) @@ -99,21 +103,11 @@ class SetupMultiversion(Subcommand): "version.") urls = self.get_urls(version) - artifacts_url = urls.get("Artifacts", "") if self.download_artifacts else None - binaries_url = urls.get("Binaries", "") if self.download_binaries else None - download_symbols_url = None - - if self.download_symbols: - download_symbols_url = urls.get(" mongo-debugsymbols.tgz", "") - if not download_symbols_url: - download_symbols_url = urls.get(" mongo-debugsymbols.zip", "") - bin_suffix = self._get_bin_suffix(version, urls["project_id"]) # Give each version a unique install dir install_dir = os.path.join(self.install_dir, version) - self.setup_mongodb(artifacts_url, binaries_url, download_symbols_url, install_dir, - bin_suffix, self.link_dir) + self.download_and_extract_from_urls(urls, bin_suffix, install_dir) except (github_conn.GithubConnError, evergreen_conn.EvergreenConnError, download.DownloadError) as ex: @@ -124,6 +118,20 @@ class SetupMultiversion(Subcommand): LOGGER.info("Setup version completed.", version=version) LOGGER.info("-" * 50) + def download_and_extract_from_urls(self, urls, bin_suffix, install_dir): + """Download and extract values indicated in `urls`.""" + artifacts_url = urls.get("Artifacts", "") if self.download_artifacts else None + binaries_url = urls.get("Binaries", "") if self.download_binaries else None + download_symbols_url = None + + if self.download_symbols: + download_symbols_url = urls.get(" mongo-debugsymbols.tgz", None) + if not download_symbols_url: + download_symbols_url = urls.get(" mongo-debugsymbols.zip", None) + + self.setup_mongodb(artifacts_url, binaries_url, download_symbols_url, install_dir, + bin_suffix, self.link_dir) + def get_latest_urls(self, version): """Return latest urls.""" urls = {} @@ -149,34 +157,48 @@ class SetupMultiversion(Subcommand): if buildvariant_name not in evg_version.build_variants_map: buildvariant_name = self.fallback_to_generic_buildvariant(major_minor_version) - curr_urls = evergreen_conn.get_compile_artifact_urls(self.evg_api, evg_version, - buildvariant_name) + curr_urls = evergreen_conn.get_compile_artifact_urls( + self.evg_api, evg_version, buildvariant_name, + ignore_failed_push=self.ignore_failed_push) if "Binaries" in curr_urls: urls = curr_urls break return urls - def get_urls(self, version): - """Return urls.""" - git_tag, commit_hash = github_conn.get_git_tag_and_commit(self.github_oauth_token, version) - LOGGER.info("Found git attributes.", git_tag=git_tag, commit_hash=commit_hash) + def get_urls(self, binary_version=None, evergreen_version=None, buildvariant_name=None): + """Return multiversion urls for a given binary version or (Evergreen version + variant).""" + if (binary_version and evergreen_version) or not (binary_version or evergreen_version): + raise ValueError("Must specify exactly one of `version` and `evergreen_version`") + + if binary_version: + git_tag, commit_hash = github_conn.get_git_tag_and_commit(self.github_oauth_token, + binary_version) + LOGGER.info("Found git attributes.", git_tag=git_tag, commit_hash=commit_hash) + + evg_project, evg_version = evergreen_conn.get_evergreen_project_and_version( + self.config, self.evg_api, commit_hash) + else: + evg_project, evg_version = evergreen_conn.get_evergreen_project( + self.config, self.evg_api, evergreen_version) - evg_project, evg_version = evergreen_conn.get_evergreen_project_and_version( - self.config, self.evg_api, commit_hash) LOGGER.debug("Found evergreen project.", evergreen_project=evg_project) + try: major_minor_version = re.findall(r"\d+\.\d+", evg_project)[-1] except IndexError: major_minor_version = "master" - buildvariant_name = self.get_buildvariant_name(major_minor_version) + if not buildvariant_name: + buildvariant_name = self.get_buildvariant_name(major_minor_version) + LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name) if buildvariant_name not in evg_version.build_variants_map: buildvariant_name = self.fallback_to_generic_buildvariant(major_minor_version) urls = evergreen_conn.get_compile_artifact_urls(self.evg_api, evg_version, - buildvariant_name) + buildvariant_name, + ignore_failed_push=self.ignore_failed_push) return urls @@ -203,8 +225,6 @@ class SetupMultiversion(Subcommand): try_download(url) if binaries_url is not None: - if not link_dir: - raise ValueError("link_dir must be specified if downloading binaries") download.symlink_version(bin_suffix, install_dir, link_dir) def get_buildvariant_name(self, major_minor_version): @@ -227,21 +247,36 @@ class SetupMultiversion(Subcommand): major_minor_version=major_minor_version) +class _DownloadOptions(object): + def __init__(self, db, ds, da): + self.download_binaries = db + self.download_symbols = ds + self.download_artifacts = da + + class SetupMultiversionPlugin(PluginInterface): """Integration point for setup-multiversion-mongodb.""" def parse(self, subcommand, parser, parsed_args, **kwargs): """Parse command-line options.""" + if subcommand != SUBCOMMAND: + return None - if subcommand == SUBCOMMAND: - return SetupMultiversion(parsed_args) + # Shorthand for brevity. + args = parsed_args - return None + download_options = _DownloadOptions(db=args.download_binaries, ds=args.download_symbols, + da=args.download_artifacts) - def add_subcommand(self, subparsers): - """Create and add the parser for the subcommand.""" - parser = subparsers.add_parser(SUBCOMMAND, help=__doc__) + return SetupMultiversion(install_dir=args.install_dir, link_dir=args.link_dir, + platform=args.platform, edition=args.edition, + architecture=args.architecture, use_latest=args.use_latest, + versions=args.versions, download_options=download_options, + evergreen_config=args.evergreen_config, + github_oauth_token=args.github_oauth_token, debug=args.debug) + @classmethod + def _add_args_to_parser(cls, parser): parser.add_argument("-i", "--installDir", dest="install_dir", required=True, help="Directory to install the download archive. [REQUIRED]") parser.add_argument( @@ -292,3 +327,8 @@ class SetupMultiversionPlugin(PluginInterface): "https://developer.github.com/v3/#rate-limiting") parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False, help="Set DEBUG logging level.") + + def add_subcommand(self, subparsers): + """Create and add the parser for the subcommand.""" + parser = subparsers.add_parser(SUBCOMMAND, help=__doc__) + self._add_args_to_parser(parser) diff --git a/buildscripts/resmokelib/symbolizer/__init__.py b/buildscripts/resmokelib/symbolizer/__init__.py new file mode 100644 index 00000000000..29e286a5fd7 --- /dev/null +++ b/buildscripts/resmokelib/symbolizer/__init__.py @@ -0,0 +1,235 @@ +"""Wrapper around mongosym to download everything required.""" +import argparse +import logging +import os +import shutil +import subprocess +import sys + +import structlog + +from buildscripts import mongosymb +from buildscripts.resmokelib.plugin import PluginInterface, Subcommand +from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, _DownloadOptions +from buildscripts.resmokelib.utils import evergreen_conn + +LOGGER = structlog.getLogger(__name__) + +_HELP = """ +Symbolize a backtrace JSON file given an Evergreen Task ID. +""" + +_MESSAGE = """TODO""" + +_COMMAND = "symbolize" + +DEFAULT_SYMBOLIZER_LOCATION = "/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" + + +def setup_logging(debug=False): + """Enable logging.""" + log_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", + level=log_level, + stream=sys.stdout, + ) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("github").setLevel(logging.WARNING) + structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) + + +class Symbolizer(Subcommand): + """Interact with Symbolizer.""" + + def __init__(self, task_id, execution_num, bin_name, mongosym_fwd_args): + """Constructor.""" + self.execution_num = execution_num + self.bin_name = bin_name + self.mongosym_args = mongosym_fwd_args + + self.evg_api: evergreen_conn.RetryingEvergreenApi = evergreen_conn.get_evergreen_api() + self.multiversion_setup = self._get_multiversion_setup() + self.task_info = self.evg_api.task_by_id(task_id) + + self.dest_dir = None # Populated later. + + @staticmethod + def _get_multiversion_setup(): + # Add the args we care about. + download_options = _DownloadOptions(db=True, ds=True, da=False) + return SetupMultiversion(download_options=download_options, ignore_failed_push=True) + + def _get_compile_artifacts(self): + version_id = self.task_info.version_id + buildvariant_name = self.task_info.build_variant + + urls = self.multiversion_setup.get_urls(binary_version=None, evergreen_version=version_id, + buildvariant_name=buildvariant_name) + + self.multiversion_setup.download_and_extract_from_urls(urls, bin_suffix=None, + install_dir=self.dest_dir) + + def _patch_diff_by_id(self): + version_id = self.task_info.version_id + module_diffs = evergreen_conn.get_patch_module_diffs(self.evg_api, version_id) + + # Not a patch build. + if not module_diffs: + return + + for module_name, diff in module_diffs.items(): + # TODO enterprise + if "mongodb-mongo-" in module_name: + with open(os.path.join(self.dest_dir, "patch.diff"), 'w') as git_diff_file: + git_diff_file.write(diff) + subprocess.run(["git", "apply", "patch.diff"], cwd=self.dest_dir, check=True) + + def _get_source(self): + revision = self.task_info.revision + source_url = f"https://github.com/mongodb/mongo/archive/{revision}.zip" + # TODO: enterprise. + + try: + # Get source for community. No need for entire repo to use `git apply [patch]`. + src_parent_dir = os.path.dirname(self.dest_dir) + try: + os.makedirs(src_parent_dir) + except FileExistsError: + pass + + subprocess.run(["curl", "-L", "-o", "source.zip", source_url], cwd=src_parent_dir, + check=True) + subprocess.run(["unzip", "-q", "source.zip"], cwd=src_parent_dir, check=True) + subprocess.run(["rm", "source.zip"], cwd=src_parent_dir, check=True) + + # Do a little dance to get the downloaded source into `self.dest_dir` + src_dir = os.listdir(src_parent_dir) + if len(src_dir) != 1: + raise ValueError( + f"expected exactly 1 directory containing source file, got {src_dir}") + src_dir = src_dir[0] + os.rename(os.path.join(src_parent_dir, src_dir), self.dest_dir) + + except subprocess.CalledProcessError as err: + LOGGER.error(err.stdout) + LOGGER.error(err.stderr) + raise + + def _setup_symbols(self): + try: + self.dest_dir = os.path.join("build", "multiversion", self.task_info.build_id) + + if os.path.isdir(self.dest_dir): + LOGGER.info( + "directory for build already exists, skipping fetching source and symbols") + return + + LOGGER.info("Getting source from GitHub...") + self._get_source() + LOGGER.info("Downloading debug symbols and binaries, this may take a few minutes...") + self._get_compile_artifacts() + LOGGER.info("Applying patch diff (if any)...") + self._patch_diff_by_id() + + except: # pylint: disable=bare-except + if self.dest_dir is not None: + LOGGER.warning("Removing downloaded directory due to error", + directory=self.dest_dir) + shutil.rmtree(self.dest_dir) + raise + + def _parse_mongosymb_args(self): + symbolizer_path = self.mongosym_args.symbolizer_path + if symbolizer_path: + raise ValueError("Must use the default symbolizer from the toolchain," + f"not {symbolizer_path}") + self.mongosym_args.symbolizer_path = DEFAULT_SYMBOLIZER_LOCATION + + sym_search_path = self.mongosym_args.path_to_executable + if sym_search_path: + raise ValueError(f"Must not specify path_to_executable, the original path that " + f"generated the symbols will be used: {sym_search_path}") + # TODO: support non-hygienic builds. + self.mongosym_args.path_to_executable = os.path.join(self.dest_dir, "dist-test", "bin", + self.bin_name) + + self.mongosym_args.src_dir_to_move = self.dest_dir + + def execute(self) -> None: + """ + Work your magic. + + :return: None + """ + self._setup_symbols() + self._parse_mongosymb_args() + LOGGER.info("Invoking mongosymb...") + mongosymb.main(self.mongosym_args) + + +class SymbolizerPlugin(PluginInterface): + """Symbolizer for MongoDB stacktraces.""" + + def add_subcommand(self, subparsers): + """ + Add 'symbolize' subcommand. + + :param subparsers: argparse parser to add to + :return: None + """ + parser = subparsers.add_parser(_COMMAND, help=_HELP) + parser.add_argument( + "--task-id", '-t', action="store", type=str, required=True, + help="Fetch corresponding binaries and symbols given an Evergreen task ID") + # TODO: support multiple Evergreen executions. + parser.add_argument("--execution", "-e", action="store", type=int, default=0, + help=argparse.SUPPRESS) + parser.add_argument( + "--binary-name", "-b", action="store", type=str, default="mongod", + help="Base name of the binary that generated the stacktrace; e.g. `mongod` or `mongos`") + parser.add_argument("--debug", "-d", dest="debug", action="store_true", default=False, + help="Set DEBUG logging level.") + + group = parser.add_argument_group( + "Verbatim mongosymb.py options for advanced usages", + description="Compatibility not guaranteed, use at your own risk") + mongosymb.make_argument_parser(group) + + def parse(self, subcommand, parser, parsed_args, **kwargs): + """ + Return Symbolizer if command is one we recognize. + + :param subcommand: equivalent to parsed_args.command + :param parser: parser used + :param parsed_args: output of parsing + :param kwargs: additional args + :return: None or a Subcommand + """ + + if subcommand != _COMMAND: + return None + + setup_logging(parsed_args.debug) + + task_id = parsed_args.task_id + binary_name = parsed_args.binary_name + + if not task_id: + raise ValueError( + "A valid Evergreen Task ID is required. You can get it by double clicking the" + " Evergreen URL after `/task/` on any task page") + + if not binary_name: + raise ValueError("A binary base name is required. This is usually `mongod` or `mongos`") + + # Check is Linux. + if not os.path.isfile(DEFAULT_SYMBOLIZER_LOCATION): + raise ValueError("llvm-symbolizer in MongoDB toolchain not found. Please run this on a " + "virtual workstation or install the toolchain manually") + + if not os.access("/data/mci", os.W_OK): + raise ValueError("Please ensure you have write access to /data/mci. " + "E.g. with `sudo mkdir -p /data/mci; sudo chown $USER /data/mci`") + + return Symbolizer(task_id, parsed_args.execution, binary_name, parsed_args) diff --git a/buildscripts/resmokelib/utils/evergreen_conn.py b/buildscripts/resmokelib/utils/evergreen_conn.py new file mode 100644 index 00000000000..5cdb6774dc1 --- /dev/null +++ b/buildscripts/resmokelib/utils/evergreen_conn.py @@ -0,0 +1,233 @@ +"""Helper functions to interact with evergreen.""" +import os + +import requests +import structlog +from requests import HTTPError + +from evergreen import RetryingEvergreenApi, Patch + +EVERGREEN_HOST = "https://evergreen.mongodb.com" +EVERGREEN_CONFIG_LOCATIONS = ( + # Common for machines in Evergreen + os.path.join(os.getcwd(), ".evergreen.yml"), + # Common for local machines + os.path.expanduser(os.path.join("~", ".evergreen.yml")), +) + +GENERIC_EDITION = "base" +GENERIC_PLATFORM = "linux_x86_64" +GENERIC_ARCHITECTURE = "x86_64" + +LOGGER = structlog.getLogger(__name__) + + +class EvergreenConnError(Exception): + """Errors in evergreen_conn.py.""" + + pass + + +def get_evergreen_api(evergreen_config=None): + """Return evergreen API.""" + config_to_pass = evergreen_config + if not config_to_pass: + # Pickup the first config file found in common locations. + for file in EVERGREEN_CONFIG_LOCATIONS: + if os.path.isfile(file): + config_to_pass = file + break + try: + evg_api = RetryingEvergreenApi.get_api(config_file=config_to_pass) + except Exception as ex: + LOGGER.error("Most likely something is wrong with evergreen config file.", + config_file=config_to_pass) + raise ex + else: + return evg_api + + +def get_buildvariant_name(config, edition, platform, architecture, major_minor_version): + """Return Evergreen buildvariant name.""" + + buildvariant_name = "" + evergreen_buildvariants = config.evergreen_buildvariants + + for buildvariant in evergreen_buildvariants: + if (buildvariant.edition == edition and buildvariant.platform == platform + and buildvariant.architecture == architecture): + versions = buildvariant.versions + if major_minor_version in versions: + buildvariant_name = buildvariant.name + break + elif not versions: + buildvariant_name = buildvariant.name + + return buildvariant_name + + +# pylint: disable=protected-access +def get_patch_module_diffs(evg_api, version_id): + """Get the raw git diffs for all modules.""" + evg_url = evg_api._create_url(f"/patches/{version_id}") + try: + res = evg_api._call_api(evg_url) + except requests.exceptions.HTTPError as err: + err_res = err.response + if err_res.status_code == 400: + LOGGER.debug("Not a patch build task, skipping applying patch", + version_id_of_task=version_id) + return None + else: + raise + + patch = Patch(res.json(), evg_api) + + res = {} + for module_code_change in patch.module_code_changes: + git_diff_link = module_code_change.raw_link + raw = evg_api._call_api(git_diff_link) + diff = raw.text + res[module_code_change.branch_name] = diff + + return res + + +def get_generic_buildvariant_name(config, major_minor_version): + """Return Evergreen buildvariant name for generic platform.""" + + LOGGER.info("Falling back to generic architecture.", edition=GENERIC_EDITION, + platform=GENERIC_PLATFORM, architecture=GENERIC_ARCHITECTURE) + + generic_buildvariant_name = get_buildvariant_name( + config=config, edition=GENERIC_EDITION, platform=GENERIC_PLATFORM, + architecture=GENERIC_ARCHITECTURE, major_minor_version=major_minor_version) + + if not generic_buildvariant_name: + raise EvergreenConnError("Generic architecture buildvariant not found.") + + return generic_buildvariant_name + + +def get_evergreen_project_and_version(config, evg_api, commit_hash): + """Return evergreen project and version by commit hash.""" + + for evg_project in config.evergreen_projects: + try: + version_id = evg_project.replace("-", "_") + "_" + commit_hash + evg_version = evg_api.version_by_id(version_id) + except HTTPError: + continue + else: + LOGGER.debug("Found evergreen version.", + evergreen_version=f"{EVERGREEN_HOST}/version/{evg_version.version_id}") + return evg_project, evg_version + + raise EvergreenConnError(f"Evergreen version for commit hash {commit_hash} not found.") + + +def get_evergreen_project(config, evg_api, evergreen_version_id): + """Return evergreen project for a given Evergreen version.""" + + for evg_project in config.evergreen_projects: + try: + evg_version = evg_api.version_by_id(evergreen_version_id) + except HTTPError: + continue + else: + LOGGER.debug("Found evergreen version.", + evergreen_version=f"{EVERGREEN_HOST}/version/{evg_version.version_id}") + return evg_project, evg_version + + raise EvergreenConnError(f"Evergreen version {evergreen_version_id} not found.") + + +def get_evergreen_versions(evg_api, evg_project): + """Return the list of evergreen versions by evergreen project name.""" + return evg_api.versions_by_project(evg_project) + + +def get_compile_artifact_urls(evg_api, evg_version, buildvariant_name, ignore_failed_push=False): + """Return compile urls from buildvariant in Evergreen version.""" + try: + build_id = evg_version.build_variants_map[buildvariant_name] + except KeyError: + raise EvergreenConnError(f"Buildvariant {buildvariant_name} not found.") + + evg_build = evg_api.build_by_id(build_id) + LOGGER.debug("Found evergreen build.", evergreen_build=f"{EVERGREEN_HOST}/build/{build_id}") + evg_tasks = evg_build.get_tasks() + tasks_wrapper = _filter_successful_tasks(evg_tasks) + + # Ignore push tasks if specified as such, else return no results if push does not exist. + if ignore_failed_push: + tasks_wrapper.push_task = None + elif tasks_wrapper.push_task is None: + return {} + + return _get_multiversion_urls(tasks_wrapper) + + +def _get_multiversion_urls(tasks_wrapper): + compile_artifact_urls = {} + + binary = tasks_wrapper.binary_task + push = tasks_wrapper.push_task + symbols = tasks_wrapper.symbols_task + + required_tasks = [binary, push] if push is not None else [binary] + + if all(task and task.status == "success" for task in required_tasks): + LOGGER.info("Required evergreen task(s) were successful.", + required_tasks=f"{required_tasks}", + task_id=f"{EVERGREEN_HOST}/task/{required_tasks[0].task_id}") + evg_artifacts = binary.artifacts + for artifact in evg_artifacts: + compile_artifact_urls[artifact.name] = artifact.url + + if symbols and symbols.status == "success": + for artifact in symbols.artifacts: + compile_artifact_urls[artifact.name] = artifact.url + elif symbols and symbols.task_id: + LOGGER.warning("debug symbol archive was unsuccessful", + archive_symbols_task=f"{EVERGREEN_HOST}/task/{symbols.task_id}") + + # Tack on the project id for generating a friendly decompressed name for the artifacts. + compile_artifact_urls["project_id"] = binary.project_id + + elif all(task for task in required_tasks): + LOGGER.warning("Required Evergreen task(s) were not successful.", + required_tasks=f"{required_tasks}", + task_id=f"{EVERGREEN_HOST}/task/{required_tasks[0].task_id}") + else: + LOGGER.error("There are no `compile` and/or 'push' tasks in the evergreen build") + + return compile_artifact_urls + + +class _MultiversionTasks(object): + """Tasks relevant for multiversion setup.""" + + def __init__(self, symbols, binary, push): + """Init function.""" + self.symbols_task = symbols + self.binary_task = binary + self.push_task = push + + +def _filter_successful_tasks(evg_tasks) -> _MultiversionTasks: + compile_task = None + archive_symbols_task = None + push_task = None + for evg_task in evg_tasks: + # Only set the compile task if there isn't one already, otherwise + # newer tasks like "archive_dist_test_debug" take precedence. + if evg_task.display_name in ("compile", "archive_dist_test") and compile_task is None: + compile_task = evg_task + elif evg_task.display_name == "push": + push_task = evg_task + elif evg_task.display_name == "archive_dist_test_debug": + archive_symbols_task = evg_task + if compile_task and push_task and archive_symbols_task: + break + return _MultiversionTasks(symbols=archive_symbols_task, binary=compile_task, push=push_task) diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py index aa0975e2381..24786de4ac7 100644 --- a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py +++ b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py @@ -5,9 +5,9 @@ from argparse import Namespace from mock import patch -from buildscripts.resmokelib.setup_multiversion import evergreen_conn +from buildscripts.resmokelib.utils import evergreen_conn from buildscripts.resmokelib.setup_multiversion.config import SetupMultiversionConfig -from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion +from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, _DownloadOptions class TestSetupMultiversionBase(unittest.TestCase): @@ -36,6 +36,9 @@ class TestSetupMultiversionBase(unittest.TestCase): }, ] } + + download_options = _DownloadOptions(db=True, ds=False, da=False) + options = Namespace( install_dir="install", link_dir="link", @@ -44,17 +47,15 @@ class TestSetupMultiversionBase(unittest.TestCase): architecture=architecture, use_latest=False, versions=["4.2.1"], - download_symbols=False, - download_binaries=True, - download_artifacts=False, evergreen_config=None, github_oauth_token=None, + download_options=download_options, debug=False, ) with patch("buildscripts.resmokelib.setup_multiversion.config.SetupMultiversionConfig" ) as mock_config: mock_config.return_value = SetupMultiversionConfig(raw_yaml_config) - self.setup_multiversion = SetupMultiversion(options) + self.setup_multiversion = SetupMultiversion(**vars(options)) class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): @@ -66,7 +67,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): @patch("evergreen.version.Version") @patch("evergreen.api.EvergreenApi.versions_by_project") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_no_compile_artifacts(self, mock_get_compile_artifact_urls, mock_versions_by_project, mock_version): mock_version.build_variants_map = {self.buildvariant_name: "build_id"} @@ -78,7 +79,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): @patch("evergreen.version.Version") @patch("evergreen.api.EvergreenApi.versions_by_project") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_urls_found_on_last_version(self, mock_get_compile_artifact_urls, mock_versions_by_project, mock_version): expected_urls = { @@ -96,7 +97,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): @patch("evergreen.version.Version") @patch("evergreen.version.Version") @patch("evergreen.api.EvergreenApi.versions_by_project") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_urls_found_on_not_last_version(self, mock_get_compile_artifact_urls, mock_versions_by_project, mock_version, mock_expected_version): @@ -110,18 +111,21 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): evg_versions = [mock_version for _ in range(3)] evg_versions.append(mock_expected_version) mock_versions_by_project.return_value = evg_versions - mock_get_compile_artifact_urls.side_effect = lambda evg_api, evg_version, buildvariant_name: { - (self.setup_multiversion.evg_api, mock_version, self.buildvariant_name): {}, - (self.setup_multiversion.evg_api, mock_expected_version, self.buildvariant_name): + print(self.setup_multiversion.evg_api) + print(self.buildvariant_name) + print(mock_version) + mock_get_compile_artifact_urls.side_effect = lambda evg_api, evg_version, buildvariant_name, ignore_failed_push: { + (self.setup_multiversion.evg_api, mock_version, self.buildvariant_name, False): {}, + (self.setup_multiversion.evg_api, mock_expected_version, self.buildvariant_name, False): expected_urls, - }[evg_api, evg_version, buildvariant_name] + }[evg_api, evg_version, buildvariant_name, ignore_failed_push] urls = self.setup_multiversion.get_latest_urls("4.4") self.assertEqual(urls, expected_urls) @patch("evergreen.version.Version") @patch("evergreen.api.EvergreenApi.versions_by_project") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_fallback_to_generic_buildvariant(self, mock_get_compile_artifact_urls, mock_versions_by_project, mock_version): expected_urls = { @@ -139,11 +143,9 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): class TestSetupMultiversionGetUrls(TestSetupMultiversionBase): @patch("evergreen.version.Version") - @patch( - "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" - ) + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_evergreen_project_and_version") @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_urls_found(self, mock_get_compile_artifact_urls, mock_get_git_tag_and_commit, mock_get_evergreen_project_and_version, mock_version): expected_urls = { @@ -160,11 +162,9 @@ class TestSetupMultiversionGetUrls(TestSetupMultiversionBase): self.assertEqual(urls, expected_urls) @patch("evergreen.version.Version") - @patch( - "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" - ) + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_evergreen_project_and_version") @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_urls_not_found(self, mock_get_compile_artifact_urls, mock_get_git_tag_and_commit, mock_get_evergreen_project_and_version, mock_version): mock_get_git_tag_and_commit.return_value = ("git_tag", "commit_hash") @@ -176,11 +176,9 @@ class TestSetupMultiversionGetUrls(TestSetupMultiversionBase): self.assertEqual(urls, {}) @patch("evergreen.version.Version") - @patch( - "buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_evergreen_project_and_version" - ) + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_evergreen_project_and_version") @patch("buildscripts.resmokelib.setup_multiversion.github_conn.get_git_tag_and_commit") - @patch("buildscripts.resmokelib.setup_multiversion.evergreen_conn.get_compile_artifact_urls") + @patch("buildscripts.resmokelib.utils.evergreen_conn.get_compile_artifact_urls") def test_fallback_to_generic_buildvariant(self, mock_get_compile_artifact_urls, mock_get_git_tag_and_commit, mock_get_evergreen_project_and_version, mock_version): diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py index 568d37f74d1..e95841b0caa 100644 --- a/buildscripts/tests/resmokelib/setup_multiversion/test_evergreen_conn.py +++ b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py @@ -1,4 +1,4 @@ -"""Unit tests for buildscripts/resmokelib/setup_multiversion/evergreen_conn.py.""" +"""Unit tests for buildscripts/resmokelib/utils/evergreen_conn.py.""" # pylint: disable=missing-docstring,too-many-arguments import unittest @@ -6,7 +6,7 @@ from mock import patch from requests import HTTPError from evergreen import RetryingEvergreenApi -from buildscripts.resmokelib.setup_multiversion import evergreen_conn +from buildscripts.resmokelib.utils import evergreen_conn from buildscripts.resmokelib.setup_multiversion.config import SetupMultiversionConfig |