diff options
author | Mikhail Shchatko <mikhail.shchatko@mongodb.com> | 2022-05-23 18:27:46 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-23 14:28:56 +0000 |
commit | f6e237006a85a670592763d5a13d61775261c7de (patch) | |
tree | 8c546acc7bd1a25ecb0fcf66b180474e7023bbc3 | |
parent | d69a2bf1c49f3517cb0838326370f31b2894d519 (diff) | |
download | mongo-f6e237006a85a670592763d5a13d61775261c7de.tar.gz |
SERVER-66613 Send binary version instead of evergreen version to symbolizer service
(cherry picked from commit c2f3bb6b7f263d91f9a5f212faec2417adadea97)
-rw-r--r-- | buildscripts/debugsymb_mapper.py | 193 | ||||
-rwxr-xr-x | buildscripts/mongosymb.py | 20 | ||||
-rw-r--r-- | buildscripts/tests/test_debugsymb_mapper.py | 115 | ||||
-rw-r--r-- | buildscripts/tests/test_mongosymb.py | 22 |
4 files changed, 279 insertions, 71 deletions
diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py index 977716a7c0f..4e9df04398e 100644 --- a/buildscripts/debugsymb_mapper.py +++ b/buildscripts/debugsymb_mapper.py @@ -4,11 +4,13 @@ import json import logging import os import pathlib +import re import shutil import subprocess import sys import time -import typing +from json import JSONDecoder +from typing import Optional, Tuple, Generator, Dict, List, NamedTuple import requests @@ -20,30 +22,95 @@ from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download from buildscripts.build_system_options import PathOptions +BUILD_INFO_RE = re.compile(r"Build Info: ({(\n.*)*})") +MONGOD = "mongod" -class LinuxBuildIDExtractor: - """Parse readlef command output & extract Build ID.""" - default_executable_path = "readelf" +class CmdClient: + """Client to run commands.""" - def __init__(self, executable_path: str = None): - """Initialize instance.""" + @staticmethod + def run(args: List[str]) -> str: + """ + Run command with args. + + :param args: Argument list. + :return: Command output. + """ + + out = subprocess.run(args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False) + return out.stdout.strip().decode() + + +class BuildIdOutput(NamedTuple): + """ + Build ID and command output. + + * build_id: Build ID or None. + * cmd_output: Command output. + """ + + build_id: Optional[str] + cmd_output: str + + +class BinVersionOutput(NamedTuple): + """ + Mongodb bin version and command output. + + * mongodb_version: Bin version. + * cmd_output: Command output. + """ + + mongodb_version: Optional[str] + cmd_output: str + + +class CmdOutputExtractor: + """Data extractor from command output.""" + + def __init__(self, cmd_client: Optional[CmdClient] = None, + json_decoder: Optional[JSONDecoder] = None) -> None: + """ + Initialize. - self.executable_path = executable_path or self.default_executable_path + :param cmd_client: Client to run commands. + :param json_decoder: JSONDecoder object. + """ + self.cmd_client = cmd_client if cmd_client is not None else CmdClient() + self.json_decoder = json_decoder if json_decoder is not None else JSONDecoder() - def callreadelf(self, binary_path: str) -> str: - """Call readelf command for given binary & return string output.""" + def get_build_id(self, bin_path: str) -> BuildIdOutput: + """ + Get build ID from readelf command. - args = [self.executable_path, "-n", binary_path] - process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - process.wait() - return process.stdout.read().decode() + :param bin_path: Path to binary of the build. + :return: Build ID or None and command output. + """ + out = self.cmd_client.run(["readelf", "-n", bin_path]) + build_id = self._extract_build_id(out) + return BuildIdOutput(build_id, out) + + def get_bin_version(self, bin_path: str) -> BinVersionOutput: + """ + Get mongodb bin version from `{bin} --version` command. + + :param bin_path: Path to mongodb binary. + :return: Bin version or None and command output. + """ + out = self.cmd_client.run([os.path.abspath(bin_path), "--version"]) + mongodb_version = self._get_mongodb_version(out) + return BinVersionOutput(mongodb_version, out) @staticmethod - def extractbuildid(out: str) -> typing.Optional[str]: - """Parse readelf output and extract Build ID from it.""" + def _extract_build_id(out: str) -> Optional[str]: + """ + Parse readelf output and extract Build ID from it. + :param out: readelf command output. + :return: Build ID on None. + """ build_id = None for line in out.splitlines(): line = line.strip() @@ -53,13 +120,21 @@ class LinuxBuildIDExtractor: build_id = line.split(': ')[1] return build_id - def run(self, binary_path: str) -> typing.Tuple[str, str]: - """Perform all necessary actions to get Build ID.""" + def _get_mongodb_version(self, out: str) -> Optional[str]: + """ + Parse version command output and extract mongodb version. + + :param out: Version command output. + :return: Version or None. + """ + mongodb_version = None - readelfout = self.callreadelf(binary_path) - buildid = self.extractbuildid(readelfout) + search = BUILD_INFO_RE.search(out) + if search: + build_info = self.json_decoder.decode(search.group(1)) + mongodb_version = build_info.get("version") - return buildid, readelfout + return mongodb_version class DownloadOptions(object): @@ -88,19 +163,22 @@ class Mapper: default_client_credentials_user_name = "client-user" default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') - def __init__(self, version: str, client_id: str, client_secret: str, variant: str, + def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str, cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. - :param version: version string - :param variant: build variant string - :param cache_dir: full path to cache directory as a string - :param web_service_base_url: URL of symbolizer web service + :param evg_version: Evergreen version ID. + :param evg_variant: Evergreen build variant name. + :param client_id: Client id for Okta Oauth. + :param client_secret: Secret key for Okta Oauth. + :param cache_dir: Full path to cache directory as a string. + :param web_service_base_url: URL of symbolizer web service. + :param logger: Debug symbols mapper logger. """ - self.version = version - self.variant = variant + self.evg_version = evg_version + self.evg_variant = evg_variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url @@ -113,8 +191,8 @@ class Mapper: self.http_client = requests.Session() self.multiversion_setup = SetupMultiversion( - DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant, - ignore_failed_push=True) + DownloadOptions(download_symbols=True, download_binaries=True), + variant=self.evg_variant, ignore_failed_push=True) self.debug_symbols_url = None self.url = None self.configs = Configs( @@ -139,7 +217,7 @@ class Mapper: data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get("expire_time") if time.time() < expire_time: - # credentials hasn't expired yet + # credentials haven't expired yet self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) return @@ -147,7 +225,7 @@ class Mapper: configs=self.configs) self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) - # write credentials to local file for further useage + # write credentials to local file for further usage with open(self.default_creds_file_path, "w") as cfile: cfile.write( json.dumps({ @@ -184,7 +262,7 @@ class Mapper: def setup_urls(self): """Set up URLs using multiversion.""" - urlinfo = self.multiversion_setup.get_urls(self.version, self.variant) + urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant) download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) binaries_url = urlinfo.urls.get("Binaries", "") @@ -194,7 +272,7 @@ class Mapper: if not download_symbols_url: self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", - self.version, urlinfo.urls) + self.evg_version, urlinfo.urls) raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}") self.debug_symbols_url = download_symbols_url @@ -233,14 +311,14 @@ class Mapper: tarball_full_path = download.download_from_s3(url) return tarball_full_path - def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]: + def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]: """ Extract build id from binaries and creates new dict using them. :return: mapped data as dict """ - readelf_extractor = LinuxBuildIDExtractor() + extractor = CmdOutputExtractor() debug_symbols_path = self.download(self.debug_symbols_url) debug_symbols_unpacked_path = self.unpack(debug_symbols_path) @@ -262,6 +340,17 @@ class Mapper: self.logger.info("INSIDE unpacked binaries/dist-test: %s", os.listdir(binaries_unpacked_path)) + mongod_bin = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name, + MONGOD) + bin_version_output = extractor.get_bin_version(mongod_bin) + + if bin_version_output.mongodb_version is None: + self.logger.error("mongodb version could not be extracted. \n`%s --version` output: %s", + mongod_bin, bin_version_output.cmd_output) + return + else: + self.logger.info("Extracted mongodb version: %s", bin_version_output.mongodb_version) + # start with main binary folder for binary in self.selected_binaries: full_bin_path = os.path.join(debug_symbols_unpacked_path, @@ -271,16 +360,21 @@ class Mapper: self.logger.error("Could not find binary at %s", full_bin_path) return - build_id, readelf_out = readelf_extractor.run(full_bin_path) + build_id_output = extractor.get_build_id(full_bin_path) - if not build_id: + if not build_id_output.build_id: self.logger.error("Build ID couldn't be extracted. \nReadELF output %s", - readelf_out) + build_id_output.cmd_output) return + else: + self.logger.info("Extracted build ID: %s", build_id_output.build_id) yield { - 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, - 'file_name': binary, 'version': self.version + 'url': self.url, + 'debug_symbols_url': self.debug_symbols_url, + 'build_id': build_id_output.build_id, + 'file_name': binary, + 'version': bin_version_output.mongodb_version, } # move to shared libraries folder. @@ -306,18 +400,21 @@ class Mapper: self.logger.error("Could not find binary at %s", sofile_path) return - build_id, readelf_out = readelf_extractor.run(sofile_path) + build_id_output = extractor.get_build_id(sofile_path) - if not build_id: - self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out) + if not build_id_output.build_id: + self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", + build_id_output.cmd_output) return + else: + self.logger.info("Extracted build ID: %s", build_id_output.build_id) yield { 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, - 'build_id': build_id, + 'build_id': build_id_output.build_id, 'file_name': sofile, - 'version': self.version, + 'version': bin_version_output.mongodb_version, } def run(self): @@ -355,8 +452,8 @@ def make_argument_parser(parser=None, **kwargs): def main(options): """Execute mapper here. Main entry point.""" - mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id, - client_secret=options.client_secret, + mapper = Mapper(evg_version=options.version, evg_variant=options.variant, + client_id=options.client_id, client_secret=options.client_secret, web_service_base_url=options.web_service_base_url) # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job. diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 3224ede0dfa..035b00f3a07 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -362,6 +362,7 @@ class PathResolver(DbgFileResolver): search_parameters = {"build_id": build_id} if version: search_parameters["version"] = version + print(f"Getting data from service... Search parameters: {search_parameters}") response = self.http_client.get(f"{self.host}/find_by_id", params=search_parameters) if response.status_code != 200: sys.stderr.write( @@ -416,7 +417,7 @@ def parse_input(trace_doc, dbg_path_resolver): return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry} base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"]) - version = parse_version(trace_doc) + version = get_version(trace_doc) frames = [] for frame in trace_doc["backtrace"]: @@ -449,19 +450,14 @@ def parse_input(trace_doc, dbg_path_resolver): return frames -def parse_version(trace_doc: Dict[str, Any]) -> Optional[str]: - """Parse version from trace doc. - - In evergreen patches `mongodbVersion` is appended by `-patch-{version}`. - We want to use this version value to distinguish patch builds. +def get_version(trace_doc: Dict[str, Any]) -> Optional[str]: + """ + Get version from trace doc. - :param trace_doc: traceback object - :return: version string or None + :param trace_doc: Traceback dict. + :return: Version string or None. """ - version = trace_doc.get("processInfo", {}).get("mongodbVersion") - if version and "patch" in version: - return version.split("-")[-1] - return None + return trace_doc.get("processInfo", {}).get("mongodbVersion") def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format, diff --git a/buildscripts/tests/test_debugsymb_mapper.py b/buildscripts/tests/test_debugsymb_mapper.py new file mode 100644 index 00000000000..b1b063f5331 --- /dev/null +++ b/buildscripts/tests/test_debugsymb_mapper.py @@ -0,0 +1,115 @@ +"""Unit tests for debugsymb_mapper.py.""" +# pylint: disable=missing-docstring +import unittest +from unittest.mock import MagicMock + +import buildscripts.debugsymb_mapper as under_test + + +def mock_cmd_client(): + cmd_client = MagicMock(spec_set=under_test.CmdClient) + return cmd_client + + +class TestCmdOutputExtractor(unittest.TestCase): + def setUp(self): + self.cmd_client_mock = mock_cmd_client() + self.cmd_output_extractor = under_test.CmdOutputExtractor(self.cmd_client_mock) + + +class TestGetBuildId(TestCmdOutputExtractor): + def test_get_build_id_returns_build_id(self): + readelf_output = ( + "Displaying notes found in: .note.gnu.build-id\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot\n" + " Owner Data size\tDescription\n" + " GA$<version>3h864 0x00000010\tOPEN\n" + " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)\n" + " GA$<version>3h864 0x00000010\tOPEN\n" + " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)") + self.cmd_client_mock.run.return_value = readelf_output + + build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") + self.assertEqual(build_id_output.build_id, "74c2322104428836f3d94af6cd7471ee7cb5c4ee") + self.assertEqual(build_id_output.cmd_output, readelf_output) + + def test_get_build_id_raises_error(self): + readelf_output = ( + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot") + self.cmd_client_mock.run.return_value = readelf_output + + self.assertRaises(ValueError, self.cmd_output_extractor.get_build_id, "path/to/bin") + + def test_get_build_id_returns_none(self): + readelf_output = ( + "Displaying notes found in: .note.gnu.build-id\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)") + self.cmd_client_mock.run.return_value = readelf_output + + build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") + self.assertIsNone(build_id_output.build_id) + self.assertEqual(build_id_output.cmd_output, readelf_output) + + +class TestGetBinVersion(TestCmdOutputExtractor): + def test_get_bin_version_returns_version(self): + # Newer versions command output + version_cmd_output = ('db version v4.4.14-25-gb0475e2\n' + 'Build Info: {\n' + ' "version": "4.4.14-25-gb0475e2",\n' + ' "gitVersion": "b0475e2657c3351b25499971d3340f054ea85b98",\n' + ' "openSSLVersion": "OpenSSL 1.1.1 11 Sep 2018",\n' + ' "modules": [\n' + ' "enterprise"\n' + ' ],\n' + ' "allocator": "tcmalloc",\n' + ' "environment": {\n' + ' "distmod": "ubuntu1804",\n' + ' "distarch": "x86_64",\n' + ' "target_arch": "x86_64"\n' + ' }\n' + '}') + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertEqual(bin_version_output.mongodb_version, "4.4.14-25-gb0475e2") + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) + + def test_get_bin_version_unsupported_output(self): + # Versions prior to 5.0 are not supported + version_cmd_output = ('db version v4.2.20-7-g5a81409\n' + 'git version: 5a81409faf16f30f1189af6367eb3ceee50a02b5\n' + 'OpenSSL version: OpenSSL 1.1.1 11 Sep 2018\n' + 'allocator: tcmalloc\n' + 'modules: enterprise \n' + 'build environment:\n' + ' distmod: ubuntu1804\n' + ' distarch: x86_64\n' + ' target_arch: x86_64') + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertIsNone(bin_version_output.mongodb_version) + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) + + def test_get_bin_version_returns_none(self): + version_cmd_output = "error: unrecognized arguments: --version" + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertIsNone(bin_version_output.mongodb_version) + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) diff --git a/buildscripts/tests/test_mongosymb.py b/buildscripts/tests/test_mongosymb.py index 4065f9f3e96..188ea385c5f 100644 --- a/buildscripts/tests/test_mongosymb.py +++ b/buildscripts/tests/test_mongosymb.py @@ -5,29 +5,29 @@ import unittest from buildscripts import mongosymb as under_test -class TestParseVersion(unittest.TestCase): - def test_parse_version_with_patch(self): +class TestGetVersion(unittest.TestCase): + def test_get_version_with_patch(self): trace_doc = { "processInfo": { "mongodbVersion": "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169" } } - version = under_test.parse_version(trace_doc) - self.assertEqual(version, "6257e60a32f417196bc25169") + version = under_test.get_version(trace_doc) + self.assertEqual(version, "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169") - def test_parse_version_without_patch(self): + def test_get_version_without_patch(self): trace_doc = {"processInfo": {"mongodbVersion": "6.1.0-alpha-504-g0c8a142"}} - version = under_test.parse_version(trace_doc) - self.assertEqual(version, None) + version = under_test.get_version(trace_doc) + self.assertEqual(version, "6.1.0-alpha-504-g0c8a142") - def test_parse_version_no_mongodb_version(self): + def test_get_version_no_mongodb_version(self): trace_doc = {"processInfo": {}} - version = under_test.parse_version(trace_doc) + version = under_test.get_version(trace_doc) self.assertEqual(version, None) - def test_parse_version_no_process_info(self): + def test_get_version_no_process_info(self): trace_doc = {} - version = under_test.parse_version(trace_doc) + version = under_test.get_version(trace_doc) self.assertEqual(version, None) |