diff options
author | Jasur Nurboyev <bluestacks6523@gmail.com> | 2021-12-08 17:20:55 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-12-08 17:56:35 +0000 |
commit | a86cafd2a920f0887a1cdb8e5bc6157e224a1771 (patch) | |
tree | 666e76b6e9a64967c4c6188000649cb93f03df96 | |
parent | ad3ba4b6ae53962de7b943bac4c4c50bd5d85002 (diff) | |
download | mongo-a86cafd2a920f0887a1cdb8e5bc6157e224a1771.tar.gz |
SERVER-61242 Add logic to the CI to add BUILD ID info to the symbolizer service.
-rw-r--r-- | buildscripts/debugsymb_mapper.py | 360 | ||||
-rwxr-xr-x | buildscripts/mongosymb.py | 11 | ||||
-rw-r--r-- | etc/evergreen.yml | 54 | ||||
-rw-r--r-- | evergreen/generate_buildid_debug_symbols_mapping.sh | 15 |
4 files changed, 437 insertions, 3 deletions
diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py new file mode 100644 index 00000000000..25435421554 --- /dev/null +++ b/buildscripts/debugsymb_mapper.py @@ -0,0 +1,360 @@ +"""Script to generate & upload 'buildId -> debug symbols URL' mappings to symbolizer service.""" +import argparse +import json +import logging +import os +import pathlib +import shutil +import subprocess +import sys +import time +import typing + +import requests + +# register parent directory in sys.path, so 'buildscripts' is detected no matter where the script is called from +sys.path.append(str(pathlib.Path(os.path.join(os.getcwd(), __file__)).parent.parent)) + +# pylint: disable=wrong-import-position +from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs +from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download + + +class LinuxBuildIDExtractor: + """Parse readlef command output & extract Build ID.""" + + default_executable_path = "readelf" + + def __init__(self, executable_path: str = None): + """Initialize instance.""" + + self.executable_path = executable_path or self.default_executable_path + + def callreadelf(self, binary_path: str) -> str: + """Call readelf command for given binary & return string output.""" + + args = [self.executable_path, "-n", binary_path] + process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + process.wait() + return process.stdout.read().decode() + + @staticmethod + def extractbuildid(out: str) -> typing.Optional[str]: + """Parse readelf output and extract Build ID from it.""" + + build_id = None + for line in out.splitlines(): + line = line.strip() + if line.startswith('Build ID'): + if build_id is not None: + raise ValueError("Found multiple Build ID values.") + build_id = line.split(': ')[1] + return build_id + + def run(self, binary_path: str) -> typing.Tuple[str, str]: + """Perform all necessary actions to get Build ID.""" + + readelfout = self.callreadelf(binary_path) + buildid = self.extractbuildid(readelfout) + + return buildid, readelfout + + +class DownloadOptions(object): + """A class to collect download option configurations.""" + + def __init__(self, download_binaries=False, download_symbols=False, download_artifacts=False, + download_python_venv=False): + """Initialize instance.""" + + self.download_binaries = download_binaries + self.download_symbols = download_symbols + self.download_artifacts = download_artifacts + self.download_python_venv = download_python_venv + + +class Mapper: + """A class to to basically all of the work.""" + + # pylint: disable=too-many-instance-attributes + # This amount of attributes are necessary. + + default_web_service_base_url: str = "https://symbolizer-service.server-tig.staging.corp.mongodb.com" + default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache') + selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug') + default_client_credentials_scope = "servertig-symbolizer-fullaccess" + default_client_credentials_user_name = "client-user" + default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') + + def __init__(self, version: str, client_id: str, client_secret: str, variant: str, + cache_dir: str = None, web_service_base_url: str = None, + logger: logging.Logger = None): + """ + Initialize instance. + + :param version: version string + :param variant: build variant string + :param cache_dir: full path to cache directory as a string + :param web_service_base_url: URL of symbolizer web service + """ + self.version = version + self.variant = variant + self.cache_dir = cache_dir or self.default_cache_dir + self.web_service_base_url = web_service_base_url or self.default_web_service_base_url + + if not logger: + logging.basicConfig() + logger = logging.getLogger('symbolizer') + logger.setLevel(logging.INFO) + self.logger = logger + + self.http_client = requests.Session() + + self.multiversion_setup = SetupMultiversion( + DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant, + ignore_failed_push=True) + self.debug_symbols_url = None + self.url = None + self.configs = Configs( + client_credentials_scope=self.default_client_credentials_scope, + client_credentials_user_name=self.default_client_credentials_user_name) + self.client_id = client_id + self.client_secret = client_secret + + if not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + self.authenticate() + self.setup_urls() + + def authenticate(self): + """Login & get credentials for further requests to web service.""" + + # try to read from file + if os.path.exists(self.default_creds_file_path): + with open(self.default_creds_file_path) as cfile: + data = json.loads(cfile.read()) + access_token, expire_time = data.get("access_token"), data.get("expire_time") + if time.time() < expire_time: + # credentials hasn't expired yet + self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) + return + + credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret, + configs=self.configs) + self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) + + # write credentials to local file for further useage + with open(self.default_creds_file_path, "w") as cfile: + cfile.write( + json.dumps({ + "access_token": credentials.access_token, + "expire_time": time.time() + credentials.expires_in + })) + + def __enter__(self): + """Return instance when used as a context manager.""" + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Do cleaning process when used as a context manager.""" + + self.cleanup() + + def cleanup(self): + """Remove temporary files & folders.""" + + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + + @staticmethod + def url_to_filename(url: str) -> str: + """ + Convert URL to local filename. + + :param url: download URL + :return: full name for local file + """ + return url.split('/')[-1] + + def setup_urls(self): + """Set up URLs using multiversion.""" + + urlinfo = self.multiversion_setup.get_urls(self.version, self.variant) + + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) + binaries_url = urlinfo.urls.get("Binaries", "") + + if not download_symbols_url: + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None) + + if not download_symbols_url: + self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", + self.version, urlinfo.urls) + raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}") + + self.debug_symbols_url = download_symbols_url + self.url = binaries_url + + def unpack(self, path: str) -> str: + """ + Use to untar/unzip files. + + :param path: full path of file + :return: full path of directory of unpacked file + """ + foldername = path.replace('.tgz', '', 1).split('/')[-1] + out_dir = os.path.join(self.cache_dir, foldername) + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + download.extract_archive(path, out_dir) + + # extracted everything, we don't need the original tar file anymore and it should be deleted + if os.path.exists(path): + os.remove(path) + + return out_dir + + @staticmethod + def download(url: str) -> str: + """ + Use to download file from URL. + + :param url: URL of file to download + :return: full path of downloaded file in local filesystem + """ + + tarball_full_path = download.download_from_s3(url) + return tarball_full_path + + def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]: + """ + Extract build id from binaries and creates new dict using them. + + :return: mapped data as dict + """ + + readelf_extractor = LinuxBuildIDExtractor() + + debug_symbols_path = self.download(self.debug_symbols_url) + debug_symbols_unpacked_path = self.unpack(debug_symbols_path) + + binaries_path = self.download(self.url) + binaries_unpacked_path = self.unpack(binaries_path) + + # we need to analyze two directories: bin inside debug-symbols and lib inside binaries. + # bin holds main binaries, like mongos, mongod, mongo ... + # lib holds shared libraries, tons of them. some build variants do not contain shared libraries. + + debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test') + binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test') + + self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s", + os.listdir(debug_symbols_unpacked_path)) + self.logger.info("INSIDE unpacked binaries/dist-test: %s", + os.listdir(binaries_unpacked_path)) + + # start with 'bin' folder + for binary in self.selected_binaries: + full_bin_path = os.path.join(debug_symbols_unpacked_path, 'bin', binary) + + if not os.path.exists(full_bin_path): + self.logger.error("Could not find binary at %s", full_bin_path) + return + + build_id, readelf_out = readelf_extractor.run(full_bin_path) + + if not build_id: + self.logger.error("Build ID couldn't be extracted. \nReadELF output %s", + readelf_out) + return + + yield { + 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, + 'file_name': binary, 'version': self.version + } + + # move to 'lib' folder. + # it contains all shared library binary files, + # we run readelf on each of them. + lib_folder_path = os.path.join(binaries_unpacked_path, 'lib') + + if not os.path.exists(lib_folder_path): + # sometimes we don't get lib folder, which means there is no shared libraries for current build variant. + sofiles = [] + else: + _, _, sofiles = os.walk(lib_folder_path) + + for sofile in sofiles: + sofile_path = os.path.join(lib_folder_path, sofile) + + if not os.path.exists(sofile_path): + self.logger.error("Could not find binary at %s", sofile_path) + return + + build_id, readelf_out = readelf_extractor.run(sofile_path) + + if not build_id: + self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out) + return + + yield { + 'url': self.url, + 'debug_symbols_url': self.debug_symbols_url, + 'build_id': build_id, + 'file_name': sofile, + 'version': self.version, + } + + def run(self): + """Run all necessary processes.""" + + mappings = self.generate_build_id_mapping() + if not mappings: + self.logger.error("Could not generate mapping") + return + + # mappings is a generator, we iterate over to generate mappings on the go + for mapping in mappings: + response = self.http_client.post('/'.join((self.web_service_base_url, 'add')), + json=mapping) + if response.status_code != 200: + self.logger.error( + "Could not store mapping, web service returned status code %s from URL %s. " + "Response: %s", response.status_code, response.url, response.text) + + +def make_argument_parser(parser=None, **kwargs): + """Make and return an argparse.""" + + if parser is None: + parser = argparse.ArgumentParser(**kwargs) + + parser.add_argument('--version') + parser.add_argument('--client-id') + parser.add_argument('--client-secret') + parser.add_argument('--variant') + parser.add_argument('--web-service-base-url', default="") + return parser + + +def main(options): + """Execute mapper here. Main entry point.""" + + mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id, + client_secret=options.client_secret, + web_service_base_url=options.web_service_base_url) + + # when used as a context manager, mapper instance automatically cleans files/folders after finishing its job. + # in other cases, mapper.cleanup() method should be called manually. + with mapper: + mapper.run() + + +if __name__ == '__main__': + mapper_options = make_argument_parser(description=__doc__).parse_args() + main(mapper_options) diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 1f17c7b7485..0c70352eff4 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -321,7 +321,7 @@ class PathResolver(object): return None else: data = response.json().get('data', {}) - path, binary_name = data.get('debug_symbols_url'), data.get('binary_name') + path, binary_name = data.get('debug_symbols_url'), data.get('file_name') except Exception as err: # noqa pylint: disable=broad-except sys.stderr.write(f"Error occurred while trying to get response from server " f"for buildId({build_id}): {err}\n") @@ -344,7 +344,14 @@ class PathResolver(object): path = self.unpack(dl_path) except Exception as err: # noqa pylint: disable=broad-except sys.stderr.write(f"Failed to download & unpack file: {err}\n") - return os.path.join(path, f'{binary_name}.debug') + # we may have '<name>.debug', '<name>.so' or just executable binary file which may not have file 'extension'. + # if file has extension, it is good. if not, we should append .debug, because those without extension are + # from release builds, and their debug symbol files contain .debug extension. + # we need to map those 2 different file names ('<name>' becomes '<name>.debug'). + if not binary_name.endswith('.debug') and not binary_name.endswith('.so'): + binary_name = f'{binary_name}.debug' + + return os.path.join(path, binary_name) def parse_input(trace_doc, dbg_path_resolver): diff --git a/etc/evergreen.yml b/etc/evergreen.yml index c3d58fbcfb8..092216dc802 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -6921,6 +6921,22 @@ tasks: content_type: text/plain display_name: Last Push Date (seconds since epoch) +- name: generate_buildid_to_debug_symbols_mapping + tags: ["symbolizer"] + stepback: false + patchable: true + depends_on: + - archive_dist_test_debug + commands: + - *f_expansions_write + - func: "do setup" + - func: "configure evergreen api credentials" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/generate_buildid_debug_symbols_mapping.sh" + ####################################### # Task Groups # ####################################### @@ -7184,6 +7200,7 @@ buildvariants: ########################################### # Linux buildvariants # ########################################### + - &linux-64-debug-required-template name: linux-64-debug-required display_name: "! Shared Library Linux DEBUG" @@ -7245,6 +7262,7 @@ buildvariants: - name: server_selection_json_test_TG distros: - rhel80-large + - name: generate_buildid_to_debug_symbols_mapping - <<: *linux-64-debug-required-template name: linux-64-debug-wtdevelop @@ -7290,6 +7308,7 @@ buildvariants: - name: failpoints_auth - name: .jscore .common !.sharding !.decimal !.txns - name: .jstestfuzz .common !.sharding !.repl + - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu1804 display_name: Ubuntu 18.04 @@ -7348,6 +7367,7 @@ buildvariants: - name: .publish distros: - ubuntu1804-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu1804-64 display_name: Enterprise Ubuntu 18.04 @@ -7421,6 +7441,7 @@ buildvariants: - name: .publish distros: - ubuntu1804-small + - name: generate_buildid_to_debug_symbols_mapping - name: tla-plus display_name: TLA+ @@ -7486,6 +7507,7 @@ buildvariants: - name: .publish distros: - ubuntu1804-test + - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu1804-arm64 display_name: Ubuntu 18.04 arm64 @@ -7522,6 +7544,7 @@ buildvariants: - name: .publish distros: - ubuntu1804-test + - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu2004 display_name: Ubuntu 20.04 @@ -7574,6 +7597,7 @@ buildvariants: - name: .publish distros: - ubuntu2004-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu2004-64 display_name: Enterprise Ubuntu 20.04 @@ -7627,6 +7651,7 @@ buildvariants: - name: .publish distros: - ubuntu2004-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-ubuntu2004-arm64 display_name: Enterprise Ubuntu 20.04 arm64 @@ -7680,6 +7705,7 @@ buildvariants: - name: .publish distros: - ubuntu2004-test + - name: generate_buildid_to_debug_symbols_mapping - name: ubuntu2004-arm64 display_name: Ubuntu 20.04 arm64 @@ -7715,6 +7741,7 @@ buildvariants: - name: .publish distros: - ubuntu2004-test + - name: generate_buildid_to_debug_symbols_mapping - &enterprise-rhel80-dynamic-v4gcc-debug-experimental-template name: enterprise-rhel80-dynamic-v4gcc-debug-experimental @@ -8035,6 +8062,7 @@ buildvariants: - name: .publish distros: - amazon1-2018-small + - name: generate_buildid_to_debug_symbols_mapping - name: amazon display_name: Amazon Linux @@ -8090,6 +8118,7 @@ buildvariants: - name: .publish distros: - amazon1-2018-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-amazon2 display_name: "Enterprise Amazon Linux 2" @@ -8147,6 +8176,7 @@ buildvariants: - name: .publish distros: - amazon2-small + - name: generate_buildid_to_debug_symbols_mapping - name: amazon2 display_name: Amazon Linux 2 @@ -8202,6 +8232,7 @@ buildvariants: - name: .publish distros: - amazon2-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-amazon2-arm64 display_name: "Enterprise Amazon Linux 2 arm64" @@ -8259,6 +8290,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping - name: amazon2-arm64 display_name: Amazon Linux 2 arm64 @@ -8309,6 +8341,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping - name: stm-daily-cron modules: @@ -9168,6 +9201,7 @@ buildvariants: - ubuntu2004-package - name: selinux_rhel8_enterprise - name: .publish + - name: generate_buildid_to_debug_symbols_mapping - &enterprise-rhel-80-64-bit-dynamic-required-template name: enterprise-rhel-80-64-bit-dynamic-required @@ -9615,6 +9649,7 @@ buildvariants: - name: sharded_multi_stmt_txn_jscore_passthrough distros: - rhel80-medium + - name: generate_buildid_to_debug_symbols_mapping # This build variant is used to run multiversion tests as part of burn_in_tags as these tests are # currently only run on our daily builders. @@ -9756,6 +9791,7 @@ buildvariants: - name: .publish distros: - rhel70-small + - name: generate_buildid_to_debug_symbols_mapping - name: ubi8 display_name: "UBI 8" @@ -9839,6 +9875,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-82-arm64 display_name: "Enterprise RHEL 8.2 arm64" @@ -9894,6 +9931,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping # This variant is to intentionally test uncommon features nightly - <<: *enterprise-rhel-70-64-bit-template @@ -10077,6 +10115,7 @@ buildvariants: - name: .publish distros: - rhel70-small + - name: generate_buildid_to_debug_symbols_mapping - name: rhel80 display_name: RHEL 8.0 @@ -10129,6 +10168,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping - name: rhel-82-arm64 display_name: RHEL 8.2 arm64 @@ -10178,6 +10218,7 @@ buildvariants: - name: .publish distros: - rhel80-small + - name: generate_buildid_to_debug_symbols_mapping # This variant compiles on RHEL 7.0 and runs tests on RHEL 7.6 - name: rhel76_compile_rhel70 @@ -10246,6 +10287,7 @@ buildvariants: - name: .publish distros: - rhel70-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-72-s390x-compile display_name: Enterprise RHEL 7.2 s390x Compile @@ -10276,6 +10318,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-72-s390x display_name: Enterprise RHEL 7.2 s390x @@ -10327,6 +10370,7 @@ buildvariants: - name: .publish distros: - rhel70-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-rhel-72-s390x-all-feature-flags display_name: Enterprise RHEL 7.2 s390x (all feature flags) @@ -10361,7 +10405,7 @@ buildvariants: - name: build_variant_gen - name: jsCore - name: replica_sets_jscore_passthrough - + - name: generate_buildid_to_debug_symbols_mapping ########################################### # Ubuntu buildvariants # @@ -10439,6 +10483,7 @@ buildvariants: - name: .publish distros: - suse12-small + - name: generate_buildid_to_debug_symbols_mapping - name: suse12 display_name: SUSE 12 @@ -10490,6 +10535,7 @@ buildvariants: - name: .publish distros: - suse12-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-suse15-64 display_name: Enterprise SLES 15 @@ -10532,6 +10578,7 @@ buildvariants: - name: .publish distros: - suse15-small + - name: generate_buildid_to_debug_symbols_mapping - name: suse15 display_name: SUSE 15 @@ -10582,6 +10629,7 @@ buildvariants: - name: .publish distros: - suse15-small + - name: generate_buildid_to_debug_symbols_mapping ########################################### # Debian buildvariants # @@ -10635,6 +10683,7 @@ buildvariants: - name: .publish distros: - debian92-small + - name: generate_buildid_to_debug_symbols_mapping - name: debian92 display_name: Debian 9.2 @@ -10688,6 +10737,7 @@ buildvariants: - name: .publish distros: - debian92-small + - name: generate_buildid_to_debug_symbols_mapping - name: enterprise-debian10-64 display_name: Enterprise Debian 10 @@ -10741,6 +10791,7 @@ buildvariants: - name: .publish distros: - debian10-small + - name: generate_buildid_to_debug_symbols_mapping - name: debian10 display_name: Debian 10 @@ -10795,6 +10846,7 @@ buildvariants: - name: .publish distros: - debian10-small + - name: generate_buildid_to_debug_symbols_mapping ################################ # storage engine buildvariants # diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh new file mode 100644 index 00000000000..d1866e961a5 --- /dev/null +++ b/evergreen/generate_buildid_debug_symbols_mapping.sh @@ -0,0 +1,15 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +activate_venv + +$python buildscripts/debugsymb_mapper.py \ + --version "${version_id}" \ + --client-id "${symbolizer_client_id}" \ + --client-secret "${symbolizer_client_secret}" \ + --variant "${build_variant}" |