diff options
author | Mark Benvenuto <mark.benvenuto@mongodb.com> | 2020-10-12 15:19:03 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-10-12 21:50:32 +0000 |
commit | bfd4d609f71b3ef7e7ef6947486ceb54da26ec2b (patch) | |
tree | 953b98f096630565a5e64247ec2624c74c2e38a4 /buildscripts/blackduck_hub.py | |
parent | 3a5d55eb20d7e5593e77827c9a24aafab6d0952b (diff) | |
download | mongo-bfd4d609f71b3ef7e7ef6947486ceb54da26ec2b.tar.gz |
SERVER-51034 Write Upgrade Software Report for Black Duck
Diffstat (limited to 'buildscripts/blackduck_hub.py')
-rw-r--r-- | buildscripts/blackduck_hub.py | 339 |
1 files changed, 306 insertions, 33 deletions
diff --git a/buildscripts/blackduck_hub.py b/buildscripts/blackduck_hub.py index dfffde6eb5b..3227f3c772e 100644 --- a/buildscripts/blackduck_hub.py +++ b/buildscripts/blackduck_hub.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 """Utility script to run Black Duck scans and query Black Duck database.""" +#pylint: disable=too-many-lines import argparse import datetime +import functools import io import json import logging @@ -46,6 +48,17 @@ BLACKDUCK_TIMEOUT_SECS = 600 # Black Duck hub api uses this file to get settings BLACKDUCK_RESTCONFIG = ".restconfig.json" +# Wiki page where we document more information about Black Duck +BLACKDUCK_WIKI_PAGE = "https://wiki.corp.mongodb.com/display/KERNEL/Black+Duck" + +# Black Duck failed report prefix +BLACKDUCK_FAILED_PREFIX = "A Black Duck scan was run and failed" + +############################################################################ + +# Globals +BLACKDUCK_PROJECT_URL = None + ############################################################################ # Build Logger constants @@ -96,7 +109,8 @@ class HTTPHandler(object): self.url_root = url_root - def _make_url(self, endpoint): + def make_url(self, endpoint): + """Generate a url to post to.""" return "%s/%s/" % (self.url_root.rstrip("/"), endpoint.strip("/")) def post(self, endpoint, data=None, headers=None, timeout_secs=BUILD_LOGGER_TIMEOUT_SECS): @@ -113,7 +127,7 @@ class HTTPHandler(object): headers = default_if_none(headers, {}) headers["Content-Type"] = "application/json; charset=utf-8" - url = self._make_url(endpoint) + url = self.make_url(endpoint) LOGGER.info("POSTING to %s", url) @@ -225,6 +239,8 @@ class BuildloggerServer(object): except: # pylint: disable=bare-except raise ValueError("Encountered an error.") + return self.handler.make_url(endpoint) + def _to_dict(items, func): dm = {} @@ -250,6 +266,154 @@ def _compute_security_risk(security_risk_profile): return "OK" +@functools.total_ordering +class VersionInfo: + """Parse and break apart version strings so they can be compared.""" + + def __init__(self, ver_str): + """Parse a version string input a tuple of ints or mark it as a beta release.""" + try: + + self.ver_str = ver_str + self.production_version = True + + # Abseil has an empty string for one version + if self.ver_str == "": + self.production_version = False + return + + # Special case Intel's Decimal library since it is just too weird + if ver_str == "v2.0 U1": + self.ver_array = [2, 0] + return + + # "git" is an abseil version + # "unknown_version" comes from this script when components do not have versions + # icu has cldr, release, snv, milestone, latest + # zlib has alt and task + # boost has ubuntu, fc, old-boost, start, .... + # BlackDuck thinks boost 1.70.0 was released on 2007 which means we have to check hundreds of versions + bad_keywords = [ + "unknown_version", "rc", "alpha", "beta", "git", "release", "cldr", "svn", "cvs", + "milestone", "latest", "alt", "task", "ubuntu", "fc", "old-boost", "start", "split", + "unofficial", "(", "ctp", "before", "review", "develop", "master", "filesystem", + "geometry", "icl", "intrusive", "old", "optional", "super", "docs", "+b", "-b", + "b1", ".0a", "system", "html", "interprocess" + ] + if [bad for bad in bad_keywords if bad in self.ver_str]: + self.production_version = False + return + + # Clean the version information + # Some versions start with 'v'. Some components have a mix of 'v' and not 'v' prefixed versions so trim the 'v' + # MongoDB versions start with 'r' + if ver_str[0] == 'v' or ver_str[0] == 'r': + self.ver_str = ver_str[1:] + + # Git hashes are not valid versions + if len(self.ver_str) == 40 and bytes.fromhex(self.ver_str): + self.production_version = False + return + + # Clean out Mozilla's suffix + self.ver_str = self.ver_str.replace("esr", "") + + # Clean out GPerfTool's prefix + self.ver_str = self.ver_str.replace("gperftools-", "") + + # Clean out Yaml Cpp's prefix + self.ver_str = self.ver_str.replace("yaml-cpp-", "") + + # Clean out Boosts's prefix + self.ver_str = self.ver_str.replace("boost-", "") + self.ver_str = self.ver_str.replace("asio-", "") + + if self.ver_str.endswith('-'): + self.ver_str = self.ver_str[0:-1] + + # Some versions end with "-\d", change the "-" since it just means a patch release from a debian/rpm package + # yaml-cpp has this problem where Black Duck sourced the wrong version information + self.ver_str = self.ver_str.replace("-", ".") + + # Versions are generally a multi-part integer tuple + self.ver_array = [int(part) for part in self.ver_str.split(".")] + + except: + LOGGER.error("Failed to parse version '%s', exception", ver_str) + raise + + def __repr__(self): + return self.__str__() + + def __str__(self): + return ".".join([str(val) for val in self.ver_array]) + + def __eq__(self, other): + return (self.production_version, self.ver_array) == (other.production_version, + other.ver_array) + + def __gt__(self, other): + if self.production_version != other.production_version: + return self.production_version + + return self.ver_array > other.ver_array + + +def _test_version_info(): + VersionInfo("v2.0 U1") + VersionInfo("60.7.0-esr") + VersionInfo("v1.1") + VersionInfo("0.4.2-1") + VersionInfo("7.0.2") + VersionInfo("gperftools-2.8") + VersionInfo("v1.5-rc2") + VersionInfo("r4.7.0-alpha") + VersionInfo("r4.2.10") + VersionInfo("2.0.0.1") + VersionInfo("7.0.2-2") + VersionInfo("git") + VersionInfo("20200225.2") + VersionInfo('release-68-alpha') + VersionInfo('cldr/2020-09-22') + VersionInfo('release-67-rc') + VersionInfo('66.1~rc') + VersionInfo('release-66-rc') + VersionInfo('release-66-preview') + VersionInfo('65.1') + VersionInfo('release-65-rc') + VersionInfo('64.2-rc') + VersionInfo('release-64-rc2') + VersionInfo('release-63-rc') + VersionInfo('last-cvs-commit') + VersionInfo('last-svn-commit') + VersionInfo('release-62-rc') + VersionInfo('cldr-32-beta2') + VersionInfo('release-60-rc') + VersionInfo('milestone-60-0-1') + VersionInfo('release-59-rc') + VersionInfo('milestone-59-0-1') + VersionInfo('release-58-2-eclipse-20170118') + VersionInfo('tools-release-58') + VersionInfo('icu-latest') + VersionInfo('icu4j-latest') + VersionInfo('icu4j-release-58-1') + VersionInfo('icu4j-release-58-rc') + VersionInfo('icu-release-58-rc') + VersionInfo('icu-milestone-58-0-1') + VersionInfo('icu4j-milestone-58-0-1') + + VersionInfo('yaml-cpp-0.6.3') + + VersionInfo('gb-c8-task188949.100') + VersionInfo('1.2.8-alt1.M80C.1') + VersionInfo('1.2.8-alt2') + + assert VersionInfo('7.0.2.2') > VersionInfo('7.0.0.1') + assert VersionInfo('7.0.2.2') > VersionInfo('7.0.2') + assert VersionInfo('7.0.2.2') > VersionInfo('3.1') + assert VersionInfo('7.0.2.2') <= VersionInfo('8.0.2') + + class Component: """ Black Duck Component description. @@ -257,7 +421,7 @@ class Component: Contains a subset of information about a component extracted from Black Duck for a given project and version """ - def __init__(self, name, version, licenses, policy_status, security_risk, newer_releases): + def __init__(self, name, version, licenses, policy_status, security_risk, newest_release): # pylint: disable=too-many-arguments """Initialize Black Duck component.""" self.name = name @@ -265,10 +429,10 @@ class Component: self.licenses = licenses self.policy_status = policy_status self.security_risk = security_risk - self.newer_releases = newer_releases + self.newest_release = newest_release @staticmethod - def parse(component): + def parse(hub, component): """Parse a Black Duck component from a dictionary.""" name = component["componentName"] cversion = component.get("componentVersionName", "unknown_version") @@ -277,9 +441,49 @@ class Component: policy_status = component["policyStatus"] security_risk = _compute_security_risk(component['securityRiskProfile']) - newer_releases = component["activityData"].get("newerReleases", None) + newer_releases = component["activityData"].get("newerReleases", 0) + + LOGGER.info("Retrievinng version information for Comp %s - %s Releases %s", name, cversion, + newer_releases) + cver = VersionInfo(cversion) + newest_release = None + + # Blackduck's newerReleases is based on "releasedOn" date. This means that if a upstream component releases a beta or rc, + # it counts as newer but we do not consider those newer for our purposes + # Missing newerReleases means we do not have to upgrade + # TODO - remove skip of FireFox since it has soooo many versions + #if newer_releases > 0 and name not in ("Mozilla Firefox", "Boost C++ Libraries - boost"): - return Component(name, cversion, licenses, policy_status, security_risk, newer_releases) + if newer_releases > 0: + limit = newer_releases + 1 + versions_url = component["component"] + f"/versions?sort=releasedon%20desc&limit={limit}" + + LOGGER.info("Retrieving version information via %s", versions_url) + vjson = hub.execute_get(versions_url).json() + + versions = [(ver["versionName"], ver["releasedOn"]) for ver in vjson["items"]] + + LOGGER.info("Known versions: %s ", versions) + + versions = [ver["versionName"] for ver in vjson["items"]] + + # For Firefox, only examine Extended Service Releases (i.e. esr), their long term support releases + if name == "Mozilla Firefox": + versions = [ver for ver in versions if "esr" in ver] + + ver_info = [VersionInfo(ver) for ver in versions] + ver_info = [ver for ver in ver_info if ver.production_version] + LOGGER.info("Filtered versions: %s ", ver_info) + + ver_info = sorted([ver for ver in ver_info if ver.production_version and ver > cver], + reverse=True) + + LOGGER.info("Sorted versions: %s ", ver_info) + + if ver_info: + newest_release = ver_info[0] + + return Component(name, cversion, licenses, policy_status, security_risk, newest_release) class BlackDuckConfig: @@ -323,6 +527,8 @@ def _scan_cmd_args(args): def _query_blackduck(): + # pylint: disable=global-statement + global BLACKDUCK_PROJECT_URL hub = HubInstance() @@ -335,7 +541,9 @@ def _query_blackduck(): LOGGER.info("Getting version components from blackduck") bom_components = hub.get_version_components(version) - components = [Component.parse(comp) for comp in bom_components["items"]] + components = [Component.parse(hub, comp) for comp in bom_components["items"]] + + BLACKDUCK_PROJECT_URL = version["_meta"]["href"] return components @@ -352,7 +560,7 @@ class TestResultEncoder(json.JSONEncoder): class TestResult: """A single test result in the Evergreen report.json format.""" - def __init__(self, name, status): + def __init__(self, name, status, url): """Init test result.""" # This matches the report.json schema # See https://github.com/evergreen-ci/evergreen/blob/789bee107d3ffb9f0f82ae344d72502945bdc914/model/task/task.go#L264-L284 @@ -362,6 +570,10 @@ class TestResult: self.status = status self.exit_code = 1 + if url: + self.url = url + self.url_raw = url + "?raw=1" + if status == "pass": self.exit_code = 0 @@ -388,7 +600,7 @@ class ReportLogger(object, metaclass=ABCMeta): """Base Class for all report loggers.""" @abstractmethod - def log_report(self, name: str, content: str): + def log_report(self, name: str, content: str) -> Optional[str]: """Get the command to run a linter.""" pass @@ -401,7 +613,7 @@ class LocalReportLogger(ReportLogger): if not os.path.exists(LOCAL_REPORTS_DIR): os.mkdir(LOCAL_REPORTS_DIR) - def log_report(self, name: str, content: str): + def log_report(self, name: str, content: str) -> Optional[str]: """Log report to a local file.""" file_name = os.path.join(LOCAL_REPORTS_DIR, name + ".log") @@ -418,12 +630,12 @@ class BuildLoggerReportLogger(ReportLogger): self.build_id = self.build_logger.new_build_id("bdh") - def log_report(self, name: str, content: str): + def log_report(self, name: str, content: str) -> Optional[str]: """Log report to a build logger.""" content = content.split("\n") - self.build_logger.post_new_file(self.build_id, name, content) + return self.build_logger.post_new_file(self.build_id, name, content) def _get_default(list1, idx, default): @@ -449,9 +661,10 @@ class TableWriter: def _write_row(col_sizes: [int], row: [str], writer: io.StringIO): writer.write("|") for idx, row_value in enumerate(row): + writer.write(" ") writer.write(row_value) writer.write(" " * (col_sizes[idx] - len(row_value))) - writer.write("|") + writer.write(" |") writer.write("\n") def print(self, writer: io.StringIO): @@ -507,35 +720,44 @@ class ReportManager: self._results = TestResults() self._data = TableData() + @staticmethod + def _get_norm_comp_name(comp_name: str): + return comp_name.replace(" ", "_").replace("/", "_").lower() + def write_report(self, comp_name: str, report_name: str, status: str, content: str): """ Write a report about a test to the build logger. status is a string of "pass" or "fail" """ - comp_name = comp_name.replace(" ", "_").replace("/", "_") + comp_name = ReportManager._get_norm_comp_name(comp_name) name = comp_name + "_" + report_name LOGGER.info("Writing Report %s - %s", name, status) - self._results.add_result(TestResult(name, status)) self._data.add_value(comp_name, status) - # TODO - evaluate whether to wrap lines if that would look better in BFs - # The textwrap module strips empty lines by default + url = self._logger.log_report(name, content) + + self._results.add_result(TestResult(name, status, url)) + + def add_report_metric(self, comp_name: str, metric: str): + """Add a column to be included in the pretty table.""" + comp_name = ReportManager._get_norm_comp_name(comp_name) - self._logger.log_report(name, content) + self._data.add_value(comp_name, metric) def finish(self, reports_file: Optional[str]): """Generate final summary of all reports run.""" if reports_file: - self.results.write(reports_file) + self._results.write(reports_file) stream = io.StringIO() - self._data.write(["Component", "Vulnerability"], stream) + self._data.write( + ["Component", "Vulnerability", "Upgrade", "Current Version", "Newest Version"], stream) print(stream.getvalue()) @@ -553,6 +775,7 @@ class ThirdPartyComponent: # optional fields self.vulnerability_suppression = None + self.upgrade_suppression = None def _get_field(name, ymap, field: str): @@ -576,12 +799,46 @@ def _read_third_party_components(): _get_field(comp, cmap, 'team_owner')) tp.vulnerability_suppression = cmap.get("vulnerability_suppression", None) + tp.upgrade_suppression = cmap.get("upgrade_suppression", None) third_party.append(tp) return third_party +def _generate_report_upgrade(mgr: ReportManager, comp: Component, mcomp: ThirdPartyComponent, + fail: bool): + if not fail: + mgr.write_report(comp.name, "upgrade_check", "pass", "Blackduck run passed") + else: + mgr.write_report( + comp.name, "upgrade_check", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The {comp.name} library at {mcomp.local_path} is out of date. The current version is +"{comp.version}" and the newest version is "{comp.newest_release}" according to Black Duck. + +MongoDB policy requires all third-party software to be updated to the latest version on the master +branch. + +Next Steps: + +Build Baron: A BF ticket should be generated and assigned to "Service Development Platform" with this text. + +Developer: To address this build failure, the next steps are as follows: +1. File a SERVER ticket to update the software if one already does not exist. +2. Add a “upgrade_supression” to {THIRD_PARTY_COMPONENTS_FILE} with the SERVER ticket to acknowledge + this report. Note that you do not need to immediately update the library, just file a ticket. + +If you believe the library is already up-to-date but Black Duck has the wrong version, please update +version information for this component at {BLACKDUCK_PROJECT_URL}. + +For more information, {BLACKDUCK_WIKI_PAGE}. +""") + + mgr.add_report_metric(comp.name, str(comp.version)) + mgr.add_report_metric(comp.name, str(comp.newest_release)) + + def _generate_report_vulnerability(mgr: ReportManager, comp: Component, mcomp: ThirdPartyComponent, fail: bool): if not fail: @@ -589,25 +846,28 @@ def _generate_report_vulnerability(mgr: ReportManager, comp: Component, mcomp: T return mgr.write_report( - comp.name, "vulnerability_check", "fail", f"""A Black Duck scan was run and failed. + comp.name, "vulnerability_check", "fail", f"""{BLACKDUCK_FAILED_PREFIX} -The ${comp.name} library had HIGH and/or CRITICAL security issues. The current version in Black Duck is ${comp.version}. +The {comp.name} library at {mcomp.local_path} had HIGH and/or CRITICAL security issues. The current +version in Black Duck is "{comp.version}". -MongoDB policy requires all third-party software to be updated to a version clean of HIGH and CRITICAL vulnerabilities on the master branch. +MongoDB policy requires all third-party software to be updated to a version clean of HIGH and +CRITICAL vulnerabilities on the master branch. Next Steps: -Build Baron: -A BF ticket should be generated and assigned to ${mcomp.team_owner} with this text. +Build Baron: A BF ticket should be generated and assigned to "Service Development Platform" with this text. -Developer: -To address this build failure, the next steps are as follows: -1. File a SERVER ticket to update the software if one already does not exist. -2. Add a “vulnerability_supression” to etc/third_party_components.yml with the SERVER ticket +Developer: To address this build failure, the next steps are as follows: +1. File a SERVER ticket to update the software if one already does not exist. Note that you do not + need to immediately update the library, just file a ticket. +2. Add a “vulnerability_supression” to {THIRD_PARTY_COMPONENTS_FILE} with the SERVER ticket to + acknowledge this report. -If you believe the library is already up-to-date but Black Duck has the wrong version, you will need to update the Black Duck configuration. +If you believe the library is already up-to-date but Black Duck has the wrong version, please update +version information for this component at {BLACKDUCK_PROJECT_URL}. -Note that you do not need to immediately update the library. For more information, https://wiki.corp.mongodb.com/Black Duck. + For more information, {BLACKDUCK_WIKI_PAGE}. """) @@ -631,10 +891,21 @@ class Analyzer: # 1. Validate there are no security issues self._verify_vulnerability_status(comp) + # 2. Check for upgrade issues + self._verify_upgrade_status(comp) + + def _verify_upgrade_status(self, comp: Component): + mcomp = self._get_mongo_component(comp) + + if comp.newest_release and not mcomp.upgrade_suppression: + _generate_report_upgrade(self.mgr, comp, mcomp, True) + else: + _generate_report_upgrade(self.mgr, comp, mcomp, False) + def _verify_vulnerability_status(self, comp: Component): mcomp = self._get_mongo_component(comp) - if comp.security_risk in ["HIGH", "CRITICAL"]: + if comp.security_risk in ["HIGH", "CRITICAL"] and not mcomp.vulnerability_suppression: _generate_report_vulnerability(self.mgr, comp, mcomp, True) else: _generate_report_vulnerability(self.mgr, comp, mcomp, False) @@ -768,6 +1039,8 @@ def main() -> None: args = parser.parse_args() + _test_version_info() + if args.debug: logging.basicConfig(level=logging.DEBUG) elif args.verbose: |