diff options
author | Daniel Moody <daniel.moody@mongodb.com> | 2020-12-30 18:12:51 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-02-24 17:04:59 +0000 |
commit | a6f50f0c9d2c7afecf13cad922635aa9a2a15cc2 (patch) | |
tree | b86f660738bad4d22b53bf25439876b479bc26fb | |
parent | 6ca44ce30689b3fe93d3afe75f07c70a0fdc3d8d (diff) | |
download | mongo-a6f50f0c9d2c7afecf13cad922635aa9a2a15cc2.tar.gz |
SERVER-53531 added evergreen task for libdeps graph generation
-rwxr-xr-x | buildscripts/libdeps/gacli.py | 62 | ||||
-rw-r--r-- | buildscripts/libdeps/graph_analyzer.py | 7 | ||||
-rw-r--r-- | etc/evergreen.yml | 64 |
3 files changed, 98 insertions, 35 deletions
diff --git a/buildscripts/libdeps/gacli.py b/buildscripts/libdeps/gacli.py index 9121702c5bd..b01ec5b0d4f 100755 --- a/buildscripts/libdeps/gacli.py +++ b/buildscripts/libdeps/gacli.py @@ -34,7 +34,7 @@ from pathlib import Path import networkx import graph_analyzer - +from libdeps_graph_enums import CountTypes, LinterTypes class LinterSplitArgs(argparse.Action): """Custom argument action for checking multiple choice comma separated list.""" @@ -58,17 +58,13 @@ class LinterSplitArgs(argparse.Action): class CountSplitArgs(LinterSplitArgs): """Special case of common custom arg action for Count types.""" - valid_choices = [ - name[0].replace('_', '-') for name in graph_analyzer.CountTypes.__members__.items() - ] + valid_choices = [name[0].replace('_', '-') for name in CountTypes.__members__.items()] class LintSplitArgs(LinterSplitArgs): """Special case of common custom arg action for Count types.""" - valid_choices = [ - name[0].replace('_', '-') for name in graph_analyzer.LinterTypes.__members__.items() - ] + valid_choices = [name[0].replace('_', '-') for name in LinterTypes.__members__.items()] class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter): @@ -85,30 +81,30 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe def _get_help_string(self, action): if isinstance(action, CountSplitArgs): - count_help = self._get_help_length(graph_analyzer.CountTypes) + count_help = self._get_help_length(CountTypes) return textwrap.dedent(f"""\ {action.help} default: all, choices: - {count_help[graph_analyzer.CountTypes.all.name]}perform all counts - {count_help[graph_analyzer.CountTypes.node.name]}count nodes - {count_help[graph_analyzer.CountTypes.edge.name]}count edges - {count_help[graph_analyzer.CountTypes.dir_edge.name]}count edges declared directly on a node - {count_help[graph_analyzer.CountTypes.trans_edge.name]}count edges induced by direct public edges - {count_help[graph_analyzer.CountTypes.dir_pub_edge.name]}count edges that are directly public - {count_help[graph_analyzer.CountTypes.pub_edge.name]}count edges that are public - {count_help[graph_analyzer.CountTypes.priv_edge.name]}count edges that are private - {count_help[graph_analyzer.CountTypes.if_edge.name]}count edges that are interface - {count_help[graph_analyzer.CountTypes.shim.name]}count shim nodes - {count_help[graph_analyzer.CountTypes.lib.name]}count library nodes - {count_help[graph_analyzer.CountTypes.prog.name]}count program nodes + {count_help[CountTypes.all.name]}perform all counts + {count_help[CountTypes.node.name]}count nodes + {count_help[CountTypes.edge.name]}count edges + {count_help[CountTypes.dir_edge.name]}count edges declared directly on a node + {count_help[CountTypes.trans_edge.name]}count edges induced by direct public edges + {count_help[CountTypes.dir_pub_edge.name]}count edges that are directly public + {count_help[CountTypes.pub_edge.name]}count edges that are public + {count_help[CountTypes.priv_edge.name]}count edges that are private + {count_help[CountTypes.if_edge.name]}count edges that are interface + {count_help[CountTypes.shim.name]}count shim nodes + {count_help[CountTypes.lib.name]}count library nodes + {count_help[CountTypes.prog.name]}count program nodes """) elif isinstance(action, LintSplitArgs): - count_help = self._get_help_length(graph_analyzer.LinterTypes) + count_help = self._get_help_length(LinterTypes) return textwrap.dedent(f"""\ {action.help} default: all, choices: - {count_help[graph_analyzer.LinterTypes.all.name]}perform all linters - {count_help[graph_analyzer.LinterTypes.node.name]}find unnecessary public libdeps + {count_help[LinterTypes.all.name]}perform all linters + {count_help[LinterTypes.node.name]}find unnecessary public libdeps """) return super()._get_help_string(action) @@ -124,23 +120,27 @@ def setup_args_parser(): parser.add_argument('--format', choices=['pretty', 'json'], default='pretty', help="The output format type.") - parser.add_argument('--build-data', action='store_true', + parser.add_argument('--build-data', choices=['on', 'off'], default='on', help="Print the invocation and git hash used to build the graph") - parser.add_argument('--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs, - help="Output various counts from the graph. Comma separated list.") + parser.add_argument( + '--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs, + default=[name[0] for name in CountTypes.__members__.items() if name[0] != 'all'], + help="Output various counts from the graph. Comma separated list.") - parser.add_argument('--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs, - help="Perform various linters on the graph. Comma separated list.") + parser.add_argument( + '--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs, + default=[name[0] for name in LinterTypes.__members__.items() if name[0] != 'all'], + help="Perform various linters on the graph. Comma separated list.") - parser.add_argument('--direct-depends', action='append', + parser.add_argument('--direct-depends', action='append', default=[], help="Print the nodes which depends on a given node.") - parser.add_argument('--common-depends', nargs='+', action='append', + parser.add_argument('--common-depends', nargs='+', action='append', default=[], help="Print the nodes which have a common dependency on all N nodes.") parser.add_argument( - '--exclude-depends', nargs='+', action='append', help= + '--exclude-depends', nargs='+', action='append', default=[], help= "Print nodes which depend on the first node of N nodes, but exclude all nodes listed there after." ) diff --git a/buildscripts/libdeps/graph_analyzer.py b/buildscripts/libdeps/graph_analyzer.py index 52defa1ab32..c9138b2dc63 100644 --- a/buildscripts/libdeps/graph_analyzer.py +++ b/buildscripts/libdeps/graph_analyzer.py @@ -486,10 +486,9 @@ class UnusedPublicLinter(Analyzer): if (edge_attribs.get(EdgeProps.direct.name) and edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public) - and self.graph[edge[0]].get(NodeProps.shim.name) - and self.graph[edge[0]].get(NodeProps.bin_type.name) == 'SharedLibrary'): - - # First we will get all the transitive libdeps the dependnet node + and not self.graph.nodes()[edge[0]].get(NodeProps.shim.name) and + self.graph.nodes()[edge[1]].get(NodeProps.bin_type.name) == 'SharedLibrary'): + # First we will get all the transitive libdeps the dependent node # induces, while we are getting those we also check if the depender # node has any symbol dependencies to that transitive libdep. trans_pub_nodes = set([edge[0]]) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 7896d83c0d1..17e2768860a 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -3428,6 +3428,62 @@ tasks: targets: install-core +- name: libdeps_graph_linting + commands: + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + ${activate_virtualenv} + python -m pip install networkx + + - func: "scons compile" + vars: + task_install_action: + default + task_compile_flags: >- + --link-model=dynamic + --build-tools=next + targets: + generate-libdeps-graph + + - command: shell.exec + params: + working_dir: src + shell: bash + script: | + set -o errexit + set -o verbose + + ${activate_virtualenv} + GRAPH_FILE=$(find build -name "libdeps.graphml") + python buildscripts/libdeps/gacli.py --graph-file $GRAPH_FILE | gzip > results.txt.gz + gzip $GRAPH_FILE + mv $GRAPH_FILE.gz . + + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/results.txt.gz + remote_file: ${project}/${build_variant}/${revision}/sources/libdeps-${build_id}-results.txt.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: libdeps results tarball + + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/libdeps.graphml.gz + remote_file: ${project}/${build_variant}/${revision}/sources/libdeps-${build_id}-graph.graphml.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: libdeps graph tarball + ## compile_all - build all scons targets ## - name: compile_all depends_on: @@ -8265,6 +8321,11 @@ task_groups: - compile_build_tools_next - <<: *compile_task_group_template + name: libdeps_graph_linting_TG + tasks: + - libdeps_graph_linting + +- <<: *compile_task_group_template name: compile_ninja_TG tasks: - compile_ninja @@ -8685,6 +8746,9 @@ buildvariants: - name: compile_ninja_next_TG distros: - ubuntu1804-build + - name: libdeps_graph_linting_TG + distros: + - ubuntu1804-build - name: compile_ninja_TG - name: .aggfuzzer .common - name: audit |