diff options
author | Daniel Moody <daniel.moody@mongodb.com> | 2021-03-30 20:23:00 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-04-15 16:34:40 +0000 |
commit | 0782c59e71146e316e7d61ac61d76e2330f29eb4 (patch) | |
tree | 75eebd697434aaaeacf0771cc49e43e30857dcc3 /buildscripts/libdeps | |
parent | 53230099fc21fe4c4bf60e26420ca2e839ec22f1 (diff) | |
download | mongo-0782c59e71146e316e7d61ac61d76e2330f29eb4.tar.gz |
SERVER-52568 Added GraphPaths, CriticalEdges, and PublicWieghts quieries to libdeps graph analyzer.
Diffstat (limited to 'buildscripts/libdeps')
-rw-r--r-- | buildscripts/libdeps/analyzer_unittests.py | 448 | ||||
-rwxr-xr-x | buildscripts/libdeps/gacli.py | 121 | ||||
-rw-r--r-- | buildscripts/libdeps/graph_visualizer.py | 6 | ||||
-rw-r--r-- | buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py | 75 | ||||
-rw-r--r-- | buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js | 41 | ||||
-rw-r--r-- | buildscripts/libdeps/libdeps/analyzer.py | 486 | ||||
-rw-r--r-- | buildscripts/libdeps/libdeps/graph.py | 131 |
7 files changed, 1056 insertions, 252 deletions
diff --git a/buildscripts/libdeps/analyzer_unittests.py b/buildscripts/libdeps/analyzer_unittests.py new file mode 100644 index 00000000000..8061f23e27d --- /dev/null +++ b/buildscripts/libdeps/analyzer_unittests.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +"""Unittests for the graph analyzer.""" + +import json +import sys +import unittest + +import networkx + +import libdeps.analyzer +from libdeps.graph import LibdepsGraph, EdgeProps, NodeProps, CountTypes + + +def add_node(graph, node, builder, shim): + """Add a node to the graph.""" + + graph.add_nodes_from([(node, {NodeProps.bin_type.name: builder, NodeProps.shim.name: shim})]) + + +def add_edge(graph, from_node, to_node, **kwargs): + """Add an edge to the graph.""" + + edge_props = { + EdgeProps.direct.name: kwargs[EdgeProps.direct.name], + EdgeProps.visibility.name: int(kwargs[EdgeProps.visibility.name]), + } + + graph.add_edges_from([(from_node, to_node, edge_props)]) + + +def get_double_diamond_mock_graph(): + """Construct a mock graph which covers a double diamond structure.""" + + graph = LibdepsGraph() + graph.graph['build_dir'] = '.' + graph.graph['graph_schema_version'] = 2 + graph.graph['deptypes'] = '''{ + "Global": 0, + "Public": 1, + "Private": 2, + "Interface": 3, + "Typeinfo": 4 + }''' + + # builds a graph of mostly public edges that looks like this: + # + # + # /lib3.so /lib7.so + # | \ | \ + # <-lib1.so--lib2.so lib5.so--lib6.so lib9.so + # | / | / + # \lib4.so \lib8.so + # + + add_node(graph, 'lib1.so', 'SharedLibrary', False) + add_node(graph, 'lib2.so', 'SharedLibrary', False) + add_node(graph, 'lib3.so', 'SharedLibrary', False) + add_node(graph, 'lib4.so', 'SharedLibrary', False) + add_node(graph, 'lib5.so', 'SharedLibrary', False) + add_node(graph, 'lib6.so', 'SharedLibrary', False) + add_node(graph, 'lib7.so', 'SharedLibrary', False) + add_node(graph, 'lib8.so', 'SharedLibrary', False) + add_node(graph, 'lib9.so', 'SharedLibrary', False) + + add_edge(graph, 'lib1.so', 'lib2.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib3.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib4.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib5.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib6.so', 'lib7.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib6.so', 'lib8.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib7.so', 'lib9.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib8.so', 'lib9.so', direct=True, visibility=graph.get_deptype('Public')) + + # trans for 3 and 4 + add_edge(graph, 'lib1.so', 'lib3.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib1.so', 'lib4.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 5 + add_edge(graph, 'lib1.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 6 + add_edge(graph, 'lib1.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 7 + add_edge(graph, 'lib1.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib5.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 8 + add_edge(graph, 'lib1.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib5.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 9 + add_edge(graph, 'lib1.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib5.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib6.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public')) + + return graph + + +def get_basic_mock_graph(): + """Construct a mock graph which covers most cases and is easy to understand.""" + + graph = LibdepsGraph() + graph.graph['build_dir'] = '.' + graph.graph['graph_schema_version'] = 2 + graph.graph['deptypes'] = '''{ + "Global": 0, + "Public": 1, + "Private": 2, + "Interface": 3, + "Typeinfo": 4 + }''' + + # builds a graph of mostly public edges: + # + # /-lib5.so + # /lib3.so + # | \-lib6.so + # <-lib1.so--lib2.so + # | /-lib5.so (private) + # \lib4.so + # \-lib6.so + + # nodes + add_node(graph, 'lib1.so', 'SharedLibrary', False) + add_node(graph, 'lib2.so', 'SharedLibrary', False) + add_node(graph, 'lib3.so', 'SharedLibrary', False) + add_node(graph, 'lib4.so', 'SharedLibrary', False) + add_node(graph, 'lib5.so', 'SharedLibrary', False) + add_node(graph, 'lib6.so', 'SharedLibrary', False) + + # direct edges + add_edge(graph, 'lib1.so', 'lib2.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib3.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib2.so', 'lib4.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib3.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib4.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Private')) + + # trans for 3 + add_edge(graph, 'lib1.so', 'lib3.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 4 + add_edge(graph, 'lib1.so', 'lib4.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 5 + add_edge(graph, 'lib2.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib1.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public')) + + # trans for 6 + add_edge(graph, 'lib2.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + add_edge(graph, 'lib1.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public')) + + return graph + + +class Tests(unittest.TestCase): + """Common unittest for the libdeps graph analyzer module.""" + + def run_analysis(self, expected, graph, algo, *args): + """Check results of analysis generically.""" + + analysis = [algo(graph, *args)] + ga = libdeps.analyzer.LibdepsGraphAnalysis(graph, analysis) + printer = libdeps.analyzer.GaJsonPrinter(ga) + result = json.loads(printer.get_json()) + self.assertEqual(result, expected) + + def run_counts(self, expected, graph): + """Check results of counts generically.""" + + analysis = libdeps.analyzer.counter_factory( + graph, + [name[0] for name in CountTypes.__members__.items() if name[0] != CountTypes.ALL.name]) + ga = libdeps.analyzer.LibdepsGraphAnalysis(graph, analysis) + printer = libdeps.analyzer.GaJsonPrinter(ga) + result = json.loads(printer.get_json()) + self.assertEqual(result, expected) + + def test_graph_paths_basic(self): + """Test for the GraphPaths analyzer on a basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = { + "GRAPH_PATHS": { + "('lib1.so', 'lib6.so')": [["lib1.so", "lib2.so", "lib3.so", "lib6.so"], + ["lib1.so", "lib2.so", "lib4.so", "lib6.so"]] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib1.so', + 'lib6.so') + + expected_result = {"GRAPH_PATHS": {"('lib4.so', 'lib5.so')": []}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib4.so', + 'lib5.so') + + expected_result = { + "GRAPH_PATHS": {"('lib2.so', 'lib5.so')": [['lib2.so', 'lib3.so', 'lib5.so']]} + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib2.so', + 'lib5.so') + + def test_graph_paths_double_diamond(self): + """Test path algorithm on the double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = { + "GRAPH_PATHS": { + "('lib1.so', 'lib9.so')": + [["lib1.so", "lib2.so", "lib3.so", "lib5.so", "lib6.so", "lib7.so", "lib9.so"], + ["lib1.so", "lib2.so", "lib3.so", "lib5.so", "lib6.so", "lib8.so", "lib9.so"], + ["lib1.so", "lib2.so", "lib4.so", "lib5.so", "lib6.so", "lib7.so", "lib9.so"], + ["lib1.so", "lib2.so", "lib4.so", "lib5.so", "lib6.so", "lib8.so", "lib9.so"]] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib1.so', + 'lib9.so') + + expected_result = { + "GRAPH_PATHS": { + "('lib5.so', 'lib9.so')": [["lib5.so", "lib6.so", "lib7.so", "lib9.so"], + ["lib5.so", "lib6.so", "lib8.so", "lib9.so"]] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib5.so', + 'lib9.so') + + expected_result = { + "GRAPH_PATHS": { + "('lib2.so', 'lib6.so')": [["lib2.so", "lib3.so", "lib5.so", "lib6.so"], + ["lib2.so", "lib4.so", "lib5.so", "lib6.so"]] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib2.so', + 'lib6.so') + + def test_critical_paths_basic(self): + """Test for the CriticalPaths for basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib6.so')": [["lib1.so", "lib2.so"]]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so', + 'lib6.so') + + expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib5.so')": [["lib1.so", "lib2.so"]]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so', + 'lib5.so') + + def test_critical_paths_double_diamond(self): + """Test for the CriticalPaths for double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib9.so')": [["lib1.so", "lib2.so"]]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so', + 'lib9.so') + + expected_result = {"CRITICAL_EDGES": {"('lib2.so', 'lib9.so')": [["lib5.so", "lib6.so"]]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib2.so', + 'lib9.so') + + def test_direct_depends_basic(self): + """Test for the DirectDependents for basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = {"DIRECT_DEPENDS": {"lib6.so": ["lib3.so", "lib4.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents, + 'lib6.so') + + expected_result = {'DIRECT_DEPENDS': {'lib1.so': []}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents, + 'lib1.so') + + def test_direct_depends_double_diamond(self): + """Test for the DirectDependents for double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = {"DIRECT_DEPENDS": {"lib9.so": ["lib7.so", "lib8.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents, + 'lib9.so') + + expected_result = {"DIRECT_DEPENDS": {"lib6.so": ["lib5.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents, + 'lib6.so') + + def test_common_depends_basic(self): + """Test for the CommonDependents for basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = { + "COMMON_DEPENDS": { + "('lib6.so', 'lib5.so')": ["lib1.so", "lib2.so", "lib3.so", "lib4.so"] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib6.so', 'lib5.so']) + + expected_result = { + "COMMON_DEPENDS": { + "('lib5.so', 'lib6.so')": ["lib1.so", "lib2.so", "lib3.so", "lib4.so"] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib5.so', 'lib6.so']) + + expected_result = {"COMMON_DEPENDS": {"('lib5.so', 'lib6.so', 'lib2.so')": ["lib1.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib5.so', 'lib6.so', 'lib2.so']) + + def test_common_depends_double_diamond(self): + """Test for the CommonDependents for double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = { + "COMMON_DEPENDS": { + "('lib9.so',)": [ + "lib1.so", "lib2.so", "lib3.so", "lib4.so", "lib5.so", "lib6.so", "lib7.so", + "lib8.so" + ] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib9.so']) + + expected_result = {"COMMON_DEPENDS": {"('lib9.so', 'lib2.so')": ["lib1.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib9.so', 'lib2.so']) + + expected_result = {"COMMON_DEPENDS": {"('lib1.so', 'lib4.so', 'lib3.so')": []}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents, + ['lib1.so', 'lib4.so', 'lib3.so']) + + def test_exclude_depends_basic(self): + """Test for the ExcludeDependents for basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = {"EXCLUDE_DEPENDS": {"('lib6.so', 'lib5.so')": []}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib6.so', 'lib5.so']) + + expected_result = {"EXCLUDE_DEPENDS": {"('lib3.so', 'lib1.so')": ["lib1.so", "lib2.so"]}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib3.so', 'lib1.so']) + + expected_result = { + "EXCLUDE_DEPENDS": { + "('lib6.so', 'lib1.so', 'lib2.so')": ["lib2.so", "lib3.so", "lib4.so"] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib6.so', 'lib1.so', 'lib2.so']) + + def test_exclude_depends_double_diamond(self): + """Test for the ExcludeDependents for double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = { + "EXCLUDE_DEPENDS": {"('lib6.so', 'lib4.so')": ["lib3.so", "lib4.so", "lib5.so"]} + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib6.so', 'lib4.so']) + + expected_result = {"EXCLUDE_DEPENDS": {"('lib2.so', 'lib9.so')": []}} + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib2.so', 'lib9.so']) + + expected_result = { + "EXCLUDE_DEPENDS": { + "('lib8.so', 'lib1.so', 'lib2.so', 'lib3.so', 'lib4.so', 'lib5.so')": [ + "lib5.so", "lib6.so" + ] + } + } + self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents, + ['lib8.so', 'lib1.so', 'lib2.so', 'lib3.so', 'lib4.so', 'lib5.so']) + + def test_counts_basic(self): + """Test counts on basic graph.""" + + libdeps_graph = LibdepsGraph(get_basic_mock_graph()) + + expected_result = { + "NODE": 6, "EDGE": 13, "DIR_EDGE": 7, "TRANS_EDGE": 6, "DIR_PUB_EDGE": 6, + "PUB_EDGE": 12, "PRIV_EDGE": 1, "IF_EDGE": 0, "SHIM": 0, "PROG": 0, "LIB": 6 + } + self.run_counts(expected_result, libdeps_graph) + + def test_counts_double_diamond(self): + """Test counts on double diamond graph.""" + + libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph()) + + expected_result = { + "NODE": 9, "EDGE": 34, "DIR_EDGE": 10, "TRANS_EDGE": 24, "DIR_PUB_EDGE": 10, + "PUB_EDGE": 34, "PRIV_EDGE": 0, "IF_EDGE": 0, "SHIM": 0, "PROG": 0, "LIB": 9 + } + self.run_counts(expected_result, libdeps_graph) + + +if __name__ == '__main__': + unittest.main() diff --git a/buildscripts/libdeps/gacli.py b/buildscripts/libdeps/gacli.py index 23168518853..6fcc0d23400 100755 --- a/buildscripts/libdeps/gacli.py +++ b/buildscripts/libdeps/gacli.py @@ -31,10 +31,12 @@ import argparse import textwrap import sys from pathlib import Path +import copy import networkx -import libdeps.analyzer -from libdeps.graph import CountTypes, LinterTypes + +import libdeps.analyzer as libdeps_analyzer +from libdeps.graph import LibdepsGraph, CountTypes, LinterTypes class LinterSplitArgs(argparse.Action): @@ -43,17 +45,18 @@ class LinterSplitArgs(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): """Create a multi choice comma separated list.""" - selected_choices = [v for v in ''.join(values).split(',') if v] + selected_choices = [v.upper() for v in ''.join(values).split(',') if v] invalid_choices = [ choice for choice in selected_choices if choice not in self.valid_choices ] if invalid_choices: raise Exception( f"Invalid choices: {invalid_choices}\nMust use choices from {self.valid_choices}") - - if 'all' in selected_choices or selected_choices == []: - selected_choices = self.valid_choices - selected_choices.remove('all') + if CountTypes.ALL.name in selected_choices: + selected_choices = copy.copy(self.valid_choices) + selected_choices.remove(CountTypes.ALL.name) + if selected_choices == []: + selected_choices = copy.copy(self.default_choices) setattr(namespace, self.dest, [opt.replace('-', '_') for opt in selected_choices]) @@ -61,12 +64,16 @@ class CountSplitArgs(LinterSplitArgs): """Special case of common custom arg action for Count types.""" valid_choices = [name[0].replace('_', '-') for name in CountTypes.__members__.items()] + default_choices = [ + name[0] for name in CountTypes.__members__.items() if name[0] != CountTypes.ALL.name + ] class LintSplitArgs(LinterSplitArgs): """Special case of common custom arg action for Count types.""" valid_choices = [name[0].replace('_', '-') for name in LinterTypes.__members__.items()] + default_choices = [LinterTypes.PUBLIC_UNUSED.name] class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter): @@ -77,7 +84,7 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe max_length = max([len(name[0]) for name in enum_type.__members__.items()]) help_text = {} for name in enum_type.__members__.items(): - help_text[name[0]] = name[0] + ('-' * (max_length - len(name[0]))) + ": " + help_text[name[0]] = name[0].lower() + ('-' * (max_length - len(name[0]))) + ": " return help_text def _get_help_string(self, action): @@ -87,26 +94,26 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe return textwrap.dedent(f"""\ {action.help} default: all, choices: - {help_text[CountTypes.all.name]}perform all counts - {help_text[CountTypes.node.name]}count nodes - {help_text[CountTypes.edge.name]}count edges - {help_text[CountTypes.dir_edge.name]}count edges declared directly on a node - {help_text[CountTypes.trans_edge.name]}count edges induced by direct public edges - {help_text[CountTypes.dir_pub_edge.name]}count edges that are directly public - {help_text[CountTypes.pub_edge.name]}count edges that are public - {help_text[CountTypes.priv_edge.name]}count edges that are private - {help_text[CountTypes.if_edge.name]}count edges that are interface - {help_text[CountTypes.shim.name]}count shim nodes - {help_text[CountTypes.lib.name]}count library nodes - {help_text[CountTypes.prog.name]}count program nodes + {help_text[CountTypes.ALL.name]}perform all counts + {help_text[CountTypes.NODE.name]}count nodes + {help_text[CountTypes.EDGE.name]}count edges + {help_text[CountTypes.DIR_EDGE.name]}count edges declared directly on a node + {help_text[CountTypes.TRANS_EDGE.name]}count edges induced by direct public edges + {help_text[CountTypes.DIR_PUB_EDGE.name]}count edges that are directly public + {help_text[CountTypes.PUB_EDGE.name]}count edges that are public + {help_text[CountTypes.PRIV_EDGE.name]}count edges that are private + {help_text[CountTypes.IF_EDGE.name]}count edges that are interface + {help_text[CountTypes.SHIM.name]}count shim nodes + {help_text[CountTypes.LIB.name]}count library nodes + {help_text[CountTypes.PROG.name]}count program nodes """) elif isinstance(action, LintSplitArgs): help_text = self._get_help_length(LinterTypes) return textwrap.dedent(f"""\ {action.help} default: all, choices: - {help_text[LinterTypes.all.name]}perform all linters - {help_text[LinterTypes.public_unused.name]}find unnecessary public libdeps + {help_text[LinterTypes.ALL.name]}perform all linters + {help_text[LinterTypes.PUBLIC_UNUSED.name]}find unnecessary public libdeps """) return super()._get_help_string(action) @@ -125,15 +132,13 @@ def setup_args_parser(): parser.add_argument('--build-data', choices=['on', 'off'], default='on', help="Print the invocation and git hash used to build the graph") - parser.add_argument( - '--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs, - default=[name[0] for name in CountTypes.__members__.items() if name[0] != 'all'], - help="Output various counts from the graph. Comma separated list.") + parser.add_argument('--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs, + default=CountSplitArgs.default_choices, + help="Output various counts from the graph. Comma separated list.") - parser.add_argument( - '--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs, - default=[name[0] for name in LinterTypes.__members__.items() if name[0] != 'all'], - help="Perform various linters on the graph. Comma separated list.") + parser.add_argument('--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs, + default=LintSplitArgs.default_choices, + help="Perform various linters on the graph. Comma separated list.") parser.add_argument('--direct-depends', action='append', default=[], help="Print the nodes which depends on a given node.") @@ -146,11 +151,31 @@ def setup_args_parser(): "Print nodes which depend on the first node of N nodes, but exclude all nodes listed there after." ) + parser.add_argument('--graph-paths', nargs='+', action='append', default=[], + help="[from_node] [to_node]: Print all paths between 2 nodes.") + + parser.add_argument( + '--critical-edges', nargs='+', action='append', default=[], help= + "[from_node] [to_node]: Print edges between two nodes, which if removed would break the dependency between those " + + "nodes,.") + + args = parser.parse_args() + + for arg_list in args.graph_paths: + if len(arg_list) != 2: + parser.error( + f'Must pass two args for --graph-paths, [from_node] [to_node], not {arg_list}') + + for arg_list in args.critical_edges: + if len(arg_list) != 2: + parser.error( + f'Must pass two args for --critical-edges, [from_node] [to_node], not {arg_list}') + return parser.parse_args() def load_graph_data(graph_file, output_format): - """Load a graphml file into a LibdepsGraph.""" + """Load a graphml file.""" if output_format == "pretty": sys.stdout.write("Loading graph data...") @@ -166,30 +191,38 @@ def main(): args = setup_args_parser() graph = load_graph_data(args.graph_file, args.format) - libdeps_graph = libdeps.graph.LibdepsGraph(graph) + libdeps_graph = LibdepsGraph(graph=graph) + + analysis = libdeps_analyzer.counter_factory(libdeps_graph, args.counts) + + for analyzer_args in args.direct_depends: + analysis.append(libdeps_analyzer.DirectDependents(libdeps_graph, analyzer_args)) - analysis = libdeps.analyzer.counter_factory(libdeps_graph, args.counts) + for analyzer_args in args.common_depends: + analysis.append(libdeps_analyzer.CommonDependents(libdeps_graph, analyzer_args)) - for depends in args.direct_depends: - analysis.append(libdeps.analyzer.DirectDependencies(libdeps_graph, depends)) + for analyzer_args in args.exclude_depends: + analysis.append(libdeps_analyzer.ExcludeDependents(libdeps_graph, analyzer_args)) - for depends in args.common_depends: - analysis.append(libdeps.analyzer.CommonDependencies(libdeps_graph, depends)) + for analyzer_args in args.graph_paths: + analysis.append( + libdeps_analyzer.GraphPaths(libdeps_graph, analyzer_args[0], analyzer_args[1])) - for depends in args.exclude_depends: - analysis.append(libdeps.analyzer.ExcludeDependencies(libdeps_graph, depends)) + for analyzer_args in args.critical_edges: + analysis.append( + libdeps_analyzer.CriticalEdges(libdeps_graph, analyzer_args[0], analyzer_args[1])) - analysis += libdeps.analyzer.linter_factory(libdeps_graph, args.lint) + analysis += libdeps_analyzer.linter_factory(libdeps_graph, args.lint) if args.build_data: - analysis.append(libdeps.analyzer.BuildDataReport(libdeps_graph)) + analysis.append(libdeps_analyzer.BuildDataReport(libdeps_graph)) - ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=libdeps_graph, analysis=analysis) + ga = libdeps_analyzer.LibdepsGraphAnalysis(libdeps_graph=libdeps_graph, analysis=analysis) if args.format == 'pretty': - ga_printer = libdeps.analyzer.GaPrettyPrinter(ga) + ga_printer = libdeps_analyzer.GaPrettyPrinter(ga) elif args.format == 'json': - ga_printer = libdeps.analyzer.GaJsonPrinter(ga) + ga_printer = libdeps_analyzer.GaJsonPrinter(ga) else: return diff --git a/buildscripts/libdeps/graph_visualizer.py b/buildscripts/libdeps/graph_visualizer.py index 510edf8d97c..8d601355a56 100644 --- a/buildscripts/libdeps/graph_visualizer.py +++ b/buildscripts/libdeps/graph_visualizer.py @@ -106,7 +106,11 @@ def check_node(node_check, cwd): node_modules = cwd / 'node_modules' if not node_modules.exists(): - print(f"{node_modules} not found, you need to run 'npm install' in {cwd}") + print( + textwrap.dedent(f"""\ + {node_modules} not found, you need to run 'npm install' in {cwd} + Perhaps run 'source {cwd}/setup_node_env.sh install'""")) + exit(1) def start_backend(web_service_info, debug): diff --git a/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py b/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py index 333e714d8dc..09f7141560d 100644 --- a/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py +++ b/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py @@ -28,7 +28,7 @@ The backend interacts with the graph_analyzer to perform queries on various libd """ from pathlib import Path -from collections import namedtuple +from collections import namedtuple, OrderedDict import flask import networkx @@ -57,7 +57,6 @@ class BackendServer: self.socketio.on_event('git_hash_selected', self.git_hash_selected) self.socketio.on_event('row_selected', self.row_selected) - self.current_graph = networkx.DiGraph() self.loaded_graphs = {} self.current_selected_rows = {} self.graphml_dir = Path(graphml_dir) @@ -66,6 +65,28 @@ class BackendServer: self.graph_file_tuple = namedtuple('GraphFile', ['version', 'git_hash', 'graph_file']) self.graph_files = self.get_graphml_files() + try: + default_selected_graph = list(self.graph_files.items())[0][1].graph_file + self.load_graph_from_file(default_selected_graph) + self._dependents_graph = networkx.reverse_view(self._dependency_graph) + except (IndexError, AttributeError) as ex: + print(ex) + print( + f"Failed to load read a graph file from {list(self.graph_files.items())} for graphml_dir '{self.graphml_dir}'" + ) + exit(1) + + def load_graph_from_file(self, file_path): + """Load a graph file from disk and handle version.""" + + graph = libdeps.graph.LibdepsGraph(networkx.read_graphml(file_path)) + if graph.graph['graph_schema_version'] == 1: + self._dependents_graph = graph + self._dependency_graph = networkx.reverse_view(self._dependents_graph) + else: + self._dependency_graph = graph + self._dependents_graph = networkx.reverse_view(self._dependency_graph) + def get_app(self): """Return the app and socketio instances.""" @@ -91,7 +112,7 @@ class BackendServer: def get_graphml_files(self): """Find all graphml files in the target graphml dir.""" - graph_files = {} + graph_files = OrderedDict() for graph_file in self.graphml_dir.glob("**/*.graphml"): graph_file_tuple = self.get_graph_build_data(graph_file) graph_files[graph_file_tuple.git_hash[:7]] = graph_file_tuple @@ -124,21 +145,21 @@ class BackendServer: str(node), 'name': node.name, - 'attribs': - [{'name': key, 'value': value} - for key, value in self.current_graph.nodes(data=True)[str(node)].items()], + 'attribs': [{ + 'name': key, 'value': value + } for key, value in self._dependents_graph.nodes(data=True)[str(node)].items()], 'dependers': [{ 'node': depender, 'symbols': - self.current_graph[str(node)][depender].get('symbols', - '').split(' ') - } for depender in self.current_graph[str(node)]], + self._dependents_graph[str(node)][depender].get('symbols', + '').split(' ') + } for depender in self._dependents_graph[str(node)]], 'dependencies': [{ 'node': dependency, 'symbols': - self.current_graph[dependency][str(node)].get('symbols', - '').split(' ') - } for dependency in self.current_graph.rgraph[str(node)]], + self._dependents_graph[dependency][str(node)].get('symbols', + '').split(' ') + } for dependency in self._dependency_graph[str(node)]], }) self.socketio.emit("node_infos", nodeinfo_data) @@ -154,18 +175,21 @@ class BackendServer: for node, _ in self.current_selected_rows.items(): nodes.add( tuple({ - 'id': str(node), 'name': node.name, 'type': self.current_graph.nodes() + 'id': str(node), 'name': node.name, 'type': self._dependents_graph.nodes() [str(node)]['bin_type'] }.items())) - for depender in self.current_graph.rgraph[str(node)]: + for depender in self._dependency_graph[str(node)]: depender_path = Path(depender) - if self.current_graph[depender][str(node)].get('direct'): + if self._dependents_graph[depender][str(node)].get('direct'): nodes.add( tuple({ - 'id': str(depender_path), 'name': depender_path.name, - 'type': self.current_graph.nodes()[str(depender_path)]['bin_type'] + 'id': + str(depender_path), 'name': + depender_path.name, 'type': + self._dependents_graph.nodes()[str(depender_path)] + ['bin_type'] }.items())) links.add( tuple({'source': str(node), 'target': str(depender_path)}.items())) @@ -203,9 +227,9 @@ class BackendServer: with self.app.test_request_context(): analysis = libdeps.analyzer.counter_factory( - self.current_graph, + self._dependents_graph, [name[0] for name in libdeps.analyzer.CountTypes.__members__.items()]) - ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=self.current_graph, + ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=self._dependents_graph, analysis=analysis) results = ga.get_results() @@ -223,7 +247,7 @@ class BackendServer: "selectedNodes": [str(node) for node in list(self.current_selected_rows.keys())] } - for node in self.current_graph.nodes(): + for node in self._dependents_graph.nodes(): node_path = Path(node) node_data['graphData']['nodes'].append( {'id': str(node_path), 'name': node_path.name}) @@ -234,20 +258,19 @@ class BackendServer: with self.app.test_request_context(): - current_hash = self.current_graph.graph.get('git_hash', 'NO_HASH')[:7] + current_hash = self._dependents_graph.graph.get('git_hash', 'NO_HASH')[:7] if current_hash != message['hash']: self.current_selected_rows = {} if message['hash'] in self.loaded_graphs: - self.current_graph = self.loaded_graphs[message['hash']] + self._dependents_graph = self.loaded_graphs[message['hash']] + self._dependents_graph = networkx.reverse_view(self._dependency_graph) else: print( f'loading new graph {current_hash} because different than {message["hash"]}' ) - graph = networkx.read_graphml(self.graph_files[message['hash']].graph_file) - - self.current_graph = libdeps.graph.LibdepsGraph(graph) - self.loaded_graphs[message['hash']] = self.current_graph + self.load_graph_from_file(self.graph_files[message['hash']].graph_file) + self.loaded_graphs[message['hash']] = self._dependents_graph self.socketio.start_background_task(self.analyze_counts) self.socketio.start_background_task(self.send_node_list) diff --git a/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js b/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js index d1da8c4cc0d..3975ffe9f3a 100644 --- a/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js +++ b/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js @@ -12,6 +12,39 @@ import { getRows } from "./redux/store"; import { updateSelected } from "./redux/nodes"; import { socket } from "./connect"; +function componentToHex(c) { + var hex = c.toString(16); + return hex.length == 1 ? "0" + hex : hex; +} + +function rgbToHex(r, g, b) { + return "#" + componentToHex(r) + componentToHex(g) + componentToHex(b); +} + +function hexToRgb(hex) { + // Expand shorthand form (e.g. "03F") to full form (e.g. "0033FF") + var shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i; + hex = hex.replace(shorthandRegex, function(m, r, g, b) { + return r + r + g + g + b + b; + }); + + var result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); + return result ? { + r: parseInt(result[1], 16), + g: parseInt(result[2], 16), + b: parseInt(result[3], 16) + } : null; +} + +function incrementPallete(palleteColor, increment){ + var rgb = hexToRgb(palleteColor); + rgb.r += increment; + rgb.g += increment; + rgb.b += increment; + return rgbToHex(rgb.r, rgb.g, rgb.b); + +} + const styles = (theme) => ({ flexContainer: { display: "flex", @@ -26,12 +59,14 @@ const styles = (theme) => ({ }, }, tableRowOdd: { - backgroundColor: "#4d4d4d", + backgroundColor: incrementPallete(theme.palette.grey[800], 10), + }, + tableRowEven: { + backgroundColor: theme.palette.grey[800], }, - tableRowEven: {}, tableRowHover: { "&:hover": { - backgroundColor: theme.palette.grey[700], + backgroundColor: theme.palette.grey[600], }, }, tableCell: { diff --git a/buildscripts/libdeps/libdeps/analyzer.py b/buildscripts/libdeps/libdeps/analyzer.py index f32f0e7a4b9..2ddc35662ca 100644 --- a/buildscripts/libdeps/libdeps/analyzer.py +++ b/buildscripts/libdeps/libdeps/analyzer.py @@ -31,15 +31,43 @@ represents the dependency information between all binaries from the build. import sys import textwrap +import copy +import json +import inspect +import functools from pathlib import Path +import networkx + from libdeps.graph import CountTypes, DependsReportTypes, LinterTypes, EdgeProps, NodeProps -sys.path.append(str(Path(__file__).parent.parent.parent)) -import scons # pylint: disable=wrong-import-position -sys.path.append(str(Path(scons.MONGODB_ROOT).joinpath('site_scons'))) -from libdeps_next import deptype # pylint: disable=wrong-import-position +class UnsupportedAnalyzer(Exception): + """Thrown when an analyzer is run on a graph with an unsupported schema.""" + + pass + + +# https://stackoverflow.com/a/25959545/1644736 +def get_class_that_defined_method(meth): + """Get the name of the class for given function.""" + + if isinstance(meth, functools.partial): + return get_class_that_defined_method(meth.func) + if inspect.ismethod(meth) or (inspect.isbuiltin(meth) + and getattr(meth, '__self__', None) is not None + and getattr(meth.__self__, '__class__', None)): + for cls in inspect.getmro(meth.__self__.__class__): + if meth.__name__ in cls.__dict__: + return cls + meth = getattr(meth, '__func__', meth) # fallback to __qualname__ parsing + if inspect.isfunction(meth): + cls = getattr( + inspect.getmodule(meth), + meth.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0], None) + if isinstance(cls, type): + return cls + return getattr(meth, '__objclass__', None) # handle special descriptor objects # newer pylints contain the fix: https://github.com/PyCQA/pylint/pull/2926/commits/35e1c61026eab90af504806ef9da6241b096e659 @@ -62,15 +90,23 @@ def schema_check(func, schema_version): """Check the version for a function against the graph.""" def check(*args, **kwargs): - if schema_version <= args[0].graph.graph.get('graph_schema_version'): + + if schema_version <= args[0].graph_schema: return func(*args, **kwargs) else: - sys.stderr.write( - f"WARNING: analysis for '{func.__name__}' requires graph schema version '{schema_version}'\n" - + - f"but detected graph schema version '{args[0].graph.graph.get('graph_schema_version')}'\n" - + f"Not running analysis for {func.__name__}\n\n") - return "GRAPH_SCHEMA_VERSION_ERR" + analyzer = get_class_that_defined_method(func) + if not analyzer: + analyzer = "UnknownAnalyzer" + else: + analyzer = analyzer.__name__ + + raise UnsupportedAnalyzer( + textwrap.dedent(f"""\ + + + ERROR: analysis for '{analyzer}' requires graph schema version '{schema_version}' + but detected graph schema version '{args[0].graph_schema}' + """)) return check @@ -78,27 +114,61 @@ def schema_check(func, schema_version): class Analyzer: """Base class for different types of analyzers.""" - def __init__(self, graph): + # pylint: disable=too-many-instance-attributes + def __init__(self, graph, progress=True): """Store the graph and extract the build_dir from the graph.""" - self.graph = graph - self.build_dir = Path(graph.graph['build_dir']) + self.graph_schema = graph.graph.get('graph_schema_version') + if self.graph_schema == 1: + self._dependents_graph = graph + else: + self._dependency_graph = graph + + self._build_dir = Path(graph.graph['build_dir']) + self.deptypes = json.loads(graph.graph.get('deptypes', "{}")) + self.set_progress(progress) + + @property + def _dependents_graph(self): + if not hasattr(self, 'rgraph'): + setattr(self, 'rgraph', networkx.reverse_view(self._dependency_graph)) + return self.rgraph + + @_dependents_graph.setter + def _dependents_graph(self, value): + self.rgraph = value + + @property + def _dependency_graph(self): + if not hasattr(self, 'graph'): + setattr(self, 'graph', networkx.reverse_view(self._dependents_graph)) + return self.graph + + @_dependency_graph.setter + def _dependency_graph(self, value): + self.graph = value + + def get_deptype(self, deptype): + """Call down to loaded graph to get the deptype from name.""" + + return int(self._dependency_graph.get_deptype(deptype)) def _strip_build_dir(self, node): """Small util function for making args match the graph paths.""" - node = Path(node) - if str(node).startswith(str(self.build_dir)): - return str(node.relative_to(self.build_dir)) - else: - raise Exception( - f"build path not in node path: node: {node} build_dir: {self.build_dir}") + return str(Path(node).relative_to(self._build_dir)) def _strip_build_dirs(self, nodes): """Small util function for making a list of nodes match graph paths.""" return [self._strip_build_dir(node) for node in nodes] + def set_progress(self, value=None): + """Get a progress bar from the loaded graph.""" + + self._progressbar = self._dependency_graph.get_progress(value) + return self._progressbar + class Counter(Analyzer): """Base Counter Analyzer class for various counters.""" @@ -106,19 +176,23 @@ class Counter(Analyzer): def number_of_edge_types(self, edge_type, value): """Count the graphs edges based on type.""" - return len( - [edge for edge in self.graph.edges(data=True) if edge[2].get(edge_type) == value]) + return len([ + edge for edge in self._dependency_graph.edges(data=True) + if edge[2].get(edge_type) == value + ]) def node_type_count(self, node_type, value): """Count the graphs nodes based on type.""" - return len( - [node for node in self.graph.nodes(data=True) if node[1].get(node_type) == value]) + return len([ + node for node in self._dependency_graph.nodes(data=True) + if node[1].get(node_type) == value + ]) def report(self, report): """Report the results for the current type.""" - report[self.count_type] = self.run() + report[self._count_type] = self.run() class NodeCounter(Counter): @@ -128,13 +202,13 @@ class NodeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.node.name + self._count_type = CountTypes.NODE.name @schema_check(schema_version=1) def run(self): """Count the graphs nodes.""" - return self.graph.number_of_nodes() + return self._dependency_graph.number_of_nodes() class EdgeCounter(Counter): @@ -144,13 +218,13 @@ class EdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.edge.name + self._count_type = CountTypes.EDGE.name @schema_check(schema_version=1) def run(self): """Count the graphs edges.""" - return self.graph.number_of_edges() + return self._dependency_graph.number_of_edges() class DirectEdgeCounter(Counter): @@ -160,7 +234,7 @@ class DirectEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.dir_edge.name + self._count_type = CountTypes.DIR_EDGE.name @schema_check(schema_version=1) def run(self): @@ -176,7 +250,7 @@ class TransEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.trans_edge.name + self._count_type = CountTypes.TRANS_EDGE.name @schema_check(schema_version=1) def run(self): @@ -192,15 +266,15 @@ class DirectPubEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.dir_pub_edge.name + self._count_type = CountTypes.DIR_PUB_EDGE.name @schema_check(schema_version=1) def run(self): """Count the graphs direct public edges.""" - return len([ - edge for edge in self.graph.edges(data=True) if edge[2].get(EdgeProps.direct.name) - and edge[2].get(EdgeProps.visibility.name) == int(deptype.Public) + edge for edge in self._dependency_graph.edges(data=True) + if edge[2].get(EdgeProps.direct.name) + and edge[2].get(EdgeProps.visibility.name) == int(self.get_deptype('Public')) ]) @@ -211,13 +285,13 @@ class PublicEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.pub_edge.name + self._count_type = CountTypes.PUB_EDGE.name @schema_check(schema_version=1) def run(self): """Count the graphs public edges.""" - return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Public)) + return self.number_of_edge_types(EdgeProps.visibility.name, int(self.get_deptype('Public'))) class PrivateEdgeCounter(Counter): @@ -227,13 +301,14 @@ class PrivateEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.priv_edge.name + self._count_type = CountTypes.PRIV_EDGE.name @schema_check(schema_version=1) def run(self): """Count the graphs private edges.""" - return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Private)) + return self.number_of_edge_types(EdgeProps.visibility.name, int( + self.get_deptype('Private'))) class InterfaceEdgeCounter(Counter): @@ -243,13 +318,14 @@ class InterfaceEdgeCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.if_edge.name + self._count_type = CountTypes.IF_EDGE.name @schema_check(schema_version=1) def run(self): """Count the graphs interface edges.""" - return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Interface)) + return self.number_of_edge_types(EdgeProps.visibility.name, + int(self.get_deptype('Interface'))) class ShimCounter(Counter): @@ -259,7 +335,7 @@ class ShimCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.shim.name + self._count_type = CountTypes.SHIM.name @schema_check(schema_version=1) def run(self): @@ -275,7 +351,7 @@ class LibCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.lib.name + self._count_type = CountTypes.LIB.name @schema_check(schema_version=1) def run(self): @@ -291,7 +367,7 @@ class ProgCounter(Counter): """Store graph and set type.""" super().__init__(graph) - self.count_type = CountTypes.prog.name + self._count_type = CountTypes.PROG.name @schema_check(schema_version=1) def run(self): @@ -300,21 +376,21 @@ class ProgCounter(Counter): return self.node_type_count(NodeProps.bin_type.name, 'Program') -def counter_factory(graph, counters): +def counter_factory(graph, counters, progressbar=True): """Construct counters from a list of strings.""" counter_map = { - CountTypes.node.name: NodeCounter, - CountTypes.edge.name: EdgeCounter, - CountTypes.dir_edge.name: DirectEdgeCounter, - CountTypes.trans_edge.name: TransEdgeCounter, - CountTypes.dir_pub_edge.name: DirectPubEdgeCounter, - CountTypes.pub_edge.name: PublicEdgeCounter, - CountTypes.priv_edge.name: PrivateEdgeCounter, - CountTypes.if_edge.name: InterfaceEdgeCounter, - CountTypes.shim.name: ShimCounter, - CountTypes.lib.name: LibCounter, - CountTypes.prog.name: ProgCounter, + CountTypes.NODE.name: NodeCounter, + CountTypes.EDGE.name: EdgeCounter, + CountTypes.DIR_EDGE.name: DirectEdgeCounter, + CountTypes.TRANS_EDGE.name: TransEdgeCounter, + CountTypes.DIR_PUB_EDGE.name: DirectPubEdgeCounter, + CountTypes.PUB_EDGE.name: PublicEdgeCounter, + CountTypes.PRIV_EDGE.name: PrivateEdgeCounter, + CountTypes.IF_EDGE.name: InterfaceEdgeCounter, + CountTypes.SHIM.name: ShimCounter, + CountTypes.LIB.name: LibCounter, + CountTypes.PROG.name: ProgCounter, } if not isinstance(counters, list): @@ -323,71 +399,74 @@ def counter_factory(graph, counters): counter_objs = [] for counter in counters: if counter in counter_map: - counter_objs.append(counter_map[counter](graph)) + counter_obj = counter_map[counter](graph) + counter_obj.set_progress(progressbar) + counter_objs.append(counter_obj) + else: print(f"Skipping unknown counter: {counter}") return counter_objs -class CommonDependencies(Analyzer): - """Finds common dependencies for a set of given nodes.""" +class CommonDependents(Analyzer): + """Finds common dependent nodes for a set of given dependency nodes.""" def __init__(self, graph, nodes): """Store graph and strip the nodes.""" super().__init__(graph) - self.nodes = self._strip_build_dirs(nodes) + self._nodes = self._strip_build_dirs(nodes) @schema_check(schema_version=1) def run(self): """For a given set of nodes, report what nodes depend on all nodes from that set.""" - neighbor_sets = [set(self.graph[node]) for node in self.nodes] - return list(set.intersection(*neighbor_sets)) + neighbor_sets = [set(self._dependents_graph[node]) for node in self._nodes] + return sorted(list(set.intersection(*neighbor_sets))) def report(self, report): """Add the common depends list for this tuple of nodes.""" - if DependsReportTypes.common_depends.name not in report: - report[DependsReportTypes.common_depends.name] = {} - report[DependsReportTypes.common_depends.name][tuple(self.nodes)] = self.run() + if DependsReportTypes.COMMON_DEPENDS.name not in report: + report[DependsReportTypes.COMMON_DEPENDS.name] = {} + report[DependsReportTypes.COMMON_DEPENDS.name][tuple(self._nodes)] = self.run() -class DirectDependencies(Analyzer): - """Finds direct dependencies for a given node.""" +class DirectDependents(Analyzer): + """Finds direct dependent nodes for a given dependency node.""" def __init__(self, graph, node): """Store graph and strip the node.""" super().__init__(graph) - self.node = self._strip_build_dir(node) + self._node = self._strip_build_dir(node) @schema_check(schema_version=1) def run(self): """For given nodes, report what nodes depend directly on that node.""" - return [ - depender for depender in self.graph[self.node] - if self.graph[self.node][depender].get(EdgeProps.direct.name) - ] + return sorted([ + depender for depender in self._dependents_graph[self._node] + if self._dependents_graph[self._node][depender].get(EdgeProps.direct.name) + ]) def report(self, report): """Add the direct depends list for this node.""" - if DependsReportTypes.direct_depends.name not in report: - report[DependsReportTypes.direct_depends.name] = {} - report[DependsReportTypes.direct_depends.name][self.node] = self.run() + if DependsReportTypes.DIRECT_DEPENDS.name not in report: + report[DependsReportTypes.DIRECT_DEPENDS.name] = {} + report[DependsReportTypes.DIRECT_DEPENDS.name][self._node] = self.run() -class ExcludeDependencies(Analyzer): - """Finds finds dependencies which include one node, but exclude others.""" +class ExcludeDependents(Analyzer): + """Finds dependents which depend on the first input node, but exclude the other input nodes.""" def __init__(self, graph, nodes): """Store graph and strip the nodes.""" super().__init__(graph) - self.nodes = self._strip_build_dirs(nodes) + self._nodes = self._strip_build_dirs(nodes) @schema_check(schema_version=1) def run(self): @@ -398,19 +477,89 @@ class ExcludeDependencies(Analyzer): """ valid_depender_nodes = [] - for depender_node in set(self.graph[self.nodes[0]]): + for depender_node in set(self._dependents_graph[self._nodes[0]]): if all( - bool(excludes_node not in set(self.graph.rgraph[depender_node])) - for excludes_node in self.nodes[1:]): + bool(excludes_node not in set(self._dependency_graph[depender_node])) + for excludes_node in self._nodes[1:]): valid_depender_nodes.append(depender_node) - return valid_depender_nodes + return sorted(valid_depender_nodes) def report(self, report): """Add the exclude depends list for this tuple of nodes.""" - if DependsReportTypes.exclude_depends.name not in report: - report[DependsReportTypes.exclude_depends.name] = {} - report[DependsReportTypes.exclude_depends.name][tuple(self.nodes)] = self.run() + if DependsReportTypes.EXCLUDE_DEPENDS.name not in report: + report[DependsReportTypes.EXCLUDE_DEPENDS.name] = {} + report[DependsReportTypes.EXCLUDE_DEPENDS.name][tuple(self._nodes)] = self.run() + + +class GraphPaths(Analyzer): + """Finds all paths between two nodes in the graph.""" + + def __init__(self, graph, from_node, to_node): + """Store graph and strip the nodes.""" + + super().__init__(graph) + self._from_node, self._to_node = self._strip_build_dirs([from_node, to_node]) + + @schema_check(schema_version=1) + def run(self): + """Find all paths between the two nodes in the graph.""" + + # We can really help out networkx path finding algorithm by striping the graph down to + # just a graph containing only paths between the source and target node. This is done by + # getting a subtree from the target down, and then getting a subtree of that tree from the + # source up. + dependents_tree = self._dependents_graph.get_direct_nonprivate_graph().get_node_tree( + self._to_node) + + if self._from_node not in dependents_tree: + return [] + + path_tree = networkx.reverse_view(dependents_tree).get_node_tree(self._from_node) + return list( + networkx.all_simple_paths(G=path_tree, source=self._from_node, target=self._to_node)) + + def report(self, report): + """Add the path list to the report.""" + + if DependsReportTypes.GRAPH_PATHS.name not in report: + report[DependsReportTypes.GRAPH_PATHS.name] = {} + report[DependsReportTypes.GRAPH_PATHS.name][tuple([self._from_node, + self._to_node])] = self.run() + + +class CriticalEdges(Analyzer): + """Finds all edges between two nodes, where removing those edges disconnects the two nodes.""" + + def __init__(self, graph, from_node, to_node): + """Store graph and strip the nodes.""" + + super().__init__(graph) + self._from_node, self._to_node = self._strip_build_dirs([from_node, to_node]) + + @schema_check(schema_version=1) + def run(self): + """Use networkx min cut algorithm to find a set of edges.""" + + from networkx.algorithms.connectivity import minimum_st_edge_cut + + # The min cut algorithm will get the min cut nearest the end + # of the direction of the graph, so we we use the reverse graph + # so that we get a cut nearest our from_node, or the first cut we + # would encounter on a given path from the from_node to the to_node. + min_cut_edges = list( + minimum_st_edge_cut( + G=self._dependents_graph.get_direct_nonprivate_graph().get_node_tree(self._to_node), + s=self._to_node, t=self._from_node)) + return [(edge[1], edge[0]) for edge in min_cut_edges] + + def report(self, report): + """Add the critical edges to report.""" + + if DependsReportTypes.CRITICAL_EDGES.name not in report: + report[DependsReportTypes.CRITICAL_EDGES.name] = {} + report[DependsReportTypes.CRITICAL_EDGES.name][tuple([self._from_node, + self._to_node])] = self.run() class UnusedPublicLinter(Analyzer): @@ -424,10 +573,11 @@ class UnusedPublicLinter(Analyzer): original_node = edge[0] depender = edge[1] try: - edge_attribs = self.graph[original_node][depender] + edge_attribs = self._dependents_graph[original_node][depender] - if (edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public) - or edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Interface)): + if (edge_attribs.get(EdgeProps.visibility.name) == int(self.get_deptype('Public')) + or edge_attribs.get(EdgeProps.visibility.name) == int( + self.get_deptype('Interface'))): if not edge_attribs.get(EdgeProps.symbols.name): if not self._tree_uses_no_symbols(depender, original_nodes, checked_edges): return False @@ -445,7 +595,7 @@ class UnusedPublicLinter(Analyzer): in that tree do not have symbol dependencies. """ - for depender in self.graph[node]: + for depender in self._dependents_graph[node]: for original_node in original_nodes: edge = (original_node, depender) if not self._check_edge_no_symbols(edge, original_nodes, checked_edges): @@ -455,13 +605,13 @@ class UnusedPublicLinter(Analyzer): def _check_trans_nodes_no_symbols(self, edge, trans_pub_nodes): """Check the edge against the transitive nodes for symbols.""" - for trans_node in self.graph.rgraph[edge[0]]: - if (self.graph.rgraph[edge[0]][trans_node].get(EdgeProps.visibility.name) == int( - deptype.Public) or self.graph.rgraph[edge[0]][trans_node].get( - EdgeProps.visibility.name) == int(deptype.Interface)): + for trans_node in self._dependency_graph[edge[0]]: + if (self._dependency_graph[edge[0]][trans_node].get(EdgeProps.visibility.name) == int( + self.get_deptype('Public')) or self._dependency_graph[edge[0]][trans_node].get( + EdgeProps.visibility.name) == int(self.get_deptype('Interface'))): trans_pub_nodes.add(trans_node) try: - if self.graph[trans_node][edge[1]].get(EdgeProps.symbols.name): + if self._dependents_graph[trans_node][edge[1]].get(EdgeProps.symbols.name): return True except KeyError: pass @@ -478,13 +628,15 @@ class UnusedPublicLinter(Analyzer): unused_public_libdeps = [] checked_edges = set() - for edge in self.graph.edges: - edge_attribs = self.graph[edge[0]][edge[1]] + for edge in self._dependents_graph.edges: + edge_attribs = self._dependents_graph[edge[0]][edge[1]] + + if (edge_attribs.get(EdgeProps.direct.name) and edge_attribs.get( + EdgeProps.visibility.name) == int(self.get_deptype('Public')) + and not self._dependents_graph.nodes()[edge[0]].get(NodeProps.shim.name) + and self._dependents_graph.nodes()[edge[1]].get( + NodeProps.bin_type.name) == 'SharedLibrary'): - if (edge_attribs.get(EdgeProps.direct.name) - and edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public) - and not self.graph.nodes()[edge[0]].get(NodeProps.shim.name) and - self.graph.nodes()[edge[1]].get(NodeProps.bin_type.name) == 'SharedLibrary'): # First we will get all the transitive libdeps the dependent node # induces, while we are getting those we also check if the depender # node has any symbol dependencies to that transitive libdep. @@ -504,14 +656,14 @@ class UnusedPublicLinter(Analyzer): def report(self, report): """Report the lint issies.""" - report[LinterTypes.public_unused.name] = self.run() + report[LinterTypes.PUBLIC_UNUSED.name] = self.run() -def linter_factory(graph, linters): +def linter_factory(graph, linters, progressbar=True): """Construct linters from a list of strings.""" linter_map = { - LinterTypes.public_unused.name: UnusedPublicLinter, + LinterTypes.PUBLIC_UNUSED.name: UnusedPublicLinter, } if not isinstance(linters, list): @@ -520,7 +672,7 @@ def linter_factory(graph, linters): linters_objs = [] for linter in linters: if linter in linter_map: - linters_objs.append(linter_map[linter](graph)) + linters_objs.append(linter_map[linter](graph, progressbar)) else: print(f"Skipping unknown counter: {linter}") @@ -534,9 +686,9 @@ class BuildDataReport(Analyzer): def report(self, report): """Add the build data from the graph to the report.""" - report['invocation'] = self.graph.graph.get('invocation') - report['git_hash'] = self.graph.graph.get('git_hash') - report['graph_schema_version'] = self.graph.graph.get('graph_schema_version') + report['invocation'] = self._dependency_graph.graph.get('invocation') + report['git_hash'] = self._dependency_graph.graph.get('git_hash') + report['graph_schema_version'] = self._dependency_graph.graph.get('graph_schema_version') class LibdepsGraphAnalysis: @@ -545,22 +697,22 @@ class LibdepsGraphAnalysis: def __init__(self, libdeps_graph, analysis): """Perform analysis based off input args.""" - self.libdeps_graph = libdeps_graph + self._libdeps_graph = libdeps_graph - self.results = {} + self._results = {} for analyzer in analysis: - analyzer.report(self.results) + analyzer.report(self._results) def get_results(self): """Return the results fo the analysis.""" - return self.results + return self._results def run_linters(self, linters): """Run the various dependency reports.""" - if LinterTypes.public_unused.name in linters: - self.results[LinterTypes.public_unused.name] = \ + if LinterTypes.PUBLIC_UNUSED.name in linters: + self.results[LinterTypes.PUBLIC_UNUSED.name] = \ self.libdeps_graph.unused_public_linter() @@ -570,7 +722,7 @@ class GaPrinter: def __init__(self, libdeps_graph_analysis): """Store the graph analysis for use when printing.""" - self.libdeps_graph_analysis = libdeps_graph_analysis + self._libdeps_graph_analysis = libdeps_graph_analysis class GaJsonPrinter(GaPrinter): @@ -589,26 +741,30 @@ class GaJsonPrinter(GaPrinter): def print(self): """Print the result data.""" - import json # pylint: disable=import-outside-toplevel - results = self.libdeps_graph_analysis.get_results() - print(json.dumps(self.serialize(results))) + print(self.get_json()) + + def get_json(self): + """Return the results as a JSON string.""" + + results = self._libdeps_graph_analysis.get_results() + return json.dumps(self.serialize(results)) class GaPrettyPrinter(GaPrinter): """Printer for pretty console output.""" - count_descs = { - CountTypes.node.name: "Nodes in Graph: {}", - CountTypes.edge.name: "Edges in Graph: {}", - CountTypes.dir_edge.name: "Direct Edges in Graph: {}", - CountTypes.trans_edge.name: "Transitive Edges in Graph: {}", - CountTypes.dir_pub_edge.name: "Direct Public Edges in Graph: {}", - CountTypes.pub_edge.name: "Public Edges in Graph: {}", - CountTypes.priv_edge.name: "Private Edges in Graph: {}", - CountTypes.if_edge.name: "Interface Edges in Graph: {}", - CountTypes.shim.name: "Shim Nodes in Graph: {}", - CountTypes.lib.name: "Library Nodes in Graph: {}", - CountTypes.prog.name: "Program Nodes in Graph: {}", + _count_descs = { + CountTypes.NODE.name: "Nodes in Graph: {}", + CountTypes.EDGE.name: "Edges in Graph: {}", + CountTypes.DIR_EDGE.name: "Direct Edges in Graph: {}", + CountTypes.TRANS_EDGE.name: "Transitive Edges in Graph: {}", + CountTypes.DIR_PUB_EDGE.name: "Direct Public Edges in Graph: {}", + CountTypes.PUB_EDGE.name: "Public Edges in Graph: {}", + CountTypes.PRIV_EDGE.name: "Private Edges in Graph: {}", + CountTypes.IF_EDGE.name: "Interface Edges in Graph: {}", + CountTypes.SHIM.name: "Shim Nodes in Graph: {}", + CountTypes.LIB.name: "Library Nodes in Graph: {}", + CountTypes.PROG.name: "Program Nodes in Graph: {}", } @staticmethod @@ -620,13 +776,53 @@ class GaPrettyPrinter(GaPrinter): print(f" {i}: {depender}") print("") + def _print_depends_reports(self, results): + """Print the depends reports result data.""" + + # pylint: disable=too-many-branches + if DependsReportTypes.DIRECT_DEPENDS.name in results: + print("\nNodes that directly depend on:") + for node in results[DependsReportTypes.DIRECT_DEPENDS.name]: + self._print_results_node_list(f'=>depends on {node}:', + results[DependsReportTypes.DIRECT_DEPENDS.name][node]) + + if DependsReportTypes.COMMON_DEPENDS.name in results: + print("\nNodes that commonly depend on:") + for nodes in results[DependsReportTypes.COMMON_DEPENDS.name]: + self._print_results_node_list( + f'=>depends on {nodes}:', + results[DependsReportTypes.COMMON_DEPENDS.name][nodes]) + + if DependsReportTypes.EXCLUDE_DEPENDS.name in results: + print("\nNodes that depend on a node, but exclude others:") + for nodes in results[DependsReportTypes.EXCLUDE_DEPENDS.name]: + self._print_results_node_list( + f"=>depends: {nodes[0]}, exclude: {nodes[1:]}:", + results[DependsReportTypes.EXCLUDE_DEPENDS.name][nodes]) + + if DependsReportTypes.GRAPH_PATHS.name in results: + print("\nDependency graph paths:") + for nodes in results[DependsReportTypes.GRAPH_PATHS.name]: + self._print_results_node_list(f"=>start node: {nodes[0]}, end node: {nodes[1]}:", [ + f"{' -> '.join(path)}" + for path in results[DependsReportTypes.GRAPH_PATHS.name][nodes] + ]) + + if DependsReportTypes.CRITICAL_EDGES.name in results: + print("\nCritical Edges:") + for nodes in results[DependsReportTypes.CRITICAL_EDGES.name]: + self._print_results_node_list( + f"=>critical edges between {nodes[0]} and {nodes[1]}:", + results[DependsReportTypes.CRITICAL_EDGES.name][nodes]) + def print(self): """Print the result data.""" - results = self.libdeps_graph_analysis.get_results() + results = self._libdeps_graph_analysis.get_results() if 'invocation' in results: print( textwrap.dedent(f"""\ + Graph built from git hash: {results['git_hash']} @@ -638,33 +834,15 @@ class GaPrettyPrinter(GaPrinter): """)) for count_type in CountTypes.__members__.items(): - if count_type[0] in self.count_descs and count_type[0] in results: - print(self.count_descs[count_type[0]].format(results[count_type[0]])) + if count_type[0] in self._count_descs and count_type[0] in results: + print(self._count_descs[count_type[0]].format(results[count_type[0]])) - if DependsReportTypes.direct_depends.name in results: - print("\nNodes that directly depend on:") - for node in results[DependsReportTypes.direct_depends.name]: - self._print_results_node_list(f'=>depends on {node}:', - results[DependsReportTypes.direct_depends.name][node]) - - if DependsReportTypes.common_depends.name in results: - print("\nNodes that commonly depend on:") - for nodes in results[DependsReportTypes.common_depends.name]: - self._print_results_node_list( - f'=>depends on {nodes}:', - results[DependsReportTypes.common_depends.name][nodes]) - - if DependsReportTypes.exclude_depends.name in results: - print("\nNodes that depend on a node, but exclude others:") - for nodes in results[DependsReportTypes.exclude_depends.name]: - self._print_results_node_list( - f"=>depends: {nodes[0]}, exclude: {nodes[1:]}:", - results[DependsReportTypes.exclude_depends.name][nodes]) + self._print_depends_reports(results) - if LinterTypes.public_unused.name in results: + if LinterTypes.PUBLIC_UNUSED.name in results: print( - f"\nLibdepsLinter: PUBLIC libdeps that could be PRIVATE: {len(results[LinterTypes.public_unused.name])}" + f"\nLibdepsLinter: PUBLIC libdeps that could be PRIVATE: {len(results[LinterTypes.PUBLIC_UNUSED.name])}" ) - for issue in sorted(results[LinterTypes.public_unused.name], + for issue in sorted(results[LinterTypes.PUBLIC_UNUSED.name], key=lambda item: item[1] + item[0]): print(f" {issue[1]}: PUBLIC -> {issue[0]} -> PRIVATE") diff --git a/buildscripts/libdeps/libdeps/graph.py b/buildscripts/libdeps/libdeps/graph.py index 3f3f444a2ff..3bd047cc79f 100644 --- a/buildscripts/libdeps/libdeps/graph.py +++ b/buildscripts/libdeps/libdeps/graph.py @@ -26,44 +26,54 @@ Libdeps Graph Enums. These are used for attributing data across the build scripts and analyzer scripts. """ - from enum import Enum, auto +from pathlib import Path +import json import networkx -# pylint: disable=invalid-name +try: + import progressbar +except ImportError: + pass +# We need to disable invalid name here because it break backwards compatibility with +# our graph schemas. Possibly we could use lower case conversion process to maintain +# backwards compatibility and make pylint happy. +# pylint: disable=invalid-name class CountTypes(Enum): """Enums for the different types of counts to perform on a graph.""" - all = auto() - node = auto() - edge = auto() - dir_edge = auto() - trans_edge = auto() - dir_pub_edge = auto() - pub_edge = auto() - priv_edge = auto() - if_edge = auto() - shim = auto() - prog = auto() - lib = auto() + ALL = auto() + NODE = auto() + EDGE = auto() + DIR_EDGE = auto() + TRANS_EDGE = auto() + DIR_PUB_EDGE = auto() + PUB_EDGE = auto() + PRIV_EDGE = auto() + IF_EDGE = auto() + SHIM = auto() + PROG = auto() + LIB = auto() class DependsReportTypes(Enum): """Enums for the different type of depends reports to perform on a graph.""" - direct_depends = auto() - common_depends = auto() - exclude_depends = auto() + DIRECT_DEPENDS = auto() + COMMON_DEPENDS = auto() + EXCLUDE_DEPENDS = auto() + GRAPH_PATHS = auto() + CRITICAL_EDGES = auto() class LinterTypes(Enum): """Enums for the different types of counts to perform on a graph.""" - all = auto() - public_unused = auto() + ALL = auto() + PUBLIC_UNUSED = auto() class EdgeProps(Enum): @@ -72,7 +82,6 @@ class EdgeProps(Enum): direct = auto() visibility = auto() symbols = auto() - shim = auto() class NodeProps(Enum): @@ -82,14 +91,88 @@ class NodeProps(Enum): bin_type = auto() +def null_progressbar(items): + """Fake stand-in for normal progressbar.""" + for item in items: + yield item + + class LibdepsGraph(networkx.DiGraph): """Class for analyzing the graph.""" def __init__(self, graph=networkx.DiGraph()): """Load the graph data.""" - super().__init__(incoming_graph_data=graph) + self._progressbar = None + self._deptypes = None + + def get_deptype(self, deptype): + """Convert graphs deptypes from json string to dict, and return requested value.""" + + if not self._deptypes: + self._deptypes = json.loads(self.graph.get('deptypes', "{}")) + if self.graph['graph_schema_version'] == 1: + # get and set the legacy values + self._deptypes['Global'] = self._deptypes.get('Global', 0) + self._deptypes['Public'] = self._deptypes.get('Public', 1) + self._deptypes['Private'] = self._deptypes.get('Private', 2) + self._deptypes['Interface'] = self._deptypes.get('Interface', 3) + self._deptypes['Typeinfo'] = self._deptypes.get('Typeinfo', 4) + + return self._deptypes[deptype] + + def _strip_build_dir(self, node): + """Small util function for making args match the graph paths.""" + + return str(Path(node).resolve().relative_to(Path(self.graph['build_dir']).resolve())) + + def get_direct_nonprivate_graph(self): + """Get a graph view of direct nonprivate edges.""" + + def filter_direct_nonprivate_edges(n1, n2): + return (self[n1][n2].get(EdgeProps.direct.name) and + (self[n1][n2].get(EdgeProps.visibility.name) == self.get_deptype('Public') or + self[n1][n2].get(EdgeProps.visibility.name) == self.get_deptype('Interface'))) + + return networkx.subgraph_view(self, filter_edge=filter_direct_nonprivate_edges) + + def get_node_tree(self, node): + """Get a tree with the passed node as the single root.""" + + direct_nonprivate_graph = self.get_direct_nonprivate_graph() + substree_set = networkx.descendants(direct_nonprivate_graph, node) + + def subtree(n1): + return n1 in substree_set or n1 == node + + return networkx.subgraph_view(direct_nonprivate_graph, filter_node=subtree) + + def get_progress(self, value=None): + """ + Set if a progress bar should be used or not. + + No args means use progress bar if available. + """ + + if value is None: + value = ('progressbar' in globals()) + + if self._progressbar: + return self._progressbar + + if value: + + def get_progress_bar(title, *args): + custom_bar = progressbar.ProgressBar(widgets=[ + title, + progressbar.Counter(format='[%(value)d/%(max_value)d]'), + progressbar.Timer(format=" Time: %(elapsed)s "), + progressbar.Bar(marker='>', fill=' ', left='|', right='|') + ]) + return custom_bar(*args) + + self._progressbar = get_progress_bar + else: + self._progressbar = null_progressbar - # Load in the graph and store a reversed version as well for quick look ups - # the in directions. - self.rgraph = graph.reverse() + return self._progressbar |