summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorDaniel Moody <daniel.moody@mongodb.com>2021-03-30 20:23:00 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2021-04-15 16:34:40 +0000
commit0782c59e71146e316e7d61ac61d76e2330f29eb4 (patch)
tree75eebd697434aaaeacf0771cc49e43e30857dcc3 /buildscripts
parent53230099fc21fe4c4bf60e26420ca2e839ec22f1 (diff)
downloadmongo-0782c59e71146e316e7d61ac61d76e2330f29eb4.tar.gz
SERVER-52568 Added GraphPaths, CriticalEdges, and PublicWieghts quieries to libdeps graph analyzer.
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/libdeps/analyzer_unittests.py448
-rwxr-xr-xbuildscripts/libdeps/gacli.py121
-rw-r--r--buildscripts/libdeps/graph_visualizer.py6
-rw-r--r--buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py75
-rw-r--r--buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js41
-rw-r--r--buildscripts/libdeps/libdeps/analyzer.py486
-rw-r--r--buildscripts/libdeps/libdeps/graph.py131
7 files changed, 1056 insertions, 252 deletions
diff --git a/buildscripts/libdeps/analyzer_unittests.py b/buildscripts/libdeps/analyzer_unittests.py
new file mode 100644
index 00000000000..8061f23e27d
--- /dev/null
+++ b/buildscripts/libdeps/analyzer_unittests.py
@@ -0,0 +1,448 @@
+#!/usr/bin/env python3
+#
+# Copyright 2021 MongoDB Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+"""Unittests for the graph analyzer."""
+
+import json
+import sys
+import unittest
+
+import networkx
+
+import libdeps.analyzer
+from libdeps.graph import LibdepsGraph, EdgeProps, NodeProps, CountTypes
+
+
+def add_node(graph, node, builder, shim):
+ """Add a node to the graph."""
+
+ graph.add_nodes_from([(node, {NodeProps.bin_type.name: builder, NodeProps.shim.name: shim})])
+
+
+def add_edge(graph, from_node, to_node, **kwargs):
+ """Add an edge to the graph."""
+
+ edge_props = {
+ EdgeProps.direct.name: kwargs[EdgeProps.direct.name],
+ EdgeProps.visibility.name: int(kwargs[EdgeProps.visibility.name]),
+ }
+
+ graph.add_edges_from([(from_node, to_node, edge_props)])
+
+
+def get_double_diamond_mock_graph():
+ """Construct a mock graph which covers a double diamond structure."""
+
+ graph = LibdepsGraph()
+ graph.graph['build_dir'] = '.'
+ graph.graph['graph_schema_version'] = 2
+ graph.graph['deptypes'] = '''{
+ "Global": 0,
+ "Public": 1,
+ "Private": 2,
+ "Interface": 3,
+ "Typeinfo": 4
+ }'''
+
+ # builds a graph of mostly public edges that looks like this:
+ #
+ #
+ # /lib3.so /lib7.so
+ # | \ | \
+ # <-lib1.so--lib2.so lib5.so--lib6.so lib9.so
+ # | / | /
+ # \lib4.so \lib8.so
+ #
+
+ add_node(graph, 'lib1.so', 'SharedLibrary', False)
+ add_node(graph, 'lib2.so', 'SharedLibrary', False)
+ add_node(graph, 'lib3.so', 'SharedLibrary', False)
+ add_node(graph, 'lib4.so', 'SharedLibrary', False)
+ add_node(graph, 'lib5.so', 'SharedLibrary', False)
+ add_node(graph, 'lib6.so', 'SharedLibrary', False)
+ add_node(graph, 'lib7.so', 'SharedLibrary', False)
+ add_node(graph, 'lib8.so', 'SharedLibrary', False)
+ add_node(graph, 'lib9.so', 'SharedLibrary', False)
+
+ add_edge(graph, 'lib1.so', 'lib2.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib3.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib4.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib5.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib6.so', 'lib7.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib6.so', 'lib8.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib7.so', 'lib9.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib8.so', 'lib9.so', direct=True, visibility=graph.get_deptype('Public'))
+
+ # trans for 3 and 4
+ add_edge(graph, 'lib1.so', 'lib3.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib1.so', 'lib4.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 5
+ add_edge(graph, 'lib1.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 6
+ add_edge(graph, 'lib1.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 7
+ add_edge(graph, 'lib1.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib5.so', 'lib7.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 8
+ add_edge(graph, 'lib1.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib5.so', 'lib8.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 9
+ add_edge(graph, 'lib1.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib5.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib6.so', 'lib9.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ return graph
+
+
+def get_basic_mock_graph():
+ """Construct a mock graph which covers most cases and is easy to understand."""
+
+ graph = LibdepsGraph()
+ graph.graph['build_dir'] = '.'
+ graph.graph['graph_schema_version'] = 2
+ graph.graph['deptypes'] = '''{
+ "Global": 0,
+ "Public": 1,
+ "Private": 2,
+ "Interface": 3,
+ "Typeinfo": 4
+ }'''
+
+ # builds a graph of mostly public edges:
+ #
+ # /-lib5.so
+ # /lib3.so
+ # | \-lib6.so
+ # <-lib1.so--lib2.so
+ # | /-lib5.so (private)
+ # \lib4.so
+ # \-lib6.so
+
+ # nodes
+ add_node(graph, 'lib1.so', 'SharedLibrary', False)
+ add_node(graph, 'lib2.so', 'SharedLibrary', False)
+ add_node(graph, 'lib3.so', 'SharedLibrary', False)
+ add_node(graph, 'lib4.so', 'SharedLibrary', False)
+ add_node(graph, 'lib5.so', 'SharedLibrary', False)
+ add_node(graph, 'lib6.so', 'SharedLibrary', False)
+
+ # direct edges
+ add_edge(graph, 'lib1.so', 'lib2.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib3.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib2.so', 'lib4.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib3.so', 'lib6.so', direct=True, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib4.so', 'lib5.so', direct=True, visibility=graph.get_deptype('Private'))
+
+ # trans for 3
+ add_edge(graph, 'lib1.so', 'lib3.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 4
+ add_edge(graph, 'lib1.so', 'lib4.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 5
+ add_edge(graph, 'lib2.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib1.so', 'lib5.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ # trans for 6
+ add_edge(graph, 'lib2.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+ add_edge(graph, 'lib1.so', 'lib6.so', direct=False, visibility=graph.get_deptype('Public'))
+
+ return graph
+
+
+class Tests(unittest.TestCase):
+ """Common unittest for the libdeps graph analyzer module."""
+
+ def run_analysis(self, expected, graph, algo, *args):
+ """Check results of analysis generically."""
+
+ analysis = [algo(graph, *args)]
+ ga = libdeps.analyzer.LibdepsGraphAnalysis(graph, analysis)
+ printer = libdeps.analyzer.GaJsonPrinter(ga)
+ result = json.loads(printer.get_json())
+ self.assertEqual(result, expected)
+
+ def run_counts(self, expected, graph):
+ """Check results of counts generically."""
+
+ analysis = libdeps.analyzer.counter_factory(
+ graph,
+ [name[0] for name in CountTypes.__members__.items() if name[0] != CountTypes.ALL.name])
+ ga = libdeps.analyzer.LibdepsGraphAnalysis(graph, analysis)
+ printer = libdeps.analyzer.GaJsonPrinter(ga)
+ result = json.loads(printer.get_json())
+ self.assertEqual(result, expected)
+
+ def test_graph_paths_basic(self):
+ """Test for the GraphPaths analyzer on a basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {
+ "GRAPH_PATHS": {
+ "('lib1.so', 'lib6.so')": [["lib1.so", "lib2.so", "lib3.so", "lib6.so"],
+ ["lib1.so", "lib2.so", "lib4.so", "lib6.so"]]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib1.so',
+ 'lib6.so')
+
+ expected_result = {"GRAPH_PATHS": {"('lib4.so', 'lib5.so')": []}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib4.so',
+ 'lib5.so')
+
+ expected_result = {
+ "GRAPH_PATHS": {"('lib2.so', 'lib5.so')": [['lib2.so', 'lib3.so', 'lib5.so']]}
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib2.so',
+ 'lib5.so')
+
+ def test_graph_paths_double_diamond(self):
+ """Test path algorithm on the double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {
+ "GRAPH_PATHS": {
+ "('lib1.so', 'lib9.so')":
+ [["lib1.so", "lib2.so", "lib3.so", "lib5.so", "lib6.so", "lib7.so", "lib9.so"],
+ ["lib1.so", "lib2.so", "lib3.so", "lib5.so", "lib6.so", "lib8.so", "lib9.so"],
+ ["lib1.so", "lib2.so", "lib4.so", "lib5.so", "lib6.so", "lib7.so", "lib9.so"],
+ ["lib1.so", "lib2.so", "lib4.so", "lib5.so", "lib6.so", "lib8.so", "lib9.so"]]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib1.so',
+ 'lib9.so')
+
+ expected_result = {
+ "GRAPH_PATHS": {
+ "('lib5.so', 'lib9.so')": [["lib5.so", "lib6.so", "lib7.so", "lib9.so"],
+ ["lib5.so", "lib6.so", "lib8.so", "lib9.so"]]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib5.so',
+ 'lib9.so')
+
+ expected_result = {
+ "GRAPH_PATHS": {
+ "('lib2.so', 'lib6.so')": [["lib2.so", "lib3.so", "lib5.so", "lib6.so"],
+ ["lib2.so", "lib4.so", "lib5.so", "lib6.so"]]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.GraphPaths, 'lib2.so',
+ 'lib6.so')
+
+ def test_critical_paths_basic(self):
+ """Test for the CriticalPaths for basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib6.so')": [["lib1.so", "lib2.so"]]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so',
+ 'lib6.so')
+
+ expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib5.so')": [["lib1.so", "lib2.so"]]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so',
+ 'lib5.so')
+
+ def test_critical_paths_double_diamond(self):
+ """Test for the CriticalPaths for double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {"CRITICAL_EDGES": {"('lib1.so', 'lib9.so')": [["lib1.so", "lib2.so"]]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib1.so',
+ 'lib9.so')
+
+ expected_result = {"CRITICAL_EDGES": {"('lib2.so', 'lib9.so')": [["lib5.so", "lib6.so"]]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CriticalEdges, 'lib2.so',
+ 'lib9.so')
+
+ def test_direct_depends_basic(self):
+ """Test for the DirectDependents for basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {"DIRECT_DEPENDS": {"lib6.so": ["lib3.so", "lib4.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents,
+ 'lib6.so')
+
+ expected_result = {'DIRECT_DEPENDS': {'lib1.so': []}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents,
+ 'lib1.so')
+
+ def test_direct_depends_double_diamond(self):
+ """Test for the DirectDependents for double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {"DIRECT_DEPENDS": {"lib9.so": ["lib7.so", "lib8.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents,
+ 'lib9.so')
+
+ expected_result = {"DIRECT_DEPENDS": {"lib6.so": ["lib5.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.DirectDependents,
+ 'lib6.so')
+
+ def test_common_depends_basic(self):
+ """Test for the CommonDependents for basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {
+ "COMMON_DEPENDS": {
+ "('lib6.so', 'lib5.so')": ["lib1.so", "lib2.so", "lib3.so", "lib4.so"]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib6.so', 'lib5.so'])
+
+ expected_result = {
+ "COMMON_DEPENDS": {
+ "('lib5.so', 'lib6.so')": ["lib1.so", "lib2.so", "lib3.so", "lib4.so"]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib5.so', 'lib6.so'])
+
+ expected_result = {"COMMON_DEPENDS": {"('lib5.so', 'lib6.so', 'lib2.so')": ["lib1.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib5.so', 'lib6.so', 'lib2.so'])
+
+ def test_common_depends_double_diamond(self):
+ """Test for the CommonDependents for double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {
+ "COMMON_DEPENDS": {
+ "('lib9.so',)": [
+ "lib1.so", "lib2.so", "lib3.so", "lib4.so", "lib5.so", "lib6.so", "lib7.so",
+ "lib8.so"
+ ]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib9.so'])
+
+ expected_result = {"COMMON_DEPENDS": {"('lib9.so', 'lib2.so')": ["lib1.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib9.so', 'lib2.so'])
+
+ expected_result = {"COMMON_DEPENDS": {"('lib1.so', 'lib4.so', 'lib3.so')": []}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.CommonDependents,
+ ['lib1.so', 'lib4.so', 'lib3.so'])
+
+ def test_exclude_depends_basic(self):
+ """Test for the ExcludeDependents for basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {"EXCLUDE_DEPENDS": {"('lib6.so', 'lib5.so')": []}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib6.so', 'lib5.so'])
+
+ expected_result = {"EXCLUDE_DEPENDS": {"('lib3.so', 'lib1.so')": ["lib1.so", "lib2.so"]}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib3.so', 'lib1.so'])
+
+ expected_result = {
+ "EXCLUDE_DEPENDS": {
+ "('lib6.so', 'lib1.so', 'lib2.so')": ["lib2.so", "lib3.so", "lib4.so"]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib6.so', 'lib1.so', 'lib2.so'])
+
+ def test_exclude_depends_double_diamond(self):
+ """Test for the ExcludeDependents for double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {
+ "EXCLUDE_DEPENDS": {"('lib6.so', 'lib4.so')": ["lib3.so", "lib4.so", "lib5.so"]}
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib6.so', 'lib4.so'])
+
+ expected_result = {"EXCLUDE_DEPENDS": {"('lib2.so', 'lib9.so')": []}}
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib2.so', 'lib9.so'])
+
+ expected_result = {
+ "EXCLUDE_DEPENDS": {
+ "('lib8.so', 'lib1.so', 'lib2.so', 'lib3.so', 'lib4.so', 'lib5.so')": [
+ "lib5.so", "lib6.so"
+ ]
+ }
+ }
+ self.run_analysis(expected_result, libdeps_graph, libdeps.analyzer.ExcludeDependents,
+ ['lib8.so', 'lib1.so', 'lib2.so', 'lib3.so', 'lib4.so', 'lib5.so'])
+
+ def test_counts_basic(self):
+ """Test counts on basic graph."""
+
+ libdeps_graph = LibdepsGraph(get_basic_mock_graph())
+
+ expected_result = {
+ "NODE": 6, "EDGE": 13, "DIR_EDGE": 7, "TRANS_EDGE": 6, "DIR_PUB_EDGE": 6,
+ "PUB_EDGE": 12, "PRIV_EDGE": 1, "IF_EDGE": 0, "SHIM": 0, "PROG": 0, "LIB": 6
+ }
+ self.run_counts(expected_result, libdeps_graph)
+
+ def test_counts_double_diamond(self):
+ """Test counts on double diamond graph."""
+
+ libdeps_graph = LibdepsGraph(get_double_diamond_mock_graph())
+
+ expected_result = {
+ "NODE": 9, "EDGE": 34, "DIR_EDGE": 10, "TRANS_EDGE": 24, "DIR_PUB_EDGE": 10,
+ "PUB_EDGE": 34, "PRIV_EDGE": 0, "IF_EDGE": 0, "SHIM": 0, "PROG": 0, "LIB": 9
+ }
+ self.run_counts(expected_result, libdeps_graph)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/buildscripts/libdeps/gacli.py b/buildscripts/libdeps/gacli.py
index 23168518853..6fcc0d23400 100755
--- a/buildscripts/libdeps/gacli.py
+++ b/buildscripts/libdeps/gacli.py
@@ -31,10 +31,12 @@ import argparse
import textwrap
import sys
from pathlib import Path
+import copy
import networkx
-import libdeps.analyzer
-from libdeps.graph import CountTypes, LinterTypes
+
+import libdeps.analyzer as libdeps_analyzer
+from libdeps.graph import LibdepsGraph, CountTypes, LinterTypes
class LinterSplitArgs(argparse.Action):
@@ -43,17 +45,18 @@ class LinterSplitArgs(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
"""Create a multi choice comma separated list."""
- selected_choices = [v for v in ''.join(values).split(',') if v]
+ selected_choices = [v.upper() for v in ''.join(values).split(',') if v]
invalid_choices = [
choice for choice in selected_choices if choice not in self.valid_choices
]
if invalid_choices:
raise Exception(
f"Invalid choices: {invalid_choices}\nMust use choices from {self.valid_choices}")
-
- if 'all' in selected_choices or selected_choices == []:
- selected_choices = self.valid_choices
- selected_choices.remove('all')
+ if CountTypes.ALL.name in selected_choices:
+ selected_choices = copy.copy(self.valid_choices)
+ selected_choices.remove(CountTypes.ALL.name)
+ if selected_choices == []:
+ selected_choices = copy.copy(self.default_choices)
setattr(namespace, self.dest, [opt.replace('-', '_') for opt in selected_choices])
@@ -61,12 +64,16 @@ class CountSplitArgs(LinterSplitArgs):
"""Special case of common custom arg action for Count types."""
valid_choices = [name[0].replace('_', '-') for name in CountTypes.__members__.items()]
+ default_choices = [
+ name[0] for name in CountTypes.__members__.items() if name[0] != CountTypes.ALL.name
+ ]
class LintSplitArgs(LinterSplitArgs):
"""Special case of common custom arg action for Count types."""
valid_choices = [name[0].replace('_', '-') for name in LinterTypes.__members__.items()]
+ default_choices = [LinterTypes.PUBLIC_UNUSED.name]
class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
@@ -77,7 +84,7 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe
max_length = max([len(name[0]) for name in enum_type.__members__.items()])
help_text = {}
for name in enum_type.__members__.items():
- help_text[name[0]] = name[0] + ('-' * (max_length - len(name[0]))) + ": "
+ help_text[name[0]] = name[0].lower() + ('-' * (max_length - len(name[0]))) + ": "
return help_text
def _get_help_string(self, action):
@@ -87,26 +94,26 @@ class CustomFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHe
return textwrap.dedent(f"""\
{action.help}
default: all, choices:
- {help_text[CountTypes.all.name]}perform all counts
- {help_text[CountTypes.node.name]}count nodes
- {help_text[CountTypes.edge.name]}count edges
- {help_text[CountTypes.dir_edge.name]}count edges declared directly on a node
- {help_text[CountTypes.trans_edge.name]}count edges induced by direct public edges
- {help_text[CountTypes.dir_pub_edge.name]}count edges that are directly public
- {help_text[CountTypes.pub_edge.name]}count edges that are public
- {help_text[CountTypes.priv_edge.name]}count edges that are private
- {help_text[CountTypes.if_edge.name]}count edges that are interface
- {help_text[CountTypes.shim.name]}count shim nodes
- {help_text[CountTypes.lib.name]}count library nodes
- {help_text[CountTypes.prog.name]}count program nodes
+ {help_text[CountTypes.ALL.name]}perform all counts
+ {help_text[CountTypes.NODE.name]}count nodes
+ {help_text[CountTypes.EDGE.name]}count edges
+ {help_text[CountTypes.DIR_EDGE.name]}count edges declared directly on a node
+ {help_text[CountTypes.TRANS_EDGE.name]}count edges induced by direct public edges
+ {help_text[CountTypes.DIR_PUB_EDGE.name]}count edges that are directly public
+ {help_text[CountTypes.PUB_EDGE.name]}count edges that are public
+ {help_text[CountTypes.PRIV_EDGE.name]}count edges that are private
+ {help_text[CountTypes.IF_EDGE.name]}count edges that are interface
+ {help_text[CountTypes.SHIM.name]}count shim nodes
+ {help_text[CountTypes.LIB.name]}count library nodes
+ {help_text[CountTypes.PROG.name]}count program nodes
""")
elif isinstance(action, LintSplitArgs):
help_text = self._get_help_length(LinterTypes)
return textwrap.dedent(f"""\
{action.help}
default: all, choices:
- {help_text[LinterTypes.all.name]}perform all linters
- {help_text[LinterTypes.public_unused.name]}find unnecessary public libdeps
+ {help_text[LinterTypes.ALL.name]}perform all linters
+ {help_text[LinterTypes.PUBLIC_UNUSED.name]}find unnecessary public libdeps
""")
return super()._get_help_string(action)
@@ -125,15 +132,13 @@ def setup_args_parser():
parser.add_argument('--build-data', choices=['on', 'off'], default='on',
help="Print the invocation and git hash used to build the graph")
- parser.add_argument(
- '--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs,
- default=[name[0] for name in CountTypes.__members__.items() if name[0] != 'all'],
- help="Output various counts from the graph. Comma separated list.")
+ parser.add_argument('--counts', metavar='COUNT,', nargs='*', action=CountSplitArgs,
+ default=CountSplitArgs.default_choices,
+ help="Output various counts from the graph. Comma separated list.")
- parser.add_argument(
- '--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs,
- default=[name[0] for name in LinterTypes.__members__.items() if name[0] != 'all'],
- help="Perform various linters on the graph. Comma separated list.")
+ parser.add_argument('--lint', metavar='LINTER,', nargs='*', action=LintSplitArgs,
+ default=LintSplitArgs.default_choices,
+ help="Perform various linters on the graph. Comma separated list.")
parser.add_argument('--direct-depends', action='append', default=[],
help="Print the nodes which depends on a given node.")
@@ -146,11 +151,31 @@ def setup_args_parser():
"Print nodes which depend on the first node of N nodes, but exclude all nodes listed there after."
)
+ parser.add_argument('--graph-paths', nargs='+', action='append', default=[],
+ help="[from_node] [to_node]: Print all paths between 2 nodes.")
+
+ parser.add_argument(
+ '--critical-edges', nargs='+', action='append', default=[], help=
+ "[from_node] [to_node]: Print edges between two nodes, which if removed would break the dependency between those "
+ + "nodes,.")
+
+ args = parser.parse_args()
+
+ for arg_list in args.graph_paths:
+ if len(arg_list) != 2:
+ parser.error(
+ f'Must pass two args for --graph-paths, [from_node] [to_node], not {arg_list}')
+
+ for arg_list in args.critical_edges:
+ if len(arg_list) != 2:
+ parser.error(
+ f'Must pass two args for --critical-edges, [from_node] [to_node], not {arg_list}')
+
return parser.parse_args()
def load_graph_data(graph_file, output_format):
- """Load a graphml file into a LibdepsGraph."""
+ """Load a graphml file."""
if output_format == "pretty":
sys.stdout.write("Loading graph data...")
@@ -166,30 +191,38 @@ def main():
args = setup_args_parser()
graph = load_graph_data(args.graph_file, args.format)
- libdeps_graph = libdeps.graph.LibdepsGraph(graph)
+ libdeps_graph = LibdepsGraph(graph=graph)
+
+ analysis = libdeps_analyzer.counter_factory(libdeps_graph, args.counts)
+
+ for analyzer_args in args.direct_depends:
+ analysis.append(libdeps_analyzer.DirectDependents(libdeps_graph, analyzer_args))
- analysis = libdeps.analyzer.counter_factory(libdeps_graph, args.counts)
+ for analyzer_args in args.common_depends:
+ analysis.append(libdeps_analyzer.CommonDependents(libdeps_graph, analyzer_args))
- for depends in args.direct_depends:
- analysis.append(libdeps.analyzer.DirectDependencies(libdeps_graph, depends))
+ for analyzer_args in args.exclude_depends:
+ analysis.append(libdeps_analyzer.ExcludeDependents(libdeps_graph, analyzer_args))
- for depends in args.common_depends:
- analysis.append(libdeps.analyzer.CommonDependencies(libdeps_graph, depends))
+ for analyzer_args in args.graph_paths:
+ analysis.append(
+ libdeps_analyzer.GraphPaths(libdeps_graph, analyzer_args[0], analyzer_args[1]))
- for depends in args.exclude_depends:
- analysis.append(libdeps.analyzer.ExcludeDependencies(libdeps_graph, depends))
+ for analyzer_args in args.critical_edges:
+ analysis.append(
+ libdeps_analyzer.CriticalEdges(libdeps_graph, analyzer_args[0], analyzer_args[1]))
- analysis += libdeps.analyzer.linter_factory(libdeps_graph, args.lint)
+ analysis += libdeps_analyzer.linter_factory(libdeps_graph, args.lint)
if args.build_data:
- analysis.append(libdeps.analyzer.BuildDataReport(libdeps_graph))
+ analysis.append(libdeps_analyzer.BuildDataReport(libdeps_graph))
- ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=libdeps_graph, analysis=analysis)
+ ga = libdeps_analyzer.LibdepsGraphAnalysis(libdeps_graph=libdeps_graph, analysis=analysis)
if args.format == 'pretty':
- ga_printer = libdeps.analyzer.GaPrettyPrinter(ga)
+ ga_printer = libdeps_analyzer.GaPrettyPrinter(ga)
elif args.format == 'json':
- ga_printer = libdeps.analyzer.GaJsonPrinter(ga)
+ ga_printer = libdeps_analyzer.GaJsonPrinter(ga)
else:
return
diff --git a/buildscripts/libdeps/graph_visualizer.py b/buildscripts/libdeps/graph_visualizer.py
index 510edf8d97c..8d601355a56 100644
--- a/buildscripts/libdeps/graph_visualizer.py
+++ b/buildscripts/libdeps/graph_visualizer.py
@@ -106,7 +106,11 @@ def check_node(node_check, cwd):
node_modules = cwd / 'node_modules'
if not node_modules.exists():
- print(f"{node_modules} not found, you need to run 'npm install' in {cwd}")
+ print(
+ textwrap.dedent(f"""\
+ {node_modules} not found, you need to run 'npm install' in {cwd}
+ Perhaps run 'source {cwd}/setup_node_env.sh install'"""))
+ exit(1)
def start_backend(web_service_info, debug):
diff --git a/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py b/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py
index 333e714d8dc..09f7141560d 100644
--- a/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py
+++ b/buildscripts/libdeps/graph_visualizer_web_stack/flask/flask_backend.py
@@ -28,7 +28,7 @@ The backend interacts with the graph_analyzer to perform queries on various libd
"""
from pathlib import Path
-from collections import namedtuple
+from collections import namedtuple, OrderedDict
import flask
import networkx
@@ -57,7 +57,6 @@ class BackendServer:
self.socketio.on_event('git_hash_selected', self.git_hash_selected)
self.socketio.on_event('row_selected', self.row_selected)
- self.current_graph = networkx.DiGraph()
self.loaded_graphs = {}
self.current_selected_rows = {}
self.graphml_dir = Path(graphml_dir)
@@ -66,6 +65,28 @@ class BackendServer:
self.graph_file_tuple = namedtuple('GraphFile', ['version', 'git_hash', 'graph_file'])
self.graph_files = self.get_graphml_files()
+ try:
+ default_selected_graph = list(self.graph_files.items())[0][1].graph_file
+ self.load_graph_from_file(default_selected_graph)
+ self._dependents_graph = networkx.reverse_view(self._dependency_graph)
+ except (IndexError, AttributeError) as ex:
+ print(ex)
+ print(
+ f"Failed to load read a graph file from {list(self.graph_files.items())} for graphml_dir '{self.graphml_dir}'"
+ )
+ exit(1)
+
+ def load_graph_from_file(self, file_path):
+ """Load a graph file from disk and handle version."""
+
+ graph = libdeps.graph.LibdepsGraph(networkx.read_graphml(file_path))
+ if graph.graph['graph_schema_version'] == 1:
+ self._dependents_graph = graph
+ self._dependency_graph = networkx.reverse_view(self._dependents_graph)
+ else:
+ self._dependency_graph = graph
+ self._dependents_graph = networkx.reverse_view(self._dependency_graph)
+
def get_app(self):
"""Return the app and socketio instances."""
@@ -91,7 +112,7 @@ class BackendServer:
def get_graphml_files(self):
"""Find all graphml files in the target graphml dir."""
- graph_files = {}
+ graph_files = OrderedDict()
for graph_file in self.graphml_dir.glob("**/*.graphml"):
graph_file_tuple = self.get_graph_build_data(graph_file)
graph_files[graph_file_tuple.git_hash[:7]] = graph_file_tuple
@@ -124,21 +145,21 @@ class BackendServer:
str(node),
'name':
node.name,
- 'attribs':
- [{'name': key, 'value': value}
- for key, value in self.current_graph.nodes(data=True)[str(node)].items()],
+ 'attribs': [{
+ 'name': key, 'value': value
+ } for key, value in self._dependents_graph.nodes(data=True)[str(node)].items()],
'dependers': [{
'node':
depender, 'symbols':
- self.current_graph[str(node)][depender].get('symbols',
- '').split(' ')
- } for depender in self.current_graph[str(node)]],
+ self._dependents_graph[str(node)][depender].get('symbols',
+ '').split(' ')
+ } for depender in self._dependents_graph[str(node)]],
'dependencies': [{
'node':
dependency, 'symbols':
- self.current_graph[dependency][str(node)].get('symbols',
- '').split(' ')
- } for dependency in self.current_graph.rgraph[str(node)]],
+ self._dependents_graph[dependency][str(node)].get('symbols',
+ '').split(' ')
+ } for dependency in self._dependency_graph[str(node)]],
})
self.socketio.emit("node_infos", nodeinfo_data)
@@ -154,18 +175,21 @@ class BackendServer:
for node, _ in self.current_selected_rows.items():
nodes.add(
tuple({
- 'id': str(node), 'name': node.name, 'type': self.current_graph.nodes()
+ 'id': str(node), 'name': node.name, 'type': self._dependents_graph.nodes()
[str(node)]['bin_type']
}.items()))
- for depender in self.current_graph.rgraph[str(node)]:
+ for depender in self._dependency_graph[str(node)]:
depender_path = Path(depender)
- if self.current_graph[depender][str(node)].get('direct'):
+ if self._dependents_graph[depender][str(node)].get('direct'):
nodes.add(
tuple({
- 'id': str(depender_path), 'name': depender_path.name,
- 'type': self.current_graph.nodes()[str(depender_path)]['bin_type']
+ 'id':
+ str(depender_path), 'name':
+ depender_path.name, 'type':
+ self._dependents_graph.nodes()[str(depender_path)]
+ ['bin_type']
}.items()))
links.add(
tuple({'source': str(node), 'target': str(depender_path)}.items()))
@@ -203,9 +227,9 @@ class BackendServer:
with self.app.test_request_context():
analysis = libdeps.analyzer.counter_factory(
- self.current_graph,
+ self._dependents_graph,
[name[0] for name in libdeps.analyzer.CountTypes.__members__.items()])
- ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=self.current_graph,
+ ga = libdeps.analyzer.LibdepsGraphAnalysis(libdeps_graph=self._dependents_graph,
analysis=analysis)
results = ga.get_results()
@@ -223,7 +247,7 @@ class BackendServer:
"selectedNodes": [str(node) for node in list(self.current_selected_rows.keys())]
}
- for node in self.current_graph.nodes():
+ for node in self._dependents_graph.nodes():
node_path = Path(node)
node_data['graphData']['nodes'].append(
{'id': str(node_path), 'name': node_path.name})
@@ -234,20 +258,19 @@ class BackendServer:
with self.app.test_request_context():
- current_hash = self.current_graph.graph.get('git_hash', 'NO_HASH')[:7]
+ current_hash = self._dependents_graph.graph.get('git_hash', 'NO_HASH')[:7]
if current_hash != message['hash']:
self.current_selected_rows = {}
if message['hash'] in self.loaded_graphs:
- self.current_graph = self.loaded_graphs[message['hash']]
+ self._dependents_graph = self.loaded_graphs[message['hash']]
+ self._dependents_graph = networkx.reverse_view(self._dependency_graph)
else:
print(
f'loading new graph {current_hash} because different than {message["hash"]}'
)
- graph = networkx.read_graphml(self.graph_files[message['hash']].graph_file)
-
- self.current_graph = libdeps.graph.LibdepsGraph(graph)
- self.loaded_graphs[message['hash']] = self.current_graph
+ self.load_graph_from_file(self.graph_files[message['hash']].graph_file)
+ self.loaded_graphs[message['hash']] = self._dependents_graph
self.socketio.start_background_task(self.analyze_counts)
self.socketio.start_background_task(self.send_node_list)
diff --git a/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js b/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js
index d1da8c4cc0d..3975ffe9f3a 100644
--- a/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js
+++ b/buildscripts/libdeps/graph_visualizer_web_stack/src/DataGrid.js
@@ -12,6 +12,39 @@ import { getRows } from "./redux/store";
import { updateSelected } from "./redux/nodes";
import { socket } from "./connect";
+function componentToHex(c) {
+ var hex = c.toString(16);
+ return hex.length == 1 ? "0" + hex : hex;
+}
+
+function rgbToHex(r, g, b) {
+ return "#" + componentToHex(r) + componentToHex(g) + componentToHex(b);
+}
+
+function hexToRgb(hex) {
+ // Expand shorthand form (e.g. "03F") to full form (e.g. "0033FF")
+ var shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i;
+ hex = hex.replace(shorthandRegex, function(m, r, g, b) {
+ return r + r + g + g + b + b;
+ });
+
+ var result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
+ return result ? {
+ r: parseInt(result[1], 16),
+ g: parseInt(result[2], 16),
+ b: parseInt(result[3], 16)
+ } : null;
+}
+
+function incrementPallete(palleteColor, increment){
+ var rgb = hexToRgb(palleteColor);
+ rgb.r += increment;
+ rgb.g += increment;
+ rgb.b += increment;
+ return rgbToHex(rgb.r, rgb.g, rgb.b);
+
+}
+
const styles = (theme) => ({
flexContainer: {
display: "flex",
@@ -26,12 +59,14 @@ const styles = (theme) => ({
},
},
tableRowOdd: {
- backgroundColor: "#4d4d4d",
+ backgroundColor: incrementPallete(theme.palette.grey[800], 10),
+ },
+ tableRowEven: {
+ backgroundColor: theme.palette.grey[800],
},
- tableRowEven: {},
tableRowHover: {
"&:hover": {
- backgroundColor: theme.palette.grey[700],
+ backgroundColor: theme.palette.grey[600],
},
},
tableCell: {
diff --git a/buildscripts/libdeps/libdeps/analyzer.py b/buildscripts/libdeps/libdeps/analyzer.py
index f32f0e7a4b9..2ddc35662ca 100644
--- a/buildscripts/libdeps/libdeps/analyzer.py
+++ b/buildscripts/libdeps/libdeps/analyzer.py
@@ -31,15 +31,43 @@ represents the dependency information between all binaries from the build.
import sys
import textwrap
+import copy
+import json
+import inspect
+import functools
from pathlib import Path
+import networkx
+
from libdeps.graph import CountTypes, DependsReportTypes, LinterTypes, EdgeProps, NodeProps
-sys.path.append(str(Path(__file__).parent.parent.parent))
-import scons # pylint: disable=wrong-import-position
-sys.path.append(str(Path(scons.MONGODB_ROOT).joinpath('site_scons')))
-from libdeps_next import deptype # pylint: disable=wrong-import-position
+class UnsupportedAnalyzer(Exception):
+ """Thrown when an analyzer is run on a graph with an unsupported schema."""
+
+ pass
+
+
+# https://stackoverflow.com/a/25959545/1644736
+def get_class_that_defined_method(meth):
+ """Get the name of the class for given function."""
+
+ if isinstance(meth, functools.partial):
+ return get_class_that_defined_method(meth.func)
+ if inspect.ismethod(meth) or (inspect.isbuiltin(meth)
+ and getattr(meth, '__self__', None) is not None
+ and getattr(meth.__self__, '__class__', None)):
+ for cls in inspect.getmro(meth.__self__.__class__):
+ if meth.__name__ in cls.__dict__:
+ return cls
+ meth = getattr(meth, '__func__', meth) # fallback to __qualname__ parsing
+ if inspect.isfunction(meth):
+ cls = getattr(
+ inspect.getmodule(meth),
+ meth.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0], None)
+ if isinstance(cls, type):
+ return cls
+ return getattr(meth, '__objclass__', None) # handle special descriptor objects
# newer pylints contain the fix: https://github.com/PyCQA/pylint/pull/2926/commits/35e1c61026eab90af504806ef9da6241b096e659
@@ -62,15 +90,23 @@ def schema_check(func, schema_version):
"""Check the version for a function against the graph."""
def check(*args, **kwargs):
- if schema_version <= args[0].graph.graph.get('graph_schema_version'):
+
+ if schema_version <= args[0].graph_schema:
return func(*args, **kwargs)
else:
- sys.stderr.write(
- f"WARNING: analysis for '{func.__name__}' requires graph schema version '{schema_version}'\n"
- +
- f"but detected graph schema version '{args[0].graph.graph.get('graph_schema_version')}'\n"
- + f"Not running analysis for {func.__name__}\n\n")
- return "GRAPH_SCHEMA_VERSION_ERR"
+ analyzer = get_class_that_defined_method(func)
+ if not analyzer:
+ analyzer = "UnknownAnalyzer"
+ else:
+ analyzer = analyzer.__name__
+
+ raise UnsupportedAnalyzer(
+ textwrap.dedent(f"""\
+
+
+ ERROR: analysis for '{analyzer}' requires graph schema version '{schema_version}'
+ but detected graph schema version '{args[0].graph_schema}'
+ """))
return check
@@ -78,27 +114,61 @@ def schema_check(func, schema_version):
class Analyzer:
"""Base class for different types of analyzers."""
- def __init__(self, graph):
+ # pylint: disable=too-many-instance-attributes
+ def __init__(self, graph, progress=True):
"""Store the graph and extract the build_dir from the graph."""
- self.graph = graph
- self.build_dir = Path(graph.graph['build_dir'])
+ self.graph_schema = graph.graph.get('graph_schema_version')
+ if self.graph_schema == 1:
+ self._dependents_graph = graph
+ else:
+ self._dependency_graph = graph
+
+ self._build_dir = Path(graph.graph['build_dir'])
+ self.deptypes = json.loads(graph.graph.get('deptypes', "{}"))
+ self.set_progress(progress)
+
+ @property
+ def _dependents_graph(self):
+ if not hasattr(self, 'rgraph'):
+ setattr(self, 'rgraph', networkx.reverse_view(self._dependency_graph))
+ return self.rgraph
+
+ @_dependents_graph.setter
+ def _dependents_graph(self, value):
+ self.rgraph = value
+
+ @property
+ def _dependency_graph(self):
+ if not hasattr(self, 'graph'):
+ setattr(self, 'graph', networkx.reverse_view(self._dependents_graph))
+ return self.graph
+
+ @_dependency_graph.setter
+ def _dependency_graph(self, value):
+ self.graph = value
+
+ def get_deptype(self, deptype):
+ """Call down to loaded graph to get the deptype from name."""
+
+ return int(self._dependency_graph.get_deptype(deptype))
def _strip_build_dir(self, node):
"""Small util function for making args match the graph paths."""
- node = Path(node)
- if str(node).startswith(str(self.build_dir)):
- return str(node.relative_to(self.build_dir))
- else:
- raise Exception(
- f"build path not in node path: node: {node} build_dir: {self.build_dir}")
+ return str(Path(node).relative_to(self._build_dir))
def _strip_build_dirs(self, nodes):
"""Small util function for making a list of nodes match graph paths."""
return [self._strip_build_dir(node) for node in nodes]
+ def set_progress(self, value=None):
+ """Get a progress bar from the loaded graph."""
+
+ self._progressbar = self._dependency_graph.get_progress(value)
+ return self._progressbar
+
class Counter(Analyzer):
"""Base Counter Analyzer class for various counters."""
@@ -106,19 +176,23 @@ class Counter(Analyzer):
def number_of_edge_types(self, edge_type, value):
"""Count the graphs edges based on type."""
- return len(
- [edge for edge in self.graph.edges(data=True) if edge[2].get(edge_type) == value])
+ return len([
+ edge for edge in self._dependency_graph.edges(data=True)
+ if edge[2].get(edge_type) == value
+ ])
def node_type_count(self, node_type, value):
"""Count the graphs nodes based on type."""
- return len(
- [node for node in self.graph.nodes(data=True) if node[1].get(node_type) == value])
+ return len([
+ node for node in self._dependency_graph.nodes(data=True)
+ if node[1].get(node_type) == value
+ ])
def report(self, report):
"""Report the results for the current type."""
- report[self.count_type] = self.run()
+ report[self._count_type] = self.run()
class NodeCounter(Counter):
@@ -128,13 +202,13 @@ class NodeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.node.name
+ self._count_type = CountTypes.NODE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs nodes."""
- return self.graph.number_of_nodes()
+ return self._dependency_graph.number_of_nodes()
class EdgeCounter(Counter):
@@ -144,13 +218,13 @@ class EdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.edge.name
+ self._count_type = CountTypes.EDGE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs edges."""
- return self.graph.number_of_edges()
+ return self._dependency_graph.number_of_edges()
class DirectEdgeCounter(Counter):
@@ -160,7 +234,7 @@ class DirectEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.dir_edge.name
+ self._count_type = CountTypes.DIR_EDGE.name
@schema_check(schema_version=1)
def run(self):
@@ -176,7 +250,7 @@ class TransEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.trans_edge.name
+ self._count_type = CountTypes.TRANS_EDGE.name
@schema_check(schema_version=1)
def run(self):
@@ -192,15 +266,15 @@ class DirectPubEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.dir_pub_edge.name
+ self._count_type = CountTypes.DIR_PUB_EDGE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs direct public edges."""
-
return len([
- edge for edge in self.graph.edges(data=True) if edge[2].get(EdgeProps.direct.name)
- and edge[2].get(EdgeProps.visibility.name) == int(deptype.Public)
+ edge for edge in self._dependency_graph.edges(data=True)
+ if edge[2].get(EdgeProps.direct.name)
+ and edge[2].get(EdgeProps.visibility.name) == int(self.get_deptype('Public'))
])
@@ -211,13 +285,13 @@ class PublicEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.pub_edge.name
+ self._count_type = CountTypes.PUB_EDGE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs public edges."""
- return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Public))
+ return self.number_of_edge_types(EdgeProps.visibility.name, int(self.get_deptype('Public')))
class PrivateEdgeCounter(Counter):
@@ -227,13 +301,14 @@ class PrivateEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.priv_edge.name
+ self._count_type = CountTypes.PRIV_EDGE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs private edges."""
- return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Private))
+ return self.number_of_edge_types(EdgeProps.visibility.name, int(
+ self.get_deptype('Private')))
class InterfaceEdgeCounter(Counter):
@@ -243,13 +318,14 @@ class InterfaceEdgeCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.if_edge.name
+ self._count_type = CountTypes.IF_EDGE.name
@schema_check(schema_version=1)
def run(self):
"""Count the graphs interface edges."""
- return self.number_of_edge_types(EdgeProps.visibility.name, int(deptype.Interface))
+ return self.number_of_edge_types(EdgeProps.visibility.name,
+ int(self.get_deptype('Interface')))
class ShimCounter(Counter):
@@ -259,7 +335,7 @@ class ShimCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.shim.name
+ self._count_type = CountTypes.SHIM.name
@schema_check(schema_version=1)
def run(self):
@@ -275,7 +351,7 @@ class LibCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.lib.name
+ self._count_type = CountTypes.LIB.name
@schema_check(schema_version=1)
def run(self):
@@ -291,7 +367,7 @@ class ProgCounter(Counter):
"""Store graph and set type."""
super().__init__(graph)
- self.count_type = CountTypes.prog.name
+ self._count_type = CountTypes.PROG.name
@schema_check(schema_version=1)
def run(self):
@@ -300,21 +376,21 @@ class ProgCounter(Counter):
return self.node_type_count(NodeProps.bin_type.name, 'Program')
-def counter_factory(graph, counters):
+def counter_factory(graph, counters, progressbar=True):
"""Construct counters from a list of strings."""
counter_map = {
- CountTypes.node.name: NodeCounter,
- CountTypes.edge.name: EdgeCounter,
- CountTypes.dir_edge.name: DirectEdgeCounter,
- CountTypes.trans_edge.name: TransEdgeCounter,
- CountTypes.dir_pub_edge.name: DirectPubEdgeCounter,
- CountTypes.pub_edge.name: PublicEdgeCounter,
- CountTypes.priv_edge.name: PrivateEdgeCounter,
- CountTypes.if_edge.name: InterfaceEdgeCounter,
- CountTypes.shim.name: ShimCounter,
- CountTypes.lib.name: LibCounter,
- CountTypes.prog.name: ProgCounter,
+ CountTypes.NODE.name: NodeCounter,
+ CountTypes.EDGE.name: EdgeCounter,
+ CountTypes.DIR_EDGE.name: DirectEdgeCounter,
+ CountTypes.TRANS_EDGE.name: TransEdgeCounter,
+ CountTypes.DIR_PUB_EDGE.name: DirectPubEdgeCounter,
+ CountTypes.PUB_EDGE.name: PublicEdgeCounter,
+ CountTypes.PRIV_EDGE.name: PrivateEdgeCounter,
+ CountTypes.IF_EDGE.name: InterfaceEdgeCounter,
+ CountTypes.SHIM.name: ShimCounter,
+ CountTypes.LIB.name: LibCounter,
+ CountTypes.PROG.name: ProgCounter,
}
if not isinstance(counters, list):
@@ -323,71 +399,74 @@ def counter_factory(graph, counters):
counter_objs = []
for counter in counters:
if counter in counter_map:
- counter_objs.append(counter_map[counter](graph))
+ counter_obj = counter_map[counter](graph)
+ counter_obj.set_progress(progressbar)
+ counter_objs.append(counter_obj)
+
else:
print(f"Skipping unknown counter: {counter}")
return counter_objs
-class CommonDependencies(Analyzer):
- """Finds common dependencies for a set of given nodes."""
+class CommonDependents(Analyzer):
+ """Finds common dependent nodes for a set of given dependency nodes."""
def __init__(self, graph, nodes):
"""Store graph and strip the nodes."""
super().__init__(graph)
- self.nodes = self._strip_build_dirs(nodes)
+ self._nodes = self._strip_build_dirs(nodes)
@schema_check(schema_version=1)
def run(self):
"""For a given set of nodes, report what nodes depend on all nodes from that set."""
- neighbor_sets = [set(self.graph[node]) for node in self.nodes]
- return list(set.intersection(*neighbor_sets))
+ neighbor_sets = [set(self._dependents_graph[node]) for node in self._nodes]
+ return sorted(list(set.intersection(*neighbor_sets)))
def report(self, report):
"""Add the common depends list for this tuple of nodes."""
- if DependsReportTypes.common_depends.name not in report:
- report[DependsReportTypes.common_depends.name] = {}
- report[DependsReportTypes.common_depends.name][tuple(self.nodes)] = self.run()
+ if DependsReportTypes.COMMON_DEPENDS.name not in report:
+ report[DependsReportTypes.COMMON_DEPENDS.name] = {}
+ report[DependsReportTypes.COMMON_DEPENDS.name][tuple(self._nodes)] = self.run()
-class DirectDependencies(Analyzer):
- """Finds direct dependencies for a given node."""
+class DirectDependents(Analyzer):
+ """Finds direct dependent nodes for a given dependency node."""
def __init__(self, graph, node):
"""Store graph and strip the node."""
super().__init__(graph)
- self.node = self._strip_build_dir(node)
+ self._node = self._strip_build_dir(node)
@schema_check(schema_version=1)
def run(self):
"""For given nodes, report what nodes depend directly on that node."""
- return [
- depender for depender in self.graph[self.node]
- if self.graph[self.node][depender].get(EdgeProps.direct.name)
- ]
+ return sorted([
+ depender for depender in self._dependents_graph[self._node]
+ if self._dependents_graph[self._node][depender].get(EdgeProps.direct.name)
+ ])
def report(self, report):
"""Add the direct depends list for this node."""
- if DependsReportTypes.direct_depends.name not in report:
- report[DependsReportTypes.direct_depends.name] = {}
- report[DependsReportTypes.direct_depends.name][self.node] = self.run()
+ if DependsReportTypes.DIRECT_DEPENDS.name not in report:
+ report[DependsReportTypes.DIRECT_DEPENDS.name] = {}
+ report[DependsReportTypes.DIRECT_DEPENDS.name][self._node] = self.run()
-class ExcludeDependencies(Analyzer):
- """Finds finds dependencies which include one node, but exclude others."""
+class ExcludeDependents(Analyzer):
+ """Finds dependents which depend on the first input node, but exclude the other input nodes."""
def __init__(self, graph, nodes):
"""Store graph and strip the nodes."""
super().__init__(graph)
- self.nodes = self._strip_build_dirs(nodes)
+ self._nodes = self._strip_build_dirs(nodes)
@schema_check(schema_version=1)
def run(self):
@@ -398,19 +477,89 @@ class ExcludeDependencies(Analyzer):
"""
valid_depender_nodes = []
- for depender_node in set(self.graph[self.nodes[0]]):
+ for depender_node in set(self._dependents_graph[self._nodes[0]]):
if all(
- bool(excludes_node not in set(self.graph.rgraph[depender_node]))
- for excludes_node in self.nodes[1:]):
+ bool(excludes_node not in set(self._dependency_graph[depender_node]))
+ for excludes_node in self._nodes[1:]):
valid_depender_nodes.append(depender_node)
- return valid_depender_nodes
+ return sorted(valid_depender_nodes)
def report(self, report):
"""Add the exclude depends list for this tuple of nodes."""
- if DependsReportTypes.exclude_depends.name not in report:
- report[DependsReportTypes.exclude_depends.name] = {}
- report[DependsReportTypes.exclude_depends.name][tuple(self.nodes)] = self.run()
+ if DependsReportTypes.EXCLUDE_DEPENDS.name not in report:
+ report[DependsReportTypes.EXCLUDE_DEPENDS.name] = {}
+ report[DependsReportTypes.EXCLUDE_DEPENDS.name][tuple(self._nodes)] = self.run()
+
+
+class GraphPaths(Analyzer):
+ """Finds all paths between two nodes in the graph."""
+
+ def __init__(self, graph, from_node, to_node):
+ """Store graph and strip the nodes."""
+
+ super().__init__(graph)
+ self._from_node, self._to_node = self._strip_build_dirs([from_node, to_node])
+
+ @schema_check(schema_version=1)
+ def run(self):
+ """Find all paths between the two nodes in the graph."""
+
+ # We can really help out networkx path finding algorithm by striping the graph down to
+ # just a graph containing only paths between the source and target node. This is done by
+ # getting a subtree from the target down, and then getting a subtree of that tree from the
+ # source up.
+ dependents_tree = self._dependents_graph.get_direct_nonprivate_graph().get_node_tree(
+ self._to_node)
+
+ if self._from_node not in dependents_tree:
+ return []
+
+ path_tree = networkx.reverse_view(dependents_tree).get_node_tree(self._from_node)
+ return list(
+ networkx.all_simple_paths(G=path_tree, source=self._from_node, target=self._to_node))
+
+ def report(self, report):
+ """Add the path list to the report."""
+
+ if DependsReportTypes.GRAPH_PATHS.name not in report:
+ report[DependsReportTypes.GRAPH_PATHS.name] = {}
+ report[DependsReportTypes.GRAPH_PATHS.name][tuple([self._from_node,
+ self._to_node])] = self.run()
+
+
+class CriticalEdges(Analyzer):
+ """Finds all edges between two nodes, where removing those edges disconnects the two nodes."""
+
+ def __init__(self, graph, from_node, to_node):
+ """Store graph and strip the nodes."""
+
+ super().__init__(graph)
+ self._from_node, self._to_node = self._strip_build_dirs([from_node, to_node])
+
+ @schema_check(schema_version=1)
+ def run(self):
+ """Use networkx min cut algorithm to find a set of edges."""
+
+ from networkx.algorithms.connectivity import minimum_st_edge_cut
+
+ # The min cut algorithm will get the min cut nearest the end
+ # of the direction of the graph, so we we use the reverse graph
+ # so that we get a cut nearest our from_node, or the first cut we
+ # would encounter on a given path from the from_node to the to_node.
+ min_cut_edges = list(
+ minimum_st_edge_cut(
+ G=self._dependents_graph.get_direct_nonprivate_graph().get_node_tree(self._to_node),
+ s=self._to_node, t=self._from_node))
+ return [(edge[1], edge[0]) for edge in min_cut_edges]
+
+ def report(self, report):
+ """Add the critical edges to report."""
+
+ if DependsReportTypes.CRITICAL_EDGES.name not in report:
+ report[DependsReportTypes.CRITICAL_EDGES.name] = {}
+ report[DependsReportTypes.CRITICAL_EDGES.name][tuple([self._from_node,
+ self._to_node])] = self.run()
class UnusedPublicLinter(Analyzer):
@@ -424,10 +573,11 @@ class UnusedPublicLinter(Analyzer):
original_node = edge[0]
depender = edge[1]
try:
- edge_attribs = self.graph[original_node][depender]
+ edge_attribs = self._dependents_graph[original_node][depender]
- if (edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public)
- or edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Interface)):
+ if (edge_attribs.get(EdgeProps.visibility.name) == int(self.get_deptype('Public'))
+ or edge_attribs.get(EdgeProps.visibility.name) == int(
+ self.get_deptype('Interface'))):
if not edge_attribs.get(EdgeProps.symbols.name):
if not self._tree_uses_no_symbols(depender, original_nodes, checked_edges):
return False
@@ -445,7 +595,7 @@ class UnusedPublicLinter(Analyzer):
in that tree do not have symbol dependencies.
"""
- for depender in self.graph[node]:
+ for depender in self._dependents_graph[node]:
for original_node in original_nodes:
edge = (original_node, depender)
if not self._check_edge_no_symbols(edge, original_nodes, checked_edges):
@@ -455,13 +605,13 @@ class UnusedPublicLinter(Analyzer):
def _check_trans_nodes_no_symbols(self, edge, trans_pub_nodes):
"""Check the edge against the transitive nodes for symbols."""
- for trans_node in self.graph.rgraph[edge[0]]:
- if (self.graph.rgraph[edge[0]][trans_node].get(EdgeProps.visibility.name) == int(
- deptype.Public) or self.graph.rgraph[edge[0]][trans_node].get(
- EdgeProps.visibility.name) == int(deptype.Interface)):
+ for trans_node in self._dependency_graph[edge[0]]:
+ if (self._dependency_graph[edge[0]][trans_node].get(EdgeProps.visibility.name) == int(
+ self.get_deptype('Public')) or self._dependency_graph[edge[0]][trans_node].get(
+ EdgeProps.visibility.name) == int(self.get_deptype('Interface'))):
trans_pub_nodes.add(trans_node)
try:
- if self.graph[trans_node][edge[1]].get(EdgeProps.symbols.name):
+ if self._dependents_graph[trans_node][edge[1]].get(EdgeProps.symbols.name):
return True
except KeyError:
pass
@@ -478,13 +628,15 @@ class UnusedPublicLinter(Analyzer):
unused_public_libdeps = []
checked_edges = set()
- for edge in self.graph.edges:
- edge_attribs = self.graph[edge[0]][edge[1]]
+ for edge in self._dependents_graph.edges:
+ edge_attribs = self._dependents_graph[edge[0]][edge[1]]
+
+ if (edge_attribs.get(EdgeProps.direct.name) and edge_attribs.get(
+ EdgeProps.visibility.name) == int(self.get_deptype('Public'))
+ and not self._dependents_graph.nodes()[edge[0]].get(NodeProps.shim.name)
+ and self._dependents_graph.nodes()[edge[1]].get(
+ NodeProps.bin_type.name) == 'SharedLibrary'):
- if (edge_attribs.get(EdgeProps.direct.name)
- and edge_attribs.get(EdgeProps.visibility.name) == int(deptype.Public)
- and not self.graph.nodes()[edge[0]].get(NodeProps.shim.name) and
- self.graph.nodes()[edge[1]].get(NodeProps.bin_type.name) == 'SharedLibrary'):
# First we will get all the transitive libdeps the dependent node
# induces, while we are getting those we also check if the depender
# node has any symbol dependencies to that transitive libdep.
@@ -504,14 +656,14 @@ class UnusedPublicLinter(Analyzer):
def report(self, report):
"""Report the lint issies."""
- report[LinterTypes.public_unused.name] = self.run()
+ report[LinterTypes.PUBLIC_UNUSED.name] = self.run()
-def linter_factory(graph, linters):
+def linter_factory(graph, linters, progressbar=True):
"""Construct linters from a list of strings."""
linter_map = {
- LinterTypes.public_unused.name: UnusedPublicLinter,
+ LinterTypes.PUBLIC_UNUSED.name: UnusedPublicLinter,
}
if not isinstance(linters, list):
@@ -520,7 +672,7 @@ def linter_factory(graph, linters):
linters_objs = []
for linter in linters:
if linter in linter_map:
- linters_objs.append(linter_map[linter](graph))
+ linters_objs.append(linter_map[linter](graph, progressbar))
else:
print(f"Skipping unknown counter: {linter}")
@@ -534,9 +686,9 @@ class BuildDataReport(Analyzer):
def report(self, report):
"""Add the build data from the graph to the report."""
- report['invocation'] = self.graph.graph.get('invocation')
- report['git_hash'] = self.graph.graph.get('git_hash')
- report['graph_schema_version'] = self.graph.graph.get('graph_schema_version')
+ report['invocation'] = self._dependency_graph.graph.get('invocation')
+ report['git_hash'] = self._dependency_graph.graph.get('git_hash')
+ report['graph_schema_version'] = self._dependency_graph.graph.get('graph_schema_version')
class LibdepsGraphAnalysis:
@@ -545,22 +697,22 @@ class LibdepsGraphAnalysis:
def __init__(self, libdeps_graph, analysis):
"""Perform analysis based off input args."""
- self.libdeps_graph = libdeps_graph
+ self._libdeps_graph = libdeps_graph
- self.results = {}
+ self._results = {}
for analyzer in analysis:
- analyzer.report(self.results)
+ analyzer.report(self._results)
def get_results(self):
"""Return the results fo the analysis."""
- return self.results
+ return self._results
def run_linters(self, linters):
"""Run the various dependency reports."""
- if LinterTypes.public_unused.name in linters:
- self.results[LinterTypes.public_unused.name] = \
+ if LinterTypes.PUBLIC_UNUSED.name in linters:
+ self.results[LinterTypes.PUBLIC_UNUSED.name] = \
self.libdeps_graph.unused_public_linter()
@@ -570,7 +722,7 @@ class GaPrinter:
def __init__(self, libdeps_graph_analysis):
"""Store the graph analysis for use when printing."""
- self.libdeps_graph_analysis = libdeps_graph_analysis
+ self._libdeps_graph_analysis = libdeps_graph_analysis
class GaJsonPrinter(GaPrinter):
@@ -589,26 +741,30 @@ class GaJsonPrinter(GaPrinter):
def print(self):
"""Print the result data."""
- import json # pylint: disable=import-outside-toplevel
- results = self.libdeps_graph_analysis.get_results()
- print(json.dumps(self.serialize(results)))
+ print(self.get_json())
+
+ def get_json(self):
+ """Return the results as a JSON string."""
+
+ results = self._libdeps_graph_analysis.get_results()
+ return json.dumps(self.serialize(results))
class GaPrettyPrinter(GaPrinter):
"""Printer for pretty console output."""
- count_descs = {
- CountTypes.node.name: "Nodes in Graph: {}",
- CountTypes.edge.name: "Edges in Graph: {}",
- CountTypes.dir_edge.name: "Direct Edges in Graph: {}",
- CountTypes.trans_edge.name: "Transitive Edges in Graph: {}",
- CountTypes.dir_pub_edge.name: "Direct Public Edges in Graph: {}",
- CountTypes.pub_edge.name: "Public Edges in Graph: {}",
- CountTypes.priv_edge.name: "Private Edges in Graph: {}",
- CountTypes.if_edge.name: "Interface Edges in Graph: {}",
- CountTypes.shim.name: "Shim Nodes in Graph: {}",
- CountTypes.lib.name: "Library Nodes in Graph: {}",
- CountTypes.prog.name: "Program Nodes in Graph: {}",
+ _count_descs = {
+ CountTypes.NODE.name: "Nodes in Graph: {}",
+ CountTypes.EDGE.name: "Edges in Graph: {}",
+ CountTypes.DIR_EDGE.name: "Direct Edges in Graph: {}",
+ CountTypes.TRANS_EDGE.name: "Transitive Edges in Graph: {}",
+ CountTypes.DIR_PUB_EDGE.name: "Direct Public Edges in Graph: {}",
+ CountTypes.PUB_EDGE.name: "Public Edges in Graph: {}",
+ CountTypes.PRIV_EDGE.name: "Private Edges in Graph: {}",
+ CountTypes.IF_EDGE.name: "Interface Edges in Graph: {}",
+ CountTypes.SHIM.name: "Shim Nodes in Graph: {}",
+ CountTypes.LIB.name: "Library Nodes in Graph: {}",
+ CountTypes.PROG.name: "Program Nodes in Graph: {}",
}
@staticmethod
@@ -620,13 +776,53 @@ class GaPrettyPrinter(GaPrinter):
print(f" {i}: {depender}")
print("")
+ def _print_depends_reports(self, results):
+ """Print the depends reports result data."""
+
+ # pylint: disable=too-many-branches
+ if DependsReportTypes.DIRECT_DEPENDS.name in results:
+ print("\nNodes that directly depend on:")
+ for node in results[DependsReportTypes.DIRECT_DEPENDS.name]:
+ self._print_results_node_list(f'=>depends on {node}:',
+ results[DependsReportTypes.DIRECT_DEPENDS.name][node])
+
+ if DependsReportTypes.COMMON_DEPENDS.name in results:
+ print("\nNodes that commonly depend on:")
+ for nodes in results[DependsReportTypes.COMMON_DEPENDS.name]:
+ self._print_results_node_list(
+ f'=>depends on {nodes}:',
+ results[DependsReportTypes.COMMON_DEPENDS.name][nodes])
+
+ if DependsReportTypes.EXCLUDE_DEPENDS.name in results:
+ print("\nNodes that depend on a node, but exclude others:")
+ for nodes in results[DependsReportTypes.EXCLUDE_DEPENDS.name]:
+ self._print_results_node_list(
+ f"=>depends: {nodes[0]}, exclude: {nodes[1:]}:",
+ results[DependsReportTypes.EXCLUDE_DEPENDS.name][nodes])
+
+ if DependsReportTypes.GRAPH_PATHS.name in results:
+ print("\nDependency graph paths:")
+ for nodes in results[DependsReportTypes.GRAPH_PATHS.name]:
+ self._print_results_node_list(f"=>start node: {nodes[0]}, end node: {nodes[1]}:", [
+ f"{' -> '.join(path)}"
+ for path in results[DependsReportTypes.GRAPH_PATHS.name][nodes]
+ ])
+
+ if DependsReportTypes.CRITICAL_EDGES.name in results:
+ print("\nCritical Edges:")
+ for nodes in results[DependsReportTypes.CRITICAL_EDGES.name]:
+ self._print_results_node_list(
+ f"=>critical edges between {nodes[0]} and {nodes[1]}:",
+ results[DependsReportTypes.CRITICAL_EDGES.name][nodes])
+
def print(self):
"""Print the result data."""
- results = self.libdeps_graph_analysis.get_results()
+ results = self._libdeps_graph_analysis.get_results()
if 'invocation' in results:
print(
textwrap.dedent(f"""\
+
Graph built from git hash:
{results['git_hash']}
@@ -638,33 +834,15 @@ class GaPrettyPrinter(GaPrinter):
"""))
for count_type in CountTypes.__members__.items():
- if count_type[0] in self.count_descs and count_type[0] in results:
- print(self.count_descs[count_type[0]].format(results[count_type[0]]))
+ if count_type[0] in self._count_descs and count_type[0] in results:
+ print(self._count_descs[count_type[0]].format(results[count_type[0]]))
- if DependsReportTypes.direct_depends.name in results:
- print("\nNodes that directly depend on:")
- for node in results[DependsReportTypes.direct_depends.name]:
- self._print_results_node_list(f'=>depends on {node}:',
- results[DependsReportTypes.direct_depends.name][node])
-
- if DependsReportTypes.common_depends.name in results:
- print("\nNodes that commonly depend on:")
- for nodes in results[DependsReportTypes.common_depends.name]:
- self._print_results_node_list(
- f'=>depends on {nodes}:',
- results[DependsReportTypes.common_depends.name][nodes])
-
- if DependsReportTypes.exclude_depends.name in results:
- print("\nNodes that depend on a node, but exclude others:")
- for nodes in results[DependsReportTypes.exclude_depends.name]:
- self._print_results_node_list(
- f"=>depends: {nodes[0]}, exclude: {nodes[1:]}:",
- results[DependsReportTypes.exclude_depends.name][nodes])
+ self._print_depends_reports(results)
- if LinterTypes.public_unused.name in results:
+ if LinterTypes.PUBLIC_UNUSED.name in results:
print(
- f"\nLibdepsLinter: PUBLIC libdeps that could be PRIVATE: {len(results[LinterTypes.public_unused.name])}"
+ f"\nLibdepsLinter: PUBLIC libdeps that could be PRIVATE: {len(results[LinterTypes.PUBLIC_UNUSED.name])}"
)
- for issue in sorted(results[LinterTypes.public_unused.name],
+ for issue in sorted(results[LinterTypes.PUBLIC_UNUSED.name],
key=lambda item: item[1] + item[0]):
print(f" {issue[1]}: PUBLIC -> {issue[0]} -> PRIVATE")
diff --git a/buildscripts/libdeps/libdeps/graph.py b/buildscripts/libdeps/libdeps/graph.py
index 3f3f444a2ff..3bd047cc79f 100644
--- a/buildscripts/libdeps/libdeps/graph.py
+++ b/buildscripts/libdeps/libdeps/graph.py
@@ -26,44 +26,54 @@ Libdeps Graph Enums.
These are used for attributing data across the build scripts and analyzer scripts.
"""
-
from enum import Enum, auto
+from pathlib import Path
+import json
import networkx
-# pylint: disable=invalid-name
+try:
+ import progressbar
+except ImportError:
+ pass
+# We need to disable invalid name here because it break backwards compatibility with
+# our graph schemas. Possibly we could use lower case conversion process to maintain
+# backwards compatibility and make pylint happy.
+# pylint: disable=invalid-name
class CountTypes(Enum):
"""Enums for the different types of counts to perform on a graph."""
- all = auto()
- node = auto()
- edge = auto()
- dir_edge = auto()
- trans_edge = auto()
- dir_pub_edge = auto()
- pub_edge = auto()
- priv_edge = auto()
- if_edge = auto()
- shim = auto()
- prog = auto()
- lib = auto()
+ ALL = auto()
+ NODE = auto()
+ EDGE = auto()
+ DIR_EDGE = auto()
+ TRANS_EDGE = auto()
+ DIR_PUB_EDGE = auto()
+ PUB_EDGE = auto()
+ PRIV_EDGE = auto()
+ IF_EDGE = auto()
+ SHIM = auto()
+ PROG = auto()
+ LIB = auto()
class DependsReportTypes(Enum):
"""Enums for the different type of depends reports to perform on a graph."""
- direct_depends = auto()
- common_depends = auto()
- exclude_depends = auto()
+ DIRECT_DEPENDS = auto()
+ COMMON_DEPENDS = auto()
+ EXCLUDE_DEPENDS = auto()
+ GRAPH_PATHS = auto()
+ CRITICAL_EDGES = auto()
class LinterTypes(Enum):
"""Enums for the different types of counts to perform on a graph."""
- all = auto()
- public_unused = auto()
+ ALL = auto()
+ PUBLIC_UNUSED = auto()
class EdgeProps(Enum):
@@ -72,7 +82,6 @@ class EdgeProps(Enum):
direct = auto()
visibility = auto()
symbols = auto()
- shim = auto()
class NodeProps(Enum):
@@ -82,14 +91,88 @@ class NodeProps(Enum):
bin_type = auto()
+def null_progressbar(items):
+ """Fake stand-in for normal progressbar."""
+ for item in items:
+ yield item
+
+
class LibdepsGraph(networkx.DiGraph):
"""Class for analyzing the graph."""
def __init__(self, graph=networkx.DiGraph()):
"""Load the graph data."""
-
super().__init__(incoming_graph_data=graph)
+ self._progressbar = None
+ self._deptypes = None
+
+ def get_deptype(self, deptype):
+ """Convert graphs deptypes from json string to dict, and return requested value."""
+
+ if not self._deptypes:
+ self._deptypes = json.loads(self.graph.get('deptypes', "{}"))
+ if self.graph['graph_schema_version'] == 1:
+ # get and set the legacy values
+ self._deptypes['Global'] = self._deptypes.get('Global', 0)
+ self._deptypes['Public'] = self._deptypes.get('Public', 1)
+ self._deptypes['Private'] = self._deptypes.get('Private', 2)
+ self._deptypes['Interface'] = self._deptypes.get('Interface', 3)
+ self._deptypes['Typeinfo'] = self._deptypes.get('Typeinfo', 4)
+
+ return self._deptypes[deptype]
+
+ def _strip_build_dir(self, node):
+ """Small util function for making args match the graph paths."""
+
+ return str(Path(node).resolve().relative_to(Path(self.graph['build_dir']).resolve()))
+
+ def get_direct_nonprivate_graph(self):
+ """Get a graph view of direct nonprivate edges."""
+
+ def filter_direct_nonprivate_edges(n1, n2):
+ return (self[n1][n2].get(EdgeProps.direct.name) and
+ (self[n1][n2].get(EdgeProps.visibility.name) == self.get_deptype('Public') or
+ self[n1][n2].get(EdgeProps.visibility.name) == self.get_deptype('Interface')))
+
+ return networkx.subgraph_view(self, filter_edge=filter_direct_nonprivate_edges)
+
+ def get_node_tree(self, node):
+ """Get a tree with the passed node as the single root."""
+
+ direct_nonprivate_graph = self.get_direct_nonprivate_graph()
+ substree_set = networkx.descendants(direct_nonprivate_graph, node)
+
+ def subtree(n1):
+ return n1 in substree_set or n1 == node
+
+ return networkx.subgraph_view(direct_nonprivate_graph, filter_node=subtree)
+
+ def get_progress(self, value=None):
+ """
+ Set if a progress bar should be used or not.
+
+ No args means use progress bar if available.
+ """
+
+ if value is None:
+ value = ('progressbar' in globals())
+
+ if self._progressbar:
+ return self._progressbar
+
+ if value:
+
+ def get_progress_bar(title, *args):
+ custom_bar = progressbar.ProgressBar(widgets=[
+ title,
+ progressbar.Counter(format='[%(value)d/%(max_value)d]'),
+ progressbar.Timer(format=" Time: %(elapsed)s "),
+ progressbar.Bar(marker='>', fill=' ', left='|', right='|')
+ ])
+ return custom_bar(*args)
+
+ self._progressbar = get_progress_bar
+ else:
+ self._progressbar = null_progressbar
- # Load in the graph and store a reversed version as well for quick look ups
- # the in directions.
- self.rgraph = graph.reverse()
+ return self._progressbar