SERVER-52568 Added GraphPaths, CriticalEdges, and PublicWieghts quieries to libdeps graph analyzer.

1 files changed, 48 insertions, 41 deletions

diff --git a/site_scons/libdeps_next.py b/site_scons/libdeps_next.py
index 455131e6f38..194613d3e8a 100644
--- a/site_scons/libdeps_next.py
+++ b/site_scons/libdeps_next.py
@@ -55,9 +55,20 @@ from collections import defaultdict
 from functools import partial
 import enum
 import copy
+import json
 import os
 import sys
+import glob
 import textwrap
+import hashlib
+import json
+import fileinput
+
+try:
+    import networkx
+    from buildscripts.libdeps.libdeps.graph import EdgeProps, NodeProps, LibdepsGraph
+except ImportError:
+    pass
 
 import SCons.Errors
 import SCons.Scanner
@@ -948,7 +959,6 @@ def _get_node_with_ixes(env, node, node_builder_type):
 _get_node_with_ixes.node_type_ixes = dict()
 
 def add_node_from(env, node):
-    from buildscripts.libdeps.libdeps.graph import NodeProps
 
     env.GetLibdepsGraph().add_nodes_from([(
         str(node.abspath),
@@ -957,12 +967,11 @@ def add_node_from(env, node):
             NodeProps.shim.name: getattr(node.attributes, "is_shim", False)
         })])
 
-def add_edge_from(env, depender_node, dependent_node, visibility, direct):
-    from buildscripts.libdeps.libdeps.graph import EdgeProps
+def add_edge_from(env, from_node, to_node, visibility, direct):
 
     env.GetLibdepsGraph().add_edges_from([(
-        dependent_node,
-        depender_node,
+        from_node,
+        to_node,
         {
             EdgeProps.direct.name: direct,
             EdgeProps.visibility.name: int(visibility)
@@ -1184,34 +1193,30 @@ def expand_libdeps_with_flags(source, target, env, for_signature):
 
 def generate_libdeps_graph(env):
     if env.get('SYMBOLDEPSSUFFIX', None):
-        import glob
-        find_symbols = env.Dir("$BUILD_DIR").path + "/libdeps/find_symbols"
 
-        env.GetLibdepsGraph().graph['invocation'] = " ".join([env['ESCAPE'](str(sys.executable))] + [env['ESCAPE'](arg) for arg in sys.argv])
-        env.GetLibdepsGraph().graph['git_hash'] = env['MONGO_GIT_HASH']
-        env.GetLibdepsGraph().graph['graph_schema_version'] = env['LIBDEPS_GRAPH_SCHEMA_VERSION']
-        env.GetLibdepsGraph().graph['build_dir'] = env.Dir('$BUILD_DIR').path
+        find_symbols = env.Dir("$BUILD_DIR").path + "/libdeps/find_symbols"
+        libdeps_graph = env.GetLibdepsGraph()
 
         symbol_deps = []
-        for target, source in env.get('LIBDEPS_SYMBOL_DEP_FILES', []):
+        for symbols_file, target_node in env.get('LIBDEPS_SYMBOL_DEP_FILES', []):
 
             direct_libdeps = []
-            for direct_libdep in _get_sorted_direct_libdeps(source):
+            for direct_libdep in _get_sorted_direct_libdeps(target_node):
                 add_node_from(env, direct_libdep.target_node)
                 add_edge_from(
                     env,
-                    str(source.abspath),
+                    str(target_node.abspath),
                     str(direct_libdep.target_node.abspath),
                     visibility=int(direct_libdep.dependency_type),
                     direct=True)
                 direct_libdeps.append(direct_libdep.target_node.abspath)
 
-            for libdep in _get_libdeps(source):
+            for libdep in _get_libdeps(target_node):
                 if libdep.abspath not in direct_libdeps:
                     add_node_from(env, libdep)
                     add_edge_from(
                         env,
-                        str(source.abspath),
+                        str(target_node.abspath),
                         str(libdep.abspath),
                         visibility=int(deptype.Public),
                         direct=False)
@@ -1219,18 +1224,16 @@ def generate_libdeps_graph(env):
                 sep = ' '
             else:
                 sep = ':'
-            ld_path = sep.join([os.path.dirname(str(libdep)) for libdep in _get_libdeps(source)])
+            ld_path = sep.join([os.path.dirname(str(libdep)) for libdep in _get_libdeps(target_node)])
             symbol_deps.append(env.Command(
-                target=target,
-                source=source,
+                target=symbols_file,
+                source=target_node,
                 action=SCons.Action.Action(
                     f'{find_symbols} $SOURCE "{ld_path}" $TARGET',
                     "Generating $SOURCE symbol dependencies" if not env['VERBOSE'] else "")))
 
         def write_graph_hash(env, target, source):
-            import networkx
-            import hashlib
-            import json
+
             with open(target[0].path, 'w') as f:
                 json_str = json.dumps(networkx.readwrite.json_graph.node_link_data(env.GetLibdepsGraph()), sort_keys=True).encode('utf-8')
                 f.write(hashlib.sha256(json_str).hexdigest())
@@ -1253,7 +1256,7 @@ def generate_libdeps_graph(env):
                 generate_graph,
                 {"cmdstr": "Generating libdeps graph"}))
 
-        env.Depends(graph_node, graph_hash)
+        env.Depends(graph_node, [graph_hash] + env.Glob("#buildscripts/libdeps/libdeps/*"))
 
 def get_typeinfo_link_command():
     if LibdepLinter.skip_linting:
@@ -1309,10 +1312,8 @@ def get_libdeps_ld_path(source, target, env, for_signature):
 
 
 def generate_graph(env, target, source):
-    import fileinput
-    import networkx
-    import json
-    from buildscripts.libdeps.libdeps.graph import EdgeProps, NodeProps
+
+    libdeps_graph = env.GetLibdepsGraph()
 
     for symbol_deps_file in source:
         with open(str(symbol_deps_file)) as f:
@@ -1328,17 +1329,18 @@ def generate_graph(env, target, source):
             except json.JSONDecodeError:
                 env.FatalError(f"Failed processing json file: {str(symbol_deps_file)}")
 
-            for lib in symbols:
-
-                env.GetLibdepsGraph().add_edges_from([(
-                    os.path.abspath(lib).strip(),
-                    os.path.abspath(str(symbol_deps_file)[:-len(env['SYMBOLDEPSSUFFIX'])]),
-                    {EdgeProps.symbols.name: " ".join(symbols[lib]) })])
+            for libdep in symbols:
+                from_node = os.path.abspath(str(symbol_deps_file)[:-len(env['SYMBOLDEPSSUFFIX'])])
+                to_node = os.path.abspath(libdep).strip()
+                libdeps_graph.add_edges_from([(
+                    from_node,
+                    to_node,
+                    {EdgeProps.symbols.name: " ".join(symbols[libdep]) })])
                 node = env.File(str(symbol_deps_file)[:-len(env['SYMBOLDEPSSUFFIX'])])
                 add_node_from(env, node)
 
     libdeps_graph_file = f"{env.Dir('$BUILD_DIR').path}/libdeps/libdeps.graphml"
-    networkx.write_graphml(env.GetLibdepsGraph(), libdeps_graph_file, named_key_ids=True)
+    networkx.write_graphml(libdeps_graph, libdeps_graph_file, named_key_ids=True)
     with fileinput.FileInput(libdeps_graph_file, inplace=True) as file:
         for line in file:
             print(line.replace(str(env.Dir("$BUILD_DIR").abspath + os.sep), ''), end='')
@@ -1399,7 +1401,6 @@ def setup_environment(env, emitting_shared=False, debug='off', linting='on', san
         "${BUILD_DIR}/libdeps/libdeps.graphml")[0]
 
     if str(env['LIBDEPS_GRAPH_ALIAS']) in COMMAND_LINE_TARGETS:
-        import networkx
 
         # Detect if the current system has the tools to perform the generation.
         if env.GetOption('ninja') != 'disabled':
@@ -1435,16 +1436,22 @@ def setup_environment(env, emitting_shared=False, debug='off', linting='on', san
             symbol_deps.append(symbol_deps_file)
         env.AddMethod(append_symbol_deps, "AppendSymbolDeps")
 
-        libdeps_graph = networkx.DiGraph()
-        def get_libdeps_graph(env):
-            return libdeps_graph
-        env.AddMethod(get_libdeps_graph, "GetLibdepsGraph")
-
         env['LIBDEPS_SYMBOL_DEP_FILES'] = symbol_deps
         env['LIBDEPS_GRAPH_FILE'] = env.File("${BUILD_DIR}/libdeps/libdeps.graphml")
-        env['LIBDEPS_GRAPH_SCHEMA_VERSION'] = 1
+        env['LIBDEPS_GRAPH_SCHEMA_VERSION'] = 2
         env["SYMBOLDEPSSUFFIX"] = '.symbol_deps'
 
+        libdeps_graph = LibdepsGraph()
+        libdeps_graph.graph['invocation'] = " ".join([env['ESCAPE'](str(sys.executable))] + [env['ESCAPE'](arg) for arg in sys.argv])
+        libdeps_graph.graph['git_hash'] = env['MONGO_GIT_HASH']
+        libdeps_graph.graph['graph_schema_version'] = env['LIBDEPS_GRAPH_SCHEMA_VERSION']
+        libdeps_graph.graph['build_dir'] = env.Dir('$BUILD_DIR').path
+        libdeps_graph.graph['deptypes'] = json.dumps({key: value[0] for key, value in deptype.__members__.items() if isinstance(value, tuple)})
+
+        def get_libdeps_graph(env):
+            return libdeps_graph
+        env.AddMethod(get_libdeps_graph, "GetLibdepsGraph")
+
         # Now we will setup an emitter, and an additional action for several
         # of the builder involved with dynamic builds.
         def libdeps_graph_emitter(target, source, env):