SERVER-67058 Create libdeps metrics

4 files changed, 114 insertions, 40 deletions

diff --git a/site_scons/site_tools/build_metrics/__init__.py b/site_scons/site_tools/build_metrics/__init__.py
index c5c7d277e7b..c2c8b8fad94 100644
--- a/site_scons/site_tools/build_metrics/__init__.py
+++ b/site_scons/site_tools/build_metrics/__init__.py
@@ -37,6 +37,7 @@ from .per_action_metrics import PerActionMetrics
 from .artifacts import CollectArtifacts
 from .scons import SConsStats
 from .cache_dir import CacheDirCollector, CacheDirValidateWithMetrics
+from .libdeps import LibdepsCollector
 
 _SEC_TO_NANOSEC_FACTOR = 1000000000.0
 _METRICS_COLLECTORS = []
@@ -48,6 +49,7 @@ def finalize_build_metrics(env):
     for m in _METRICS_COLLECTORS:
         start_time = timer()
         sys.stdout.write(f"Processing {m.get_name()}...")
+        sys.stdout.flush()
         key, value = m.finalize()
         sys.stdout.write(f" {timer() - start_time}s\n")
         metrics[key] = value
@@ -90,7 +92,8 @@ def generate(env, **kwargs):
         PerActionMetrics(),
         CollectArtifacts(env),
         SConsStats(),
-        CacheDirCollector()
+        CacheDirCollector(),
+        LibdepsCollector(env)
     ]
 
     env['CACHEDIR_CLASS'] = CacheDirValidateWithMetrics
diff --git a/site_scons/site_tools/build_metrics/artifacts.py b/site_scons/site_tools/build_metrics/artifacts.py
index 7adf95336fe..386d996d130 100644
--- a/site_scons/site_tools/build_metrics/artifacts.py
+++ b/site_scons/site_tools/build_metrics/artifacts.py
@@ -158,7 +158,7 @@ class CollectArtifacts(BuildMetricsCollector):
         self._env = env
         self._build_dir = env.get("BUILD_METRICS_ARTIFACTS_DIR", env.Dir('#').abspath)
         self._artifacts = []
-        self._bloaty_bin = env.get("BUILD_METRICS_BLOATY", "bloaty")
+        self._bloaty_bin = env.get("BUILD_METRICS_BLOATY", env.WhereIs('bloaty'))
         self._metrics = {"total_artifact_size": 0, "num_artifacts": 0, "artifacts": []}
 
     def get_name(self):
@@ -202,7 +202,7 @@ class CollectArtifacts(BuildMetricsCollector):
 
                 if type_ == ArtifactType.UNKNOWN and any(s in magic_out for s in _TEXT_IDENTIFIERS):
                     type_ = ArtifactType.TEXT
-            except puremagic.main.PureError:
+            except (puremagic.main.PureError, ValueError):
                 # exception means that puremagic failed to id the filetype. We'll
                 # fallback to file extension in this case.
                 pass
diff --git a/site_scons/site_tools/build_metrics/build_metrics_format.schema b/site_scons/site_tools/build_metrics/build_metrics_format.schema
index faaf3384243..118028ad090 100644
--- a/site_scons/site_tools/build_metrics/build_metrics_format.schema
+++ b/site_scons/site_tools/build_metrics/build_metrics_format.schema
@@ -106,10 +106,10 @@
                         "items": { "type": "string"}
                     },
                     "action": {"type" : "string"}
-                },
-                "required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
-                "additionalProperties": false
-            }
+                }
+            },
+            "required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
+            "additionalProperties": false
         },
         "cache_metrics": {
             "type": "object",
@@ -134,49 +134,65 @@
             "required": ["cache_artifacts", "push_time", "pull_time", "cache_size"],
             "additionalProperties": false
         },
-        "scons_metrics": {
+        "libdeps_metrics": {
             "type": "object",
             "properties": {
-                "memory": {
+                "NODE": {"type": "integer"},
+                "EDGE": {"type": "integer"},
+                "DIR_EDGE": {"type": "integer"},
+                "TRANS_EDGE": {"type": "integer"},
+                "DIR_PUB_EDGE": {"type": "integer"},
+                "PUB_EDGE": {"type": "integer"},
+                "PRIV_EDGE": {"type": "integer"},
+                "IF_EDGE": {"type": "integer"},
+                "PROG": {"type": "integer"},
+                "LIB": {"type": "integer"}
+            },
+            "additionalProperties": false
+        }
+    },
+    "scons_metrics": {
+        "type": "object",
+        "properties": {
+            "memory": {
+                "type": "object",
+                "properties": {
+                    "pre_read": {"$ref": "#/$defs/bytes"},
+                    "post_read": {"$ref": "#/$defs/bytes"},
+                    "pre_build": {"$ref": "#/$defs/bytes"},
+                    "post_build": {"$ref": "#/$defs/bytes"}
+                },
+                "required": ["pre_read", "post_read", "pre_build", "post_build"],
+                "additionalProperties": false
+            },
+            "time": {
+                "type": "object",
+                "properties": {
+                    "total": {"type": "number"},
+                    "sconscript_exec": {"type": "number"},
+                    "scons_exec": {"type": "number"},
+                    "command_exec": {"type": "number"}
+                },
+                "required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
+                "additionalProperties": false
+            },
+            "counts": {
+                "type": "array",
+                "items": {
                     "type": "object",
                     "properties": {
+                        "array_index": {"type": "integer"},
+                        "item_name": {"type": "string"},
                         "pre_read": {"$ref": "#/$defs/bytes"},
                         "post_read": {"$ref": "#/$defs/bytes"},
                         "pre_build": {"$ref": "#/$defs/bytes"},
                         "post_build": {"$ref": "#/$defs/bytes"}
                     },
-                    "required": ["pre_read", "post_read", "pre_build", "post_build"],
+                    "required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
                     "additionalProperties": false
-                },
-                "time": {
-                    "type": "object",
-                    "properties": {
-                        "total": {"type": "number"},
-                        "sconscript_exec": {"type": "number"},
-                        "scons_exec": {"type": "number"},
-                        "command_exec": {"type": "number"}
-                    },
-                    "required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
-                    "additionalProperties": false
-                },
-                "counts": {
-                    "type": "array",
-                    "items": {
-                        "type": "object",
-                        "properties": {
-                            "array_index": {"type": "integer"},
-                            "item_name": {"type": "string"},
-                            "pre_read": {"$ref": "#/$defs/bytes"},
-                            "post_read": {"$ref": "#/$defs/bytes"},
-                            "pre_build": {"$ref": "#/$defs/bytes"},
-                            "post_build": {"$ref": "#/$defs/bytes"}
-                        },
-                        "required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
-                        "additionalProperties": false
-                    }
-                },
-                "additionalProperties": false
+                }
             }
-        }
+        },
+        "additionalProperties": false
     }
 }
diff --git a/site_scons/site_tools/build_metrics/libdeps.py b/site_scons/site_tools/build_metrics/libdeps.py
new file mode 100644
index 00000000000..623f77c9386
--- /dev/null
+++ b/site_scons/site_tools/build_metrics/libdeps.py
@@ -0,0 +1,55 @@
+import os
+import sys
+import json
+from .protocol import BuildMetricsCollector
+
+import networkx
+
+# libdeps analyzer does not assume the root build directory, so we need to add its own root to the path
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(dir_path, "..", "..", "..", "buildscripts", "libdeps"))
+
+from buildscripts.libdeps.libdeps.analyzer import counter_factory, LibdepsGraphAnalysis, GaJsonPrinter
+from buildscripts.libdeps.libdeps.graph import LibdepsGraph, CountTypes
+
+_ALLOWED_KEYS = set([
+    "NODE", "EDGE", "DIR_EDGE", "TRANS_EDGE", "DIR_PUB_EDGE", "PUB_EDGE", "PRIV_EDGE", "IF_EDGE",
+    "PROG", "LIB"
+])
+
+
+class LibdepsCollector(BuildMetricsCollector):
+    def __init__(self, env):
+        self._env = env
+
+    def get_name(self):
+        return "LibdepsCollector"
+
+    @staticmethod
+    def _libdeps(graph_file):
+        libdeps_graph = LibdepsGraph(graph=networkx.read_graphml(graph_file))
+
+        if libdeps_graph.graph['graph_schema_version'] == 1:
+            libdeps_graph = networkx.reverse_view(libdeps_graph)
+
+        return GaJsonPrinter(
+            LibdepsGraphAnalysis(counter_factory(libdeps_graph, CountTypes.ALL.name))).get_json()
+
+    @staticmethod
+    def _finalize(libdeps_graph_file):
+        out = {}
+        for key, value in json.loads(LibdepsCollector._libdeps(libdeps_graph_file)).items():
+            if key in _ALLOWED_KEYS:
+                out[key] = value
+        return out
+
+    def finalize(self):
+        libdeps_graph_file = self._env.get('LIBDEPS_GRAPH_FILE')
+        out = {}
+        if libdeps_graph_file is not None and os.path.exists(libdeps_graph_file.path):
+            out = self._finalize(libdeps_graph_file.path)
+        else:
+            print(
+                f"WARNING: libdeps graph file '{libdeps_graph_file}' could not be found. Skipping libdeps metrics"
+            )
+        return "libdeps_metrics", out