SERVER-67057 create scons cache metrics

3 files changed, 139 insertions, 42 deletions

diff --git a/site_scons/site_tools/build_metrics/__init__.py b/site_scons/site_tools/build_metrics/__init__.py
index 07be60782e9..c5c7d277e7b 100644
--- a/site_scons/site_tools/build_metrics/__init__.py
+++ b/site_scons/site_tools/build_metrics/__init__.py
@@ -36,6 +36,7 @@ from .memory import MemoryMonitor
 from .per_action_metrics import PerActionMetrics
 from .artifacts import CollectArtifacts
 from .scons import SConsStats
+from .cache_dir import CacheDirCollector, CacheDirValidateWithMetrics
 
 _SEC_TO_NANOSEC_FACTOR = 1000000000.0
 _METRICS_COLLECTORS = []
@@ -88,9 +89,12 @@ def generate(env, **kwargs):
         MemoryMonitor(psutil.Process().memory_info().vms),
         PerActionMetrics(),
         CollectArtifacts(env),
-        SConsStats()
+        SConsStats(),
+        CacheDirCollector()
     ]
 
+    env['CACHEDIR_CLASS'] = CacheDirValidateWithMetrics
+
 
 def exists(env):
     return True
diff --git a/site_scons/site_tools/build_metrics/build_metrics_format.schema b/site_scons/site_tools/build_metrics/build_metrics_format.schema
index 9a2544b5624..faaf3384243 100644
--- a/site_scons/site_tools/build_metrics/build_metrics_format.schema
+++ b/site_scons/site_tools/build_metrics/build_metrics_format.schema
@@ -13,16 +13,14 @@
             "description": "Size in bytes",
             "minimum": 0
         },
-        "seconds": {
-            "type": "integer",
-            "description": "Number of seconds"
-        },
         "binsize": {
             "type": "object",
             "properties": {
                 "filesize": { "$ref": "#/$defs/bytes" },
                 "vmsize": { "$ref": "#/$defs/bytes" }
-            }
+            },
+            "required": ["filesize", "vmsize"],
+            "additionalProperties": false
         }
     },
     "type" : "object",
@@ -48,7 +46,9 @@
                 "max": { "$ref": "#/$defs/memory" },
                 "arithmetic_mean": {"type": "number"},
                 "start_mem": { "$ref": "#/$defs/memory" }
-            }
+            },
+            "required": ["mem_over_time", "max", "arithmetic_mean", "start_mem"],
+            "additionalProperties": false
         },
         "artifact_metrics": {
             "type": "object",
@@ -83,66 +83,99 @@
                         }
                     }
                 }
-            }
+            },
+            "additionalProperties": false
         },
         "build_tasks" : {
             "type": "array",
-            "task": {
+            "items": {
                 "type": "object",
                 "properties": {
                     "array_index": {"type": "integer"},
                     "start_time": {"$ref": "#/$defs/timestamp"},
                     "end_time" : {"$ref": "#/$defs/timestamp"},
                     "cpu_time" : {"type" : "integer"},
-                    "builder" : {"type" : "str"},
+                    "builder" : {"type" : "string"},
                     "mem_usage": {"$ref": "#/$defs/memory"},
                     "inputs" : {
                         "type": "array",
-                        "input": { "type": "str"}
+                        "items": { "type": "string"}
                     },
                     "outputs" : {
                         "type": "array",
-                        "output": { "type": "str"}
+                        "items": { "type": "string"}
                     },
-                    "action": {"type" : "str"}
-                }
+                    "action": {"type" : "string"}
+                },
+                "required": ["array_index", "start_time", "end_time", "cpu_time", "builder", "mem_usage", "inputs", "outputs", "action"],
+                "additionalProperties": false
             }
-        }
-    },
-    "scons_metrics": {
-        "type": "object",
-        "properties": {
-            "memory": {
-                "type": "object",
-                "properties": {
-                    "pre_read": {"$ref": "#/$defs/bytes"},
-                    "post_read": {"$ref": "#/$defs/bytes"},
-                    "pre_build": {"$ref": "#/$defs/bytes"},
-                    "post_build": {"$ref": "#/$defs/bytes"}
-                }
-            },
-            "time": {
-                "type": "object",
-                "properties": {
-                    "total": {"type": "number"},
-                    "sconscript_exec": {"type": "number"},
-                    "scons_exec": {"type": "number"},
-                    "command_exec": {"type": "number"}
-                }
+        },
+        "cache_metrics": {
+            "type": "object",
+            "properties": {
+                "cache_artifacts": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "array_index": {"type": "integer"},
+                            "name": {"type": "string"},
+                            "size": {"$ref": "#/$defs/bytes"}
+                        },
+                        "required": ["array_index", "name", "size"],
+                        "additionalProperties": false
+                    }
+                },
+                "push_time": { "$ref": "#/$defs/timestamp" },
+                "pull_time": { "$ref": "#/$defs/timestamp" },
+                "cache_size": { "$ref": "#/$defs/bytes" }
             },
-            "counts": {
-                "type": "array",
-                "items": {
+            "required": ["cache_artifacts", "push_time", "pull_time", "cache_size"],
+            "additionalProperties": false
+        },
+        "scons_metrics": {
+            "type": "object",
+            "properties": {
+                "memory": {
                     "type": "object",
                     "properties": {
-                        "array_index": {"type": "integer"},
-                        "item_name": {"type": "str"},
                         "pre_read": {"$ref": "#/$defs/bytes"},
                         "post_read": {"$ref": "#/$defs/bytes"},
                         "pre_build": {"$ref": "#/$defs/bytes"},
                         "post_build": {"$ref": "#/$defs/bytes"}
+                    },
+                    "required": ["pre_read", "post_read", "pre_build", "post_build"],
+                    "additionalProperties": false
+                },
+                "time": {
+                    "type": "object",
+                    "properties": {
+                        "total": {"type": "number"},
+                        "sconscript_exec": {"type": "number"},
+                        "scons_exec": {"type": "number"},
+                        "command_exec": {"type": "number"}
+                    },
+                    "required": ["total", "sconscript_exec", "scons_exec", "command_exec"],
+                    "additionalProperties": false
+                },
+                "counts": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "array_index": {"type": "integer"},
+                            "item_name": {"type": "string"},
+                            "pre_read": {"$ref": "#/$defs/bytes"},
+                            "post_read": {"$ref": "#/$defs/bytes"},
+                            "pre_build": {"$ref": "#/$defs/bytes"},
+                            "post_build": {"$ref": "#/$defs/bytes"}
+                        },
+                        "required": ["array_index", "item_name", "pre_read", "post_read", "pre_build", "post_build"],
+                        "additionalProperties": false
                     }
-                }
+                },
+                "additionalProperties": false
             }
         }
     }
diff --git a/site_scons/site_tools/build_metrics/cache_dir.py b/site_scons/site_tools/build_metrics/cache_dir.py
new file mode 100644
index 00000000000..9751f91cd64
--- /dev/null
+++ b/site_scons/site_tools/build_metrics/cache_dir.py
@@ -0,0 +1,60 @@
+import os
+import time
+from site_tools.validate_cache_dir import CacheDirValidate
+from .protocol import BuildMetricsCollector
+from typing import List, Set
+
+from typing_extensions import TypedDict
+
+
+class CacheArtifact(TypedDict):
+    array_index: int
+    name: str
+    size: int
+
+
+class CacheMetrics(TypedDict):
+    cache_artifacts: List[CacheArtifact]
+    push_time: int
+    pull_time: int
+    cache_size: int
+
+
+class CacheDirValidateWithMetrics(CacheDirValidate):
+    DATA: CacheMetrics = CacheMetrics(push_time=0, pull_time=0, cache_artifacts=[], cache_size=0)
+    SET: Set[str] = set()
+
+    @classmethod
+    def keep_stats(cls, target):
+        if target in cls.SET:
+            return
+
+        size = os.path.getsize(target)
+        cls.DATA["cache_artifacts"].append(
+            CacheArtifact(array_index=len(cls.DATA["cache_artifacts"]), name=target, size=size))
+        cls.DATA["cache_size"] += size
+        cls.SET.add(target)
+
+    @classmethod
+    def copy_from_cache(cls, env, src, dst):
+        start = time.time_ns()
+        super().copy_from_cache(env, src, dst)
+        pull_time = time.time_ns() - start
+        cls.DATA["pull_time"] += pull_time
+        cls.keep_stats(dst)
+
+    @classmethod
+    def copy_to_cache(cls, env, src, dst):
+        start = time.time_ns()
+        super().copy_to_cache(env, src, dst)
+        push_time = time.time_ns() - start
+        cls.DATA["push_time"] += push_time
+        cls.keep_stats(src)
+
+
+class CacheDirCollector(BuildMetricsCollector):
+    def get_name(self):
+        return "CacheDirCollector"
+
+    def finalize(self):
+        return "cache_metrics", CacheDirValidateWithMetrics.DATA