SERVER-69636 Collect SCons Metrics

21 files changed, 555 insertions, 145 deletions

diff --git a/.gitignore b/.gitignore
index c7144d21d38..38ff24451e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -241,3 +241,9 @@ all_feature_flags.txt
 
 # generated by clang-tidy buildscripts
 clang_tidy_fixes
+
+#SCons runtime configuration
+scons_env.env
+
+# buildscripts_test by-product
+test_success.ninja
diff --git a/SConstruct b/SConstruct
index 23392f3c577..83d42997c8e 100644
--- a/SConstruct
+++ b/SConstruct
@@ -15,6 +15,7 @@ import subprocess
 import sys
 import textwrap
 import uuid
+from datetime import datetime
 from glob import glob
 
 from pkg_resources import parse_version
@@ -38,6 +39,8 @@ from mongo.build_profiles import BUILD_PROFILES
 EnsurePythonVersion(3, 6)
 EnsureSConsVersion(3, 1, 1)
 
+utc_starttime = datetime.utcnow()
+
 
 # Monkey patch SCons.FS.File.release_target_info to be a no-op.
 # See https://github.com/SCons/scons/issues/3454
@@ -49,6 +52,7 @@ SCons.Node.FS.File.release_target_info = release_target_info_noop
 
 from buildscripts import utils
 from buildscripts import moduleconfig
+from buildscripts.metrics.scons_tooling_metrics import setup_scons_metrics_collection_atexit
 
 import psutil
 
@@ -1030,7 +1034,7 @@ env_vars.Add(
 
 
 def validate_dwarf_version(key, val, env):
-    if val == '4' or val == '5':
+    if val == '4' or val == '5' or val == '':
         return
 
     print(f"Invalid DWARF_VERSION '{val}'. Only valid versions are 4 or 5.")
@@ -1041,7 +1045,8 @@ env_vars.Add(
     'DWARF_VERSION',
     help='Sets the DWARF version (non-Windows). Incompatible with SPLIT_DWARF=1.',
     validator=validate_dwarf_version,
-    converter=int,
+    converter=lambda val: int(val) if val != '' else '',
+    default='',
 )
 
 env_vars.Add(
@@ -1546,6 +1551,11 @@ env = Environment(variables=env_vars, **envDict)
 del envDict
 env.AddMethod(lambda env, name, **kwargs: add_option(name, **kwargs), 'AddOption')
 
+# Setup atexit method to store tooling metrics
+# The placement of this is intentional. We should only register this function atexit after
+# env, env_vars and the parser have been properly initialized.
+setup_scons_metrics_collection_atexit(utc_starttime, env_vars, env, _parser, sys.argv)
+
 if get_option('build-metrics'):
     env['BUILD_METRICS_ARTIFACTS_DIR'] = '$BUILD_ROOT/$VARIANT_DIR'
     env.Tool('build_metrics')
diff --git a/buildscripts/metrics/metrics_datatypes.py b/buildscripts/metrics/metrics_datatypes.py
index c5640dc2ca3..146afd81d38 100644
--- a/buildscripts/metrics/metrics_datatypes.py
+++ b/buildscripts/metrics/metrics_datatypes.py
@@ -1,17 +1,21 @@
 from abc import abstractmethod
+import configparser
 from datetime import datetime
 import multiprocessing
 import os
 import socket
 import sys
 import traceback
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 import distro
 import git
 from pydantic import BaseModel
 
 # pylint: disable=bare-except
 
+SCONS_ENV_FILE = "scons_env.env"
+SCONS_SECTION_HEADER = "SCONS_ENV"
+
 
 class BaseMetrics(BaseModel):
     """Base class for an metrics object."""
@@ -22,15 +26,107 @@ class BaseMetrics(BaseModel):
         raise NotImplementedError
 
 
+class BuildInfo(BaseMetrics):
+    """Class to store the Build environment, options & artifacts."""
+
+    env: Optional[Dict[str, Any]]
+    options: Optional[Dict[str, Any]]
+    build_artifacts: Optional[List[str]]
+    artifact_dir: Optional[str]
+
+    @classmethod
+    def get_scons_build_info(
+            cls,
+            utc_starttime: datetime,
+            env_vars: "SCons.Variables.Variables",
+            env: "SCons.Script.SConscript.SConsEnvironment",
+            parser: "SCons.Script.SConsOptions.SConsOptionParser",
+            args: List[str],
+    ):
+        """Get SCons build info to the best of our ability."""
+        artifact_dir = cls._get_scons_artifact_dir(env)
+        return cls(
+            env=cls._get_scons_env_vars_dict(env_vars, env),
+            options=cls._get_scons_options_dict(parser, args),
+            build_artifacts=cls._get_artifacts(utc_starttime, artifact_dir),
+            artifact_dir=artifact_dir,
+        )
+
+    @staticmethod
+    def _get_scons_env_vars_dict(
+            env_vars: "SCons.Variables.Variables",
+            env: "SCons.Script.SConscript.SConsEnvironment",
+    ) -> Optional[Dict[str, Any]]:
+        """Get the environment variables options that can be set by users."""
+
+        try:
+            # Use SCons built-in method to save environment variables to a file
+            env_vars.Save(SCONS_ENV_FILE, env)
+
+            # Add a section header to the file so we can easily parse with ConfigParser
+            with open(SCONS_ENV_FILE, 'r') as original:
+                data = original.read()
+            with open(SCONS_ENV_FILE, 'w') as modified:
+                modified.write(f"[{SCONS_SECTION_HEADER}]\n" + data)
+
+            # Parse file using config parser
+            config = configparser.ConfigParser()
+            config.read(SCONS_ENV_FILE)
+            str_dict = dict(config[SCONS_SECTION_HEADER])
+            return {key: eval(val) for key, val in str_dict.items()}  # pylint: disable=eval-used
+        except:
+            return None
+
+    @staticmethod
+    def _get_scons_options_dict(
+            parser: "SCons.Script.SConsOptions.SConsOptionParser",
+            args: List[str],
+    ) -> Optional[Dict[str, Any]]:
+        """Get the scons cli options set by users."""
+        try:
+            scons_options, _ = parser.parse_args(args)
+            return vars(scons_options)
+        except:
+            return None
+
+    @staticmethod
+    def _get_scons_artifact_dir(env: "SCons.Script.SConscript.SConsEnvironment") -> Optional[str]:
+        """Get the artifact dir for this build."""
+        try:
+            return env.Dir('$BUILD_DIR').get_abspath()
+        except:
+            return None
+
+    @staticmethod
+    def _get_artifacts(utc_starttime: datetime, artifact_dir: str) -> List[str]:
+        """Search a directory recursively for all files created after the given timestamp."""
+        try:
+            start_timestamp = datetime.timestamp(utc_starttime)
+            artifacts = []
+            for root, _, files in os.walk(artifact_dir):
+                for file in files:
+                    filepath = os.path.join(root, file)
+                    _, ext = os.path.splitext(filepath)
+                    if ext in ['.a', '.so', ''] and os.path.getmtime(filepath) >= start_timestamp:
+                        artifacts.append(filepath)
+            return artifacts
+        except:
+            return None
+
+    def is_malformed(self) -> bool:
+        """Confirm whether this instance has all expected fields."""
+        return None in [self.artifact_dir, self.env, self.options, self.build_artifacts]
+
+
 class ExitInfo(BaseMetrics):
     """Class to store tooling exit information."""
 
-    exit_code: int
+    exit_code: Optional[int]
     exception: Optional[str]
     stacktrace: Optional[str]
 
     @classmethod
-    def get_exit_info(cls):
+    def get_resmoke_exit_info(cls):
         """Get the current exit info."""
         exc = sys.exc_info()[1]
         return cls(
@@ -39,9 +135,18 @@ class ExitInfo(BaseMetrics):
             stacktrace=traceback.format_exc() if exc else None,
         )
 
+    @classmethod
+    def get_scons_exit_info(cls, exit_code):
+        """Get the current exit info using the given exit code."""
+        return cls(
+            exit_code=exit_code if isinstance(exit_code, int) else None,
+            exception=None,
+            stacktrace=None,
+        )
+
     def is_malformed(self):
-        """Return True since this object cannot be malformed if created with classmethod."""
-        return True
+        """Return True if this object is missing an exit code."""
+        return self.exit_code is None
 
 
 class HostInfo(BaseMetrics):
@@ -109,7 +214,7 @@ class GitInfo(BaseMetrics):
 
     def is_malformed(self):
         """Confirm whether this instance has all expected fields."""
-        return self.commit_hash is None or self.branch_name is None or self.repo_name is None
+        return None in [self.commit_hash, self.branch_name, self.repo_name]
 
 
 MODULES_FILEPATH = 'src/mongo/db/modules'
@@ -132,28 +237,63 @@ def _get_modules_git_info():
 class ToolingMetrics(BaseMetrics):
     """Class to store tooling metrics."""
 
+    source: str
     utc_starttime: datetime
     utc_endtime: datetime
     host_info: HostInfo
     git_info: GitInfo
     exit_info: ExitInfo
+    build_info: Optional[BuildInfo]
     command: List[str]
     module_info: List[GitInfo]
     ip_address: Optional[str]
 
     @classmethod
-    def get_tooling_metrics(cls, utc_starttime: datetime):
-        """Get tooling metrics to the best of our ability."""
+    def get_resmoke_metrics(
+            cls,
+            utc_starttime: datetime,
+    ):
+        """Get resmoke metrics to the best of our ability."""
+        try:
+            ip_address = socket.gethostbyname(socket.gethostname())
+        except:
+            ip_address = None
+        return cls(
+            source='resmoke',
+            utc_starttime=utc_starttime,
+            utc_endtime=datetime.utcnow(),
+            host_info=HostInfo.get_host_info(),
+            git_info=GitInfo.get_git_info('.'),
+            exit_info=ExitInfo.get_resmoke_exit_info(),
+            build_info=None,
+            module_info=_get_modules_git_info(),
+            command=sys.argv,
+            ip_address=ip_address,
+        )
+
+    @classmethod
+    def get_scons_metrics(
+            cls,
+            utc_starttime: datetime,
+            env_vars: "SCons.Variables.Variables",
+            env: "SCons.Script.SConscript.SConsEnvironment",
+            parser: "SCons.Script.SConsOptions.SConsOptionParser",
+            args: List[str],
+            exit_code: int,
+    ):
+        """Get scons metrics to the best of our ability."""
         try:
             ip_address = socket.gethostbyname(socket.gethostname())
         except:
             ip_address = None
         return cls(
+            source='scons',
             utc_starttime=utc_starttime,
             utc_endtime=datetime.utcnow(),
             host_info=HostInfo.get_host_info(),
             git_info=GitInfo.get_git_info('.'),
-            exit_info=ExitInfo.get_exit_info(),
+            exit_info=ExitInfo.get_scons_exit_info(exit_code),
+            build_info=BuildInfo.get_scons_build_info(utc_starttime, env_vars, env, parser, args),
             module_info=_get_modules_git_info(),
             command=sys.argv,
             ip_address=ip_address,
@@ -161,6 +301,6 @@ class ToolingMetrics(BaseMetrics):
 
     def is_malformed(self):
         """Confirm whether this instance has all expected fields."""
-        return self.ip_address is None or any(
-            info_obj.is_malformed()
-            for info_obj in self.module_info + [self.git_info] + [self.host_info])
+        sub_metrics = [self.build_info] if self.source == 'scons' else []
+        sub_metrics += self.module_info + [self.git_info] + [self.host_info] + [self.exit_info]
+        return self.ip_address is None or any(metrics.is_malformed() for metrics in sub_metrics)
diff --git a/buildscripts/metrics/resmoke_tooling_metrics.py b/buildscripts/metrics/resmoke_tooling_metrics.py
new file mode 100644
index 00000000000..1b24160d26f
--- /dev/null
+++ b/buildscripts/metrics/resmoke_tooling_metrics.py
@@ -0,0 +1,19 @@
+from datetime import datetime
+import logging
+
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+from buildscripts.metrics.tooling_metrics_utils import is_virtual_workstation, save_tooling_metrics
+
+logger = logging.getLogger('resmoke_tooling_metrics')
+
+
+def save_resmoke_tooling_metrics(utc_starttime: datetime):
+    try:
+        if not is_virtual_workstation():
+            return
+        tooling_metrics = ToolingMetrics.get_resmoke_metrics(utc_starttime)
+        save_tooling_metrics(tooling_metrics)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.warning(
+            "%s\nResmoke Metrics Collection Failed -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
+            exc)
diff --git a/buildscripts/metrics/scons_tooling_metrics.py b/buildscripts/metrics/scons_tooling_metrics.py
new file mode 100644
index 00000000000..0b02b6f55c3
--- /dev/null
+++ b/buildscripts/metrics/scons_tooling_metrics.py
@@ -0,0 +1,71 @@
+import atexit
+import datetime
+import logging
+import sys
+from typing import List
+
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+from buildscripts.metrics.tooling_metrics_utils import is_virtual_workstation, save_tooling_metrics
+
+logger = logging.getLogger('scons_tooling_metrics')
+
+
+class SConsExitHook(object):
+    """Plumb all sys.exit through this object so that we can access the exit code in atexit."""
+
+    def __init__(self):
+        self.exit_code = None
+        self._orig_exit = sys.exit
+
+    def __del__(self):
+        sys.exit = self._orig_exit
+
+    def initialize(self):
+        sys.exit = self.exit
+
+    def exit(self, code=0):
+        self.exit_code = code
+        self._orig_exit(code)
+
+
+# This method should only be used when registered on atexit
+def _save_scons_tooling_metrics(
+        utc_starttime: datetime,
+        env_vars: "SCons.Variables.Variables",
+        env: "SCons.Script.SConscript.SConsEnvironment",
+        parser: "SCons.Script.SConsOptions.SConsOptionParser",
+        args: List[str],
+        exit_hook: SConsExitHook,
+):
+    """Save SCons tooling metrics to atlas cluster."""
+    try:
+        if not is_virtual_workstation():
+            return
+        tooling_metrics = ToolingMetrics.get_scons_metrics(utc_starttime, env_vars, env, parser,
+                                                           args, exit_hook.exit_code)
+        save_tooling_metrics(tooling_metrics)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.warning(
+            "%sSCons Metrics Collection Failed -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
+            exc)
+
+
+def setup_scons_metrics_collection_atexit(
+        utc_starttime: datetime,
+        env_vars: "SCons.Variables.Variables",
+        env: "SCons.Script.SConscript.SConsEnvironment",
+        parser: "SCons.Script.SConsOptions.SConsOptionParser",
+        args: List[str],
+) -> None:
+    """Register an atexit method for scons metrics collection."""
+    scons_exit_hook = SConsExitHook()
+    scons_exit_hook.initialize()
+    atexit.register(
+        _save_scons_tooling_metrics,
+        utc_starttime,
+        env_vars,
+        env,
+        parser,
+        args,
+        scons_exit_hook,
+    )
diff --git a/buildscripts/metrics/tooling_metrics.py b/buildscripts/metrics/tooling_metrics_utils.py
index 1a890083de4..9e8b3256ecb 100644
--- a/buildscripts/metrics/tooling_metrics.py
+++ b/buildscripts/metrics/tooling_metrics_utils.py
@@ -1,5 +1,3 @@
-import dataclasses
-import datetime
 import logging
 import os
 import asyncio
@@ -9,7 +7,7 @@ import pymongo
 
 from buildscripts.metrics.metrics_datatypes import ToolingMetrics
 
-logger = logging.getLogger('tooling_metrics_collection')
+logger = logging.getLogger('tooling_metrics_utils')
 
 INTERNAL_TOOLING_METRICS_HOSTNAME = "mongodb+srv://dev-metrics-pl-0.kewhj.mongodb.net"
 INTERNAL_TOOLING_METRICS_USERNAME = "internal_tooling_user"
@@ -41,7 +39,7 @@ def _git_user_exists() -> Optional[str]:
         return None
 
 
-def _is_virtual_workstation() -> bool:
+def is_virtual_workstation() -> bool:
     """Detect whether this is a MongoDB internal virtual workstation."""
     return _toolchain_exists() and _git_user_exists()
 
@@ -52,14 +50,10 @@ async def _save_metrics(metrics: ToolingMetrics) -> None:
     client.metrics.tooling_metrics.insert_one(metrics.dict())
 
 
-def save_tooling_metrics(utc_starttime: datetime) -> None:
+def save_tooling_metrics(tooling_metrics: ToolingMetrics) -> None:
     """Persist tooling metrics data to MongoDB Internal Atlas Cluster."""
     try:
-        if not _is_virtual_workstation():
-            return
-        asyncio.run(
-            asyncio.wait_for(
-                _save_metrics(ToolingMetrics.get_tooling_metrics(utc_starttime)), timeout=1.0))
+        asyncio.run(asyncio.wait_for(_save_metrics(tooling_metrics), timeout=1.0))
     except asyncio.TimeoutError as exc:
         logger.warning(
             "%s\nTimeout: Tooling metrics collection is not available -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py
index 7dc4cbaf648..b84f07e0cf0 100644
--- a/buildscripts/resmokelib/cli.py
+++ b/buildscripts/resmokelib/cli.py
@@ -4,7 +4,7 @@ from datetime import datetime
 import time
 import os
 import psutil
-from buildscripts.metrics.tooling_metrics import save_tooling_metrics
+from buildscripts.metrics.resmoke_tooling_metrics import save_resmoke_tooling_metrics
 from buildscripts.resmokelib import parser
 
 
@@ -28,4 +28,4 @@ def main(argv):
     try:
         subcommand.execute()
     finally:
-        save_tooling_metrics(datetime.utcfromtimestamp(__start_time))
+        save_resmoke_tooling_metrics(datetime.utcfromtimestamp(__start_time))
diff --git a/buildscripts/scons.py b/buildscripts/scons.py
index 534fca3264e..f7f7fcd3d5b 100755
--- a/buildscripts/scons.py
+++ b/buildscripts/scons.py
@@ -34,5 +34,10 @@ except ImportError as import_err:
     print("ImportError:", import_err)
     sys.exit(1)
 
-if __name__ == '__main__':
+
+def entrypoint():
     SCons.Script.main()
+
+
+if __name__ == '__main__':
+    entrypoint()
diff --git a/buildscripts/tests/tooling_metrics/test_metrics_collection.py b/buildscripts/tests/tooling_metrics/test_metrics_collection.py
deleted file mode 100644
index 9d820a9c5e0..00000000000
--- a/buildscripts/tests/tooling_metrics/test_metrics_collection.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""Unit tests for tooling_metrics.py."""
-import asyncio
-from datetime import datetime
-import unittest
-from unittest.mock import patch
-import mongomock
-import pymongo
-from buildscripts.metrics.metrics_datatypes import ToolingMetrics
-import buildscripts.metrics.tooling_metrics as under_test
-from buildscripts.resmoke import entrypoint
-
-# pylint: disable=unused-argument
-# pylint: disable=protected-access
-
-TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
-CURRENT_DATE_TIME = datetime(2022, 10, 4)
-
-
-async def extended_sleep(arg):
-    await asyncio.sleep(2)
-
-
-@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME",
-       TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-class TestToolingMetricsCollection(unittest.TestCase):
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
-    def test_on_virtual_workstation(self, mock_is_virtual_workstation):
-        under_test.save_tooling_metrics(CURRENT_DATE_TIME)
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert client.metrics.tooling_metrics.find_one()
-
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=False)
-    def test_not_on_virtual_workstation(self, mock_is_virtual_workstation):
-        under_test.save_tooling_metrics(CURRENT_DATE_TIME)
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert not client.metrics.tooling_metrics.find_one()
-
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("buildscripts.metrics.tooling_metrics._save_metrics",
-           side_effect=pymongo.errors.WriteError(error="Error Information"))
-    @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
-    def test_exception_caught(self, mock_is_virtual_workstation, mock_save_metrics):
-        with self.assertLogs('tooling_metrics_collection') as cm:
-            under_test.save_tooling_metrics(CURRENT_DATE_TIME)
-        assert "Error Information" in cm.output[0]
-        assert "Unexpected: Tooling metrics collection is not available" in cm.output[0]
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert not client.metrics.tooling_metrics.find_one()
-
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("buildscripts.metrics.tooling_metrics._save_metrics", side_effect=extended_sleep)
-    @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
-    def test_timeout_caught(self, mock_is_virtual_workstation, mock_save_metrics):
-        with self.assertLogs('tooling_metrics_collection') as cm:
-            under_test.save_tooling_metrics(CURRENT_DATE_TIME)
-        assert "Timeout: Tooling metrics collection is not available" in cm.output[0]
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert not client.metrics.tooling_metrics.find_one()
-
-
-class TestIsVirtualWorkstation(unittest.TestCase):
-    @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False)
-    @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True)
-    def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
-        assert not under_test._is_virtual_workstation()
-
-    @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True)
-    @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True)
-    def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
-        assert under_test._is_virtual_workstation()
-
-    @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True)
-    @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False)
-    def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
-        assert not under_test._is_virtual_workstation()
-
-    @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False)
-    @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False)
-    def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
-        assert not under_test._is_virtual_workstation()
-
-
-@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME",
-       TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None)
-@patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
-class TestResmokeMetricsCollection(unittest.TestCase):
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test'])
-    @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests",
-           side_effect=Exception())
-    def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation):
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert not client.metrics.tooling_metrics.find_one()
-        with self.assertRaises(SystemExit):
-            entrypoint()
-        assert client.metrics.tooling_metrics.find_one()
-
-    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
-    @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites'])
-    def test_resmoke_metrics_collection(self, mock_is_virtual_workstation):
-        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-        assert not client.metrics.tooling_metrics.find_one()
-        entrypoint()
-        assert client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
index 14a17264be2..b584b77e647 100644
--- a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
+++ b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
@@ -3,20 +3,50 @@ from datetime import datetime
 import unittest
 from unittest.mock import patch
 
+from mock import MagicMock
+
 import buildscripts.metrics.metrics_datatypes as under_test
 
 # pylint: disable=unused-argument
 
 
+@patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir",
+       return_value='/test')
+class TestBuildInfo(unittest.TestCase):
+    @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict",
+           return_value={'env': 'env'})
+    @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict",
+           return_value={'opt': 'opt'})
+    def test_build_info_valid(self, mock_env, mock_options, mock_artifact_dir):
+        build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(),
+                                                               MagicMock(), MagicMock(),
+                                                               MagicMock())
+        assert not build_info.is_malformed()
+
+    def test_build_info_malformed(self, mock_artifact_dir):
+        build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(),
+                                                               MagicMock(), MagicMock(),
+                                                               MagicMock())
+        assert build_info.is_malformed()
+
+
 class TestExitInfo(unittest.TestCase):
     @patch("sys.exc_info", return_value=(None, None, None))
-    def test_no_exc_info(self, mock_exc_info):
-        exit_info = under_test.ExitInfo.get_exit_info()
-        assert exit_info.is_malformed()
+    def test_resmoke_no_exc_info(self, mock_exc_info):
+        exit_info = under_test.ExitInfo.get_resmoke_exit_info()
+        assert not exit_info.is_malformed()
 
     @patch("sys.exc_info", return_value=(None, ValueError(), None))
-    def test_with_exc_info(self, mock_exc_info):
-        exit_info = under_test.ExitInfo.get_exit_info()
+    def test_resmoke_with_exc_info(self, mock_exc_info):
+        exit_info = under_test.ExitInfo.get_resmoke_exit_info()
+        assert not exit_info.is_malformed()
+
+    def test_scons_exit_info_valid(self):
+        exit_info = under_test.ExitInfo.get_scons_exit_info(0)
+        assert not exit_info.is_malformed()
+
+    def test_scons_exit_info_malformed(self):
+        exit_info = under_test.ExitInfo.get_scons_exit_info('string')
         assert exit_info.is_malformed()
 
 
@@ -53,10 +83,29 @@ class TestGitInfo(unittest.TestCase):
 @patch("buildscripts.metrics.metrics_datatypes.HostInfo._get_memory", return_value=30)
 class TestToolingMetrics(unittest.TestCase):
     @patch("socket.gethostname", side_effect=Exception())
-    def test_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory):
-        tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow())
+    def test_resmoke_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory):
+        tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow())
         assert tooling_metrics.is_malformed()
 
-    def test_tooling_metrics_no_exc(self, mock_get_memory):
-        tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow())
+    def test_resmoke_tooling_metrics_no_exc(self, mock_get_memory):
+        tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow())
+        assert not tooling_metrics.is_malformed()
+
+    @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir",
+           return_value='/test')
+    @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict",
+           return_value={'env': 'env'})
+    @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict",
+           return_value={'opt': 'opt'})
+    def test_scons_tooling_metrics_valid(self, mock_options, mock_env, mock_artifact_dir,
+                                         mock_get_memory):
+        parser = MagicMock()
+        parser.parse_args = MagicMock(return_value={"opt1": "val1"})
+        tooling_metrics = under_test.ToolingMetrics.get_scons_metrics(
+            datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, parser, ['test1', 'test2'], 0)
         assert not tooling_metrics.is_malformed()
+
+    def test_scons_tooling_metrics_malformed(self, mock_get_memory):
+        tooling_metrics = under_test.ToolingMetrics.get_scons_metrics(
+            datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, None, [], 0)
+        assert tooling_metrics.is_malformed()
diff --git a/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py
new file mode 100644
index 00000000000..3510184eb0c
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py
@@ -0,0 +1,44 @@
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import patch
+import mongomock
+import pymongo
+
+import buildscripts.metrics.resmoke_tooling_metrics as under_test
+from buildscripts.resmoke import entrypoint as resmoke_entrypoint
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+# pylint: disable=unused-argument
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+    sys.exit()
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+       TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None)
+@patch("buildscripts.metrics.resmoke_tooling_metrics.is_virtual_workstation", return_value=True)
+class TestResmokeMetricsCollection(unittest.TestCase):
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test'])
+    @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests",
+           side_effect=Exception())
+    def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation):
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+        with self.assertRaises(SystemExit):
+            resmoke_entrypoint()
+        assert client.metrics.tooling_metrics.find_one()
+
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites'])
+    def test_resmoke_metrics_collection(self, mock_is_virtual_workstation):
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+        resmoke_entrypoint()
+        assert client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py
new file mode 100644
index 00000000000..f0e020bf306
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py
@@ -0,0 +1,64 @@
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+import mongomock
+import pymongo
+import buildscripts.metrics.scons_tooling_metrics as under_test
+from buildscripts.scons import entrypoint as scons_entrypoint
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+# pylint: disable=unused-argument
+# pylint: disable=protected-access
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+    sys.exit()
+
+
+@patch("sys.argv", [
+    'buildscripts/scons.py', "CC=/opt/mongodbtoolchain/v3/bin/gcc",
+    "CXX=/opt/mongodbtoolchain/v3/bin/g++", "NINJA_PREFIX=test_success", "--ninja"
+])
+@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True)
+@patch("atexit.register")
+class TestSconsAtExitMetricsCollection(unittest.TestCase):
+    def test_scons_at_exit_metrics_collection(self, mock_atexit_register,
+                                              mock_is_virtual_workstation):
+        with self.assertRaises(SystemExit) as context:
+            scons_entrypoint()
+        assert context.exception.code == 0
+        atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list]
+        assert "_save_scons_tooling_metrics" in atexit_functions
+
+    @patch("buildscripts.moduleconfig.get_module_sconscripts", side_effect=Exception())
+    def test_scons_at_exit_metrics_collection_exc(self, mock_method, mock_atexit_register,
+                                                  mock_is_virtual_workstation):
+        with self.assertRaises(SystemExit) as context:
+            scons_entrypoint()
+        assert context.exception.code == 2
+        atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list]
+        assert "_save_scons_tooling_metrics" in atexit_functions
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+       TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True)
+class TestSconsMetricsPersist(unittest.TestCase):
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    def test_scons_metrics_collection_success(self, mock_is_virtual_workstation):
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+        under_test._save_scons_tooling_metrics(CURRENT_DATE_TIME, None, None, None, None,
+                                               MagicMock(exit_code=0))
+        assert client.metrics.tooling_metrics.find_one()
+
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    def test_scons_metrics_collection_fail(self, mock_is_virtual_workstation):
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+        under_test._save_scons_tooling_metrics(None, None, None, None, None, None)
+        assert not client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py
new file mode 100644
index 00000000000..c1c768e50d2
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py
@@ -0,0 +1,78 @@
+"""Unit tests for tooling_metrics.py."""
+import asyncio
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import patch
+import mongomock
+import pymongo
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+import buildscripts.metrics.tooling_metrics_utils as under_test
+
+# pylint: disable=unused-argument
+# pylint: disable=protected-access
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+
+async def extended_sleep(arg):
+    await asyncio.sleep(2)
+
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+    sys.exit()
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+       TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+class TestSaveToolingMetrics(unittest.TestCase):
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    def test_on_virtual_workstation(self):
+        under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert client.metrics.tooling_metrics.find_one()
+
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics",
+           side_effect=pymongo.errors.WriteError(error="Error Information"))
+    def test_exception_caught(self, mock_save_metrics):
+        with self.assertLogs('tooling_metrics_utils') as cm:
+            under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+        assert "Error Information" in cm.output[0]
+        assert "Unexpected: Tooling metrics collection is not available" in cm.output[0]
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+
+    @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+    @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics", side_effect=extended_sleep)
+    def test_timeout_caught(self, mock_save_metrics):
+        with self.assertLogs('tooling_metrics_utils') as cm:
+            under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+        assert "Timeout: Tooling metrics collection is not available" in cm.output[0]
+        client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+        assert not client.metrics.tooling_metrics.find_one()
+
+
+class TestIsVirtualWorkstation(unittest.TestCase):
+    @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False)
+    @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True)
+    def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
+        assert not under_test.is_virtual_workstation()
+
+    @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True)
+    @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True)
+    def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
+        assert under_test.is_virtual_workstation()
+
+    @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True)
+    @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False)
+    def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
+        assert not under_test.is_virtual_workstation()
+
+    @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False)
+    @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False)
+    def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
+        assert not under_test.is_virtual_workstation()
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 9949cc7064b..023df931196 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3243,6 +3243,7 @@ buildvariants:
   - name: test_api_version_compatibility
   - name: validate_commit_message
   - name: check_feature_flag_tags
+  - name: compile_venv_deps_check
 
 - name: &windows-dynamic-visibility-test windows-dynamic-visibility-test
   display_name: "~ Shared Library Windows (visibility test)"
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index 1d6462b2a0a..8c4fd8093e0 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -2323,6 +2323,21 @@ tasks:
       content_type: application/tar
       display_name: Benchmarks
 
+- name: compile_venv_deps_check
+  commands:
+  - *f_expansions_write
+  - command: manifest.load
+  - func: "git get project and add git tag"
+  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - command: subprocess.exec
+    type: test
+    params:
+      binary: bash
+      args:
+      - "src/evergreen/compile_venv_dependency_check.sh"
+
 - name: determine_patch_tests
   commands:
   - *f_expansions_write
diff --git a/etc/pip/compile-requirements.txt b/etc/pip/compile-requirements.txt
index 74b44861e27..96fe580a637 100644
--- a/etc/pip/compile-requirements.txt
+++ b/etc/pip/compile-requirements.txt
@@ -2,3 +2,5 @@
 -r components/core.req
 
 -r components/compile.req
+
+-r components/tooling_metrics.req
diff --git a/etc/pip/components/tooling_metrics.req b/etc/pip/components/tooling_metrics.req
new file mode 100644
index 00000000000..0d8d220dad4
--- /dev/null
+++ b/etc/pip/components/tooling_metrics.req
@@ -0,0 +1,4 @@
+distro == 1.5.0
+GitPython ~= 3.1.7
+pydantic ~= 1.8.2
+dnspython == 2.1.0
diff --git a/etc/pip/dev-requirements.txt b/etc/pip/dev-requirements.txt
index ce2ebbd3de1..178f57d1cab 100644
--- a/etc/pip/dev-requirements.txt
+++ b/etc/pip/dev-requirements.txt
@@ -8,3 +8,5 @@
 -r components/evergreen.req
 -r components/aws.req
 -r components/jiraclient.req
+
+-r components/tooling_metrics.req
diff --git a/etc/pip/toolchain-requirements.txt b/etc/pip/toolchain-requirements.txt
index 3694f6bb9d2..27c792e52e1 100644
--- a/etc/pip/toolchain-requirements.txt
+++ b/etc/pip/toolchain-requirements.txt
@@ -17,3 +17,5 @@
 
 -r components/build_metrics.req
 -r components/libdeps.req
+
+-r components/tooling_metrics.req
diff --git a/evergreen/compile_venv_dependency_check.sh b/evergreen/compile_venv_dependency_check.sh
new file mode 100644
index 00000000000..b71a93e29f1
--- /dev/null
+++ b/evergreen/compile_venv_dependency_check.sh
@@ -0,0 +1,12 @@
+# Quick check to ensure all scons.py dependecies have been added to compile-requirements.txt
+set -o errexit
+set -o verbose
+
+# Create virtual env
+/opt/mongodbtoolchain/v3/bin/virtualenv --python /opt/mongodbtoolchain/v3/bin/python3 ./compile_venv
+source ./compile_venv/bin/activate
+
+# Try printing scons.py help message
+cd src
+python -m pip install -r etc/pip/compile-requirements.txt
+buildscripts/scons.py --help
diff --git a/evergreen/publish_metrics.py b/evergreen/publish_metrics.py
index d2536e0b6a8..c5db53b0220 100755
--- a/evergreen/publish_metrics.py
+++ b/evergreen/publish_metrics.py
@@ -1,5 +1,4 @@
 import datetime
-import subprocess
 import sys
 import os
 
@@ -10,7 +9,7 @@ if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from buildscripts.metrics.metrics_datatypes import ToolingMetrics
-from buildscripts.metrics.tooling_metrics import _get_internal_tooling_metrics_client
+from buildscripts.metrics.tooling_metrics_utils import _get_internal_tooling_metrics_client
 from evergreen.api import RetryingEvergreenApi
 
 # Check cluster connectivity