summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorTausif Rahman <tausif.rahman@mongodb.com>2022-11-08 15:36:04 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2022-11-08 16:32:27 +0000
commit3a4d6fdeb7cc74f1b9f2821b3fa6f6dddeca4a62 (patch)
treeb3b637a73c7014b11943b75ca501a80a32e779aa /buildscripts
parentccffa2062f09352f6c064e64090bdda42b829e73 (diff)
downloadmongo-3a4d6fdeb7cc74f1b9f2821b3fa6f6dddeca4a62.tar.gz
SERVER-69636 Collect SCons Metrics
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/metrics/metrics_datatypes.py164
-rw-r--r--buildscripts/metrics/resmoke_tooling_metrics.py19
-rw-r--r--buildscripts/metrics/scons_tooling_metrics.py71
-rw-r--r--buildscripts/metrics/tooling_metrics_utils.py (renamed from buildscripts/metrics/tooling_metrics.py)14
-rw-r--r--buildscripts/resmokelib/cli.py4
-rwxr-xr-xbuildscripts/scons.py7
-rw-r--r--buildscripts/tests/tooling_metrics/test_metrics_collection.py107
-rw-r--r--buildscripts/tests/tooling_metrics/test_metrics_datatypes.py67
-rw-r--r--buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py44
-rw-r--r--buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py64
-rw-r--r--buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py78
11 files changed, 498 insertions, 141 deletions
diff --git a/buildscripts/metrics/metrics_datatypes.py b/buildscripts/metrics/metrics_datatypes.py
index c5640dc2ca3..146afd81d38 100644
--- a/buildscripts/metrics/metrics_datatypes.py
+++ b/buildscripts/metrics/metrics_datatypes.py
@@ -1,17 +1,21 @@
from abc import abstractmethod
+import configparser
from datetime import datetime
import multiprocessing
import os
import socket
import sys
import traceback
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
import distro
import git
from pydantic import BaseModel
# pylint: disable=bare-except
+SCONS_ENV_FILE = "scons_env.env"
+SCONS_SECTION_HEADER = "SCONS_ENV"
+
class BaseMetrics(BaseModel):
"""Base class for an metrics object."""
@@ -22,15 +26,107 @@ class BaseMetrics(BaseModel):
raise NotImplementedError
+class BuildInfo(BaseMetrics):
+ """Class to store the Build environment, options & artifacts."""
+
+ env: Optional[Dict[str, Any]]
+ options: Optional[Dict[str, Any]]
+ build_artifacts: Optional[List[str]]
+ artifact_dir: Optional[str]
+
+ @classmethod
+ def get_scons_build_info(
+ cls,
+ utc_starttime: datetime,
+ env_vars: "SCons.Variables.Variables",
+ env: "SCons.Script.SConscript.SConsEnvironment",
+ parser: "SCons.Script.SConsOptions.SConsOptionParser",
+ args: List[str],
+ ):
+ """Get SCons build info to the best of our ability."""
+ artifact_dir = cls._get_scons_artifact_dir(env)
+ return cls(
+ env=cls._get_scons_env_vars_dict(env_vars, env),
+ options=cls._get_scons_options_dict(parser, args),
+ build_artifacts=cls._get_artifacts(utc_starttime, artifact_dir),
+ artifact_dir=artifact_dir,
+ )
+
+ @staticmethod
+ def _get_scons_env_vars_dict(
+ env_vars: "SCons.Variables.Variables",
+ env: "SCons.Script.SConscript.SConsEnvironment",
+ ) -> Optional[Dict[str, Any]]:
+ """Get the environment variables options that can be set by users."""
+
+ try:
+ # Use SCons built-in method to save environment variables to a file
+ env_vars.Save(SCONS_ENV_FILE, env)
+
+ # Add a section header to the file so we can easily parse with ConfigParser
+ with open(SCONS_ENV_FILE, 'r') as original:
+ data = original.read()
+ with open(SCONS_ENV_FILE, 'w') as modified:
+ modified.write(f"[{SCONS_SECTION_HEADER}]\n" + data)
+
+ # Parse file using config parser
+ config = configparser.ConfigParser()
+ config.read(SCONS_ENV_FILE)
+ str_dict = dict(config[SCONS_SECTION_HEADER])
+ return {key: eval(val) for key, val in str_dict.items()} # pylint: disable=eval-used
+ except:
+ return None
+
+ @staticmethod
+ def _get_scons_options_dict(
+ parser: "SCons.Script.SConsOptions.SConsOptionParser",
+ args: List[str],
+ ) -> Optional[Dict[str, Any]]:
+ """Get the scons cli options set by users."""
+ try:
+ scons_options, _ = parser.parse_args(args)
+ return vars(scons_options)
+ except:
+ return None
+
+ @staticmethod
+ def _get_scons_artifact_dir(env: "SCons.Script.SConscript.SConsEnvironment") -> Optional[str]:
+ """Get the artifact dir for this build."""
+ try:
+ return env.Dir('$BUILD_DIR').get_abspath()
+ except:
+ return None
+
+ @staticmethod
+ def _get_artifacts(utc_starttime: datetime, artifact_dir: str) -> List[str]:
+ """Search a directory recursively for all files created after the given timestamp."""
+ try:
+ start_timestamp = datetime.timestamp(utc_starttime)
+ artifacts = []
+ for root, _, files in os.walk(artifact_dir):
+ for file in files:
+ filepath = os.path.join(root, file)
+ _, ext = os.path.splitext(filepath)
+ if ext in ['.a', '.so', ''] and os.path.getmtime(filepath) >= start_timestamp:
+ artifacts.append(filepath)
+ return artifacts
+ except:
+ return None
+
+ def is_malformed(self) -> bool:
+ """Confirm whether this instance has all expected fields."""
+ return None in [self.artifact_dir, self.env, self.options, self.build_artifacts]
+
+
class ExitInfo(BaseMetrics):
"""Class to store tooling exit information."""
- exit_code: int
+ exit_code: Optional[int]
exception: Optional[str]
stacktrace: Optional[str]
@classmethod
- def get_exit_info(cls):
+ def get_resmoke_exit_info(cls):
"""Get the current exit info."""
exc = sys.exc_info()[1]
return cls(
@@ -39,9 +135,18 @@ class ExitInfo(BaseMetrics):
stacktrace=traceback.format_exc() if exc else None,
)
+ @classmethod
+ def get_scons_exit_info(cls, exit_code):
+ """Get the current exit info using the given exit code."""
+ return cls(
+ exit_code=exit_code if isinstance(exit_code, int) else None,
+ exception=None,
+ stacktrace=None,
+ )
+
def is_malformed(self):
- """Return True since this object cannot be malformed if created with classmethod."""
- return True
+ """Return True if this object is missing an exit code."""
+ return self.exit_code is None
class HostInfo(BaseMetrics):
@@ -109,7 +214,7 @@ class GitInfo(BaseMetrics):
def is_malformed(self):
"""Confirm whether this instance has all expected fields."""
- return self.commit_hash is None or self.branch_name is None or self.repo_name is None
+ return None in [self.commit_hash, self.branch_name, self.repo_name]
MODULES_FILEPATH = 'src/mongo/db/modules'
@@ -132,28 +237,63 @@ def _get_modules_git_info():
class ToolingMetrics(BaseMetrics):
"""Class to store tooling metrics."""
+ source: str
utc_starttime: datetime
utc_endtime: datetime
host_info: HostInfo
git_info: GitInfo
exit_info: ExitInfo
+ build_info: Optional[BuildInfo]
command: List[str]
module_info: List[GitInfo]
ip_address: Optional[str]
@classmethod
- def get_tooling_metrics(cls, utc_starttime: datetime):
- """Get tooling metrics to the best of our ability."""
+ def get_resmoke_metrics(
+ cls,
+ utc_starttime: datetime,
+ ):
+ """Get resmoke metrics to the best of our ability."""
+ try:
+ ip_address = socket.gethostbyname(socket.gethostname())
+ except:
+ ip_address = None
+ return cls(
+ source='resmoke',
+ utc_starttime=utc_starttime,
+ utc_endtime=datetime.utcnow(),
+ host_info=HostInfo.get_host_info(),
+ git_info=GitInfo.get_git_info('.'),
+ exit_info=ExitInfo.get_resmoke_exit_info(),
+ build_info=None,
+ module_info=_get_modules_git_info(),
+ command=sys.argv,
+ ip_address=ip_address,
+ )
+
+ @classmethod
+ def get_scons_metrics(
+ cls,
+ utc_starttime: datetime,
+ env_vars: "SCons.Variables.Variables",
+ env: "SCons.Script.SConscript.SConsEnvironment",
+ parser: "SCons.Script.SConsOptions.SConsOptionParser",
+ args: List[str],
+ exit_code: int,
+ ):
+ """Get scons metrics to the best of our ability."""
try:
ip_address = socket.gethostbyname(socket.gethostname())
except:
ip_address = None
return cls(
+ source='scons',
utc_starttime=utc_starttime,
utc_endtime=datetime.utcnow(),
host_info=HostInfo.get_host_info(),
git_info=GitInfo.get_git_info('.'),
- exit_info=ExitInfo.get_exit_info(),
+ exit_info=ExitInfo.get_scons_exit_info(exit_code),
+ build_info=BuildInfo.get_scons_build_info(utc_starttime, env_vars, env, parser, args),
module_info=_get_modules_git_info(),
command=sys.argv,
ip_address=ip_address,
@@ -161,6 +301,6 @@ class ToolingMetrics(BaseMetrics):
def is_malformed(self):
"""Confirm whether this instance has all expected fields."""
- return self.ip_address is None or any(
- info_obj.is_malformed()
- for info_obj in self.module_info + [self.git_info] + [self.host_info])
+ sub_metrics = [self.build_info] if self.source == 'scons' else []
+ sub_metrics += self.module_info + [self.git_info] + [self.host_info] + [self.exit_info]
+ return self.ip_address is None or any(metrics.is_malformed() for metrics in sub_metrics)
diff --git a/buildscripts/metrics/resmoke_tooling_metrics.py b/buildscripts/metrics/resmoke_tooling_metrics.py
new file mode 100644
index 00000000000..1b24160d26f
--- /dev/null
+++ b/buildscripts/metrics/resmoke_tooling_metrics.py
@@ -0,0 +1,19 @@
+from datetime import datetime
+import logging
+
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+from buildscripts.metrics.tooling_metrics_utils import is_virtual_workstation, save_tooling_metrics
+
+logger = logging.getLogger('resmoke_tooling_metrics')
+
+
+def save_resmoke_tooling_metrics(utc_starttime: datetime):
+ try:
+ if not is_virtual_workstation():
+ return
+ tooling_metrics = ToolingMetrics.get_resmoke_metrics(utc_starttime)
+ save_tooling_metrics(tooling_metrics)
+ except Exception as exc: # pylint: disable=broad-except
+ logger.warning(
+ "%s\nResmoke Metrics Collection Failed -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
+ exc)
diff --git a/buildscripts/metrics/scons_tooling_metrics.py b/buildscripts/metrics/scons_tooling_metrics.py
new file mode 100644
index 00000000000..0b02b6f55c3
--- /dev/null
+++ b/buildscripts/metrics/scons_tooling_metrics.py
@@ -0,0 +1,71 @@
+import atexit
+import datetime
+import logging
+import sys
+from typing import List
+
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+from buildscripts.metrics.tooling_metrics_utils import is_virtual_workstation, save_tooling_metrics
+
+logger = logging.getLogger('scons_tooling_metrics')
+
+
+class SConsExitHook(object):
+ """Plumb all sys.exit through this object so that we can access the exit code in atexit."""
+
+ def __init__(self):
+ self.exit_code = None
+ self._orig_exit = sys.exit
+
+ def __del__(self):
+ sys.exit = self._orig_exit
+
+ def initialize(self):
+ sys.exit = self.exit
+
+ def exit(self, code=0):
+ self.exit_code = code
+ self._orig_exit(code)
+
+
+# This method should only be used when registered on atexit
+def _save_scons_tooling_metrics(
+ utc_starttime: datetime,
+ env_vars: "SCons.Variables.Variables",
+ env: "SCons.Script.SConscript.SConsEnvironment",
+ parser: "SCons.Script.SConsOptions.SConsOptionParser",
+ args: List[str],
+ exit_hook: SConsExitHook,
+):
+ """Save SCons tooling metrics to atlas cluster."""
+ try:
+ if not is_virtual_workstation():
+ return
+ tooling_metrics = ToolingMetrics.get_scons_metrics(utc_starttime, env_vars, env, parser,
+ args, exit_hook.exit_code)
+ save_tooling_metrics(tooling_metrics)
+ except Exception as exc: # pylint: disable=broad-except
+ logger.warning(
+ "%sSCons Metrics Collection Failed -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
+ exc)
+
+
+def setup_scons_metrics_collection_atexit(
+ utc_starttime: datetime,
+ env_vars: "SCons.Variables.Variables",
+ env: "SCons.Script.SConscript.SConsEnvironment",
+ parser: "SCons.Script.SConsOptions.SConsOptionParser",
+ args: List[str],
+) -> None:
+ """Register an atexit method for scons metrics collection."""
+ scons_exit_hook = SConsExitHook()
+ scons_exit_hook.initialize()
+ atexit.register(
+ _save_scons_tooling_metrics,
+ utc_starttime,
+ env_vars,
+ env,
+ parser,
+ args,
+ scons_exit_hook,
+ )
diff --git a/buildscripts/metrics/tooling_metrics.py b/buildscripts/metrics/tooling_metrics_utils.py
index 1a890083de4..9e8b3256ecb 100644
--- a/buildscripts/metrics/tooling_metrics.py
+++ b/buildscripts/metrics/tooling_metrics_utils.py
@@ -1,5 +1,3 @@
-import dataclasses
-import datetime
import logging
import os
import asyncio
@@ -9,7 +7,7 @@ import pymongo
from buildscripts.metrics.metrics_datatypes import ToolingMetrics
-logger = logging.getLogger('tooling_metrics_collection')
+logger = logging.getLogger('tooling_metrics_utils')
INTERNAL_TOOLING_METRICS_HOSTNAME = "mongodb+srv://dev-metrics-pl-0.kewhj.mongodb.net"
INTERNAL_TOOLING_METRICS_USERNAME = "internal_tooling_user"
@@ -41,7 +39,7 @@ def _git_user_exists() -> Optional[str]:
return None
-def _is_virtual_workstation() -> bool:
+def is_virtual_workstation() -> bool:
"""Detect whether this is a MongoDB internal virtual workstation."""
return _toolchain_exists() and _git_user_exists()
@@ -52,14 +50,10 @@ async def _save_metrics(metrics: ToolingMetrics) -> None:
client.metrics.tooling_metrics.insert_one(metrics.dict())
-def save_tooling_metrics(utc_starttime: datetime) -> None:
+def save_tooling_metrics(tooling_metrics: ToolingMetrics) -> None:
"""Persist tooling metrics data to MongoDB Internal Atlas Cluster."""
try:
- if not _is_virtual_workstation():
- return
- asyncio.run(
- asyncio.wait_for(
- _save_metrics(ToolingMetrics.get_tooling_metrics(utc_starttime)), timeout=1.0))
+ asyncio.run(asyncio.wait_for(_save_metrics(tooling_metrics), timeout=1.0))
except asyncio.TimeoutError as exc:
logger.warning(
"%s\nTimeout: Tooling metrics collection is not available -- this is a non-issue.\nIf this message persists, feel free to reach out to #server-development-platform",
diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py
index 7dc4cbaf648..b84f07e0cf0 100644
--- a/buildscripts/resmokelib/cli.py
+++ b/buildscripts/resmokelib/cli.py
@@ -4,7 +4,7 @@ from datetime import datetime
import time
import os
import psutil
-from buildscripts.metrics.tooling_metrics import save_tooling_metrics
+from buildscripts.metrics.resmoke_tooling_metrics import save_resmoke_tooling_metrics
from buildscripts.resmokelib import parser
@@ -28,4 +28,4 @@ def main(argv):
try:
subcommand.execute()
finally:
- save_tooling_metrics(datetime.utcfromtimestamp(__start_time))
+ save_resmoke_tooling_metrics(datetime.utcfromtimestamp(__start_time))
diff --git a/buildscripts/scons.py b/buildscripts/scons.py
index 534fca3264e..f7f7fcd3d5b 100755
--- a/buildscripts/scons.py
+++ b/buildscripts/scons.py
@@ -34,5 +34,10 @@ except ImportError as import_err:
print("ImportError:", import_err)
sys.exit(1)
-if __name__ == '__main__':
+
+def entrypoint():
SCons.Script.main()
+
+
+if __name__ == '__main__':
+ entrypoint()
diff --git a/buildscripts/tests/tooling_metrics/test_metrics_collection.py b/buildscripts/tests/tooling_metrics/test_metrics_collection.py
deleted file mode 100644
index 9d820a9c5e0..00000000000
--- a/buildscripts/tests/tooling_metrics/test_metrics_collection.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""Unit tests for tooling_metrics.py."""
-import asyncio
-from datetime import datetime
-import unittest
-from unittest.mock import patch
-import mongomock
-import pymongo
-from buildscripts.metrics.metrics_datatypes import ToolingMetrics
-import buildscripts.metrics.tooling_metrics as under_test
-from buildscripts.resmoke import entrypoint
-
-# pylint: disable=unused-argument
-# pylint: disable=protected-access
-
-TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
-CURRENT_DATE_TIME = datetime(2022, 10, 4)
-
-
-async def extended_sleep(arg):
- await asyncio.sleep(2)
-
-
-@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME",
- TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-class TestToolingMetricsCollection(unittest.TestCase):
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
- def test_on_virtual_workstation(self, mock_is_virtual_workstation):
- under_test.save_tooling_metrics(CURRENT_DATE_TIME)
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert client.metrics.tooling_metrics.find_one()
-
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=False)
- def test_not_on_virtual_workstation(self, mock_is_virtual_workstation):
- under_test.save_tooling_metrics(CURRENT_DATE_TIME)
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert not client.metrics.tooling_metrics.find_one()
-
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("buildscripts.metrics.tooling_metrics._save_metrics",
- side_effect=pymongo.errors.WriteError(error="Error Information"))
- @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
- def test_exception_caught(self, mock_is_virtual_workstation, mock_save_metrics):
- with self.assertLogs('tooling_metrics_collection') as cm:
- under_test.save_tooling_metrics(CURRENT_DATE_TIME)
- assert "Error Information" in cm.output[0]
- assert "Unexpected: Tooling metrics collection is not available" in cm.output[0]
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert not client.metrics.tooling_metrics.find_one()
-
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("buildscripts.metrics.tooling_metrics._save_metrics", side_effect=extended_sleep)
- @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
- def test_timeout_caught(self, mock_is_virtual_workstation, mock_save_metrics):
- with self.assertLogs('tooling_metrics_collection') as cm:
- under_test.save_tooling_metrics(CURRENT_DATE_TIME)
- assert "Timeout: Tooling metrics collection is not available" in cm.output[0]
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert not client.metrics.tooling_metrics.find_one()
-
-
-class TestIsVirtualWorkstation(unittest.TestCase):
- @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False)
- @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True)
- def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
- assert not under_test._is_virtual_workstation()
-
- @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True)
- @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True)
- def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
- assert under_test._is_virtual_workstation()
-
- @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True)
- @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False)
- def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
- assert not under_test._is_virtual_workstation()
-
- @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False)
- @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False)
- def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
- assert not under_test._is_virtual_workstation()
-
-
-@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME",
- TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
-@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None)
-@patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True)
-class TestResmokeMetricsCollection(unittest.TestCase):
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test'])
- @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests",
- side_effect=Exception())
- def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation):
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert not client.metrics.tooling_metrics.find_one()
- with self.assertRaises(SystemExit):
- entrypoint()
- assert client.metrics.tooling_metrics.find_one()
-
- @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
- @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites'])
- def test_resmoke_metrics_collection(self, mock_is_virtual_workstation):
- client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
- assert not client.metrics.tooling_metrics.find_one()
- entrypoint()
- assert client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
index 14a17264be2..b584b77e647 100644
--- a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
+++ b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py
@@ -3,20 +3,50 @@ from datetime import datetime
import unittest
from unittest.mock import patch
+from mock import MagicMock
+
import buildscripts.metrics.metrics_datatypes as under_test
# pylint: disable=unused-argument
+@patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir",
+ return_value='/test')
+class TestBuildInfo(unittest.TestCase):
+ @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict",
+ return_value={'env': 'env'})
+ @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict",
+ return_value={'opt': 'opt'})
+ def test_build_info_valid(self, mock_env, mock_options, mock_artifact_dir):
+ build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(),
+ MagicMock(), MagicMock(),
+ MagicMock())
+ assert not build_info.is_malformed()
+
+ def test_build_info_malformed(self, mock_artifact_dir):
+ build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(),
+ MagicMock(), MagicMock(),
+ MagicMock())
+ assert build_info.is_malformed()
+
+
class TestExitInfo(unittest.TestCase):
@patch("sys.exc_info", return_value=(None, None, None))
- def test_no_exc_info(self, mock_exc_info):
- exit_info = under_test.ExitInfo.get_exit_info()
- assert exit_info.is_malformed()
+ def test_resmoke_no_exc_info(self, mock_exc_info):
+ exit_info = under_test.ExitInfo.get_resmoke_exit_info()
+ assert not exit_info.is_malformed()
@patch("sys.exc_info", return_value=(None, ValueError(), None))
- def test_with_exc_info(self, mock_exc_info):
- exit_info = under_test.ExitInfo.get_exit_info()
+ def test_resmoke_with_exc_info(self, mock_exc_info):
+ exit_info = under_test.ExitInfo.get_resmoke_exit_info()
+ assert not exit_info.is_malformed()
+
+ def test_scons_exit_info_valid(self):
+ exit_info = under_test.ExitInfo.get_scons_exit_info(0)
+ assert not exit_info.is_malformed()
+
+ def test_scons_exit_info_malformed(self):
+ exit_info = under_test.ExitInfo.get_scons_exit_info('string')
assert exit_info.is_malformed()
@@ -53,10 +83,29 @@ class TestGitInfo(unittest.TestCase):
@patch("buildscripts.metrics.metrics_datatypes.HostInfo._get_memory", return_value=30)
class TestToolingMetrics(unittest.TestCase):
@patch("socket.gethostname", side_effect=Exception())
- def test_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory):
- tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow())
+ def test_resmoke_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory):
+ tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow())
assert tooling_metrics.is_malformed()
- def test_tooling_metrics_no_exc(self, mock_get_memory):
- tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow())
+ def test_resmoke_tooling_metrics_no_exc(self, mock_get_memory):
+ tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow())
+ assert not tooling_metrics.is_malformed()
+
+ @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir",
+ return_value='/test')
+ @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict",
+ return_value={'env': 'env'})
+ @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict",
+ return_value={'opt': 'opt'})
+ def test_scons_tooling_metrics_valid(self, mock_options, mock_env, mock_artifact_dir,
+ mock_get_memory):
+ parser = MagicMock()
+ parser.parse_args = MagicMock(return_value={"opt1": "val1"})
+ tooling_metrics = under_test.ToolingMetrics.get_scons_metrics(
+ datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, parser, ['test1', 'test2'], 0)
assert not tooling_metrics.is_malformed()
+
+ def test_scons_tooling_metrics_malformed(self, mock_get_memory):
+ tooling_metrics = under_test.ToolingMetrics.get_scons_metrics(
+ datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, None, [], 0)
+ assert tooling_metrics.is_malformed()
diff --git a/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py
new file mode 100644
index 00000000000..3510184eb0c
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py
@@ -0,0 +1,44 @@
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import patch
+import mongomock
+import pymongo
+
+import buildscripts.metrics.resmoke_tooling_metrics as under_test
+from buildscripts.resmoke import entrypoint as resmoke_entrypoint
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+# pylint: disable=unused-argument
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+ sys.exit()
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+ TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None)
+@patch("buildscripts.metrics.resmoke_tooling_metrics.is_virtual_workstation", return_value=True)
+class TestResmokeMetricsCollection(unittest.TestCase):
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test'])
+ @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests",
+ side_effect=Exception())
+ def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation):
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+ with self.assertRaises(SystemExit):
+ resmoke_entrypoint()
+ assert client.metrics.tooling_metrics.find_one()
+
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites'])
+ def test_resmoke_metrics_collection(self, mock_is_virtual_workstation):
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+ resmoke_entrypoint()
+ assert client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py
new file mode 100644
index 00000000000..f0e020bf306
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py
@@ -0,0 +1,64 @@
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+import mongomock
+import pymongo
+import buildscripts.metrics.scons_tooling_metrics as under_test
+from buildscripts.scons import entrypoint as scons_entrypoint
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+# pylint: disable=unused-argument
+# pylint: disable=protected-access
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+ sys.exit()
+
+
+@patch("sys.argv", [
+ 'buildscripts/scons.py', "CC=/opt/mongodbtoolchain/v3/bin/gcc",
+ "CXX=/opt/mongodbtoolchain/v3/bin/g++", "NINJA_PREFIX=test_success", "--ninja"
+])
+@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True)
+@patch("atexit.register")
+class TestSconsAtExitMetricsCollection(unittest.TestCase):
+ def test_scons_at_exit_metrics_collection(self, mock_atexit_register,
+ mock_is_virtual_workstation):
+ with self.assertRaises(SystemExit) as context:
+ scons_entrypoint()
+ assert context.exception.code == 0
+ atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list]
+ assert "_save_scons_tooling_metrics" in atexit_functions
+
+ @patch("buildscripts.moduleconfig.get_module_sconscripts", side_effect=Exception())
+ def test_scons_at_exit_metrics_collection_exc(self, mock_method, mock_atexit_register,
+ mock_is_virtual_workstation):
+ with self.assertRaises(SystemExit) as context:
+ scons_entrypoint()
+ assert context.exception.code == 2
+ atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list]
+ assert "_save_scons_tooling_metrics" in atexit_functions
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+ TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True)
+class TestSconsMetricsPersist(unittest.TestCase):
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ def test_scons_metrics_collection_success(self, mock_is_virtual_workstation):
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+ under_test._save_scons_tooling_metrics(CURRENT_DATE_TIME, None, None, None, None,
+ MagicMock(exit_code=0))
+ assert client.metrics.tooling_metrics.find_one()
+
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ def test_scons_metrics_collection_fail(self, mock_is_virtual_workstation):
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+ under_test._save_scons_tooling_metrics(None, None, None, None, None, None)
+ assert not client.metrics.tooling_metrics.find_one()
diff --git a/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py
new file mode 100644
index 00000000000..c1c768e50d2
--- /dev/null
+++ b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py
@@ -0,0 +1,78 @@
+"""Unit tests for tooling_metrics.py."""
+import asyncio
+from datetime import datetime
+import os
+import sys
+import unittest
+from unittest.mock import patch
+import mongomock
+import pymongo
+from buildscripts.metrics.metrics_datatypes import ToolingMetrics
+import buildscripts.metrics.tooling_metrics_utils as under_test
+
+# pylint: disable=unused-argument
+# pylint: disable=protected-access
+
+TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017'
+CURRENT_DATE_TIME = datetime(2022, 10, 4)
+
+
+async def extended_sleep(arg):
+ await asyncio.sleep(2)
+
+
+# Metrics collection is not supported for Windows
+if os.name == "nt":
+ sys.exit()
+
+
+@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME",
+ TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+class TestSaveToolingMetrics(unittest.TestCase):
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ def test_on_virtual_workstation(self):
+ under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert client.metrics.tooling_metrics.find_one()
+
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics",
+ side_effect=pymongo.errors.WriteError(error="Error Information"))
+ def test_exception_caught(self, mock_save_metrics):
+ with self.assertLogs('tooling_metrics_utils') as cm:
+ under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+ assert "Error Information" in cm.output[0]
+ assert "Unexpected: Tooling metrics collection is not available" in cm.output[0]
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+
+ @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), ))
+ @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics", side_effect=extended_sleep)
+ def test_timeout_caught(self, mock_save_metrics):
+ with self.assertLogs('tooling_metrics_utils') as cm:
+ under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME))
+ assert "Timeout: Tooling metrics collection is not available" in cm.output[0]
+ client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME)
+ assert not client.metrics.tooling_metrics.find_one()
+
+
+class TestIsVirtualWorkstation(unittest.TestCase):
+ @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False)
+ @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True)
+ def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
+ assert not under_test.is_virtual_workstation()
+
+ @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True)
+ @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True)
+ def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists):
+ assert under_test.is_virtual_workstation()
+
+ @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True)
+ @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False)
+ def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
+ assert not under_test.is_virtual_workstation()
+
+ @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False)
+ @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False)
+ def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists):
+ assert not under_test.is_virtual_workstation()