From 3a4d6fdeb7cc74f1b9f2821b3fa6f6dddeca4a62 Mon Sep 17 00:00:00 2001 From: Tausif Rahman Date: Tue, 8 Nov 2022 15:36:04 +0000 Subject: SERVER-69636 Collect SCons Metrics --- .../tooling_metrics/test_metrics_collection.py | 107 --------------------- .../tooling_metrics/test_metrics_datatypes.py | 67 +++++++++++-- .../test_resmoke_tooling_metrics.py | 44 +++++++++ .../tooling_metrics/test_scons_tooling_metrics.py | 64 ++++++++++++ .../tooling_metrics/test_tooling_metrics_utils.py | 78 +++++++++++++++ 5 files changed, 244 insertions(+), 116 deletions(-) delete mode 100644 buildscripts/tests/tooling_metrics/test_metrics_collection.py create mode 100644 buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py create mode 100644 buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py create mode 100644 buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py (limited to 'buildscripts/tests') diff --git a/buildscripts/tests/tooling_metrics/test_metrics_collection.py b/buildscripts/tests/tooling_metrics/test_metrics_collection.py deleted file mode 100644 index 9d820a9c5e0..00000000000 --- a/buildscripts/tests/tooling_metrics/test_metrics_collection.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Unit tests for tooling_metrics.py.""" -import asyncio -from datetime import datetime -import unittest -from unittest.mock import patch -import mongomock -import pymongo -from buildscripts.metrics.metrics_datatypes import ToolingMetrics -import buildscripts.metrics.tooling_metrics as under_test -from buildscripts.resmoke import entrypoint - -# pylint: disable=unused-argument -# pylint: disable=protected-access - -TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' -CURRENT_DATE_TIME = datetime(2022, 10, 4) - - -async def extended_sleep(arg): - await asyncio.sleep(2) - - -@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME", - TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) -class TestToolingMetricsCollection(unittest.TestCase): - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True) - def test_on_virtual_workstation(self, mock_is_virtual_workstation): - under_test.save_tooling_metrics(CURRENT_DATE_TIME) - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert client.metrics.tooling_metrics.find_one() - - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=False) - def test_not_on_virtual_workstation(self, mock_is_virtual_workstation): - under_test.save_tooling_metrics(CURRENT_DATE_TIME) - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("buildscripts.metrics.tooling_metrics._save_metrics", - side_effect=pymongo.errors.WriteError(error="Error Information")) - @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True) - def test_exception_caught(self, mock_is_virtual_workstation, mock_save_metrics): - with self.assertLogs('tooling_metrics_collection') as cm: - under_test.save_tooling_metrics(CURRENT_DATE_TIME) - assert "Error Information" in cm.output[0] - assert "Unexpected: Tooling metrics collection is not available" in cm.output[0] - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("buildscripts.metrics.tooling_metrics._save_metrics", side_effect=extended_sleep) - @patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True) - def test_timeout_caught(self, mock_is_virtual_workstation, mock_save_metrics): - with self.assertLogs('tooling_metrics_collection') as cm: - under_test.save_tooling_metrics(CURRENT_DATE_TIME) - assert "Timeout: Tooling metrics collection is not available" in cm.output[0] - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - - -class TestIsVirtualWorkstation(unittest.TestCase): - @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False) - @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True) - def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists): - assert not under_test._is_virtual_workstation() - - @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True) - @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=True) - def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists): - assert under_test._is_virtual_workstation() - - @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=True) - @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False) - def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists): - assert not under_test._is_virtual_workstation() - - @patch("buildscripts.metrics.tooling_metrics._toolchain_exists", return_value=False) - @patch("buildscripts.metrics.tooling_metrics._git_user_exists", return_value=False) - def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists): - assert not under_test._is_virtual_workstation() - - -@patch("buildscripts.metrics.tooling_metrics.INTERNAL_TOOLING_METRICS_HOSTNAME", - TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) -@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None) -@patch("buildscripts.metrics.tooling_metrics._is_virtual_workstation", return_value=True) -class TestResmokeMetricsCollection(unittest.TestCase): - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test']) - @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests", - side_effect=Exception()) - def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation): - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - with self.assertRaises(SystemExit): - entrypoint() - assert client.metrics.tooling_metrics.find_one() - - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) - def test_resmoke_metrics_collection(self, mock_is_virtual_workstation): - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - entrypoint() - assert client.metrics.tooling_metrics.find_one() diff --git a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py index 14a17264be2..b584b77e647 100644 --- a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py +++ b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py @@ -3,20 +3,50 @@ from datetime import datetime import unittest from unittest.mock import patch +from mock import MagicMock + import buildscripts.metrics.metrics_datatypes as under_test # pylint: disable=unused-argument +@patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir", + return_value='/test') +class TestBuildInfo(unittest.TestCase): + @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict", + return_value={'env': 'env'}) + @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict", + return_value={'opt': 'opt'}) + def test_build_info_valid(self, mock_env, mock_options, mock_artifact_dir): + build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(), + MagicMock(), MagicMock(), + MagicMock()) + assert not build_info.is_malformed() + + def test_build_info_malformed(self, mock_artifact_dir): + build_info = under_test.BuildInfo.get_scons_build_info(datetime.utcnow(), MagicMock(), + MagicMock(), MagicMock(), + MagicMock()) + assert build_info.is_malformed() + + class TestExitInfo(unittest.TestCase): @patch("sys.exc_info", return_value=(None, None, None)) - def test_no_exc_info(self, mock_exc_info): - exit_info = under_test.ExitInfo.get_exit_info() - assert exit_info.is_malformed() + def test_resmoke_no_exc_info(self, mock_exc_info): + exit_info = under_test.ExitInfo.get_resmoke_exit_info() + assert not exit_info.is_malformed() @patch("sys.exc_info", return_value=(None, ValueError(), None)) - def test_with_exc_info(self, mock_exc_info): - exit_info = under_test.ExitInfo.get_exit_info() + def test_resmoke_with_exc_info(self, mock_exc_info): + exit_info = under_test.ExitInfo.get_resmoke_exit_info() + assert not exit_info.is_malformed() + + def test_scons_exit_info_valid(self): + exit_info = under_test.ExitInfo.get_scons_exit_info(0) + assert not exit_info.is_malformed() + + def test_scons_exit_info_malformed(self): + exit_info = under_test.ExitInfo.get_scons_exit_info('string') assert exit_info.is_malformed() @@ -53,10 +83,29 @@ class TestGitInfo(unittest.TestCase): @patch("buildscripts.metrics.metrics_datatypes.HostInfo._get_memory", return_value=30) class TestToolingMetrics(unittest.TestCase): @patch("socket.gethostname", side_effect=Exception()) - def test_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory): - tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow()) + def test_resmoke_tooling_metrics_with_exc(self, mock_gethostname, mock_get_memory): + tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow()) assert tooling_metrics.is_malformed() - def test_tooling_metrics_no_exc(self, mock_get_memory): - tooling_metrics = under_test.ToolingMetrics.get_tooling_metrics(datetime.utcnow()) + def test_resmoke_tooling_metrics_no_exc(self, mock_get_memory): + tooling_metrics = under_test.ToolingMetrics.get_resmoke_metrics(datetime.utcnow()) + assert not tooling_metrics.is_malformed() + + @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir", + return_value='/test') + @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict", + return_value={'env': 'env'}) + @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict", + return_value={'opt': 'opt'}) + def test_scons_tooling_metrics_valid(self, mock_options, mock_env, mock_artifact_dir, + mock_get_memory): + parser = MagicMock() + parser.parse_args = MagicMock(return_value={"opt1": "val1"}) + tooling_metrics = under_test.ToolingMetrics.get_scons_metrics( + datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, parser, ['test1', 'test2'], 0) assert not tooling_metrics.is_malformed() + + def test_scons_tooling_metrics_malformed(self, mock_get_memory): + tooling_metrics = under_test.ToolingMetrics.get_scons_metrics( + datetime.utcnow(), {'env': 'env'}, {'opts': 'opts'}, None, [], 0) + assert tooling_metrics.is_malformed() diff --git a/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py new file mode 100644 index 00000000000..3510184eb0c --- /dev/null +++ b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py @@ -0,0 +1,44 @@ +from datetime import datetime +import os +import sys +import unittest +from unittest.mock import patch +import mongomock +import pymongo + +import buildscripts.metrics.resmoke_tooling_metrics as under_test +from buildscripts.resmoke import entrypoint as resmoke_entrypoint + +TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' +CURRENT_DATE_TIME = datetime(2022, 10, 4) + +# pylint: disable=unused-argument + +# Metrics collection is not supported for Windows +if os.name == "nt": + sys.exit() + + +@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME", + TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) +@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None) +@patch("buildscripts.metrics.resmoke_tooling_metrics.is_virtual_workstation", return_value=True) +class TestResmokeMetricsCollection(unittest.TestCase): + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test']) + @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests", + side_effect=Exception()) + def test_resmoke_metrics_collection_exc(self, mock_executor_run, mock_is_virtual_workstation): + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + with self.assertRaises(SystemExit): + resmoke_entrypoint() + assert client.metrics.tooling_metrics.find_one() + + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) + def test_resmoke_metrics_collection(self, mock_is_virtual_workstation): + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + resmoke_entrypoint() + assert client.metrics.tooling_metrics.find_one() diff --git a/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py new file mode 100644 index 00000000000..f0e020bf306 --- /dev/null +++ b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py @@ -0,0 +1,64 @@ +from datetime import datetime +import os +import sys +import unittest +from unittest.mock import MagicMock, patch +import mongomock +import pymongo +import buildscripts.metrics.scons_tooling_metrics as under_test +from buildscripts.scons import entrypoint as scons_entrypoint + +TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' +CURRENT_DATE_TIME = datetime(2022, 10, 4) + +# pylint: disable=unused-argument +# pylint: disable=protected-access + +# Metrics collection is not supported for Windows +if os.name == "nt": + sys.exit() + + +@patch("sys.argv", [ + 'buildscripts/scons.py', "CC=/opt/mongodbtoolchain/v3/bin/gcc", + "CXX=/opt/mongodbtoolchain/v3/bin/g++", "NINJA_PREFIX=test_success", "--ninja" +]) +@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True) +@patch("atexit.register") +class TestSconsAtExitMetricsCollection(unittest.TestCase): + def test_scons_at_exit_metrics_collection(self, mock_atexit_register, + mock_is_virtual_workstation): + with self.assertRaises(SystemExit) as context: + scons_entrypoint() + assert context.exception.code == 0 + atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] + assert "_save_scons_tooling_metrics" in atexit_functions + + @patch("buildscripts.moduleconfig.get_module_sconscripts", side_effect=Exception()) + def test_scons_at_exit_metrics_collection_exc(self, mock_method, mock_atexit_register, + mock_is_virtual_workstation): + with self.assertRaises(SystemExit) as context: + scons_entrypoint() + assert context.exception.code == 2 + atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] + assert "_save_scons_tooling_metrics" in atexit_functions + + +@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME", + TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) +@patch("buildscripts.metrics.scons_tooling_metrics.is_virtual_workstation", return_value=True) +class TestSconsMetricsPersist(unittest.TestCase): + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + def test_scons_metrics_collection_success(self, mock_is_virtual_workstation): + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + under_test._save_scons_tooling_metrics(CURRENT_DATE_TIME, None, None, None, None, + MagicMock(exit_code=0)) + assert client.metrics.tooling_metrics.find_one() + + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + def test_scons_metrics_collection_fail(self, mock_is_virtual_workstation): + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + under_test._save_scons_tooling_metrics(None, None, None, None, None, None) + assert not client.metrics.tooling_metrics.find_one() diff --git a/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py new file mode 100644 index 00000000000..c1c768e50d2 --- /dev/null +++ b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py @@ -0,0 +1,78 @@ +"""Unit tests for tooling_metrics.py.""" +import asyncio +from datetime import datetime +import os +import sys +import unittest +from unittest.mock import patch +import mongomock +import pymongo +from buildscripts.metrics.metrics_datatypes import ToolingMetrics +import buildscripts.metrics.tooling_metrics_utils as under_test + +# pylint: disable=unused-argument +# pylint: disable=protected-access + +TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' +CURRENT_DATE_TIME = datetime(2022, 10, 4) + + +async def extended_sleep(arg): + await asyncio.sleep(2) + + +# Metrics collection is not supported for Windows +if os.name == "nt": + sys.exit() + + +@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME", + TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) +class TestSaveToolingMetrics(unittest.TestCase): + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + def test_on_virtual_workstation(self): + under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME)) + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert client.metrics.tooling_metrics.find_one() + + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics", + side_effect=pymongo.errors.WriteError(error="Error Information")) + def test_exception_caught(self, mock_save_metrics): + with self.assertLogs('tooling_metrics_utils') as cm: + under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME)) + assert "Error Information" in cm.output[0] + assert "Unexpected: Tooling metrics collection is not available" in cm.output[0] + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + + @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) + @patch("buildscripts.metrics.tooling_metrics_utils._save_metrics", side_effect=extended_sleep) + def test_timeout_caught(self, mock_save_metrics): + with self.assertLogs('tooling_metrics_utils') as cm: + under_test.save_tooling_metrics(ToolingMetrics.get_resmoke_metrics(CURRENT_DATE_TIME)) + assert "Timeout: Tooling metrics collection is not available" in cm.output[0] + client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) + assert not client.metrics.tooling_metrics.find_one() + + +class TestIsVirtualWorkstation(unittest.TestCase): + @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False) + @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True) + def test_no_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists): + assert not under_test.is_virtual_workstation() + + @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True) + @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=True) + def test_has_toolchain_has_email(self, mock_git_user_exists, mock_toolchain_exists): + assert under_test.is_virtual_workstation() + + @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=True) + @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False) + def test_has_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists): + assert not under_test.is_virtual_workstation() + + @patch("buildscripts.metrics.tooling_metrics_utils._toolchain_exists", return_value=False) + @patch("buildscripts.metrics.tooling_metrics_utils._git_user_exists", return_value=False) + def test_no_toolchain_no_email(self, mock_git_user_exists, mock_toolchain_exists): + assert not under_test.is_virtual_workstation() -- cgit v1.2.1