diff options
author | Tausif Rahman <tausif.rahman@mongodb.com> | 2023-01-12 02:34:38 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-01-12 03:08:02 +0000 |
commit | 08ae3cb5d125b8d2cccf5fe63a4e3d3014091749 (patch) | |
tree | 606562963ecf6c3bbd59cc47e9126c28bd27479b /buildscripts/tests | |
parent | 5e9efe141520d36adcdd8af424c210d30e955943 (diff) | |
download | mongo-08ae3cb5d125b8d2cccf5fe63a4e3d3014091749.tar.gz |
SERVER-72348 Use the new mongo-tooling-metrics library
Diffstat (limited to 'buildscripts/tests')
6 files changed, 169 insertions, 299 deletions
diff --git a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py b/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py deleted file mode 100644 index 3e181eb4971..00000000000 --- a/buildscripts/tests/tooling_metrics/test_metrics_datatypes.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Unit tests for metrics_datatypes.py.""" -from datetime import datetime -import os -import sys -import unittest -from unittest.mock import patch - -from mock import MagicMock - -import buildscripts.metrics.metrics_datatypes as under_test - -# pylint: disable=unused-argument - -# Metrics collection is not supported for Windows -if os.name == "nt": - sys.exit() - -MOCK_EXIT_HOOK = MagicMock(exit_code=0) - - -@patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir", - return_value='/test') -class TestBuildInfo(unittest.TestCase): - @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict", - return_value={'env': 'env'}) - @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict", - return_value={'opt': 'opt'}) - def test_build_info_valid(self, mock_env, mock_options, mock_artifact_dir): - build_info = under_test.BuildInfo.generate_metrics(datetime.utcnow(), MagicMock(), - MagicMock(), MagicMock(), MagicMock()) - assert not build_info.is_malformed() - - def test_build_info_malformed(self, mock_artifact_dir): - build_info = under_test.BuildInfo.generate_metrics(datetime.utcnow(), MagicMock(), - MagicMock(), MagicMock(), MagicMock()) - assert build_info.is_malformed() - - -class TestHostInfo(unittest.TestCase): - @patch("buildscripts.metrics.metrics_datatypes.HostInfo._get_memory", side_effect=Exception()) - def test_host_info_with_exc(self, mock_get_memory): - host_info = under_test.HostInfo.generate_metrics() - assert host_info.is_malformed() - - # Mock this so that it passes when running the 'buildscripts_test' suite on Windows - @patch("buildscripts.metrics.metrics_datatypes.HostInfo._get_memory", return_value=30) - def test_host_info_no_exc(self, mock_get_memory): - host_info = under_test.HostInfo.generate_metrics() - assert not host_info.is_malformed() - - -class TestGitInfo(unittest.TestCase): - @patch("git.Repo", side_effect=Exception()) - def test_git_info_with_exc(self, mock_repo): - git_info = under_test.GitInfo.generate_metrics('.') - assert git_info.is_malformed() - - def test_git_info_no_exc(self): - git_info = under_test.GitInfo.generate_metrics('.') - assert not git_info.is_malformed() - - @patch("git.refs.symbolic.SymbolicReference.is_detached", True) - def test_git_info_detached_head(self): - git_info = under_test.GitInfo.generate_metrics('.') - assert not git_info.is_malformed() - - -class TestResmokeToolingMetrics(unittest.TestCase): - @patch("socket.gethostname", side_effect=Exception()) - def test_resmoke_tooling_metrics_valid(self, mock_gethostname): - tooling_metrics = under_test.ResmokeToolingMetrics.generate_metrics( - datetime.utcnow(), - MOCK_EXIT_HOOK, - ) - assert tooling_metrics.is_malformed() - - def test_resmoke_tooling_metrics_malformed(self): - tooling_metrics = under_test.ResmokeToolingMetrics.generate_metrics( - datetime.utcnow(), - MOCK_EXIT_HOOK, - ) - assert not tooling_metrics.is_malformed() - - -class TestSConsToolingMetrics(unittest.TestCase): - @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_artifact_dir", - return_value='/test') - @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_env_vars_dict", - return_value={'env': 'env'}) - @patch("buildscripts.metrics.metrics_datatypes.BuildInfo._get_scons_options_dict", - return_value={'opt': 'opt'}) - def test_scons_tooling_metrics_valid(self, mock_options, mock_env, mock_artifact_dir): - parser = MagicMock() - parser.parse_args = MagicMock(return_value={"opt1": "val1"}) - tooling_metrics = under_test.SConsToolingMetrics.generate_metrics( - datetime.utcnow(), - {'env': 'env'}, - {'opts': 'opts'}, - parser, - ['test1', 'test2'], - MOCK_EXIT_HOOK, - ) - assert not tooling_metrics.is_malformed() - - def test_scons_tooling_metrics_malformed(self): - tooling_metrics = under_test.SConsToolingMetrics.generate_metrics( - datetime.utcnow(), - {'env': 'env'}, - {'opts': 'opts'}, - None, - [], - MOCK_EXIT_HOOK, - ) - assert tooling_metrics.is_malformed() diff --git a/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py deleted file mode 100644 index c3d7468e90a..00000000000 --- a/buildscripts/tests/tooling_metrics/test_resmoke_tooling_metrics.py +++ /dev/null @@ -1,47 +0,0 @@ -from datetime import datetime -import os -import sys -import unittest -from unittest.mock import patch - -import buildscripts.resmoke as under_test - -TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' -CURRENT_DATE_TIME = datetime(2022, 10, 4) - -# pylint: disable=unused-argument - -# Metrics collection is not supported for Windows -if os.name == "nt": - sys.exit() - - -@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None) -@patch("atexit.register") -class TestResmokeAtExitMetricsCollection(unittest.TestCase): - @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=True) - def test_resmoke_at_exit_metrics_collection(self, mock_should_collect_metrics, - mock_atexit_register): - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" in atexit_functions - - @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=False) - def test_no_resmoke_at_exit_metrics_collection(self, mock_should_collect_metrics, - mock_atexit_register): - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" not in atexit_functions - - @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test']) - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=True) - @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests", - side_effect=Exception()) - def test_resmoke_at_exit_metrics_collection_exc( - self, mock_exc_method, mock_should_collect_metrics, mock_atexit_register): - with self.assertRaises(SystemExit) as _: - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" in atexit_functions diff --git a/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py b/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py deleted file mode 100644 index a80b526ed55..00000000000 --- a/buildscripts/tests/tooling_metrics/test_scons_tooling_metrics.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys -import unittest -from unittest.mock import patch -import buildscripts.scons as under_test - -# pylint: disable=unused-argument -# pylint: disable=protected-access - -# Metrics collection is not supported for Windows -if os.name == "nt": - sys.exit() - - -@patch("sys.argv", [ - 'buildscripts/scons.py', "CC=/opt/mongodbtoolchain/v4/bin/gcc", - "CXX=/opt/mongodbtoolchain/v4/bin/g++", "NINJA_PREFIX=test_success", "--ninja" -]) -@patch("atexit.register") -class TestSconsAtExitMetricsCollection(unittest.TestCase): - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=True) - def test_scons_at_exit_metrics_collection(self, mock_should_collect_metrics, - mock_atexit_register): - with self.assertRaises(SystemExit) as _: - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" in atexit_functions - - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=False) - def test_no_scons_at_exit_metrics_collection(self, mock_should_collect_metrics, - mock_atexit_register): - with self.assertRaises(SystemExit) as _: - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" not in atexit_functions - - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=True) - @patch("buildscripts.moduleconfig.get_module_sconscripts", side_effect=Exception()) - def test_scons_at_exit_metrics_collection_exc( - self, mock_exc_method, mock_should_collect_metrics, mock_atexit_register): - with self.assertRaises(SystemExit) as _: - under_test.entrypoint() - atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] - assert "_save_metrics" in atexit_functions diff --git a/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py b/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py deleted file mode 100644 index c705059194b..00000000000 --- a/buildscripts/tests/tooling_metrics/test_tooling_metrics_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Unit tests for tooling_metrics.py.""" -from datetime import datetime -import os -import sys -import unittest -from unittest.mock import mock_open, patch -from mock import MagicMock -import mongomock -import pymongo -from buildscripts.metrics.metrics_datatypes import ResmokeToolingMetrics, SConsToolingMetrics -import buildscripts.metrics.tooling_metrics_utils as under_test - -# pylint: disable=unused-argument -# pylint: disable=protected-access - -TEST_INTERNAL_TOOLING_METRICS_HOSTNAME = 'mongodb://testing:27017' -RESMOKE_METRICS_ARGS = { - "utc_starttime": datetime(2022, 10, 4), - "exit_hook": MagicMock(exit_code=0), -} - -# Metrics collection is not supported for Windows -if os.name == "nt": - sys.exit() - - -@patch("atexit.register") -class TestRegisterMetricsCollectionAtExit(unittest.TestCase): - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=True) - def test_register_metrics_collection(self, mock_should_collect_metrics, mock_atexit): - under_test.register_metrics_collection_atexit(ResmokeToolingMetrics.generate_metrics, - RESMOKE_METRICS_ARGS) - atexit_functions = [call[0][0].__name__ for call in mock_atexit.call_args_list] - assert "_save_metrics" in atexit_functions - - @patch("buildscripts.metrics.tooling_metrics_utils._should_collect_metrics", return_value=False) - def test_no_register_metrics_collection(self, mock_should_collect_metrics, mock_atexit): - under_test.register_metrics_collection_atexit(ResmokeToolingMetrics.generate_metrics, - RESMOKE_METRICS_ARGS) - atexit_functions = [call[0][0].__name__ for call in mock_atexit.call_args_list] - assert "_save_metrics" not in atexit_functions - - -@patch("buildscripts.metrics.tooling_metrics_utils.INTERNAL_TOOLING_METRICS_HOSTNAME", - TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) -class TestSaveToolingMetrics(unittest.TestCase): - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - def test_save_resmoke_metrics(self): - under_test._save_metrics(ResmokeToolingMetrics.generate_metrics, RESMOKE_METRICS_ARGS) - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert client.metrics.tooling_metrics.find_one() - - @mongomock.patch(servers=((TEST_INTERNAL_TOOLING_METRICS_HOSTNAME), )) - @patch("buildscripts.metrics.tooling_metrics_utils._get_internal_tooling_metrics_client", - side_effect=pymongo.errors.ServerSelectionTimeoutError(message="Error Information")) - def test_save_metrics_with_exc(self, mock_save_metrics): - with self.assertLogs('tooling_metrics') as cm: - under_test._save_metrics(ResmokeToolingMetrics.generate_metrics, RESMOKE_METRICS_ARGS) - assert "Error Information" in cm.output[0] - assert "Internal Metrics Collection Failed" in cm.output[0] - client = pymongo.MongoClient(host=TEST_INTERNAL_TOOLING_METRICS_HOSTNAME) - assert not client.metrics.tooling_metrics.find_one() - - -class TestIsVirtualWorkstation(unittest.TestCase): - @patch("builtins.open", mock_open(read_data="ubuntu1804-workstation")) - def test_is_virtual_workstation(self): - assert under_test._is_virtual_workstation() is True - - @patch("builtins.open", mock_open(read_data="test")) - def test_is_not_virtual_workstation(self): - assert under_test._is_virtual_workstation() is False - - -class TestHasMetricsOptOut(unittest.TestCase): - @patch("os.environ.get", return_value='1') - def test_opt_out(self, mock_environ_get): - assert under_test._has_metrics_opt_out() - - @patch("os.environ.get", return_value=None) - def test_no_opt_out(self, mock_environ_get): - assert not under_test._has_metrics_opt_out() - - -class TestShouldCollectMetrics(unittest.TestCase): - @patch("buildscripts.metrics.tooling_metrics_utils._is_virtual_workstation", return_value=True) - @patch("buildscripts.metrics.tooling_metrics_utils._has_metrics_opt_out", return_value=False) - def test_should_collect_metrics(self, mock_opt_out, mock_is_virtual_env): - assert under_test._should_collect_metrics() - - @patch("buildscripts.metrics.tooling_metrics_utils._is_virtual_workstation", return_value=True) - @patch("buildscripts.metrics.tooling_metrics_utils._has_metrics_opt_out", return_value=True) - def test_no_collect_metrics_opt_out(self, mock_opt_out, mock_is_virtual_env): - assert not under_test._should_collect_metrics() diff --git a/buildscripts/tests/tooling_metrics_e2e/test_resmoke_tooling_metrics.py b/buildscripts/tests/tooling_metrics_e2e/test_resmoke_tooling_metrics.py new file mode 100644 index 00000000000..2bda6aa05e6 --- /dev/null +++ b/buildscripts/tests/tooling_metrics_e2e/test_resmoke_tooling_metrics.py @@ -0,0 +1,73 @@ +from datetime import datetime +import os +import sys +import unittest +from unittest.mock import patch +from mock import MagicMock +from mongo_tooling_metrics import client +from mongo_tooling_metrics.base_metrics import TopLevelMetrics + +import buildscripts.resmoke as under_test + +CURRENT_DATE_TIME = datetime(2022, 10, 4) + +# pylint: disable=unused-argument + +# Metrics collection is not supported for Windows +if os.name == "nt": + sys.exit() + + +@patch("buildscripts.resmokelib.logging.flush._FLUSH_THREAD", None) +@patch("atexit.register") +class TestResmokeAtExitMetricsCollection(unittest.TestCase): + @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + def test_resmoke_at_exit_metrics_collection(self, mock_atexit_register): + under_test.entrypoint() + + atexit_functions = [ + call for call in mock_atexit_register.call_args_list + if call[0][0].__name__ == '_verbosity_enforced_save_metrics' + ] + generate_metrics = atexit_functions[0][0][1].generate_metrics + kwargs = atexit_functions[0][1] + metrics = generate_metrics(**kwargs) + + assert not metrics.is_malformed() + + @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=False)) + def test_no_resmoke_at_exit_metrics_collection(self, mock_atexit_register): + under_test.entrypoint() + atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] + assert "_verbosity_enforced_save_metrics" not in atexit_functions + + @patch("sys.argv", ['buildscripts/resmoke.py', 'list-suites']) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=False)) + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + def test_resmoke_no_metric_collection_non_vw(self, mock_atexit_register): + under_test.entrypoint() + atexit_functions = [call[0][0].__name__ for call in mock_atexit_register.call_args_list] + assert "_verbosity_enforced_save_metrics" not in atexit_functions + + @patch("sys.argv", ['buildscripts/resmoke.py', 'run', '--suite', 'buildscripts_test']) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + @patch("buildscripts.resmokelib.testing.executor.TestSuiteExecutor._run_tests", + side_effect=Exception()) + def test_resmoke_at_exit_metrics_collection_exc(self, mock_exc_method, mock_atexit_register): + with self.assertRaises(SystemExit) as _: + under_test.entrypoint() + + atexit_functions = [ + call for call in mock_atexit_register.call_args_list + if call[0][0].__name__ == '_verbosity_enforced_save_metrics' + ] + generate_metrics = atexit_functions[0][0][1].generate_metrics + kwargs = atexit_functions[0][1] + metrics = generate_metrics(**kwargs) + + assert not metrics.is_malformed() diff --git a/buildscripts/tests/tooling_metrics_e2e/test_scons_tooling_metrics.py b/buildscripts/tests/tooling_metrics_e2e/test_scons_tooling_metrics.py new file mode 100644 index 00000000000..f08a9d3b0a3 --- /dev/null +++ b/buildscripts/tests/tooling_metrics_e2e/test_scons_tooling_metrics.py @@ -0,0 +1,96 @@ +import atexit +import os +import sys +import unittest +from unittest.mock import patch +from mock import MagicMock +from mongo_tooling_metrics import client +from mongo_tooling_metrics.lib.utils import _is_virtual_workstation +from mongo_tooling_metrics.base_metrics import TopLevelMetrics +import buildscripts.scons as under_test + +# Metrics collection is not supported for Windows +if os.name == "nt": + sys.exit() + + +class InvalidSconsConfiguration(Exception): + """Exception raised if the scons invocation itself fails.""" + pass + + +@patch("sys.argv", [ + 'buildscripts/scons.py', "CC=/opt/mongodbtoolchain/v4/bin/gcc", + "CXX=/opt/mongodbtoolchain/v4/bin/g++", "NINJA_PREFIX=test_success", "--ninja" +]) +class TestSconsAtExitMetricsCollection(unittest.TestCase): + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch.object(atexit, "register", MagicMock()) + def at_exit_metrics_collection(self): + with self.assertRaises(SystemExit) as exc_info: + under_test.entrypoint() + + if exc_info.exception.code != 0: + raise InvalidSconsConfiguration("This SCons invocation is not supported on this host.") + + atexit_functions = [ + call for call in atexit.register.call_args_list + if call[0][0].__name__ == '_verbosity_enforced_save_metrics' + ] + generate_metrics = atexit_functions[0][0][1].generate_metrics + kwargs = atexit_functions[0][1] + metrics = generate_metrics(**kwargs) + + assert not metrics.is_malformed() + + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=False)) + @patch.object(atexit, "register", MagicMock()) + def no_at_exit_metrics_collection(self): + with self.assertRaises(SystemExit) as _: + under_test.entrypoint() + atexit_functions = [call[0][0].__name__ for call in atexit.register.call_args_list] + assert "_verbosity_enforced_save_metrics" not in atexit_functions + + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=False)) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch.object(atexit, "register", MagicMock()) + def no_metrics_collection_non_vw(self): + with self.assertRaises(SystemExit) as _: + under_test.entrypoint() + atexit_functions = [call[0][0].__name__ for call in atexit.register.call_args_list] + assert "_verbosity_enforced_save_metrics" not in atexit_functions + + @patch.object(TopLevelMetrics, 'should_collect_metrics', MagicMock(return_value=True)) + @patch.object(client, 'should_collect_internal_metrics', MagicMock(return_value=True)) + @patch("buildscripts.moduleconfig.get_module_sconscripts", MagicMock(side_effect=Exception())) + @patch.object(atexit, "register", MagicMock()) + def at_exit_metrics_collection_exc(self): + with self.assertRaises(SystemExit) as _: + under_test.entrypoint() + + atexit_functions = [ + call for call in atexit.register.call_args_list + if call[0][0].__name__ == '_verbosity_enforced_save_metrics' + ] + generate_metrics = atexit_functions[0][0][1].generate_metrics + kwargs = atexit_functions[0][1] + metrics = generate_metrics(**kwargs) + + assert not metrics.is_malformed() + + def test_scons_metrics_collection_at_exit(self): + """Run all tests in this TestCase sequentially from this method.""" + + try: + # If this test fails and this is NOT a Virtual Workstation, we bail because metrics + # collection is only supported on virtual workstations + self.at_exit_metrics_collection() + except InvalidSconsConfiguration: + if not _is_virtual_workstation(): + return + raise InvalidSconsConfiguration + self.no_at_exit_metrics_collection() + self.no_metrics_collection_non_vw() + self.at_exit_metrics_collection_exc() |