SERVER-46687 Run hang-analyzer from resmoke and integrate with archival

This reverts commit 6ac765dd18a96bbe43eb22a30ddaf3a4d42ae2e3.

1 files changed, 80 insertions, 53 deletions

diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py
index a1f6a57e432..bf2df6710b3 100644
--- a/buildscripts/tests/resmoke_end2end/test_resmoke.py
+++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py
@@ -1,4 +1,4 @@
-"""Test resmoke's handling of test/task timeouts."""
+"""Test resmoke's handling of test/task timeouts and archival."""
 
 import logging
 import json
@@ -14,14 +14,16 @@ from buildscripts.resmokelib.utils import rmtree
 # pylint: disable=missing-docstring,protected-access
 
 
+@unittest.skip("TODO: update the test_dir to a temporary directory so it can run in CI")
 class _ResmokeSelftest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.test_dir = os.path.normpath("/data/db/selftest")
         cls.resmoke_const_args = ["run", "--dbpathPrefix={}".format(cls.test_dir)]
 
+        cls.resmoke_process = None
+
     def setUp(self):
-        #self.test_dir = os.path.normpath("/data/db/selftest")
         self.end2end_root = "buildscripts/tests/resmoke_end2end"
         self.suites_root = f"{self.end2end_root}/suites"
         self.testfiles_root = f"{self.end2end_root}/testfiles"
@@ -37,28 +39,29 @@ class _ResmokeSelftest(unittest.TestCase):
         rmtree(self.test_dir, ignore_errors=True)
         os.makedirs(self.test_dir, mode=0o755, exist_ok=True)
 
-    def execute_resmoke(self, resmoke_args):
+    def execute_resmoke(self, resmoke_args, **kwargs):  # pylint: disable=unused-argument
         resmoke_process = core.programs.make_process(
             self.logger,
             [sys.executable, "buildscripts/resmoke.py"] + self.resmoke_const_args + resmoke_args)
         resmoke_process.start()
+        self.resmoke_process = resmoke_process
 
-        return resmoke_process
-
-    def assert_dir_file_count(self, test_file, num_entries):
+    def assert_dir_file_count(self, test_dir, test_file, num_entries):
+        file_path = os.path.join(test_dir, test_file)
         count = 0
-        with open(test_file) as file:
+        with open(file_path) as file:
             count = sum(1 for _ in file)
         self.assertEqual(count, num_entries)
 
 
+@unittest.skip("TODO: update the test_dir to a temporary directory so it can run in CI")
 class TestArchivalOnFailure(_ResmokeSelftest):
     @classmethod
     def setUpClass(cls):
         super(TestArchivalOnFailure, cls).setUpClass()
-        cls.archival_file = os.path.join(cls.test_dir, "test_archival.txt")
 
-    @unittest.skip("Requires compile. SERVER-48969 tracks re-enabling.")
+        cls.archival_file = "test_archival.txt"
+
     def test_archival_on_task_failure(self):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_task_failure.yml",
@@ -67,28 +70,27 @@ class TestArchivalOnFailure(_ResmokeSelftest):
             "--repeatTests=2",
             "--jobs=2",
         ]
-        resmoke_process = self.execute_resmoke(resmoke_args)
-        resmoke_process.wait()
+        self.execute_resmoke(resmoke_args)
+        self.resmoke_process.wait()
 
         # test archival
         archival_dirs_to_expect = 4  # 2 tests * 2 nodes
-        self.assert_dir_file_count(self.archival_file, archival_dirs_to_expect)
+        self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
 
-    @unittest.skip("Requires compile. SERVER-48969 tracks re-enabling.")
     def test_archival_on_task_failure_no_passthrough(self):
         resmoke_args = [
-            "--suites=buildscripts/tests/resmokelib/resmoke_end2end/suites/resmoke_selftest_task_failure_no_passthrough.yml",
+            "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_task_failure_no_passthrough.yml",
             "--taskId=123",
             "--internalParam=test_archival",
             "--repeatTests=2",
             "--jobs=2",
         ]
-        resmoke_process = self.execute_resmoke(resmoke_args)
-        resmoke_process.wait()
+        self.execute_resmoke(resmoke_args)
+        self.resmoke_process.wait()
 
         # test archival
         archival_dirs_to_expect = 4  # 2 tests * 2 nodes
-        self.assert_dir_file_count(self.archival_file, archival_dirs_to_expect)
+        self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
 
     def test_no_archival_locally(self):
         # archival should not happen if --taskId is not set.
@@ -98,45 +100,55 @@ class TestArchivalOnFailure(_ResmokeSelftest):
             "--repeatTests=2",
             "--jobs=2",
         ]
-        resmoke_process = self.execute_resmoke(resmoke_args)
-        resmoke_process.wait()
+        self.execute_resmoke(resmoke_args)
+        self.resmoke_process.wait()
 
         # test that archival file wasn't created.
         self.assertFalse(os.path.exists(self.archival_file))
 
 
+@unittest.skip("TODO: update the test_dir to a temporary directory so it can run in CI")
 class TestTimeout(_ResmokeSelftest):
     @classmethod
     def setUpClass(cls):
         super(TestTimeout, cls).setUpClass()
-        cls.archival_file = os.path.join(cls.test_dir, "test_archival.txt")
-        cls.analysis_file = os.path.join(cls.test_dir, "test_analysis.txt")
 
-    @staticmethod
-    def signal_resmoke(resmoke_process):
-        resmoke_process.stop()
-        resmoke_process.wait()
+        cls.test_dir_inner = os.path.normpath("/data/db/selftest_inner")
+        cls.archival_file = "test_archival.txt"
+        cls.analysis_files = ["debugger_mongo.log", "debugger_mongod.log"]
+
+    def setUp(self):
+        super(TestTimeout, self).setUp()
+        self.logger.info("Cleaning temp directory %s", self.test_dir_inner)
+        rmtree(self.test_dir_inner, ignore_errors=True)
+        self.logger.info("Cleaning hang analyzer files %s", str(self.analysis_files))
+        for filename in self.analysis_files:
+            if os.path.exists(filename):
+                os.remove(filename)
+
+    def signal_resmoke(self):
+        hang_analyzer_options = f"-o=file -o=stdout -m=contains -d={self.resmoke_process.pid}"
+        signal_resmoke_process = core.programs.make_process(
+            self.logger, [sys.executable, "buildscripts/resmoke.py", "hang-analyzer"
+                          ] + hang_analyzer_options.split())
+        signal_resmoke_process.start()
 
-        # TODO: replace above with below after SERVER-46691.
-        # signal_resmoke_process = core.programs.make_process(
-        #     self.logger,
-        #     [sys.executable, "buildscripts/signal_resmoke.py", "run", "--pid", str(resmoke_process.pid)])
-        # signal_resmoke_process.start()
+        # Wait for resmoke_process to be killed by 'run-timeout' so this doesn't hang.
+        self.resmoke_process.wait()
 
-        # return_code = signal_resmoke_process.wait()
-        # if return_code != 0:
-        #     resmoke_process.stop()
-        # self.assertEqual(return_code, 0)
+        return_code = signal_resmoke_process.wait()
+        if return_code != 0:
+            self.resmoke_process.stop()
+        self.assertEqual(return_code, 0)
 
-    def execute_resmoke(self, resmoke_args):
-        resmoke_process = _ResmokeSelftest.execute_resmoke(self, resmoke_args)
+    def execute_resmoke(self, resmoke_args, sleep_secs=10, **kwargs):  # pylint: disable=arguments-differ
+        super(TestTimeout, self).execute_resmoke(resmoke_args, **kwargs)
 
-        time.sleep(
-            10)  # TODO: Change to more durable way of ensuring the fixtures have been set up.
+        time.sleep(sleep_secs
+                   )  # TODO: Change to more durable way of ensuring the fixtures have been set up.
 
-        TestTimeout.signal_resmoke(resmoke_process)
+        self.signal_resmoke()
 
-    @unittest.skip("Requires compile. SERVER-48969 tracks re-enabling.")
     def test_task_timeout(self):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_task_timeout.yml",
@@ -148,39 +160,54 @@ class TestTimeout(_ResmokeSelftest):
         ]
         self.execute_resmoke(resmoke_args)
 
-        # TODO: enable tests
-        # archival_dirs_to_expect = 4  # 2 tests * 2 nodes
-        # self.assert_dir_file_count(self.archival_file, archival_dirs_to_expect)
+        archival_dirs_to_expect = 4  # 2 tests * 2 nodes
+        self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
 
-        # analysis_files_to_expect = 6  # 2 tests * (2 mongod + 1 mongo)
-        # self.assert_dir_file_count(self.analysis_file, analysis_files_to_expect)
+        for filename in self.analysis_files:
+            self.assertTrue(os.path.exists(os.path.join(os.getcwd(), filename)))
 
-    @unittest.skip("Requires compile. SERVER-48969 tracks re-enabling.")
     def test_task_timeout_no_passthrough(self):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_task_timeout_no_passthrough.yml",
             "--taskId=123",
             "--internalParam=test_archival",
-            "--internalParam=test_analysis",
             "--repeatTests=2",
             "--jobs=2",
         ]
         self.execute_resmoke(resmoke_args)
 
-        # TODO: Enable tests
-        # archival_dirs_to_expect = 4  # 2 tests * 2 nodes
-        # self.assert_dir_file_count(self.archival_file, archival_dirs_to_expect)
+        archival_dirs_to_expect = 4  # 2 tests * 2 nodes
+        self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
+
+        for filename in self.analysis_files:
+            self.assertTrue(os.path.exists(os.path.join(os.getcwd(), filename)))
+
+    # Test scenarios where an resmoke-launched process launches resmoke.
+    def test_nested_timeout(self):
+        resmoke_args = [
+            "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_nested_timeout.yml",
+            "--taskId=123",
+            "--internalParam=test_archival",
+            "jstests/resmoke_selftest/end2end/timeout/nested/top_level_timeout.js",
+        ]
+
+        self.execute_resmoke(resmoke_args, sleep_secs=25)
+
+        archival_dirs_to_expect = 2  # 2 tests * 2 nodes / 2 data_file directories
+        self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
+        self.assert_dir_file_count(self.test_dir_inner, self.archival_file, archival_dirs_to_expect)
 
-        # analysis_files_to_expect = 6  # 2 tests * (2 mongod + 1 mongo)
-        # self.assert_dir_file_count(self.analysis_file, analysis_files_to_expect)
+        for filename in self.analysis_files:
+            self.assertTrue(os.path.exists(os.path.join(os.getcwd(), filename)))
 
 
+@unittest.skip("TODO: update the test_dir to a temporary directory so it can run in CI")
 class TestTestSelection(_ResmokeSelftest):
     def parse_reports_json(self):
         with open(self.report_file) as fd:
             return json.load(fd)
 
-    def execute_resmoke(self, resmoke_args):
+    def execute_resmoke(self, resmoke_args, **kwargs):  # pylint: disable=unused-argument
         resmoke_process = core.programs.make_process(
             self.logger, [sys.executable, "buildscripts/resmoke.py", "run"] + resmoke_args)
         resmoke_process.start()