From 905074c42ba4d0c5b472730b00e43a01d6a50134 Mon Sep 17 00:00:00 2001 From: "zhanyong.wan" Date: Mon, 9 Feb 2009 18:05:21 +0000 Subject: Implements the test sharding protocol. By Eric Fellheimer. git-svn-id: http://googletest.googlecode.com/svn/trunk@187 861a406c-534a-0410-8894-cb66d6ee9925 --- src/gtest-internal-inl.h | 35 +++++++- src/gtest-port.cc | 22 ++--- src/gtest.cc | 163 ++++++++++++++++++++++++++++++++-- test/gtest_filter_unittest.py | 162 ++++++++++++++++++++++++++++++--- test/gtest_filter_unittest_.cc | 19 ++++ test/gtest_output_test.py | 68 +++++++++----- test/gtest_output_test_.cc | 6 ++ test/gtest_output_test_golden_lin.txt | 24 ++++- test/gtest_output_test_golden_win.txt | 24 ++++- test/gtest_unittest.cc | 158 ++++++++++++++++++++++++++++++++ 10 files changed, 619 insertions(+), 62 deletions(-) diff --git a/src/gtest-internal-inl.h b/src/gtest-internal-inl.h index cefed20..07d0c5b 100644 --- a/src/gtest-internal-inl.h +++ b/src/gtest-internal-inl.h @@ -162,6 +162,32 @@ String WideStringToUtf8(const wchar_t* str, int num_chars); // Returns the number of active threads, or 0 when there is an error. size_t GetThreadCount(); +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded(); + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (e.g., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_str, const char* shard_index_str, + bool in_subprocess_for_death_test); + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error and +// and aborts. +Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id); + // List is a simple singly-linked list container. // // We cannot use std::list as Microsoft's implementation of STL has @@ -1111,11 +1137,18 @@ class UnitTestImpl { ad_hoc_test_result_.Clear(); } + enum ReactionToSharding { + HONOR_SHARDING_PROTOCOL, + IGNORE_SHARDING_PROTOCOL + }; + // Matches the full name of each test against the user-specified // filter to decide whether the test should run, then records the // result in each TestCase and TestInfo object. + // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests + // based on sharding variables in the environment. // Returns the number of tests that should run. - int FilterTests(); + int FilterTests(ReactionToSharding shard_tests); // Lists all the tests by name. void ListAllTests(); diff --git a/src/gtest-port.cc b/src/gtest-port.cc index 3c9ec4b..9348f55 100644 --- a/src/gtest-port.cc +++ b/src/gtest-port.cc @@ -512,17 +512,6 @@ static String FlagToEnvVar(const char* flag) { return env_var.GetString(); } -// Reads and returns the Boolean environment variable corresponding to -// the given flag; if it's not set, returns default_value. -// -// The value is considered true iff it's not "0". -bool BoolFromGTestEnv(const char* flag, bool default_value) { - const String env_var = FlagToEnvVar(flag); - const char* const string_value = GetEnv(env_var.c_str()); - return string_value == NULL ? - default_value : strcmp(string_value, "0") != 0; -} - // Parses 'str' for a 32-bit signed integer. If successful, writes // the result to *value and returns true; otherwise leaves *value // unchanged and returns false. @@ -564,6 +553,17 @@ bool ParseInt32(const Message& src_text, const char* str, Int32* value) { return true; } +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromGTestEnv(const char* flag, bool default_value) { + const String env_var = FlagToEnvVar(flag); + const char* const string_value = GetEnv(env_var.c_str()); + return string_value == NULL ? + default_value : strcmp(string_value, "0") != 0; +} + // Reads and returns a 32-bit integer stored in the environment // variable corresponding to the given flag; if it isn't set or // doesn't represent a valid 32-bit integer, returns default_value. diff --git a/src/gtest.cc b/src/gtest.cc index 8e8238a..0d161e0 100644 --- a/src/gtest.cc +++ b/src/gtest.cc @@ -145,6 +145,13 @@ static const char kUniversalFilter[] = "*"; // The default output file for XML output. static const char kDefaultOutputFile[] = "test_detail.xml"; +// The environment variable name for the test shard index. +static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; +// The environment variable name for the total number of test shards. +static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; +// The environment variable name for the test shard status file. +static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; + namespace internal { // The text used in failure messages to indicate the start of the @@ -2595,6 +2602,13 @@ void PrettyUnitTestResultPrinter::OnUnitTestStart( "Note: %s filter = %s\n", GTEST_NAME, filter); } + if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { + ColoredPrintf(COLOR_YELLOW, + "Note: This is test shard %s of %s.\n", + internal::GetEnv(kTestShardIndex), + internal::GetEnv(kTestTotalShards)); + } + const internal::UnitTestImpl* const impl = unit_test->impl(); ColoredPrintf(COLOR_GREEN, "[==========] "); printf("Running %s from %s.\n", @@ -3510,6 +3524,11 @@ int UnitTestImpl::RunAllTests() { RegisterParameterizedTests(); + // Even if sharding is not on, test runners may want to use the + // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding + // protocol. + internal::WriteToShardStatusFileIfNeeded(); + // Lists all the tests and exits if the --gtest_list_tests // flag was specified. if (GTEST_FLAG(list_tests)) { @@ -3528,9 +3547,15 @@ int UnitTestImpl::RunAllTests() { UnitTestEventListenerInterface * const printer = result_printer(); + const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, + in_subprocess_for_death_test); + // Compares the full test names with the filter to decide which // tests to run. - const bool has_tests_to_run = FilterTests() > 0; + const bool has_tests_to_run = FilterTests(should_shard + ? HONOR_SHARDING_PROTOCOL + : IGNORE_SHARDING_PROTOCOL) > 0; + // True iff at least one test has failed. bool failed = false; @@ -3586,12 +3611,126 @@ int UnitTestImpl::RunAllTests() { return failed ? 1 : 0; } +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded() { + const char* const test_shard_file = GetEnv(kTestShardStatusFile); + if (test_shard_file != NULL) { +#ifdef _MSC_VER // MSVC 8 deprecates fopen(). +#pragma warning(push) // Saves the current warning state. +#pragma warning(disable:4996) // Temporarily disables warning on + // deprecated functions. +#endif + FILE* const file = fopen(test_shard_file, "w"); +#ifdef _MSC_VER +#pragma warning(pop) // Restores the warning state. +#endif + if (file == NULL) { + ColoredPrintf(COLOR_RED, + "Could not write to the test shard status file \"%s\" " + "specified by the %s environment variable.\n", + test_shard_file, kTestShardStatusFile); + fflush(stdout); + exit(EXIT_FAILURE); + } + fclose(file); + } +} + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (i.e., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_env, + const char* shard_index_env, + bool in_subprocess_for_death_test) { + if (in_subprocess_for_death_test) { + return false; + } + + const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); + const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); + + if (total_shards == -1 && shard_index == -1) { + return false; + } else if (total_shards == -1 && shard_index != -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestShardIndex << " = " << shard_index + << ", but have left " << kTestTotalShards << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (total_shards != -1 && shard_index == -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestTotalShards << " = " << total_shards + << ", but have left " << kTestShardIndex << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (shard_index < 0 || shard_index >= total_shards) { + const Message msg = Message() + << "Invalid environment variables: we require 0 <= " + << kTestShardIndex << " < " << kTestTotalShards + << ", but you have " << kTestShardIndex << "=" << shard_index + << ", " << kTestTotalShards << "=" << total_shards << ".\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } + + return total_shards > 1; +} + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error +// and aborts. +Int32 Int32FromEnvOrDie(const char* const var, Int32 default_val) { + const char* str_val = GetEnv(var); + if (str_val == NULL) { + return default_val; + } + + Int32 result; + if (!ParseInt32(Message() << "The value of environment variable " << var, + str_val, &result)) { + exit(EXIT_FAILURE); + } + return result; +} + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { + return (test_id % total_shards) == shard_index; +} + // Compares the name of each test with the user-specified filter to // decide whether the test should be run, then records the result in // each TestCase and TestInfo object. +// If shard_tests == true, further filters tests based on sharding +// variables in the environment - see +// http://code.google.com/p/googletest/wiki/GoogleTestAdvancedGuide. // Returns the number of tests that should run. -int UnitTestImpl::FilterTests() { +int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { + const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestTotalShards, -1) : -1; + const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestShardIndex, -1) : -1; + + // num_runnable_tests are the number of tests that will + // run across all shards (i.e., match filter and are not disabled). + // num_selected_tests are the number of tests to be run on + // this shard. int num_runnable_tests = 0; + int num_selected_tests = 0; for (const internal::ListNode *test_case_node = test_cases_.Head(); test_case_node != NULL; @@ -3615,18 +3754,24 @@ int UnitTestImpl::FilterTests() { kDisableTestFilter); test_info->impl()->set_is_disabled(is_disabled); - const bool should_run = + const bool is_runnable = (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && internal::UnitTestOptions::FilterMatchesTest(test_case_name, test_name); - test_info->impl()->set_should_run(should_run); - test_case->set_should_run(test_case->should_run() || should_run); - if (should_run) { - num_runnable_tests++; - } + + const bool is_selected = is_runnable && + (shard_tests == IGNORE_SHARDING_PROTOCOL || + ShouldRunTestOnShard(total_shards, shard_index, + num_runnable_tests)); + + num_runnable_tests += is_runnable; + num_selected_tests += is_selected; + + test_info->impl()->set_should_run(is_selected); + test_case->set_should_run(test_case->should_run() || is_selected); } } - return num_runnable_tests; + return num_selected_tests; } // Lists all tests by name. diff --git a/test/gtest_filter_unittest.py b/test/gtest_filter_unittest.py index 35307a2..c3a016c 100755 --- a/test/gtest_filter_unittest.py +++ b/test/gtest_filter_unittest.py @@ -36,6 +36,9 @@ the GTEST_FILTER environment variable or the --gtest_filter flag. This script tests such functionality by invoking gtest_filter_unittest_ (a program written with Google Test) with different environments and command line flags. + +Note that test sharding may also influence which tests are filtered. Therefore, +we test that here also. """ __author__ = 'wan@google.com (Zhanyong Wan)' @@ -43,6 +46,7 @@ __author__ = 'wan@google.com (Zhanyong Wan)' import os import re import sets +import tempfile import unittest import gtest_test_utils @@ -51,6 +55,11 @@ import gtest_test_utils # The environment variable for specifying the test filters. FILTER_ENV_VAR = 'GTEST_FILTER' +# The environment variables for test sharding. +TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' +SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' +SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE' + # The command line flag for specifying the test filters. FILTER_FLAG = 'gtest_filter' @@ -103,6 +112,9 @@ ACTIVE_TESTS = [ 'BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB', + + 'HasDeathTest.Test1', + 'HasDeathTest.Test2', ] + PARAM_TESTS param_tests_present = None @@ -121,7 +133,7 @@ def SetEnvVar(env_var, value): def Run(command): """Runs a Google Test program and returns a list of full names of the - tests that were run. + tests that were run along with the test exit code. """ stdout_file = os.popen(command, 'r') @@ -137,9 +149,32 @@ def Run(command): if match is not None: test = match.group(1) tests_run += [test_case + '.' + test] - stdout_file.close() - return tests_run + exit_code = stdout_file.close() + return (tests_run, exit_code) + + +def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs): + """Runs the given function and arguments in a modified environment.""" + try: + original_env = os.environ.copy() + os.environ.update(extra_env) + return function(*args, **kwargs) + finally: + for key in extra_env.iterkeys(): + if key in original_env: + os.environ[key] = original_env[key] + else: + del os.environ[key] + + +def RunWithSharding(total_shards, shard_index, command): + """Runs the Google Test program shard and returns a list of full names of the + tests that were run along with the exit code. + """ + extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index), + TOTAL_SHARDS_ENV_VAR: str(total_shards)} + return InvokeWithModifiedEnv(extra_env, Run, command) # The unit test. @@ -160,6 +195,15 @@ class GTestFilterUnitTest(unittest.TestCase): for elem in rhs: self.assert_(elem in lhs, '%s in %s' % (elem, lhs)) + def AssertPartitionIsValid(self, set_var, list_of_sets): + """Asserts that list_of_sets is a valid partition of set_var.""" + + full_partition = [] + for slice_var in list_of_sets: + full_partition.extend(slice_var) + self.assertEqual(len(set_var), len(full_partition)) + self.assertEqual(sorted(set_var), sorted(full_partition)) + def RunAndVerify(self, gtest_filter, tests_to_run): """Runs gtest_flag_unittest_ with the given filter, and verifies that the right set of tests were run. @@ -173,7 +217,7 @@ class GTestFilterUnitTest(unittest.TestCase): # First, tests using GTEST_FILTER. SetEnvVar(FILTER_ENV_VAR, gtest_filter) - tests_run = Run(COMMAND) + tests_run = Run(COMMAND)[0] SetEnvVar(FILTER_ENV_VAR, None) self.AssertSetEqual(tests_run, tests_to_run) @@ -185,9 +229,27 @@ class GTestFilterUnitTest(unittest.TestCase): else: command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, gtest_filter) - tests_run = Run(command) + tests_run = Run(command)[0] self.AssertSetEqual(tests_run, tests_to_run) + def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run, + command=COMMAND, check_exit_0=False): + """Runs all shards of gtest_flag_unittest_ with the given filter, and + verifies that the right set of tests were run. The union of tests run + on each shard should be identical to tests_to_run, without duplicates. + If check_exit_0, make sure that all shards returned 0. + """ + SetEnvVar(FILTER_ENV_VAR, gtest_filter) + partition = [] + for i in range(0, total_shards): + (tests_run, exit_code) = RunWithSharding(total_shards, i, command) + if check_exit_0: + self.assert_(exit_code is None) + partition.append(tests_run) + + self.AssertPartitionIsValid(tests_to_run, partition) + SetEnvVar(FILTER_ENV_VAR, None) + def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run): """Runs gtest_flag_unittest_ with the given filter, and enables disabled tests. Verifies that the right set of tests were run. @@ -197,7 +259,7 @@ class GTestFilterUnitTest(unittest.TestCase): if gtest_filter is not None: command = '%s --%s=%s' % (command, FILTER_FLAG, gtest_filter) - tests_run = Run(command) + tests_run = Run(command)[0] self.AssertSetEqual(tests_run, tests_to_run) def setUp(self): @@ -214,10 +276,22 @@ class GTestFilterUnitTest(unittest.TestCase): self.RunAndVerify(None, ACTIVE_TESTS) + def testDefaultBehaviorWithShards(self): + """Tests the behavior of not specifying the filter, with sharding + enabled. + """ + self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS) + self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS) + self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS) + self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS) + self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS) + def testEmptyFilter(self): """Tests an empty filter.""" self.RunAndVerify('', []) + self.RunAndVerifyWithSharding('', 1, []) + self.RunAndVerifyWithSharding('', 2, []) def testBadFilter(self): """Tests a filter that matches nothing.""" @@ -230,12 +304,14 @@ class GTestFilterUnitTest(unittest.TestCase): self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz']) self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz']) + self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz']) def testUniversalFilters(self): """Tests filters that match everything.""" self.RunAndVerify('*', ACTIVE_TESTS) self.RunAndVerify('*.*', ACTIVE_TESTS) + self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS) self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS) self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS) @@ -289,7 +365,10 @@ class GTestFilterUnitTest(unittest.TestCase): 'BazTest.TestOne', 'BazTest.TestA', - 'BazTest.TestB' ] + PARAM_TESTS) + 'BazTest.TestB', + + 'HasDeathTest.Test1', + 'HasDeathTest.Test2', ] + PARAM_TESTS) def testWildcardInTestName(self): """Tests using wildcard in the test name.""" @@ -350,7 +429,8 @@ class GTestFilterUnitTest(unittest.TestCase): ]) def testNegativeFilters(self): - self.RunAndVerify('*-FooTest.Abc', [ + self.RunAndVerify('*-HasDeathTest.Test1', [ + 'FooTest.Abc', 'FooTest.Xyz', 'BarTest.TestOne', @@ -360,14 +440,20 @@ class GTestFilterUnitTest(unittest.TestCase): 'BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB', + + 'HasDeathTest.Test2', ] + PARAM_TESTS) - self.RunAndVerify('*-FooTest.Abc:BazTest.*', [ + self.RunAndVerify('*-FooTest.Abc:HasDeathTest.*', [ 'FooTest.Xyz', 'BarTest.TestOne', 'BarTest.TestTwo', 'BarTest.TestThree', + + 'BazTest.TestOne', + 'BazTest.TestA', + 'BazTest.TestB', ] + PARAM_TESTS) self.RunAndVerify('BarTest.*-BarTest.TestOne', [ @@ -376,7 +462,7 @@ class GTestFilterUnitTest(unittest.TestCase): ]) # Tests without leading '*'. - self.RunAndVerify('-FooTest.Abc:FooTest.Xyz', [ + self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:HasDeathTest.*', [ 'BarTest.TestOne', 'BarTest.TestTwo', 'BarTest.TestThree', @@ -412,11 +498,65 @@ class GTestFilterUnitTest(unittest.TestCase): SetEnvVar(FILTER_ENV_VAR, 'Foo*') command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, '*One') - tests_run = Run(command) + tests_run = Run(command)[0] SetEnvVar(FILTER_ENV_VAR, None) self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne']) + def testShardStatusFileIsCreated(self): + """Tests that the shard file is created if specified in the environment.""" + + test_tmpdir = tempfile.mkdtemp() + shard_status_file = os.path.join(test_tmpdir, 'shard_status_file') + self.assert_(not os.path.exists(shard_status_file)) + + extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} + stdout_file = InvokeWithModifiedEnv(extra_env, os.popen, COMMAND, 'r') + try: + stdout_file.readlines() + finally: + stdout_file.close() + self.assert_(os.path.exists(shard_status_file)) + os.remove(shard_status_file) + os.removedirs(test_tmpdir) + + def testShardStatusFileIsCreatedWithListTests(self): + """Tests that the shard file is created with --gtest_list_tests.""" + + test_tmpdir = tempfile.mkdtemp() + shard_status_file = os.path.join(test_tmpdir, 'shard_status_file2') + self.assert_(not os.path.exists(shard_status_file)) + + extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file} + stdout_file = InvokeWithModifiedEnv(extra_env, os.popen, + '%s --gtest_list_tests' % COMMAND, 'r') + try: + stdout_file.readlines() + finally: + stdout_file.close() + self.assert_(os.path.exists(shard_status_file)) + os.remove(shard_status_file) + os.removedirs(test_tmpdir) + + def testShardingWorksWithDeathTests(self): + """Tests integration with death tests and sharding.""" + gtest_filter = 'HasDeathTest.*:SeqP/*' + expected_tests = [ + 'HasDeathTest.Test1', + 'HasDeathTest.Test2', + + 'SeqP/ParamTest.TestX/0', + 'SeqP/ParamTest.TestX/1', + 'SeqP/ParamTest.TestY/0', + 'SeqP/ParamTest.TestY/1', + ] + + for command in (COMMAND + ' --gtest_death_test_style=threadsafe', + COMMAND + ' --gtest_death_test_style=fast'): + self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests, + check_exit_0=True, command=command) + self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests, + check_exit_0=True, command=command) if __name__ == '__main__': gtest_test_utils.Main() diff --git a/test/gtest_filter_unittest_.cc b/test/gtest_filter_unittest_.cc index 9961079..22638e0 100644 --- a/test/gtest_filter_unittest_.cc +++ b/test/gtest_filter_unittest_.cc @@ -91,6 +91,25 @@ TEST(BazTest, DISABLED_TestC) { FAIL() << "Expected failure."; } +// Test case HasDeathTest + +TEST(HasDeathTest, Test1) { +#ifdef GTEST_HAS_DEATH_TEST + EXPECT_DEATH({exit(1);}, + ".*"); +#endif // GTEST_HAS_DEATH_TEST +} + +// We need at least two death tests to make sure that the all death tests +// aren't on the first shard. +TEST(HasDeathTest, Test2) { +#ifdef GTEST_HAS_DEATH_TEST + EXPECT_DEATH({exit(1);}, + ".*"); +#endif // GTEST_HAS_DEATH_TEST +} + + // Test case FoobarTest TEST(DISABLED_FoobarTest, Test1) { diff --git a/test/gtest_output_test.py b/test/gtest_output_test.py index 9aaade2..f35e002 100755 --- a/test/gtest_output_test.py +++ b/test/gtest_output_test.py @@ -64,13 +64,17 @@ PROGRAM_PATH = os.path.join(gtest_test_utils.GetBuildDir(), PROGRAM) # At least one command we exercise must not have the # --gtest_internal_skip_environment_and_ad_hoc_tests flag. -COMMAND_WITH_COLOR = PROGRAM_PATH + ' --gtest_color=yes' -COMMAND_WITH_TIME = (PROGRAM_PATH + ' --gtest_print_time ' +COMMAND_WITH_COLOR = ({}, PROGRAM_PATH + ' --gtest_color=yes') +COMMAND_WITH_TIME = ({}, PROGRAM_PATH + ' --gtest_print_time ' '--gtest_internal_skip_environment_and_ad_hoc_tests ' '--gtest_filter="FatalFailureTest.*:LoggingTest.*"') -COMMAND_WITH_DISABLED = (PROGRAM_PATH + ' --gtest_also_run_disabled_tests ' +COMMAND_WITH_DISABLED = ({}, PROGRAM_PATH + ' --gtest_also_run_disabled_tests ' '--gtest_internal_skip_environment_and_ad_hoc_tests ' '--gtest_filter="*DISABLED_*"') +COMMAND_WITH_SHARDING = ({'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'}, + PROGRAM_PATH + + ' --gtest_internal_skip_environment_and_ad_hoc_tests ' + ' --gtest_filter="PassingTest.*"') GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME) @@ -136,19 +140,26 @@ def NormalizeOutput(output): return output -def IterShellCommandOutput(cmd, stdin_string=None): +def IterShellCommandOutput(env_cmd, stdin_string=None): """Runs a command in a sub-process, and iterates the lines in its STDOUT. Args: - cmd: The shell command. - stdin_string: The string to be fed to the STDIN of the sub-process; - If None, the sub-process will inherit the STDIN - from the parent process. + env_cmd: The shell command. A 2-tuple where element 0 is a dict + of extra environment variables to set, and element 1 + is a string with the command and any flags. + stdin_string: The string to be fed to the STDIN of the sub-process; + If None, the sub-process will inherit the STDIN + from the parent process. """ # Spawns cmd in a sub-process, and gets its standard I/O file objects. - stdin_file, stdout_file = os.popen2(cmd, 'b') + # Set and save the environment properly. + old_env_vars = dict(os.environ) + os.environ.update(env_cmd[0]) + stdin_file, stdout_file = os.popen2(env_cmd[1], 'b') + os.environ.clear() + os.environ.update(old_env_vars) # If the caller didn't specify a string for STDIN, gets it from the # parent process. @@ -168,39 +179,50 @@ def IterShellCommandOutput(cmd, stdin_string=None): yield line -def GetShellCommandOutput(cmd, stdin_string=None): +def GetShellCommandOutput(env_cmd, stdin_string=None): """Runs a command in a sub-process, and returns its STDOUT in a string. Args: - cmd: The shell command. - stdin_string: The string to be fed to the STDIN of the sub-process; - If None, the sub-process will inherit the STDIN - from the parent process. + env_cmd: The shell command. A 2-tuple where element 0 is a dict + of extra environment variables to set, and element 1 + is a string with the command and any flags. + stdin_string: The string to be fed to the STDIN of the sub-process; + If None, the sub-process will inherit the STDIN + from the parent process. """ - lines = list(IterShellCommandOutput(cmd, stdin_string)) + lines = list(IterShellCommandOutput(env_cmd, stdin_string)) return string.join(lines, '') -def GetCommandOutput(cmd): +def GetCommandOutput(env_cmd): """Runs a command and returns its output with all file location info stripped off. Args: - cmd: the shell command. + env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra + environment variables to set, and element 1 is a string with + the command and any flags. """ # Disables exception pop-ups on Windows. os.environ['GTEST_CATCH_EXCEPTIONS'] = '1' - return NormalizeOutput(GetShellCommandOutput(cmd, '')) + return NormalizeOutput(GetShellCommandOutput(env_cmd, '')) + + +def GetOutputOfAllCommands(): + """Returns concatenated output from several representative commands.""" + + return (GetCommandOutput(COMMAND_WITH_COLOR) + + GetCommandOutput(COMMAND_WITH_TIME) + + GetCommandOutput(COMMAND_WITH_DISABLED) + + GetCommandOutput(COMMAND_WITH_SHARDING)) class GTestOutputTest(unittest.TestCase): def testOutput(self): - output = (GetCommandOutput(COMMAND_WITH_COLOR) + - GetCommandOutput(COMMAND_WITH_TIME) + - GetCommandOutput(COMMAND_WITH_DISABLED)) + output = GetOutputOfAllCommands() golden_file = open(GOLDEN_PATH, 'rb') golden = golden_file.read() golden_file.close() @@ -214,9 +236,7 @@ class GTestOutputTest(unittest.TestCase): if __name__ == '__main__': if sys.argv[1:] == [GENGOLDEN_FLAG]: - output = (GetCommandOutput(COMMAND_WITH_COLOR) + - GetCommandOutput(COMMAND_WITH_TIME) + - GetCommandOutput(COMMAND_WITH_DISABLED)) + output = GetOutputOfAllCommands() golden_file = open(GOLDEN_PATH, 'wb') golden_file.write(output) golden_file.close() diff --git a/test/gtest_output_test_.cc b/test/gtest_output_test_.cc index 5c76609..f5748d1 100644 --- a/test/gtest_output_test_.cc +++ b/test/gtest_output_test_.cc @@ -85,6 +85,12 @@ void TryTestSubroutine() { FAIL() << "This should never be reached."; } +TEST(PassingTest, PassingTest1) { +} + +TEST(PassingTest, PassingTest2) { +} + // Tests catching a fatal failure in a subroutine. TEST(FatalFailureTest, FatalFailureInSubroutine) { printf("(expecting a failure that x should be 1)\n"); diff --git a/test/gtest_output_test_golden_lin.txt b/test/gtest_output_test_golden_lin.txt index 74ed737..46a90fb 100644 --- a/test/gtest_output_test_golden_lin.txt +++ b/test/gtest_output_test_golden_lin.txt @@ -7,7 +7,7 @@ Expected: true gtest_output_test_.cc:#: Failure Value of: 3 Expected: 2 -[==========] Running 54 tests from 22 test cases. +[==========] Running 56 tests from 23 test cases. [----------] Global test environment set-up. FooEnvironment::SetUp() called. BarEnvironment::SetUp() called. @@ -26,6 +26,11 @@ BarEnvironment::SetUp() called. [----------] 1 test from My/ATypeParamDeathTest/1, where TypeParam = double [ RUN ] My/ATypeParamDeathTest/1.ShouldRunFirst [ OK ] My/ATypeParamDeathTest/1.ShouldRunFirst +[----------] 2 tests from PassingTest +[ RUN ] PassingTest.PassingTest1 +[ OK ] PassingTest.PassingTest1 +[ RUN ] PassingTest.PassingTest2 +[ OK ] PassingTest.PassingTest2 [----------] 3 tests from FatalFailureTest [ RUN ] FatalFailureTest.FatalFailureInSubroutine (expecting a failure that x should be 1) @@ -508,8 +513,8 @@ FooEnvironment::TearDown() called. gtest_output_test_.cc:#: Failure Failed Expected fatal failure. -[==========] 54 tests from 22 test cases ran. -[ PASSED ] 19 tests. +[==========] 56 tests from 23 test cases ran. +[ PASSED ] 21 tests. [ FAILED ] 35 tests, listed below: [ FAILED ] FatalFailureTest.FatalFailureInSubroutine [ FAILED ] FatalFailureTest.FatalFailureInNestedSubroutine @@ -612,3 +617,16 @@ Note: Google Test filter = *DISABLED_* [----------] Global test environment tear-down [==========] 1 test from 1 test case ran. [ PASSED ] 1 test. +Note: Google Test filter = PassingTest.* +Note: This is test shard 1 of 2. +[==========] Running 1 test from 1 test case. +[----------] Global test environment set-up. +[----------] 1 test from PassingTest +[ RUN ] PassingTest.PassingTest2 +[ OK ] PassingTest.PassingTest2 +[----------] Global test environment tear-down +[==========] 1 test from 1 test case ran. +[ PASSED ] 1 test. + + YOU HAVE 1 DISABLED TEST + diff --git a/test/gtest_output_test_golden_win.txt b/test/gtest_output_test_golden_win.txt index 17be5c0..77903ba 100644 --- a/test/gtest_output_test_golden_win.txt +++ b/test/gtest_output_test_golden_win.txt @@ -5,10 +5,15 @@ gtest_output_test_.cc:#: error: Value of: false Expected: true gtest_output_test_.cc:#: error: Value of: 3 Expected: 2 -[==========] Running 50 tests from 20 test cases. +[==========] Running 52 tests from 21 test cases. [----------] Global test environment set-up. FooEnvironment::SetUp() called. BarEnvironment::SetUp() called. +[----------] 2 tests from PassingTest +[ RUN ] PassingTest.PassingTest1 +[ OK ] PassingTest.PassingTest1 +[ RUN ] PassingTest.PassingTest2 +[ OK ] PassingTest.PassingTest2 [----------] 3 tests from FatalFailureTest [ RUN ] FatalFailureTest.FatalFailureInSubroutine (expecting a failure that x should be 1) @@ -440,8 +445,8 @@ Expected non-fatal failure. FooEnvironment::TearDown() called. gtest_output_test_.cc:#: error: Failed Expected fatal failure. -[==========] 50 tests from 20 test cases ran. -[ PASSED ] 14 tests. +[==========] 52 tests from 21 test cases ran. +[ PASSED ] 16 tests. [ FAILED ] 36 tests, listed below: [ FAILED ] FatalFailureTest.FatalFailureInSubroutine [ FAILED ] FatalFailureTest.FatalFailureInNestedSubroutine @@ -540,3 +545,16 @@ Note: Google Test filter = *DISABLED_* [----------] Global test environment tear-down [==========] 1 test from 1 test case ran. [ PASSED ] 1 test. +Note: Google Test filter = PassingTest.* +Note: This is test shard 1 of 2. +[==========] Running 1 test from 1 test case. +[----------] Global test environment set-up. +[----------] 1 test from PassingTest +[ RUN ] PassingTest.PassingTest2 +[ OK ] PassingTest.PassingTest2 +[----------] Global test environment tear-down +[==========] 1 test from 1 test case ran. +[ PASSED ] 1 test. + + YOU HAVE 1 DISABLED TEST + diff --git a/test/gtest_unittest.cc b/test/gtest_unittest.cc index 4bac8a6..335b9e0 100644 --- a/test/gtest_unittest.cc +++ b/test/gtest_unittest.cc @@ -138,7 +138,10 @@ using testing::internal::GetTestTypeId; using testing::internal::GetTypeId; using testing::internal::GTestFlagSaver; using testing::internal::Int32; +using testing::internal::Int32FromEnvOrDie; using testing::internal::List; +using testing::internal::ShouldRunTestOnShard; +using testing::internal::ShouldShard; using testing::internal::ShouldUseColor; using testing::internal::StreamableToString; using testing::internal::String; @@ -1375,6 +1378,161 @@ TEST(ParseInt32FlagTest, ParsesAndReturnsValidValue) { EXPECT_EQ(-789, value); } +// Tests that Int32FromEnvOrDie() parses the value of the var or +// returns the correct default. +TEST(Int32FromEnvOrDieTest, ParsesAndReturnsValidValue) { + EXPECT_EQ(333, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER "UnsetVar", 333)); + SetEnv(GTEST_FLAG_PREFIX_UPPER "UnsetVar", "123"); + EXPECT_EQ(123, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER "UnsetVar", 333)); + SetEnv(GTEST_FLAG_PREFIX_UPPER "UnsetVar", "-123"); + EXPECT_EQ(-123, Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER "UnsetVar", 333)); +} + +#ifdef GTEST_HAS_DEATH_TEST + +// Tests that Int32FromEnvOrDie() aborts with an error message +// if the variable is not an Int32. +TEST(Int32FromEnvOrDieDeathTest, AbortsOnFailure) { + SetEnv(GTEST_FLAG_PREFIX_UPPER "VAR", "xxx"); + EXPECT_DEATH({Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER "VAR", 123);}, + ".*"); +} + +// Tests that Int32FromEnvOrDie() aborts with an error message +// if the variable cannot be represnted by an Int32. +TEST(Int32FromEnvOrDieDeathTest, AbortsOnInt32Overflow) { + SetEnv(GTEST_FLAG_PREFIX_UPPER "VAR", "1234567891234567891234"); + EXPECT_DEATH({Int32FromEnvOrDie(GTEST_FLAG_PREFIX_UPPER "VAR", 123);}, + ".*"); +} + +#endif // GTEST_HAS_DEATH_TEST + + +// Tests that ShouldRunTestOnShard() selects all tests +// where there is 1 shard. +TEST(ShouldRunTestOnShardTest, IsPartitionWhenThereIsOneShard) { + EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 0)); + EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 1)); + EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 2)); + EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 3)); + EXPECT_TRUE(ShouldRunTestOnShard(1, 0, 4)); +} + +class ShouldShardTest : public testing::Test { + protected: + virtual void SetUp() { + index_var_ = GTEST_FLAG_PREFIX_UPPER "INDEX"; + total_var_ = GTEST_FLAG_PREFIX_UPPER "TOTAL"; + } + + virtual void TearDown() { + SetEnv(index_var_, ""); + SetEnv(total_var_, ""); + } + + const char* index_var_; + const char* total_var_; +}; + +// Tests that sharding is disabled if neither of the environment variables +// are set. +TEST_F(ShouldShardTest, ReturnsFalseWhenNeitherEnvVarIsSet) { + SetEnv(index_var_, ""); + SetEnv(total_var_, ""); + + EXPECT_FALSE(ShouldShard(total_var_, index_var_, false)); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, true)); +} + +// Tests that sharding is not enabled if total_shards == 1. +TEST_F(ShouldShardTest, ReturnsFalseWhenTotalShardIsOne) { + SetEnv(index_var_, "0"); + SetEnv(total_var_, "1"); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, false)); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, true)); +} + +// Tests that sharding is enabled if total_shards > 1 and +// we are not in a death test subprocess. +TEST_F(ShouldShardTest, WorksWhenShardEnvVarsAreValid) { + SetEnv(index_var_, "4"); + SetEnv(total_var_, "22"); + EXPECT_TRUE(ShouldShard(total_var_, index_var_, false)); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, true)); + + SetEnv(index_var_, "8"); + SetEnv(total_var_, "9"); + EXPECT_TRUE(ShouldShard(total_var_, index_var_, false)); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, true)); + + SetEnv(index_var_, "0"); + SetEnv(total_var_, "9"); + EXPECT_TRUE(ShouldShard(total_var_, index_var_, false)); + EXPECT_FALSE(ShouldShard(total_var_, index_var_, true)); +} + +#ifdef GTEST_HAS_DEATH_TEST + +// Tests that we exit in error if the sharding values are not valid. +TEST_F(ShouldShardTest, AbortsWhenShardingEnvVarsAreInvalid) { + SetEnv(index_var_, "4"); + SetEnv(total_var_, "4"); + EXPECT_DEATH({ShouldShard(total_var_, index_var_, false);}, + ".*"); + + SetEnv(index_var_, "4"); + SetEnv(total_var_, "-2"); + EXPECT_DEATH({ShouldShard(total_var_, index_var_, false);}, + ".*"); + + SetEnv(index_var_, "5"); + SetEnv(total_var_, ""); + EXPECT_DEATH({ShouldShard(total_var_, index_var_, false);}, + ".*"); + + SetEnv(index_var_, ""); + SetEnv(total_var_, "5"); + EXPECT_DEATH({ShouldShard(total_var_, index_var_, false);}, + ".*"); +} + +#endif // GTEST_HAS_DEATH_TEST + +// Tests that ShouldRunTestOnShard is a partition when 5 +// shards are used. +TEST(ShouldRunTestOnShardTest, IsPartitionWhenThereAreFiveShards) { + // Choose an arbitrary number of tests and shards. + const int num_tests = 17; + const int num_shards = 5; + + // Check partitioning: each test should be on exactly 1 shard. + for (int test_id = 0; test_id < num_tests; test_id++) { + int prev_selected_shard_index = -1; + for (int shard_index = 0; shard_index < num_shards; shard_index++) { + if (ShouldRunTestOnShard(num_shards, shard_index, test_id)) { + if (prev_selected_shard_index < 0) { + prev_selected_shard_index = shard_index; + } else { + ADD_FAILURE() << "Shard " << prev_selected_shard_index << " and " + << shard_index << " are both selected to run test " << test_id; + } + } + } + } + + // Check balance: This is not required by the sharding protocol, but is a + // desirable property for performance. + for (int shard_index = 0; shard_index < num_shards; shard_index++) { + int num_tests_on_shard = 0; + for (int test_id = 0; test_id < num_tests; test_id++) { + num_tests_on_shard += + ShouldRunTestOnShard(num_shards, shard_index, test_id); + } + EXPECT_GE(num_tests_on_shard, num_tests / num_shards); + } +} + // For the same reason we are not explicitly testing everything in the // Test class, there are no separate tests for the following classes // (except for some trivial cases): -- cgit v1.2.1