Merge topic 'ctest-repeat-notrun' into release-3.17

bbb62dcc72 CTest: Make sure NOT_RUN tests show up in the failed test log
c503251997 Tests: Add coverage of ctest_test RETURN_VALUE and REPEAT

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4801

6 files changed, 54 insertions, 6 deletions

diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index ec549606fd..58289ea50d 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -340,6 +340,11 @@ bool cmCTestRunTest::NeedsToRepeat()
   if (this->NumberOfRunsLeft == 0) {
     return false;
   }
+  // If a test is marked as NOT_RUN it will not be repeated
+  // no matter the repeat settings, so just record it as-is.
+  if (this->TestResult.Status == cmCTestTestHandler::NOT_RUN) {
+    return false;
+  }
   // if number of runs left is not 0, and we are running until
   // we find a failed (or passed) test, then return true so the test can be
   // restarted
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index 84d1d666cb..d0a6e5ff42 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -81,17 +81,43 @@ run_TestOutputSize()
 run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
 run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)
 
-function(run_TestRepeat case)
-  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN})
-  string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[
+function(run_TestRepeat case return_value )
+  set(CASE_CTEST_TEST_ARGS RETURN_VALUE result EXCLUDE RunCMakeVersion ${ARGN})
+  string(CONCAT suffix_code [[
 add_test(NAME testRepeat
   COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
                            -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
 set_property(TEST testRepeat PROPERTY TIMEOUT 5)
   ]])
+  string(APPEND CASE_CMAKELISTS_SUFFIX_CODE "${suffix_code}")
 
   run_ctest(TestRepeat${case})
+
+  #write to end of the test file logic to Verify we get the expected
+  #return code
+  string(REPLACE "RETURN_VALUE:" "" return_value "${return_value}" )
+  file(APPEND "${RunCMake_BINARY_DIR}/TestRepeat${case}/test.cmake"
+"
+
+  set(expected_result ${return_value})
+  message(STATUS \${result})
+  if(NOT result EQUAL expected_result)
+    message(FATAL_ERROR \"expected a return value of: \${expected_result},
+                         instead got: \${result}\")
+  endif()
+"
+  )
 endfunction()
-run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3)
-run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3)
-run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3)
+
+run_TestRepeat(UntilFail RETURN_VALUE:1 REPEAT UNTIL_FAIL:3)
+run_TestRepeat(UntilPass RETURN_VALUE:0 REPEAT UNTIL_PASS:3)
+run_TestRepeat(AfterTimeout RETURN_VALUE:0 REPEAT AFTER_TIMEOUT:3)
+
+# test repeat and not run tests interact correctly
+set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME testNotRun
+  COMMAND ${CMAKE_COMMAND}/doesnt_exist)
+  set_property(TEST testNotRun PROPERTY TIMEOUT 5)
+  ]])
+run_TestRepeat(NotRun RETURN_VALUE:1 REPEAT UNTIL_PASS:3)
+unset(CASE_CMAKELISTS_SUFFIX_CODE)
diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt
new file mode 100644
index 0000000000..b57e2deb77
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt
new file mode 100644
index 0000000000..a69932d8a5
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt
@@ -0,0 +1 @@
+.*Unable to find executable.*
diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt
new file mode 100644
index 0000000000..72c98bc4a7
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt
@@ -0,0 +1,5 @@
+.*
+50% tests passed, 1 tests failed out of 2
+.*
+The following tests FAILED:
+.*testNotRun.*Not Run.*
diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake b/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake
new file mode 100644
index 0000000000..abde4f0117
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake
@@ -0,0 +1,10 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()