CTest: exit nonzero after message(SEND_ERROR|FATAL_ERROR)

Fixes: #21004

18 files changed, 57 insertions, 15 deletions

diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 70e84eee3c..abd1207345 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -819,14 +819,16 @@ void cmCTestTestHandler::CheckLabelFilter(cmCTestTestProperties& it)
   this->CheckLabelFilterExclude(it);
 }
 
-void cmCTestTestHandler::ComputeTestList()
+bool cmCTestTestHandler::ComputeTestList()
 {
   this->TestList.clear(); // clear list of test
-  this->GetListOfTests();
+  if (!this->GetListOfTests()) {
+    return false;
+  }
 
   if (this->RerunFailed) {
     this->ComputeTestListForRerunFailed();
-    return;
+    return true;
   }
 
   cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
@@ -882,6 +884,7 @@ void cmCTestTestHandler::ComputeTestList()
   this->TestList = finalList;
 
   this->UpdateMaxTestNameWidth();
+  return true;
 }
 
 void cmCTestTestHandler::ComputeTestListForRerunFailed()
@@ -1260,7 +1263,10 @@ bool cmCTestTestHandler::GetValue(const char* tag, std::string& value,
 bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
                                           std::vector<std::string>& failed)
 {
-  this->ComputeTestList();
+  if (!this->ComputeTestList()) {
+    return false;
+  }
+
   this->StartTest = this->CTest->CurrentTime();
   this->StartTestTime = std::chrono::system_clock::now();
   auto elapsed_time_start = std::chrono::steady_clock::now();
@@ -1695,7 +1701,7 @@ bool cmCTestTestHandler::ParseResourceGroupsProperty(
   return lexer.ParseString(val);
 }
 
-void cmCTestTestHandler::GetListOfTests()
+bool cmCTestTestHandler::GetListOfTests()
 {
   if (!this->IncludeLabelRegExp.empty()) {
     this->IncludeLabelRegularExpression.compile(
@@ -1748,14 +1754,15 @@ void cmCTestTestHandler::GetListOfTests()
     // does the DartTestfile.txt exist ?
     testFilename = "DartTestfile.txt";
   } else {
-    return;
+    return true;
   }
 
   if (!mf.ReadListFile(testFilename)) {
-    return;
+    return false;
   }
   if (cmSystemTools::GetErrorOccuredFlag()) {
-    return;
+    // SEND_ERROR or FATAL_ERROR in CTestTestfile or TEST_INCLUDE_FILES
+    return false;
   }
   const char* specFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE");
   if (this->ResourceSpecFile.empty() && specFile) {
@@ -1764,6 +1771,7 @@ void cmCTestTestHandler::GetListOfTests()
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                      "Done constructing a list of tests" << std::endl,
                      this->Quiet);
+  return true;
 }
 
 void cmCTestTestHandler::UseIncludeRegExp()
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index b26f8a6d25..f9e9391b92 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -286,10 +286,10 @@ private:
   /**
    * Get the list of tests in directory and subdirectories.
    */
-  void GetListOfTests();
+  bool GetListOfTests();
   // compute the lists of tests that will actually run
   // based on union regex and -I stuff
-  void ComputeTestList();
+  bool ComputeTestList();
 
   // compute the lists of tests that will actually run
   // based on LastTestFailed.log
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index a5fde897c8..f0ae8de1b7 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -704,8 +704,7 @@ bool cmCTest::UpdateCTestConfiguration()
   if (!cmSystemTools::FileExists(fileName)) {
     // No need to exit if we are not producing XML
     if (this->Impl->ProduceXML) {
-      cmCTestLog(this, ERROR_MESSAGE,
-                 "Cannot find file: " << fileName << std::endl);
+      cmCTestLog(this, WARNING, "Cannot find file: " << fileName << std::endl);
       return false;
     }
   } else {
diff --git a/Tests/RunCMake/CTest/RunCMakeTest.cmake b/Tests/RunCMake/CTest/RunCMakeTest.cmake
index 761224a6d3..62606f862d 100644
--- a/Tests/RunCMake/CTest/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTest/RunCMakeTest.cmake
@@ -18,3 +18,12 @@ function(run_CMakeCTestArguments)
   run_cmake_command(CMakeCTestArguments-test ${CMAKE_COMMAND} --build . --config Debug --target "${test}")
 endfunction()
 run_CMakeCTestArguments()
+
+function(run_TestfileErrors)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestfileErrors-build)
+  run_cmake(TestfileErrors)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(TestfileErrors-build ${CMAKE_COMMAND} --build . --config Debug)
+  run_cmake_command(TestfileErrors-test ${CMAKE_CTEST_COMMAND} -C Debug)
+endfunction()
+run_TestfileErrors()
diff --git a/Tests/RunCMake/CTest/TestfileErrors-Script.cmake b/Tests/RunCMake/CTest/TestfileErrors-Script.cmake
new file mode 100644
index 0000000000..d9fc7c89a3
--- /dev/null
+++ b/Tests/RunCMake/CTest/TestfileErrors-Script.cmake
@@ -0,0 +1,4 @@
+message(SEND_ERROR "SEND_ERROR")
+message(FATAL_ERROR "FATAL_ERROR")
+# This shouldn't get printed because the script aborts on FATAL_ERROR
+message(SEND_ERROR "reaching the unreachable")
diff --git a/Tests/RunCMake/CTest/TestfileErrors-test-result.txt b/Tests/RunCMake/CTest/TestfileErrors-test-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/CTest/TestfileErrors-test-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/CTest/TestfileErrors-test-stderr.txt b/Tests/RunCMake/CTest/TestfileErrors-test-stderr.txt
new file mode 100644
index 0000000000..a3a476bf1d
--- /dev/null
+++ b/Tests/RunCMake/CTest/TestfileErrors-test-stderr.txt
@@ -0,0 +1,11 @@
+CMake Error at [^
+]*/Tests/RunCMake/CTest/TestfileErrors-Script.cmake:1 \(message\):
+  SEND_ERROR
+Call Stack \(most recent call first\):
+  CTestTestfile.cmake:[0-9]+ \(include\)
++
+CMake Error at [^
+]*/Tests/RunCMake/CTest/TestfileErrors-Script.cmake:2 \(message\):
+  FATAL_ERROR
+Call Stack \(most recent call first\):
+  CTestTestfile.cmake:[0-9]+ \(include\)
diff --git a/Tests/RunCMake/CTest/TestfileErrors.cmake b/Tests/RunCMake/CTest/TestfileErrors.cmake
new file mode 100644
index 0000000000..676eb479d7
--- /dev/null
+++ b/Tests/RunCMake/CTest/TestfileErrors.cmake
@@ -0,0 +1,3 @@
+include(CTest)
+add_test(NAME "unreachable" COMMAND ${CMAKE_COMMAND} -E true)
+set_property(DIRECTORY PROPERTY TEST_INCLUDE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/TestfileErrors-Script.cmake)
diff --git a/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-result.txt b/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-stderr.txt b/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-stderr.txt
index d95bb337ca..97e2a10c17 100644
--- a/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-stderr.txt
+++ b/Tests/RunCMake/CTestCommandLine/BadCTestTestfile-stderr.txt
@@ -1,4 +1,4 @@
 ^CMake Error at CTestTestfile.cmake:[0-9]+ \(subdirs\):
   subdirs called with incorrect number of arguments
 +
-No tests were found!!!$
+Errors while running CTest$
diff --git a/Tests/RunCMake/CTestCommandLine/TestOutputSize-stderr.txt b/Tests/RunCMake/CTestCommandLine/TestOutputSize-stderr.txt
index ba4235defb..19310b8ed0 100644
--- a/Tests/RunCMake/CTestCommandLine/TestOutputSize-stderr.txt
+++ b/Tests/RunCMake/CTestCommandLine/TestOutputSize-stderr.txt
@@ -1 +1,2 @@
+^Cannot find file: .*/Tests/RunCMake/CTestCommandLine/TestOutputSize/DartConfiguration.tcl
 Errors while running CTest
diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-result.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-result.txt b/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-stderr.txt b/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-stderr.txt
index ec2cf2c1c0..296e4262ba 100644
--- a/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-stderr.txt
+++ b/Tests/RunCMake/add_test/CMP0110-OLD-EscapedSpecialChars-ctest-stderr.txt
@@ -2,4 +2,4 @@ CMake Error at CTestTestfile.cmake:[0-9]+:
   Parse error\.  Function missing ending "\)"\.  End of file reached\.
 
 
-No tests were found!!!
+Errors while running CTest
diff --git a/Tests/RunCMake/add_test/CMP0110-OLD-FormerInvalidSpecialCharsMC-ctest-result.txt b/Tests/RunCMake/add_test/CMP0110-OLD-FormerInvalidSpecialCharsMC-ctest-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/add_test/CMP0110-OLD-FormerInvalidSpecialCharsMC-ctest-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-result.txt b/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-stderr.txt b/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-stderr.txt
index ec2cf2c1c0..296e4262ba 100644
--- a/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-stderr.txt
+++ b/Tests/RunCMake/add_test/CMP0110-WARN-EscapedSpecialChars-ctest-stderr.txt
@@ -2,4 +2,4 @@ CMake Error at CTestTestfile.cmake:[0-9]+:
   Parse error\.  Function missing ending "\)"\.  End of file reached\.
 
 
-No tests were found!!!
+Errors while running CTest
diff --git a/Tests/RunCMake/add_test/CMP0110-WARN-FormerInvalidSpecialCharsMC-ctest-result.txt b/Tests/RunCMake/add_test/CMP0110-WARN-FormerInvalidSpecialCharsMC-ctest-result.txt
new file mode 100644
index 0000000000..d197c913c2
--- /dev/null
+++ b/Tests/RunCMake/add_test/CMP0110-WARN-FormerInvalidSpecialCharsMC-ctest-result.txt
@@ -0,0 +1 @@
+[^0]