ctest: Add support for writing test results in JUnit XML format

Addresses #18654

18 files changed, 271 insertions, 6 deletions

diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst
index b4493a03dd..9081b3f4b1 100644
--- a/Help/command/ctest_test.rst
+++ b/Help/command/ctest_test.rst
@@ -25,6 +25,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [RETURN_VALUE <result-var>]
              [CAPTURE_CMAKE_ERROR <result-var>]
              [REPEAT <mode>:<n>]
+             [OUTPUT_JUNIT <file>]
              [QUIET]
              )
 
@@ -150,6 +151,15 @@ The options are:
   Store in the ``<result-var>`` variable -1 if there are any errors running
   the command and prevent ctest from returning non-zero if an error occurs.
 
+``OUTPUT_JUNIT``
+  .. versionadded:: 3.21
+
+  Write test results to ``<file>`` in JUnit XML format. If ``<file>`` is a
+  relative path it will be placed in the build directory. If ``<file>>``
+  already exists it will be overwritten. Note that the resulting JUnit XML
+  file is **not** uploaded to CDash because it would be redundant with
+  CTest's ``Test.xml`` file.
+
 ``QUIET``
   .. versionadded:: 3.3
 
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 68409e1ed9..3d80cece28 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -134,6 +134,12 @@ Options
 
  This option tells CTest to write all its output to a ``<file>`` log file.
 
+``--output-junit <file>``
+ Write test results in JUnit format.
+
+ This option tells CTest to write test results to a ``<file>`` JUnit XML file.
+ If ``<file>`` already exists it will be overwritten.
+
 ``-N,--show-only[=<format>]``
  Disable actual execution of tests.
 
diff --git a/Help/release/dev/ctest-output-junit.rst b/Help/release/dev/ctest-output-junit.rst
new file mode 100644
index 0000000000..66df19d6e4
--- /dev/null
+++ b/Help/release/dev/ctest-output-junit.rst
@@ -0,0 +1,5 @@
+ctest-output-junit
+------------------
+
+* :manual:`ctest(1)` gained a ``--output-junit`` option to write test results
+  to a JUnit XML file.
diff --git a/Source/CTest/cmCTestMemCheckCommand.cxx b/Source/CTest/cmCTestMemCheckCommand.cxx
index d0e2974524..37b3628720 100644
--- a/Source/CTest/cmCTestMemCheckCommand.cxx
+++ b/Source/CTest/cmCTestMemCheckCommand.cxx
@@ -14,7 +14,7 @@ void cmCTestMemCheckCommand::BindArguments()
   this->Bind("DEFECT_COUNT"_s, this->DefectCount);
 }
 
-cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler()
+cmCTestTestHandler* cmCTestMemCheckCommand::InitializeActualHandler()
 {
   cmCTestMemCheckHandler* handler = this->CTest->GetMemCheckHandler();
   handler->Initialize();
diff --git a/Source/CTest/cmCTestMemCheckCommand.h b/Source/CTest/cmCTestMemCheckCommand.h
index 6544f16cfd..ee39e49a92 100644
--- a/Source/CTest/cmCTestMemCheckCommand.h
+++ b/Source/CTest/cmCTestMemCheckCommand.h
@@ -13,6 +13,7 @@
 #include "cmCommand.h"
 
 class cmCTestGenericHandler;
+class cmCTestTestHandler;
 
 /** \class cmCTestMemCheck
  * \brief Run a ctest script
@@ -36,7 +37,7 @@ public:
 protected:
   void BindArguments() override;
 
-  cmCTestGenericHandler* InitializeActualHandler() override;
+  cmCTestTestHandler* InitializeActualHandler() override;
 
   void ProcessAdditionalValues(cmCTestGenericHandler* handler) override;
 
diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx
index 886c263067..67f4986086 100644
--- a/Source/CTest/cmCTestTestCommand.cxx
+++ b/Source/CTest/cmCTestTestCommand.cxx
@@ -9,7 +9,6 @@
 #include <cmext/string_view>
 
 #include "cmCTest.h"
-#include "cmCTestGenericHandler.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
 #include "cmMakefile.h"
@@ -36,6 +35,7 @@ void cmCTestTestCommand::BindArguments()
   this->Bind("TEST_LOAD"_s, this->TestLoad);
   this->Bind("RESOURCE_SPEC_FILE"_s, this->ResourceSpecFile);
   this->Bind("STOP_ON_FAILURE"_s, this->StopOnFailure);
+  this->Bind("OUTPUT_JUNIT"_s, this->OutputJUnit);
 }
 
 cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
@@ -60,7 +60,7 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
     this->ResourceSpecFile = *resourceSpecFile;
   }
 
-  cmCTestGenericHandler* handler = this->InitializeActualHandler();
+  cmCTestTestHandler* handler = this->InitializeActualHandler();
   if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) {
     handler->SetOption(
       "TestsToRunInformation",
@@ -140,11 +140,15 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
                                        *labelsForSubprojects, this->Quiet);
   }
 
+  if (!this->OutputJUnit.empty()) {
+    handler->SetJUnitXMLFileName(this->OutputJUnit);
+  }
+
   handler->SetQuiet(this->Quiet);
   return handler;
 }
 
-cmCTestGenericHandler* cmCTestTestCommand::InitializeActualHandler()
+cmCTestTestHandler* cmCTestTestCommand::InitializeActualHandler()
 {
   cmCTestTestHandler* handler = this->CTest->GetTestHandler();
   handler->Initialize();
diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h
index 624cd91fef..24e74e2baa 100644
--- a/Source/CTest/cmCTestTestCommand.h
+++ b/Source/CTest/cmCTestTestCommand.h
@@ -13,6 +13,7 @@
 #include "cmCommand.h"
 
 class cmCTestGenericHandler;
+class cmCTestTestHandler;
 
 /** \class cmCTestTest
  * \brief Run a ctest script
@@ -40,7 +41,7 @@ public:
 
 protected:
   void BindArguments() override;
-  virtual cmCTestGenericHandler* InitializeActualHandler();
+  virtual cmCTestTestHandler* InitializeActualHandler();
   cmCTestGenericHandler* InitializeHandler() override;
 
   std::string Start;
@@ -59,5 +60,6 @@ protected:
   std::string StopTime;
   std::string TestLoad;
   std::string ResourceSpecFile;
+  std::string OutputJUnit;
   bool StopOnFailure = false;
 };
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index db5cb9c2be..1596d4ac32 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -42,6 +42,7 @@
 #include "cmStateSnapshot.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
+#include "cmTimestamp.h"
 #include "cmWorkingDirectory.h"
 #include "cmXMLWriter.h"
 #include "cmake.h"
@@ -299,6 +300,9 @@ cmCTestTestHandler::cmCTestTestHandler()
 
   this->LogFile = nullptr;
 
+  // Support for JUnit XML output.
+  this->JUnitXMLFileName = "";
+
   // regex to detect <DartMeasurement>...</DartMeasurement>
   this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)");
   // regex to detect each individual <DartMeasurement>...</DartMeasurement>
@@ -456,6 +460,10 @@ int cmCTestTestHandler::ProcessHandler()
     return 1;
   }
 
+  if (!this->WriteJUnitXML()) {
+    return 1;
+  }
+
   if (!this->PostProcessHandler()) {
     this->LogFile = nullptr;
     return -1;
@@ -2457,3 +2465,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=(
 {
   return !(*this == other);
 }
+
+void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename)
+{
+  this->JUnitXMLFileName = filename;
+}
+
+bool cmCTestTestHandler::WriteJUnitXML()
+{
+  if (this->JUnitXMLFileName.empty()) {
+    return true;
+  }
+
+  // Open new XML file for writing.
+  cmGeneratedFileStream xmlfile;
+  xmlfile.SetTempExt("tmp");
+  xmlfile.Open(this->JUnitXMLFileName);
+  if (!xmlfile) {
+    cmCTestLog(this->CTest, ERROR_MESSAGE,
+               "Problem opening file: " << this->JUnitXMLFileName
+                                        << std::endl);
+    return false;
+  }
+  cmXMLWriter xml(xmlfile);
+
+  // Iterate over the test results to get the number of tests that
+  // passed, failed, etc.
+  auto num_tests = 0;
+  auto num_passed = 0;
+  auto num_failed = 0;
+  auto num_notrun = 0;
+  auto num_disabled = 0;
+  SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
+  for (cmCTestTestResult const& result : resultsSet) {
+    num_tests++;
+    if (result.Status == cmCTestTestHandler::COMPLETED) {
+      num_passed++;
+    } else if (result.Status == cmCTestTestHandler::NOT_RUN) {
+      if (result.CompletionStatus == "Disabled") {
+        num_disabled++;
+      } else {
+        num_notrun++;
+      }
+    } else {
+      num_failed++;
+    }
+  }
+
+  // Write <testsuite> element.
+  xml.StartDocument();
+  xml.StartElement("testsuite");
+
+  xml.Attribute("name",
+                cmCTest::SafeBuildIdField(
+                  this->CTest->GetCTestConfiguration("BuildName")));
+  xml.BreakAttributes();
+
+  xml.Attribute("tests", num_tests);
+  xml.Attribute("failures", num_failed);
+
+  // CTest disabled => JUnit disabled
+  xml.Attribute("disabled", num_disabled);
+
+  // Otherwise, CTest notrun => JUnit skipped.
+  // The distinction between JUnit disabled vs. skipped is that
+  // skipped tests can have a message associated with them
+  // (why the test was skipped).
+  xml.Attribute("skipped", num_notrun);
+
+  xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site"));
+  xml.Attribute(
+    "time",
+    std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime)
+      .count());
+  const std::time_t start_test_time_t =
+    std::chrono::system_clock::to_time_t(this->StartTestTime);
+  cmTimestamp cmts;
+  xml.Attribute("timestamp",
+                cmts.CreateTimestampFromTimeT(start_test_time_t,
+                                              "%Y-%m-%dT%H:%M:%S", false));
+
+  // Write <testcase> elements.
+  for (cmCTestTestResult const& result : resultsSet) {
+    xml.StartElement("testcase");
+    xml.Attribute("name", result.Name);
+    xml.Attribute("classname", result.Name);
+    xml.Attribute("time", result.ExecutionTime.count());
+
+    std::string status;
+    if (result.Status == cmCTestTestHandler::COMPLETED) {
+      status = "run";
+    } else if (result.Status == cmCTestTestHandler::NOT_RUN) {
+      if (result.CompletionStatus == "Disabled") {
+        status = "disabled";
+      } else {
+        status = "notrun";
+      }
+    } else {
+      status = "fail";
+    }
+    xml.Attribute("status", status);
+
+    if (status == "notrun") {
+      xml.StartElement("skipped");
+      xml.Attribute("message", result.CompletionStatus);
+      xml.EndElement(); // </skipped>
+    } else if (status == "fail") {
+      xml.StartElement("failure");
+      xml.Attribute("message", result.Reason);
+      xml.EndElement(); // </failure>
+    }
+
+    // Note: compressed test output is unconditionally disabled when
+    // --output-junit is specified.
+    xml.Element("system-out", result.Output);
+    xml.EndElement(); // </testcase>
+  }
+
+  xml.EndElement(); // </testsuite>
+  xml.EndDocument();
+
+  return true;
+}
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index 6fa18a900f..68416246f8 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -209,6 +209,9 @@ public:
 
   using ListOfTests = std::vector<cmCTestTestProperties>;
 
+  // Support for writing test results in JUnit XML format.
+  void SetJUnitXMLFileName(const std::string& id);
+
 protected:
   using SetOfTests =
     std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>;
@@ -274,6 +277,11 @@ private:
    */
   virtual void GenerateDartOutput(cmXMLWriter& xml);
 
+  /**
+   * Write test results in JUnit XML format
+   */
+  bool WriteJUnitXML();
+
   void PrintLabelOrSubprojectSummary(bool isSubProject);
 
   /**
@@ -354,4 +362,6 @@ private:
   cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
   int RepeatCount = 1;
   bool RerunFailed;
+
+  std::string JUnitXMLFileName;
 };
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 643b43f6dd..79a3925509 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -2069,6 +2069,17 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     }
     i++;
     this->Impl->TestDir = std::string(args[i]);
+  } else if (this->CheckArgument(arg, "--output-junit"_s)) {
+    if (i >= args.size() - 1) {
+      errormsg = "'--output-junit' requires an argument";
+      return false;
+    }
+    i++;
+    this->Impl->TestHandler.SetJUnitXMLFileName(std::string(args[i]));
+    // Turn test output compression off.
+    // This makes it easier to include test output in the resulting
+    // JUnit XML report.
+    this->Impl->CompressTestOutput = false;
   }
 
   cm::string_view noTestsPrefix = "--no-tests=";
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index a4b85ae310..cad27fac46 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -50,6 +50,7 @@ static const char* cmDocumentationOptions[][2] = {
     "given number of jobs." },
   { "-Q,--quiet", "Make ctest quiet." },
   { "-O <file>, --output-log <file>", "Output to log file" },
+  { "--output-junit <file>", "Output test results to JUnit XML file." },
   { "-N,--show-only[=format]",
     "Disable actual execution of tests. The optional 'format' defines the "
     "format of the test information and can be 'human' for the current text "
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index 2f4d731f06..afec0114aa 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -397,3 +397,22 @@ function(run_testDir)
   run_cmake_command(testDir ${CMAKE_CTEST_COMMAND} --test-dir "${RunCMake_TEST_BINARY_DIR}/sub")
 endfunction()
 run_testDir()
+
+# Test --output-junit
+function(run_output_junit)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/output-junit)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+add_test(test1 \"${CMAKE_COMMAND}\" -E false)
+add_test(test2 \"${CMAKE_COMMAND}\" -E echo \"hello world\")
+add_test(test3 \"${CMAKE_COMMAND}\" -E true)
+set_tests_properties(test3 PROPERTIES  DISABLED \"ON\")
+add_test(test4 \"${CMAKE_COMMAND}/doesnt_exist\")
+add_test(test5 \"${CMAKE_COMMAND}\" -E echo \"please skip\")
+set_tests_properties(test5 PROPERTIES  SKIP_REGULAR_EXPRESSION \"please skip\")
+")
+  run_cmake_command(output-junit ${CMAKE_CTEST_COMMAND} --output-junit "${RunCMake_TEST_BINARY_DIR}/junit.xml")
+endfunction()
+run_output_junit()
diff --git a/Tests/RunCMake/CTestCommandLine/output-junit-check.cmake b/Tests/RunCMake/CTestCommandLine/output-junit-check.cmake
new file mode 100644
index 0000000000..b270fdf6a8
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/output-junit-check.cmake
@@ -0,0 +1,36 @@
+file(GLOB junit_xml_file "${RunCMake_TEST_BINARY_DIR}/junit.xml")
+if(junit_xml_file)
+  file(READ "${junit_xml_file}" junit_xml LIMIT 4096)
+  if(NOT "${junit_xml}" MATCHES "tests=\"5\"")
+    set(RunCMake_TEST_FAILED "tests=\"5\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "failures=\"1\"")
+    set(RunCMake_TEST_FAILED "failures=\"1\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "disabled=\"1\"")
+    set(RunCMake_TEST_FAILED "disabled=\"1\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "skipped=\"2\"")
+    set(RunCMake_TEST_FAILED "skipped=\"2\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<system-out>hello world")
+    set(RunCMake_TEST_FAILED "<system-out>hello world not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<system-out>Disabled")
+    set(RunCMake_TEST_FAILED "<system-out>Disabled not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<skipped message=\"Unable to find executable\"/>")
+    set(RunCMake_TEST_FAILED "<skipped message=\"Unable to find executable\"/> not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<system-out>Unable to find executable:")
+    set(RunCMake_TEST_FAILED "<system-out>Unable to find executable: not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<skipped message=\"SKIP_REGULAR_EXPRESSION_MATCHED\"/>")
+    set(RunCMake_TEST_FAILED "<skipped message=\"SKIP_REGULAR_EXPRESSION_MATCHED\"/> not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<system-out>please skip")
+    set(RunCMake_TEST_FAILED "<system-out>please skip not found when expected")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "junit.xml not found")
+endif()
diff --git a/Tests/RunCMake/CTestCommandLine/output-junit-result.txt b/Tests/RunCMake/CTestCommandLine/output-junit-result.txt
new file mode 100644
index 0000000000..45a4fb75db
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/output-junit-result.txt
@@ -0,0 +1 @@
+8
diff --git a/Tests/RunCMake/CTestCommandLine/output-junit-stderr.txt b/Tests/RunCMake/CTestCommandLine/output-junit-stderr.txt
new file mode 100644
index 0000000000..ce30dc80d8
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/output-junit-stderr.txt
@@ -0,0 +1 @@
+Unable to find executable: .*doesnt_exist
diff --git a/Tests/RunCMake/ctest_test/OutputJUnit-check.cmake b/Tests/RunCMake/ctest_test/OutputJUnit-check.cmake
new file mode 100644
index 0000000000..00310a4f86
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/OutputJUnit-check.cmake
@@ -0,0 +1,24 @@
+file(GLOB junit_xml_file "${RunCMake_TEST_BINARY_DIR}/junit.xml")
+if(junit_xml_file)
+  file(READ "${junit_xml_file}" junit_xml LIMIT 4096)
+  if(NOT "${junit_xml}" MATCHES "tests=\"1\"")
+    set(RunCMake_TEST_FAILED "tests=\"1\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "failures=\"0\"")
+    set(RunCMake_TEST_FAILED "failures=\"0\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "disabled=\"0\"")
+    set(RunCMake_TEST_FAILED "disabled=\"0\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "skipped=\"0\"")
+    set(RunCMake_TEST_FAILED "skipped=\"0\" not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<testcase name=\"RunCMakeVersion\" classname=\"RunCMakeVersion\"")
+    set(RunCMake_TEST_FAILED "RunCMakeVersion not found when expected")
+  endif()
+  if(NOT "${junit_xml}" MATCHES "<system-out>cmake version")
+    set(RunCMake_TEST_FAILED "<system-out>cmake version not found when expected")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "junit.xml not found")
+endif()
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index 8cf6a61415..901ac11236 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -146,3 +146,6 @@ set_property(TEST RunCMakeVersion PROPERTY ENVIRONMENT "ENV1=env1;ENV2=env2")
   run_ctest(TestEnvironment)
 endfunction()
 run_environment()
+
+# test for OUTPUT_JUNIT
+run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2)
diff --git a/Utilities/IWYU/mapping.imp b/Utilities/IWYU/mapping.imp
index f2aef3ecac..f1bdcc4cad 100644
--- a/Utilities/IWYU/mapping.imp
+++ b/Utilities/IWYU/mapping.imp
@@ -21,6 +21,7 @@
   { include: [ "<wctype.h>", public, "<cwctype>", public ] },
 
   # HACK: check whether this can be removed with next iwyu release.
+  { include: [ "<bits/cxxabi_forced.h>", private, "<ctime>", public ] },
   { include: [ "<bits/shared_ptr.h>", private, "<memory>", public ] },
   { include: [ "<bits/std_function.h>", private, "<functional>", public ] },
   { include: [ "<bits/refwrap.h>", private, "<functional>", public ] },