Merge topic 'cmake-presets-output-junit' into release-3.25

bea4ed5430 CTest: Add support for outputJUnitFile in presets
9270a02003 CMakePresets.json: Add outputJUnitFile to test presets schema
757786bb73 Tests: Add test for outputLogFile in CMakePresets.json
b68c3596e7 CMakePresets.json: Disallow extra properties in test output schema

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: scivision <michael@scivision.dev>
Merge-request: !7806

19 files changed, 229 insertions, 11 deletions

diff --git a/Help/manual/cmake-presets.7.rst b/Help/manual/cmake-presets.7.rst
index eb3e460f85..ae6dcb1e43 100644
--- a/Help/manual/cmake-presets.7.rst
+++ b/Help/manual/cmake-presets.7.rst
@@ -657,6 +657,12 @@ that may contain the following fields:
     passing :option:`--output-log <ctest --output-log>` on the command line.
     This field supports macro expansion.
 
+  ``outputJUnitFile``
+    An optional string specifying a path to a JUnit file. Equivalent to
+    passing :option:`--output-junit <ctest --output-junit>` on the command line.
+    This field supports macro expansion. This is allowed in preset files
+    specifying version ``6`` or above.
+
   ``labelSummary``
     An optional bool. If false, equivalent to passing
     :option:`--no-label-summary <ctest --no-label-summary>` on the command
diff --git a/Help/manual/presets/schema.json b/Help/manual/presets/schema.json
index b4db700754..348116b38a 100644
--- a/Help/manual/presets/schema.json
+++ b/Help/manual/presets/schema.json
@@ -83,7 +83,7 @@
         "vendor": { "$ref": "#/definitions/vendor" },
         "configurePresets": { "$ref": "#/definitions/configurePresetsV3"},
         "buildPresets": { "$ref": "#/definitions/buildPresetsV4"},
-        "testPresets": { "$ref": "#/definitions/testPresetsV5"},
+        "testPresets": { "$ref": "#/definitions/testPresetsV6"},
         "packagePresets": { "$ref": "#/definitions/packagePresetsV6"},
         "workflowPresets": { "$ref": "#/definitions/workflowPresetsV6" },
         "include": { "$ref": "#/definitions/include"}
@@ -705,6 +705,25 @@
         "additionalProperties": false
       }
     },
+    "testPresetsItemsV6": {
+      "type": "array",
+      "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 6 and higher.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "output": {
+            "type": "object",
+            "description": "An optional object specifying output options.",
+            "properties": {
+              "outputJUnitFile": {
+                "type": "string",
+                "description": "An optional string specifying a path to a JUnit file. Equivalent to passing --output-junit on the command line."
+              }
+            }
+          }
+        }
+      }
+    },
     "testPresetsItemsV5": {
       "type": "array",
       "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
@@ -1051,6 +1070,58 @@
         ]
       }
     },
+    "testPresetsV6": {
+      "type": "array",
+      "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 6 and higher.",
+      "allOf": [
+        { "$ref": "#/definitions/testPresetsItemsV2" },
+        { "$ref": "#/definitions/testPresetsItemsV3" },
+        { "$ref": "#/definitions/testPresetsItemsV5" },
+        { "$ref": "#/definitions/testPresetsItemsV6" }
+      ],
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {},
+          "hidden": {},
+          "inherits": {},
+          "configurePreset": {},
+          "vendor": {},
+          "displayName": {},
+          "description": {},
+          "inheritConfigureEnvironment": {},
+          "environment": {},
+          "configuration": {},
+          "overwriteConfigurationFile": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "outputJUnitFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {},
+              "testOutputTruncation": {}
+            },
+            "additionalProperties": false
+          },
+          "filter": {},
+          "execution": {},
+          "condition": {}
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": false
+      }
+    },
     "testPresetsV5": {
       "type": "array",
       "description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
@@ -1073,7 +1144,24 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {},
+              "testOutputTruncation": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {},
           "condition": {}
@@ -1105,7 +1193,23 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {},
           "condition": {}
@@ -1136,7 +1240,23 @@
           "environment": {},
           "configuration": {},
           "overwriteConfigurationFile": {},
-          "output": {},
+          "output": {
+            "type": "object",
+            "properties": {
+              "shortProgress": {},
+              "verbosity": {},
+              "debug": {},
+              "outputOnFailure": {},
+              "quiet": {},
+              "outputLogFile": {},
+              "labelSummary": {},
+              "subprojectSummary": {},
+              "maxPassedTestOutputSize": {},
+              "maxFailedTestOutputSize": {},
+              "maxTestNameWidth": {}
+            },
+            "additionalProperties": false
+          },
           "filter": {},
           "execution": {}
         },
diff --git a/Help/release/3.25.rst b/Help/release/3.25.rst
index 9c1d96b0eb..c119faee4a 100644
--- a/Help/release/3.25.rst
+++ b/Help/release/3.25.rst
@@ -21,6 +21,9 @@ Presets
 * The :manual:`cmake-presets(7)` format now supports a
   ``workflowPresets`` field to specify presets for :option:`cmake --workflow`.
 
+* The :manual:`cmake-presets(7)` format now supports a
+  ``outputJUnitFile`` field to specify JUnit output in test presets.
+
 Languages
 ---------
 
diff --git a/Source/cmCMakePresetsGraph.cxx b/Source/cmCMakePresetsGraph.cxx
index fb3d04217d..7325e447df 100644
--- a/Source/cmCMakePresetsGraph.cxx
+++ b/Source/cmCMakePresetsGraph.cxx
@@ -265,6 +265,8 @@ bool ExpandMacros(const cmCMakePresetsGraph& graph, const TestPreset& preset,
   if (out->Output) {
     CHECK_EXPAND(out, out->Output->OutputLogFile, macroExpanders,
                  graph.GetVersion(preset));
+    CHECK_EXPAND(out, out->Output->OutputJUnitFile, macroExpanders,
+                 graph.GetVersion(preset));
   }
 
   if (out->Filter) {
@@ -851,6 +853,7 @@ cmCMakePresetsGraph::TestPreset::VisitPresetInherit(
                            parentOutput.OutputOnFailure);
       InheritOptionalValue(output.Quiet, parentOutput.Quiet);
       InheritString(output.OutputLogFile, parentOutput.OutputLogFile);
+      InheritString(output.OutputJUnitFile, parentOutput.OutputJUnitFile);
       InheritOptionalValue(output.LabelSummary, parentOutput.LabelSummary);
       InheritOptionalValue(output.SubprojectSummary,
                            parentOutput.SubprojectSummary);
@@ -1253,6 +1256,8 @@ const char* cmCMakePresetsGraph::ResultToString(ReadFileResult result)
       return "Invalid workflow steps";
     case ReadFileResult::WORKFLOW_STEP_UNREACHABLE_FROM_FILE:
       return "Workflow step is unreachable from preset's file";
+    case ReadFileResult::CTEST_JUNIT_UNSUPPORTED:
+      return "File version must be 6 or higher for CTest JUnit output support";
   }
 
   return "Unknown error";
diff --git a/Source/cmCMakePresetsGraph.h b/Source/cmCMakePresetsGraph.h
index 5b3e812e11..17c902ba62 100644
--- a/Source/cmCMakePresetsGraph.h
+++ b/Source/cmCMakePresetsGraph.h
@@ -54,6 +54,7 @@ public:
     TEST_OUTPUT_TRUNCATION_UNSUPPORTED,
     INVALID_WORKFLOW_STEPS,
     WORKFLOW_STEP_UNREACHABLE_FROM_FILE,
+    CTEST_JUNIT_UNSUPPORTED,
   };
 
   std::string errors;
@@ -230,6 +231,7 @@ public:
       cm::optional<bool> OutputOnFailure;
       cm::optional<bool> Quiet;
       std::string OutputLogFile;
+      std::string OutputJUnitFile;
       cm::optional<bool> LabelSummary;
       cm::optional<bool> SubprojectSummary;
       cm::optional<int> MaxPassedTestOutputSize;
diff --git a/Source/cmCMakePresetsGraphReadJSON.cxx b/Source/cmCMakePresetsGraphReadJSON.cxx
index 5aa4284d75..eec53c1662 100644
--- a/Source/cmCMakePresetsGraphReadJSON.cxx
+++ b/Source/cmCMakePresetsGraphReadJSON.cxx
@@ -588,6 +588,11 @@ cmCMakePresetsGraph::ReadFileResult cmCMakePresetsGraph::ReadJSONFile(
       return ReadFileResult::TEST_OUTPUT_TRUNCATION_UNSUPPORTED;
     }
 
+    // Support for outputJUnitFile added in version 6.
+    if (v < 6 && preset.Output && !preset.Output->OutputJUnitFile.empty()) {
+      return ReadFileResult::CTEST_JUNIT_UNSUPPORTED;
+    }
+
     this->TestPresetOrder.push_back(preset.Name);
   }
 
diff --git a/Source/cmCMakePresetsGraphReadJSONTestPresets.cxx b/Source/cmCMakePresetsGraphReadJSONTestPresets.cxx
index c07d380260..3856f630c5 100644
--- a/Source/cmCMakePresetsGraphReadJSONTestPresets.cxx
+++ b/Source/cmCMakePresetsGraphReadJSONTestPresets.cxx
@@ -104,6 +104,8 @@ auto const TestPresetOptionalOutputHelper =
             cmCMakePresetsGraphInternal::PresetOptionalBoolHelper, false)
       .Bind("outputLogFile"_s, &TestPreset::OutputOptions::OutputLogFile,
             cmCMakePresetsGraphInternal::PresetStringHelper, false)
+      .Bind("outputJUnitFile"_s, &TestPreset::OutputOptions::OutputJUnitFile,
+            cmCMakePresetsGraphInternal::PresetStringHelper, false)
       .Bind("labelSummary"_s, &TestPreset::OutputOptions::LabelSummary,
             cmCMakePresetsGraphInternal::PresetOptionalBoolHelper, false)
       .Bind("subprojectSummary"_s,
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 66507a73e9..f60a1e9999 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -2116,11 +2116,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       return false;
     }
     i++;
-    this->Impl->TestHandler.SetJUnitXMLFileName(std::string(args[i]));
-    // Turn test output compression off.
-    // This makes it easier to include test output in the resulting
-    // JUnit XML report.
-    this->Impl->CompressTestOutput = false;
+    this->SetOutputJUnitFileName(std::string(args[i]));
   }
 
   cm::string_view noTestsPrefix = "--no-tests=";
@@ -2458,6 +2454,9 @@ bool cmCTest::SetArgsFromPreset(const std::string& presetName,
     if (!expandedPreset->Output->OutputLogFile.empty()) {
       this->SetOutputLogFileName(expandedPreset->Output->OutputLogFile);
     }
+    if (!expandedPreset->Output->OutputJUnitFile.empty()) {
+      this->SetOutputJUnitFileName(expandedPreset->Output->OutputJUnitFile);
+    }
 
     this->Impl->LabelSummary =
       expandedPreset->Output->LabelSummary.value_or(true);
@@ -3541,6 +3540,15 @@ void cmCTest::SetOutputLogFileName(const std::string& name)
   }
 }
 
+void cmCTest::SetOutputJUnitFileName(const std::string& name)
+{
+  this->Impl->TestHandler.SetJUnitXMLFileName(name);
+  // Turn test output compression off.
+  // This makes it easier to include test output in the resulting
+  // JUnit XML report.
+  this->Impl->CompressTestOutput = false;
+}
+
 static const char* cmCTestStringLogType[] = { "DEBUG",
                                               "OUTPUT",
                                               "HANDLER_OUTPUT",
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 551c116ada..0017b3e19e 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -359,6 +359,9 @@ public:
   /** Set the output log file name */
   void SetOutputLogFileName(const std::string& name);
 
+  /** Set the output JUnit file name */
+  void SetOutputJUnitFileName(const std::string& name);
+
   /** Set the visual studio or Xcode config type */
   void SetConfigType(const std::string& ct);
 
diff --git a/Tests/RunCMake/CMakePresetsTest/Good-test-outputJUnit-check.cmake b/Tests/RunCMake/CMakePresetsTest/Good-test-outputJUnit-check.cmake
new file mode 100644
index 0000000000..e1788cb833
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/Good-test-outputJUnit-check.cmake
@@ -0,0 +1,4 @@
+include("${CMAKE_CURRENT_LIST_DIR}/check.cmake")
+if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/default/output.xml")
+  string(APPEND RunCMake_TEST_FAILED "Expected ${RunCMake_TEST_BINARY_DIR}/default/output.xml to exist but it does not\n")
+endif()
diff --git a/Tests/RunCMake/CMakePresetsTest/Good-test-outputLog-check.cmake b/Tests/RunCMake/CMakePresetsTest/Good-test-outputLog-check.cmake
new file mode 100644
index 0000000000..e860d42183
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/Good-test-outputLog-check.cmake
@@ -0,0 +1,4 @@
+include("${CMAKE_CURRENT_LIST_DIR}/check.cmake")
+if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/default/output.log")
+  string(APPEND RunCMake_TEST_FAILED "Expected ${RunCMake_TEST_BINARY_DIR}/default/output.log to exist but it does not\n")
+endif()
diff --git a/Tests/RunCMake/CMakePresetsTest/Good.json.in b/Tests/RunCMake/CMakePresetsTest/Good.json.in
index d484a199f2..a4b875a1d8 100644
--- a/Tests/RunCMake/CMakePresetsTest/Good.json.in
+++ b/Tests/RunCMake/CMakePresetsTest/Good.json.in
@@ -1,5 +1,5 @@
 {
-    "version": 5,
+    "version": 6,
     "configurePresets": [
         {
             "name": "default",
@@ -171,6 +171,20 @@
             "execution": {
                 "showOnly": "human"
             }
+        },
+        {
+            "name": "outputLog",
+            "inherits": "minimal",
+            "output": {
+                "outputLogFile": "${sourceDir}/build/default/output.log"
+            }
+        },
+        {
+            "name": "outputJUnit",
+            "inherits": "minimal",
+            "output": {
+                "outputJUnitFile": "${sourceDir}/build/default/output.xml"
+            }
         }
     ]
 }
diff --git a/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-result.txt b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-stderr.txt b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-stderr.txt
new file mode 100644
index 0000000000..acd5785489
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported-test-x-stderr.txt
@@ -0,0 +1,2 @@
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported: File version must be 6 or higher for CTest JUnit output support$
diff --git a/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported.json.in b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported.json.in
new file mode 100644
index 0000000000..93f6b0c200
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/OutputJUnitUnsupported.json.in
@@ -0,0 +1,17 @@
+{
+  "version": 5,
+  "configurePresets": [
+    {
+      "name": "default"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "unsupported",
+      "configurePreset": "default",
+      "output": {
+        "outputJUnitFile": "junit.xml"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake b/Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake
index bec0dd9464..1c7b8360f7 100644
--- a/Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CMakePresetsTest/RunCMakeTest.cmake
@@ -78,7 +78,7 @@ set(CMakePresetsTest_ASSETS "Good-indexFile.txt")
 set(GoodTestPresets
   "minimal;defaults;noEnvironment;withEnvironment"
   "config-debug;config-release"
-  "exclude;index;indexFile;showOnly")
+  "exclude;index;indexFile;showOnly;outputLog;outputJUnit")
 run_cmake_test_presets(Good
                        "default"
                        ""
@@ -105,6 +105,8 @@ run_cmake_test_presets(Invalid "" "" "hidden;vendorMacro")
 set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 run_cmake_test_presets(PresetsUnsupported "" "" "x")
 run_cmake_test_presets(ConditionFuture "" "" "x")
+run_cmake_test_presets(TestOutputTruncationUnsupported "" "" "x")
+run_cmake_test_presets(OutputJUnitUnsupported "" "" "x")
 set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
 run_cmake_test_presets(ConfigurePresetUnreachable "" "" "x")
 set(CMakePresetsTest_NO_CONFIGURE 0)
diff --git a/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-result.txt b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-stderr.txt b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-stderr.txt
new file mode 100644
index 0000000000..90ea7c3b28
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported-test-x-stderr.txt
@@ -0,0 +1,2 @@
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported: File version must be 5 or higher for testOutputTruncation preset support\.$
diff --git a/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported.json.in b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported.json.in
new file mode 100644
index 0000000000..c116039afd
--- /dev/null
+++ b/Tests/RunCMake/CMakePresetsTest/TestOutputTruncationUnsupported.json.in
@@ -0,0 +1,17 @@
+{
+  "version": 4,
+  "configurePresets": [
+    {
+      "name": "default"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "default",
+      "configurePreset": "default",
+      "output": {
+        "testOutputTruncation": "tail"
+      }
+    }
+  ]
+}