CMakePresets.json: Properly report macro expansion errors

Make a distinction between strings which simply use the
$vendor{<...>} macro, which is valid but makes it unusable by CMake,
and strings which actually contain invalid macro expansions.

Fixes: #21308

13 files changed, 78 insertions, 46 deletions

diff --git a/Tests/RunCMake/CMakePresets/CMakePresets.json.in b/Tests/RunCMake/CMakePresets/CMakePresets.json.in
index 8bfc602082..68b2e3590e 100644
--- a/Tests/RunCMake/CMakePresets/CMakePresets.json.in
+++ b/Tests/RunCMake/CMakePresets/CMakePresets.json.in
@@ -306,16 +306,6 @@
       "binaryDir": "${sourceDir}/build"
     },
     {
-      "name": "UnclosedMacro",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir"
-    },
-    {
-      "name": "NoSuchMacro",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${noexist}"
-    },
-    {
       "name": "VendorMacro",
       "generator": "@RunCMake_GENERATOR@",
       "binaryDir": "$vendor{unknown.unknownMacro}"
@@ -326,31 +316,6 @@
       "binaryDir": "${sourceDir}/build"
     },
     {
-      "name": "EnvCycle",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "environment": {
-        "ENV_1": "$env{ENV_2}",
-        "ENV_2": "$env{ENV_1}"
-      }
-    },
-    {
-      "name": "EmptyEnv",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "cacheVariables": {
-        "MY_VAR": "$env{}"
-      }
-    },
-    {
-      "name": "EmptyPenv",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "cacheVariables": {
-        "MY_VAR": "$penv{}"
-      }
-    },
-    {
       "name": "UseHiddenPreset",
       "generator": "@RunCMake_GENERATOR@",
       "binaryDir": "${sourceDir}/build",
diff --git a/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt b/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
index 9c9c0252b7..723ac2161b 100644
--- a/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EmptyEnv": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EmptyEnv: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EmptyEnv.json.in b/Tests/RunCMake/CMakePresets/EmptyEnv.json.in
new file mode 100644
index 0000000000..ef0d5758b2
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EmptyEnv.json.in
@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EmptyEnv",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "MY_VAR": "$env{}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt b/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
index 395c7b4cfd..880cee6516 100644
--- a/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EmptyPenv": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EmptyPenv: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EmptyPenv.json.in b/Tests/RunCMake/CMakePresets/EmptyPenv.json.in
new file mode 100644
index 0000000000..9081fe5138
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EmptyPenv.json.in
@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EmptyPenv",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "MY_VAR": "$penv{}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt b/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
index c8568f1dc2..1d22b87232 100644
--- a/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EnvCycle": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EnvCycle: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EnvCycle.json.in b/Tests/RunCMake/CMakePresets/EnvCycle.json.in
new file mode 100644
index 0000000000..25a1349652
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EnvCycle.json.in
@@ -0,0 +1,14 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EnvCycle",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "environment": {
+        "ENV_1": "$env{ENV_2}",
+        "ENV_2": "$env{ENV_1}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/ListPresets.json.in b/Tests/RunCMake/CMakePresets/ListPresets.json.in
index c7646c6a42..2ef3797daf 100644
--- a/Tests/RunCMake/CMakePresets/ListPresets.json.in
+++ b/Tests/RunCMake/CMakePresets/ListPresets.json.in
@@ -26,7 +26,7 @@
     {
       "name": "invalid-macro",
       "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${noexist}"
+      "binaryDir": "$vendor{noexist}"
     },
     {
       "name": "ListPresetsHidden",
diff --git a/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt b/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
index 08fa87bb2d..7dafe623df 100644
--- a/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "NoSuchMacro": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/NoSuchMacro: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in b/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in
new file mode 100644
index 0000000000..94d0b764b8
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in
@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "NoSuchMacro",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${noexist}"
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake b/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
index e0e54f4beb..dddf05fcf5 100644
--- a/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
@@ -121,6 +121,13 @@ run_cmake_presets(InvalidCMakeGeneratorConfig)
 run_cmake_presets(UnknownCMakeGeneratorConfig)
 run_cmake_presets(EmptyCacheKey)
 run_cmake_presets(EmptyEnvKey)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
+run_cmake_presets(UnclosedMacro)
+run_cmake_presets(NoSuchMacro)
+run_cmake_presets(EnvCycle)
+run_cmake_presets(EmptyEnv)
+run_cmake_presets(EmptyPenv)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 
 # Test cmakeMinimumRequired field
 run_cmake_presets(MinimumRequiredInvalid)
@@ -170,13 +177,8 @@ run_cmake_presets(GoodInheritanceMultiSecond)
 run_cmake_presets(GoodInheritanceMacro)
 
 # Test bad preset arguments
-run_cmake_presets(UnclosedMacro)
-run_cmake_presets(NoSuchMacro)
 run_cmake_presets(VendorMacro)
 run_cmake_presets(InvalidGenerator)
-run_cmake_presets(EnvCycle)
-run_cmake_presets(EmptyEnv)
-run_cmake_presets(EmptyPenv)
 
 # Test Visual Studio-specific stuff
 if(RunCMake_GENERATOR MATCHES "^Visual Studio ")
diff --git a/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt b/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
index 248510d3c7..f9481f091d 100644
--- a/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "UnclosedMacro": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/UnclosedMacro: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in b/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in
new file mode 100644
index 0000000000..ad6cf7dc9b
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in
@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "UnclosedMacro",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir"
+    }
+  ]
+}