Ninja: Match showIncludes dependencies using console output code page

Generalize the fix from commit 37a279f8d1 (Ninja: Write msvc_deps_prefix
as UTF-8 when console codepage is UTF-8, 2020-07-31, v3.19.0-rc1~349^2).
`cl /showIncludes` output is encoded using the console output code page,
so this is the byte sequence that Ninja must use to match its lines.

Fixes: #24068

14 files changed, 110 insertions, 22 deletions

diff --git a/Modules/CMakeDetermineCompilerId.cmake b/Modules/CMakeDetermineCompilerId.cmake
index 6b5854941f..73c775aeb3 100644
--- a/Modules/CMakeDetermineCompilerId.cmake
+++ b/Modules/CMakeDetermineCompilerId.cmake
@@ -1127,7 +1127,7 @@ function(CMAKE_DETERMINE_MSVC_SHOWINCLUDES_PREFIX lang userflags)
     OUTPUT_VARIABLE out
     ERROR_VARIABLE err
     RESULT_VARIABLE res
-    ENCODING AUTO # cl prints in current code page
+    ENCODING AUTO # cl prints in console output code page
     )
   if(res EQUAL 0 AND "${out}" MATCHES "(^|\n)([^:\n]*:[^:\n]*:[ \t]*)")
     set(CMAKE_${lang}_CL_SHOWINCLUDES_PREFIX "${CMAKE_MATCH_2}" PARENT_SCOPE)
diff --git a/Source/cmLocalNinjaGenerator.cxx b/Source/cmLocalNinjaGenerator.cxx
index 106f76bf8e..c11f5b4138 100644
--- a/Source/cmLocalNinjaGenerator.cxx
+++ b/Source/cmLocalNinjaGenerator.cxx
@@ -88,27 +88,11 @@ void cmLocalNinjaGenerator::Generate()
       cmGlobalNinjaGenerator::WriteComment(this->GetRulesFileStream(),
                                            "localized /showIncludes string");
       this->GetRulesFileStream() << "msvc_deps_prefix = ";
-#ifdef _WIN32
-      // Ninja uses the ANSI Windows APIs, so strings in the rules file
-      // typically need to be ANSI encoded. However, in this case the compiler
-      // is being invoked using the UTF-8 codepage so the /showIncludes prefix
-      // will be UTF-8 encoded on stdout. Ninja can't successfully compare this
-      // UTF-8 encoded prefix to the ANSI encoded msvc_deps_prefix if it
-      // contains any non-ASCII characters and dependency checking will fail.
-      // As a workaround, leave the msvc_deps_prefix UTF-8 encoded even though
-      // the rest of the file is ANSI encoded.
-      if (GetConsoleOutputCP() == CP_UTF8 && GetACP() != CP_UTF8 &&
-          this->GetGlobalGenerator()->GetMakefileEncoding() != codecvt::None) {
-        this->GetRulesFileStream().WriteRaw(showIncludesPrefix);
-      } else {
-        // Ninja 1.11 and above uses the UTF-8 code page if it's supported, so
-        // in that case we can write it normally without using raw bytes.
-        this->GetRulesFileStream() << showIncludesPrefix;
-      }
-#else
-      // It's safe to use the standard encoding on other platforms.
-      this->GetRulesFileStream() << showIncludesPrefix;
-#endif
+      // 'cl /showIncludes' encodes output in the console output code page.
+      // It may differ from the encoding used for file paths in 'build.ninja'.
+      // Ninja matches the showIncludes prefix using its raw byte sequence.
+      this->GetRulesFileStream().WriteAltEncoding(
+        showIncludesPrefix, cmGeneratedFileStream::Encoding::ConsoleOutput);
       this->GetRulesFileStream() << "\n\n";
     }
   }
diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index 6c5ab7fdc3..3038ed8468 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -197,6 +197,10 @@ if(CMAKE_GENERATOR MATCHES "Ninja")
       ${ninja_qt_args}
     )
   endif()
+  if(WIN32)
+    add_executable(showIncludes showIncludes.c)
+    list(APPEND Ninja_ARGS -DshowIncludes=$<TARGET_FILE:showIncludes>)
+  endif()
   add_RunCMake_test(Ninja)
   set(NinjaMultiConfig_ARGS
     -DCYGWIN=${CYGWIN} -DMSYS=${MSYS}
diff --git a/Tests/RunCMake/Ninja/RunCMakeTest.cmake b/Tests/RunCMake/Ninja/RunCMakeTest.cmake
index 44a7ba1d32..b94466ce8c 100644
--- a/Tests/RunCMake/Ninja/RunCMakeTest.cmake
+++ b/Tests/RunCMake/Ninja/RunCMakeTest.cmake
@@ -42,6 +42,15 @@ function(run_Intl)
 endfunction()
 run_Intl()
 
+if(WIN32)
+  if(RunCMake_MAKE_PROGRAM)
+    set(maybe_MAKE_PROGRAM "-DRunCMake_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+  endif()
+  run_cmake_script(ShowIncludes-54936 -DshowIncludes=${showIncludes} ${maybe_MAKE_PROGRAM})
+  run_cmake_script(ShowIncludes-65001 -DshowIncludes=${showIncludes} ${maybe_MAKE_PROGRAM})
+  unset(maybe_MAKE_PROGRAM)
+endif()
+
 function(run_NoWorkToDo)
   run_cmake(NoWorkToDo)
   set(RunCMake_TEST_NO_CLEAN 1)
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-54936-check.cmake b/Tests/RunCMake/Ninja/ShowIncludes-54936-check.cmake
new file mode 100644
index 0000000000..40bb68f6d6
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-54936-check.cmake
@@ -0,0 +1,3 @@
+# 'cl /showIncludes' prefix with 'VSLANG=2052' and 'chcp 54936'.
+string(ASCII 215 162 210 226 58 32 176 252 186 172 206 196 188 254 58 expect)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-check.cmake)
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-54936-stdout.txt b/Tests/RunCMake/Ninja/ShowIncludes-54936-stdout.txt
new file mode 100644
index 0000000000..42a2f35aca
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-54936-stdout.txt
@@ -0,0 +1 @@
+-- showIncludes='注意: 包含文件:'
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-54936.cmake b/Tests/RunCMake/Ninja/ShowIncludes-54936.cmake
new file mode 100644
index 0000000000..07b4192c77
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-54936.cmake
@@ -0,0 +1,2 @@
+set(CODEPAGE 54936)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes.cmake)
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-65001-check.cmake b/Tests/RunCMake/Ninja/ShowIncludes-65001-check.cmake
new file mode 100644
index 0000000000..c73b7344b5
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-65001-check.cmake
@@ -0,0 +1,3 @@
+# 'cl /showIncludes' prefix with 'VSLANG=2052' and 'chcp 65001'.
+string(ASCII 230 179 168 230 132 143 58 32 229 140 133 229 144 171 230 150 135 228 187 182 58 expect)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-check.cmake)
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-65001-stdout.txt b/Tests/RunCMake/Ninja/ShowIncludes-65001-stdout.txt
new file mode 100644
index 0000000000..42a2f35aca
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-65001-stdout.txt
@@ -0,0 +1 @@
+-- showIncludes='注意: 包含文件:'
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-65001.cmake b/Tests/RunCMake/Ninja/ShowIncludes-65001.cmake
new file mode 100644
index 0000000000..0eebd61659
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-65001.cmake
@@ -0,0 +1,2 @@
+set(CODEPAGE 65001)
+include(${CMAKE_CURRENT_LIST_DIR}/ShowIncludes.cmake)
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-check.cmake b/Tests/RunCMake/Ninja/ShowIncludes-check.cmake
new file mode 100644
index 0000000000..304a7f689c
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-check.cmake
@@ -0,0 +1,17 @@
+set(rules_ninja "${RunCMake_TEST_BINARY_DIR}/CMakeFiles/rules.ninja")
+if(NOT EXISTS "${rules_ninja}")
+  set(RunCMake_TEST_FAILED "Generator output file is missing:\n ${rules_ninja}")
+  return()
+endif()
+
+file(READ "${rules_ninja}" rules_ninja)
+if(rules_ninja MATCHES "msvc_deps_prefix = ([^\r\n]*)\n")
+  set(actual "${CMAKE_MATCH_1}")
+endif()
+
+if(NOT actual STREQUAL expect)
+  string(HEX "${actual}" actual_hex)
+  string(HEX "${expect}" expect_hex)
+  set(RunCMake_TEST_FAILED "Expected byte sequence\n '${expect}' (${expect_hex})\nbut got\n '${actual}' (${actual_hex})")
+  return()
+endif()
diff --git a/Tests/RunCMake/Ninja/ShowIncludes-cmake.cmake b/Tests/RunCMake/Ninja/ShowIncludes-cmake.cmake
new file mode 100644
index 0000000000..672a89f38c
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes-cmake.cmake
@@ -0,0 +1,7 @@
+# Set the console code page.
+execute_process(COMMAND cmd /c chcp "${CODEPAGE}")
+
+if(RunCMake_MAKE_PROGRAM)
+  set(maybe_MAKE_PROGRAM "-DCMAKE_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+endif()
+execute_process(COMMAND "${CMAKE_COMMAND}" . -G Ninja ${maybe_MAKE_PROGRAM})
diff --git a/Tests/RunCMake/Ninja/ShowIncludes.cmake b/Tests/RunCMake/Ninja/ShowIncludes.cmake
new file mode 100644
index 0000000000..b9f89fe241
--- /dev/null
+++ b/Tests/RunCMake/Ninja/ShowIncludes.cmake
@@ -0,0 +1,22 @@
+# Create a project to do showIncludes prefix detection.
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt" "
+cmake_minimum_required(VERSION 3.25)
+project(ShowIncludes NONE)
+include(CMakeDetermineCompilerId)
+set(CMAKE_dummy_COMPILER \"${showIncludes}\")
+CMAKE_DETERMINE_MSVC_SHOWINCLUDES_PREFIX(dummy \"\")
+set(CMAKE_CL_SHOWINCLUDES_PREFIX \"\${CMAKE_dummy_CL_SHOWINCLUDES_PREFIX}\")
+file(WRITE \"\${CMAKE_CURRENT_BINARY_DIR}/showIncludes.txt\" \"\${CMAKE_CL_SHOWINCLUDES_PREFIX}\")
+")
+
+if(RunCMake_MAKE_PROGRAM)
+  set(maybe_MAKE_PROGRAM "-DRunCMake_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
+endif()
+
+# Run cmake in a new Window to isolate its console code page.
+execute_process(COMMAND cmd /c start /min /wait ""
+  ${CMAKE_COMMAND} -DCODEPAGE=${CODEPAGE} ${maybe_MAKE_PROGRAM} -P ${CMAKE_CURRENT_LIST_DIR}/ShowIncludes-cmake.cmake)
+
+# Print our internal UTF-8 representation of the showIncludes prefix.
+file(READ "${CMAKE_CURRENT_BINARY_DIR}/showIncludes.txt" showIncludes_txt)
+message(STATUS "showIncludes='${showIncludes_txt}'")
diff --git a/Tests/RunCMake/showIncludes.c b/Tests/RunCMake/showIncludes.c
new file mode 100644
index 0000000000..23b38458dc
--- /dev/null
+++ b/Tests/RunCMake/showIncludes.c
@@ -0,0 +1,33 @@
+#if defined(_MSC_VER) && _MSC_VER >= 1928
+#  pragma warning(disable : 5105) /* macro expansion warning in windows.h */
+#endif
+#include <windows.h>
+
+#include <stdio.h>
+
+int main()
+{
+  /* 'cl /showIncludes' encodes output in the console output code page.  */
+  unsigned int cp = GetConsoleOutputCP();
+  printf("Console output code page: %u\n", cp);
+  printf("Console input code page: %u\n", GetConsoleCP());
+  printf("ANSI code page: %u\n", GetACP());
+  printf("OEM code page: %u\n", GetOEMCP());
+
+  if (cp == 54936 || cp == 936) {
+    /* VSLANG=2052 */
+    printf("\xd7\xa2\xd2\xe2: "
+           "\xb0\xfc\xba\xac\xce\xc4\xbc\xfe:\n");
+    return 0;
+  }
+
+  if (cp == 65001) {
+    /* VSLANG=2052  */
+    printf("\xe6\xb3\xa8\xe6\x84\x8f: "
+           "\xe5\x8c\x85\xe5\x90\xab\xe6\x96\x87\xe4\xbb\xb6:\n");
+    return 0;
+  }
+
+  fprintf(stderr, "No example showIncludes for console's output code page.\n");
+  return 1;
+}