feat: Support depend mode for MSVC (#992)

Based on -showIncludes, which prints included files on stdout with a
certain text prefix. Otherwise pretty similar to depend mode handling
for GCC.

This makes MSVC building way faster.

Co-authored-by: Luboš Luňák <l.lunak@centrum.cz>

12 files changed, 272 insertions, 12 deletions

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 7ba37a04..37e0226a 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -808,6 +808,13 @@ file in `/etc/rsyslog.d`:
     Gi, Ti (binary). The default suffix is G. See also
     _<<Cache size management>>_.
 
+[#config_msvc_dep_prefix]
+*msvc_dep_prefix* (*CCACHE_MSVC_DEP_PREFIX*)::
+
+    This option specifies the prefix of included files output for MSVC compiler.
+    The default prefix is "Note: including file:", which is the output for English,
+    but if you use a localized compiler, this should be set accordingly.
+
 [#config_namespace]
 *namespace* (*CCACHE_NAMESPACE*)::
 
@@ -1549,7 +1556,8 @@ The direct mode will be disabled if any of the following holds:
 If the depend mode is enabled, ccache will not use the preprocessor at all. The
 hash used to identify results in the cache will be based on the direct mode
 hash described above plus information about include files read from the
-dependency file generated by the compiler with `-MD` or `-MMD`.
+dependency list generated by MSVC with /showIncludes, or the dependency file
+generated by other compilers with `-MD` or `-MMD`.
 
 Advantages:
 
@@ -1566,7 +1574,7 @@ Disadvantages:
   to some types of changes of compiler options and source code changes.
 * If -MD is used, the manifest entries will include system header files as
   well, thus slowing down cache hits slightly, just as using -MD slows down
-  make.
+  make. This is also the case for MSVC with /showIncludes.
 * If -MMD is used, the manifest entries will not include system header files,
   which means ccache will ignore changes in them.
 
@@ -1574,7 +1582,8 @@ The depend mode will be disabled if any of the following holds:
 
 * <<config_depend_mode,*depend_mode*>> is false.
 * <<config_run_second_cpp,*run_second_cpp*>> is false.
-* The compiler is not generating dependencies using `-MD` or `-MMD`.
+* The compiler is not generating dependencies using `-MD` or `-MMD` (for MSVC,
+  /showIncludes).
 
 
 == Handling of newly created header files
diff --git a/src/ArgsInfo.hpp b/src/ArgsInfo.hpp
index 92d089b5..ce96ceed 100644
--- a/src/ArgsInfo.hpp
+++ b/src/ArgsInfo.hpp
@@ -75,6 +75,9 @@ struct ArgsInfo
   // Is the compiler being asked to output dependencies?
   bool generating_dependencies = false;
 
+  // Is the compiler being asked to output includes (MSVC -showIncludes)?
+  bool generating_includes = false;
+
   // The dependency target in the dependency file (the object file unless
   // overridden via e.g. -MT or -MQ).
   std::optional<std::string> dependency_target;
diff --git a/src/Config.cpp b/src/Config.cpp
index 520557e1..3cec222f 100644
--- a/src/Config.cpp
+++ b/src/Config.cpp
@@ -83,6 +83,7 @@ enum class ConfigItem {
   log_file,
   max_files,
   max_size,
+  msvc_dep_prefix,
   namespace_,
   path,
   pch_external_checksum,
@@ -138,6 +139,7 @@ const std::unordered_map<std::string, ConfigKeyTableEntry> k_config_key_table =
     {"log_file", {ConfigItem::log_file}},
     {"max_files", {ConfigItem::max_files}},
     {"max_size", {ConfigItem::max_size}},
+    {"msvc_dep_prefix", {ConfigItem::msvc_dep_prefix}},
     {"namespace", {ConfigItem::namespace_}},
     {"path", {ConfigItem::path}},
     {"pch_external_checksum", {ConfigItem::pch_external_checksum}},
@@ -187,6 +189,7 @@ const std::unordered_map<std::string, std::string> k_env_variable_table = {
   {"LOGFILE", "log_file"},
   {"MAXFILES", "max_files"},
   {"MAXSIZE", "max_size"},
+  {"MSVC_DEP_PREFIX", "msvc_dep_prefix"},
   {"NAMESPACE", "namespace"},
   {"PATH", "path"},
   {"PCH_EXTSUM", "pch_external_checksum"},
@@ -759,6 +762,9 @@ Config::get_string_value(const std::string& key) const
   case ConfigItem::max_size:
     return format_cache_size(m_max_size);
 
+  case ConfigItem::msvc_dep_prefix:
+    return m_msvc_dep_prefix;
+
   case ConfigItem::namespace_:
     return m_namespace;
 
@@ -1007,6 +1013,10 @@ Config::set_item(const std::string& key,
     m_max_size = Util::parse_size(value);
     break;
 
+  case ConfigItem::msvc_dep_prefix:
+    m_msvc_dep_prefix = Util::expand_environment_variables(value);
+    break;
+
   case ConfigItem::namespace_:
     m_namespace = Util::expand_environment_variables(value);
     break;
diff --git a/src/Config.hpp b/src/Config.hpp
index 7a01b35e..6399253f 100644
--- a/src/Config.hpp
+++ b/src/Config.hpp
@@ -77,6 +77,7 @@ public:
   const std::string& log_file() const;
   uint64_t max_files() const;
   uint64_t max_size() const;
+  const std::string& msvc_dep_prefix() const;
   const std::string& path() const;
   bool pch_external_checksum() const;
   const std::string& prefix_command() const;
@@ -185,6 +186,7 @@ private:
   std::string m_log_file;
   uint64_t m_max_files = 0;
   uint64_t m_max_size = 5ULL * 1000 * 1000 * 1000;
+  std::string m_msvc_dep_prefix = "Note: including file:";
   std::string m_path;
   bool m_pch_external_checksum = false;
   std::string m_prefix_command;
@@ -386,6 +388,12 @@ Config::max_size() const
 }
 
 inline const std::string&
+Config::msvc_dep_prefix() const
+{
+  return m_msvc_dep_prefix;
+}
+
+inline const std::string&
 Config::path() const
 {
   return m_path;
diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp
index 412787c5..e1ec1121 100644
--- a/src/argprocessing.cpp
+++ b/src/argprocessing.cpp
@@ -684,6 +684,12 @@ process_option_arg(const Context& ctx,
     return Statistic::none;
   }
 
+  if (args[i] == "-showIncludes") {
+    args_info.generating_includes = true;
+    state.dep_args.push_back(args[i]);
+    return Statistic::none;
+  }
+
   if (args[i] == "-fprofile-arcs") {
     args_info.profile_arcs = true;
     state.common_args.push_back(args[i]);
diff --git a/src/ccache.cpp b/src/ccache.cpp
index f6841265..6effb648 100644
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -46,6 +46,7 @@
 #include <core/Manifest.hpp>
 #include <core/Result.hpp>
 #include <core/ResultRetriever.hpp>
+#include <core/ShowIncludesParser.hpp>
 #include <core/Statistics.hpp>
 #include <core/StatsLog.hpp>
 #include <core/exceptions.hpp>
@@ -692,6 +693,34 @@ struct DoExecuteResult
   util::Bytes stderr_data;
 };
 
+// Extract the used includes from -showIncludes output in stdout. Note that we
+// cannot distinguish system headers from other includes here.
+static std::optional<Digest>
+result_key_from_includes(Context& ctx, Hash& hash, std::string_view stdout_data)
+{
+  for (std::string_view token : core::ShowIncludesParser::tokenize(
+         stdout_data, ctx.config.msvc_dep_prefix())) {
+    const std::string path = Util::make_relative_path(ctx, token);
+    remember_include_file(ctx, path, hash, false, &hash);
+  }
+
+  // Explicitly check the .pch file as it is not mentioned in the
+  // includes output.
+  if (!ctx.args_info.included_pch_file.empty()) {
+    std::string pch_path =
+      Util::make_relative_path(ctx, ctx.args_info.included_pch_file);
+    hash.hash(pch_path);
+    remember_include_file(ctx, pch_path, hash, false, nullptr);
+  }
+
+  const bool debug_included = getenv("CCACHE_DEBUG_INCLUDED");
+  if (debug_included) {
+    print_included_files(ctx, stdout);
+  }
+
+  return hash.digest();
+}
+
 // Execute the compiler/preprocessor, with logic to retry without requesting
 // colored diagnostics messages if that fails.
 static nonstd::expected<DoExecuteResult, Failure>
@@ -935,10 +964,10 @@ rewrite_stdout_from_compiler(const Context& ctx, util::Bytes&& stdout_data)
       // to check if a file needs to be recompiled.
       else if (ctx.config.compiler_type() == CompilerType::msvc
                && !ctx.config.base_dir().empty()
-               && util::starts_with(line, "Note: including file:")) {
+               && util::starts_with(line, ctx.config.msvc_dep_prefix())) {
         std::string orig_line(line.data(), line.length());
         std::string abs_inc_path =
-          util::replace_first(orig_line, "Note: including file:", "");
+          util::replace_first(orig_line, ctx.config.msvc_dep_prefix(), "");
         abs_inc_path = util::strip_whitespace(abs_inc_path);
         std::string rel_inc_path = Util::make_relative_path(
           ctx, Util::normalize_concrete_absolute_path(abs_inc_path));
@@ -1055,7 +1084,14 @@ to_cache(Context& ctx,
 
   if (ctx.config.depend_mode()) {
     ASSERT(depend_mode_hash);
-    result_key = result_key_from_depfile(ctx, *depend_mode_hash);
+    if (ctx.args_info.generating_dependencies) {
+      result_key = result_key_from_depfile(ctx, *depend_mode_hash);
+    } else if (ctx.args_info.generating_includes) {
+      result_key = result_key_from_includes(
+        ctx, *depend_mode_hash, util::to_string_view(result->stdout_data));
+    } else {
+      ASSERT(false);
+    }
     if (!result_key) {
       return nonstd::make_unexpected(Statistic::internal_error);
     }
@@ -2300,12 +2336,14 @@ do_cache_compilation(Context& ctx, const char* const* argv)
     ctx.config.set_run_second_cpp(true);
   }
 
-  if (ctx.config.depend_mode()
-      && (!ctx.args_info.generating_dependencies
-          || ctx.args_info.output_dep == "/dev/null"
-          || !ctx.config.run_second_cpp())) {
-    LOG_RAW("Disabling depend mode");
-    ctx.config.set_depend_mode(false);
+  if (ctx.config.depend_mode()) {
+    const bool deps = ctx.args_info.generating_dependencies
+                      && ctx.args_info.output_dep != "/dev/null";
+    const bool includes = ctx.args_info.generating_includes;
+    if (!ctx.config.run_second_cpp() || (!deps && !includes)) {
+      LOG_RAW("Disabling depend mode");
+      ctx.config.set_depend_mode(false);
+    }
   }
 
   if (ctx.storage.has_remote_storage()) {
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index d2431be3..6932336f 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -6,6 +6,7 @@ set(
   ResultExtractor.cpp
   ResultInspector.cpp
   ResultRetriever.cpp
+  ShowIncludesParser.cpp
   Statistics.cpp
   StatisticsCounters.cpp
   StatsLog.cpp
diff --git a/src/core/ShowIncludesParser.cpp b/src/core/ShowIncludesParser.cpp
new file mode 100644
index 00000000..a25cf76a
--- /dev/null
+++ b/src/core/ShowIncludesParser.cpp
@@ -0,0 +1,52 @@
+// Copyright (C) 2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ShowIncludesParser.hpp"
+
+#include <Context.hpp>
+#include <Util.hpp>
+#include <util/string.hpp>
+
+namespace core::ShowIncludesParser {
+
+std::vector<std::string_view>
+tokenize(std::string_view file_content, std::string_view prefix)
+{
+  // -showIncludes output is written to stdout together with other messages.
+  // Every line of it is '<prefix> <spaces> <file>', prefix is 'Note: including
+  // file:' in English but can be localized.
+
+  std::vector<std::string_view> result;
+  // This will split at each \r or \n, but that simply means there will be empty
+  // "lines".
+  for (std::string_view line : Util::split_into_views(file_content, "\r\n")) {
+    if (util::starts_with(line, prefix)) {
+      size_t pos = prefix.size();
+      while (pos < line.size() && isspace(line[pos])) {
+        ++pos;
+      }
+      std::string_view include = line.substr(pos);
+      if (!include.empty()) {
+        result.push_back(include);
+      }
+    }
+  }
+  return result;
+}
+
+} // namespace core::ShowIncludesParser
diff --git a/src/core/ShowIncludesParser.hpp b/src/core/ShowIncludesParser.hpp
new file mode 100644
index 00000000..a977cec0
--- /dev/null
+++ b/src/core/ShowIncludesParser.hpp
@@ -0,0 +1,31 @@
+// Copyright (C) 2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <string_view>
+#include <vector>
+
+class Context;
+
+namespace core::ShowIncludesParser {
+
+std::vector<std::string_view> tokenize(std::string_view file_content,
+                                       std::string_view prefix);
+
+} // namespace core::ShowIncludesParser
diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt
index 8b7c8d30..7b9a58dd 100644
--- a/unittest/CMakeLists.txt
+++ b/unittest/CMakeLists.txt
@@ -13,6 +13,7 @@ set(
   test_ccache.cpp
   test_compopt.cpp
   test_compression_types.cpp
+  test_core_ShowIncludesParser
   test_core_Statistics.cpp
   test_core_StatisticsCounters.cpp
   test_core_StatsLog.cpp
diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp
index 0226101f..559cc1b1 100644
--- a/unittest/test_Config.cpp
+++ b/unittest/test_Config.cpp
@@ -64,6 +64,7 @@ TEST_CASE("Config: default values")
   CHECK(config.log_file().empty());
   CHECK(config.max_files() == 0);
   CHECK(config.max_size() == static_cast<uint64_t>(5) * 1000 * 1000 * 1000);
+  CHECK(config.msvc_dep_prefix() == "Note: including file:");
   CHECK(config.path().empty());
   CHECK_FALSE(config.pch_external_checksum());
   CHECK(config.prefix_command().empty());
@@ -122,6 +123,7 @@ TEST_CASE("Config::update_from_file")
     "log_file = $USER${USER} \n"
     "max_files = 17\n"
     "max_size = 123M\n"
+    "msvc_dep_prefix = Some other prefix:\n"
     "path = $USER.x\n"
     "pch_external_checksum = true\n"
     "prefix_command = x$USER\n"
@@ -162,6 +164,7 @@ TEST_CASE("Config::update_from_file")
   CHECK(config.log_file() == FMT("{0}{0}", user));
   CHECK(config.max_files() == 17);
   CHECK(config.max_size() == 123 * 1000 * 1000);
+  CHECK(config.msvc_dep_prefix() == "Some other prefix:");
   CHECK(config.path() == FMT("{}.x", user));
   CHECK(config.pch_external_checksum());
   CHECK(config.prefix_command() == FMT("x{}", user));
@@ -406,6 +409,7 @@ TEST_CASE("Config::visit_items")
     "log_file = lf\n"
     "max_files = 4711\n"
     "max_size = 98.7M\n"
+    "msvc_dep_prefix = mdp\n"
     "namespace = ns\n"
     "path = p\n"
     "pch_external_checksum = true\n"
@@ -467,6 +471,7 @@ TEST_CASE("Config::visit_items")
     "(test.conf) log_file = lf",
     "(test.conf) max_files = 4711",
     "(test.conf) max_size = 98.7M",
+    "(test.conf) msvc_dep_prefix = mdp",
     "(test.conf) namespace = ns",
     "(test.conf) path = p",
     "(test.conf) pch_external_checksum = true",
diff --git a/unittest/test_core_ShowIncludesParser.cpp b/unittest/test_core_ShowIncludesParser.cpp
new file mode 100644
index 00000000..c3ba58c5
--- /dev/null
+++ b/unittest/test_core_ShowIncludesParser.cpp
@@ -0,0 +1,96 @@
+// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/core/ShowIncludesParser.hpp"
+#include "TestUtil.hpp"
+
+#include "third_party/doctest.h"
+
+static const std::string defaultPrefix = "Note: including file:";
+
+TEST_SUITE_BEGIN("ShowIncludesParser");
+
+TEST_CASE("ShowIncludesParser::tokenize")
+{
+  SUBCASE("Parse empty output")
+  {
+    std::string contents;
+    const auto result =
+      core::ShowIncludesParser::tokenize(contents, defaultPrefix);
+    CHECK(result.size() == 0);
+  }
+
+  SUBCASE("Parse real output")
+  {
+    std::string contents = R"(Just a line
+Note: including file: F:/Projects/ccache/build-msvc/config.h
+Note: including file: F:\Projects\ccache\unittest\../src/Context.hpp
+Note: including file:  F:\Projects\ccache\src\Args.hpp
+Note: including file:   F:\Projects\ccache\src\NonCopyable.hpp
+Note: including file:   C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.33.31629\include\deque
+)";
+    const auto result =
+      core::ShowIncludesParser::tokenize(contents, defaultPrefix);
+    REQUIRE(result.size() == 5);
+    CHECK(result[0] == "F:/Projects/ccache/build-msvc/config.h");
+    CHECK(result[1] == R"(F:\Projects\ccache\unittest\../src/Context.hpp)");
+    CHECK(result[2] == R"(F:\Projects\ccache\src\Args.hpp)");
+    CHECK(result[3] == R"(F:\Projects\ccache\src\NonCopyable.hpp)");
+    CHECK(
+      result[4]
+      == R"(C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.33.31629\include\deque)");
+  }
+
+  SUBCASE("Parse output with CRLF")
+  {
+    std::string contents =
+      "Note: including file: foo\r\n"
+      "Note: including file: bar\r\n";
+    const auto result =
+      core::ShowIncludesParser::tokenize(contents, defaultPrefix);
+    REQUIRE(result.size() == 2);
+    CHECK(result[0] == "foo");
+    CHECK(result[1] == "bar");
+  }
+
+  SUBCASE("Parse output with an empty entry")
+  {
+    std::string contents =
+      "Note: including file: foo\n"
+      "Note: including file: \n"
+      "Note: including file:  bar\n";
+    const auto result =
+      core::ShowIncludesParser::tokenize(contents, defaultPrefix);
+    REQUIRE(result.size() == 2);
+    CHECK(result[0] == "foo");
+    CHECK(result[1] == "bar");
+  }
+
+  SUBCASE("Parse output with a custom prefix")
+  {
+    std::string contents = R"(custom foo
+custom   bar
+Just a line with custom in the middle)";
+    const auto result = core::ShowIncludesParser::tokenize(contents, "custom");
+    REQUIRE(result.size() == 2);
+    CHECK(result[0] == "foo");
+    CHECK(result[1] == "bar");
+  }
+}
+
+TEST_SUITE_END();