Merge pull request #1685 from jhasse/restat

Add restat tool which recalculates all mtimes in the build log

9 files changed, 205 insertions, 24 deletions

diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index e49d26d..8f42efb 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -283,6 +283,9 @@ target, show just the target's dependencies. _Available since Ninja 1.4._
 
 `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._
 
+`restat`:: updates all recorded file modification timestamps in the `.ninja_log`
+file. _Available since Ninja 1.10._
+
 `rules`:: output the list of all rules (eventually with their description
 if they have one).  It can be used to know which rule name to pass to
 +ninja -t targets rule _name_+ or +ninja -t compdb+.
diff --git a/misc/output_test.py b/misc/output_test.py
index 966417d..3fd9c32 100755
--- a/misc/output_test.py
+++ b/misc/output_test.py
@@ -102,5 +102,10 @@ red
 \x1b[31mred\x1b[0m
 ''')
 
+    def test_pr_1685(self):
+        # Running those tools without .ninja_deps and .ninja_log shouldn't fail.
+        self.assertEqual(run('', flags='-t recompact'), '')
+        self.assertEqual(run('', flags='-t restat'), '')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/src/build_log.cc b/src/build_log.cc
index c4a08a0..e2a9344 100644
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -21,6 +21,7 @@
 #endif
 
 #include "build_log.h"
+#include "disk_interface.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -241,14 +242,14 @@ struct LineReader {
   char* line_end_;
 };
 
-bool BuildLog::Load(const string& path, string* err) {
+LoadStatus BuildLog::Load(const string& path, string* err) {
   METRIC_RECORD(".ninja_log load");
   FILE* file = fopen(path.c_str(), "r");
   if (!file) {
     if (errno == ENOENT)
-      return true;
+      return LOAD_NOT_FOUND;
     *err = strerror(errno);
-    return false;
+    return LOAD_ERROR;
   }
 
   int log_version = 0;
@@ -269,7 +270,7 @@ bool BuildLog::Load(const string& path, string* err) {
         unlink(path.c_str());
         // Don't report this as a failure.  An empty build log will cause
         // us to rebuild the outputs anyway.
-        return true;
+        return LOAD_SUCCESS;
       }
     }
 
@@ -339,7 +340,7 @@ bool BuildLog::Load(const string& path, string* err) {
   fclose(file);
 
   if (!line_start) {
-    return true; // file was empty
+    return LOAD_SUCCESS; // file was empty
   }
 
   // Decide whether it's time to rebuild the log:
@@ -354,7 +355,7 @@ bool BuildLog::Load(const string& path, string* err) {
     needs_recompaction_ = true;
   }
 
-  return true;
+  return LOAD_SUCCESS;
 }
 
 BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
@@ -418,3 +419,50 @@ bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
 
   return true;
 }
+
+bool BuildLog::Restat(const StringPiece path,
+                      const DiskInterface& disk_interface,
+                      std::string* const err) {
+  METRIC_RECORD(".ninja_log restat");
+
+  Close();
+  std::string temp_path = path.AsString() + ".restat";
+  FILE* f = fopen(temp_path.c_str(), "wb");
+  if (!f) {
+    *err = strerror(errno);
+    return false;
+  }
+
+  if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
+    *err = strerror(errno);
+    fclose(f);
+    return false;
+  }
+  for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
+    const TimeStamp mtime = disk_interface.Stat(i->second->output, err);
+    if (mtime == -1) {
+      fclose(f);
+      return false;
+    }
+    i->second->mtime = mtime;
+
+    if (!WriteEntry(f, *i->second)) {
+      *err = strerror(errno);
+      fclose(f);
+      return false;
+    }
+  }
+
+  fclose(f);
+  if (unlink(path.str_) < 0) {
+    *err = strerror(errno);
+    return false;
+  }
+
+  if (rename(temp_path.c_str(), path.str_) < 0) {
+    *err = strerror(errno);
+    return false;
+  }
+
+  return true;
+}
diff --git a/src/build_log.h b/src/build_log.h
index 5268fab..ed59d79 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -20,9 +20,11 @@
 using namespace std;
 
 #include "hash_map.h"
+#include "load_status.h"
 #include "timestamp.h"
 #include "util.h"  // uint64_t
 
+struct DiskInterface;
 struct Edge;
 
 /// Can answer questions about the manifest for the BuildLog.
@@ -49,7 +51,7 @@ struct BuildLog {
   void Close();
 
   /// Load the on-disk log.
-  bool Load(const string& path, string* err);
+  LoadStatus Load(const string& path, string* err);
 
   struct LogEntry {
     string output;
@@ -81,6 +83,10 @@ struct BuildLog {
   /// Rewrite the known log entries, throwing away old data.
   bool Recompact(const string& path, const BuildLogUser& user, string* err);
 
+  /// Restat all outputs in the log
+  bool Restat(StringPiece path, const DiskInterface& disk_interface,
+              std::string* err);
+
   typedef ExternalStringHashMap<LogEntry*>::Type Entries;
   const Entries& entries() const { return entries_; }
 
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index ad30380..48ece23 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -25,6 +25,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 #endif
+#include <cassert>
 
 namespace {
 
@@ -150,7 +151,7 @@ TEST_F(BuildLogTest, Truncate) {
 
     BuildLog log3;
     err.clear();
-    ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty());
+    ASSERT_TRUE(log3.Load(kTestFilename, &err) == LOAD_SUCCESS || !err.empty());
   }
 }
 
@@ -216,6 +217,47 @@ TEST_F(BuildLogTest, DuplicateVersionHeader) {
   ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
 }
 
+struct TestDiskInterface : public DiskInterface {
+  virtual TimeStamp Stat(const string& path, string* err) const {
+    return 4;
+  }
+  virtual bool WriteFile(const string& path, const string& contents) {
+    assert(false);
+    return true;
+  }
+  virtual bool MakeDir(const string& path) {
+    assert(false);
+    return false;
+  }
+  virtual Status ReadFile(const string& path, string* contents, string* err) {
+    assert(false);
+    return NotFound;
+  }
+  virtual int RemoveFile(const string& path) {
+    assert(false);
+    return 0;
+  }
+};
+
+TEST_F(BuildLogTest, Restat) {
+  FILE* f = fopen(kTestFilename, "wb");
+  fprintf(f, "# ninja log v4\n"
+             "1\t2\t3\tout\tcommand\n");
+  fclose(f);
+  std::string err;
+  BuildLog log;
+  EXPECT_TRUE(log.Load(kTestFilename, &err));
+  ASSERT_EQ("", err);
+  BuildLog::LogEntry* e = log.LookupByOutput("out");
+  ASSERT_EQ(3, e->mtime);
+
+  TestDiskInterface testDiskInterface;
+  EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, &err));
+  ASSERT_EQ("", err);
+  e = log.LookupByOutput("out");
+  ASSERT_EQ(4, e->mtime);
+}
+
 TEST_F(BuildLogTest, VeryLongInputLine) {
   // Ninja's build log buffer is currently 256kB. Lines longer than that are
   // silently ignored, but don't affect parsing of other lines.
diff --git a/src/deps_log.cc b/src/deps_log.cc
index 4aaffeb..cf55194 100644
--- a/src/deps_log.cc
+++ b/src/deps_log.cc
@@ -167,15 +167,15 @@ void DepsLog::Close() {
   file_ = NULL;
 }
 
-bool DepsLog::Load(const string& path, State* state, string* err) {
+LoadStatus DepsLog::Load(const string& path, State* state, string* err) {
   METRIC_RECORD(".ninja_deps load");
   char buf[kMaxRecordSize + 1];
   FILE* f = fopen(path.c_str(), "rb");
   if (!f) {
     if (errno == ENOENT)
-      return true;
+      return LOAD_NOT_FOUND;
     *err = strerror(errno);
-    return false;
+    return LOAD_ERROR;
   }
 
   bool valid_header = true;
@@ -196,7 +196,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) {
     unlink(path.c_str());
     // Don't report this as a failure.  An empty deps log will cause
     // us to rebuild the outputs anyway.
-    return true;
+    return LOAD_SUCCESS;
   }
 
   long offset;
@@ -284,12 +284,12 @@ bool DepsLog::Load(const string& path, State* state, string* err) {
     fclose(f);
 
     if (!Truncate(path, offset, err))
-      return false;
+      return LOAD_ERROR;
 
     // The truncate succeeded; we'll just report the load error as a
     // warning because the build can proceed.
     *err += "; recovering";
-    return true;
+    return LOAD_SUCCESS;
   }
 
   fclose(f);
@@ -302,7 +302,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) {
     needs_recompaction_ = true;
   }
 
-  return true;
+  return LOAD_SUCCESS;
 }
 
 DepsLog::Deps* DepsLog::GetDeps(Node* node) {
diff --git a/src/deps_log.h b/src/deps_log.h
index 3812a28..e7974a1 100644
--- a/src/deps_log.h
+++ b/src/deps_log.h
@@ -21,6 +21,7 @@ using namespace std;
 
 #include <stdio.h>
 
+#include "load_status.h"
 #include "timestamp.h"
 
 struct Node;
@@ -84,7 +85,7 @@ struct DepsLog {
     int node_count;
     Node** nodes;
   };
-  bool Load(const string& path, State* state, string* err);
+  LoadStatus Load(const string& path, State* state, string* err);
   Deps* GetDeps(Node* node);
 
   /// Rewrite the known log entries, throwing away old data.
diff --git a/src/load_status.h b/src/load_status.h
new file mode 100644
index 0000000..0b16b1a
--- /dev/null
+++ b/src/load_status.h
@@ -0,0 +1,24 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_LOAD_STATUS_H_
+#define NINJA_LOAD_STATUS_H_
+
+enum LoadStatus {
+  LOAD_ERROR,
+  LOAD_SUCCESS,
+  LOAD_NOT_FOUND,
+};
+
+#endif  // NINJA_LOAD_STATUS_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index 19646d9..0c90e74 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -17,6 +17,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <cstdlib>
 
 #ifdef _WIN32
 #include "getopt.h"
@@ -125,15 +126,16 @@ struct NinjaMain : public BuildLogUser {
   int ToolClean(const Options* options, int argc, char* argv[]);
   int ToolCompilationDatabase(const Options* options, int argc, char* argv[]);
   int ToolRecompact(const Options* options, int argc, char* argv[]);
+  int ToolRestat(const Options* options, int argc, char* argv[]);
   int ToolUrtle(const Options* options, int argc, char** argv);
   int ToolRules(const Options* options, int argc, char* argv[]);
 
   /// Open the build log.
-  /// @return false on error.
+  /// @return LOAD_ERROR on error.
   bool OpenBuildLog(bool recompact_only = false);
 
   /// Open the deps log: load it, then open for writing.
-  /// @return false on error.
+  /// @return LOAD_ERROR on error.
   bool OpenDepsLog(bool recompact_only = false);
 
   /// Ensure the build directory exists, creating it if necessary.
@@ -847,13 +849,53 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) {
   if (!EnsureBuildDirExists())
     return 1;
 
-  if (!OpenBuildLog(/*recompact_only=*/true) ||
-      !OpenDepsLog(/*recompact_only=*/true))
+  if (OpenBuildLog(/*recompact_only=*/true) == LOAD_ERROR ||
+      OpenDepsLog(/*recompact_only=*/true) == LOAD_ERROR)
     return 1;
 
   return 0;
 }
 
+int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) {
+  if (!EnsureBuildDirExists())
+    return 1;
+
+  string log_path = ".ninja_log";
+  if (!build_dir_.empty())
+    log_path = build_dir_ + "/" + log_path;
+
+  string err;
+  const LoadStatus status = build_log_.Load(log_path, &err);
+  if (status == LOAD_ERROR) {
+    Error("loading build log %s: %s", log_path.c_str(), err.c_str());
+    return EXIT_FAILURE;
+  }
+  if (status == LOAD_NOT_FOUND) {
+    // Nothing to restat, ignore this
+    return EXIT_SUCCESS;
+  }
+  if (!err.empty()) {
+    // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
+    Warning("%s", err.c_str());
+    err.clear();
+  }
+
+  bool success = build_log_.Restat(log_path, disk_interface_, &err);
+  if (!success) {
+    Error("failed recompaction: %s", err.c_str());
+    return EXIT_FAILURE;
+  }
+
+  if (!config_.dry_run) {
+    if (!build_log_.OpenForWrite(log_path, *this, &err)) {
+      Error("opening build log: %s", err.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+
+  return EXIT_SUCCESS;
+}
+
 int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) {
   // RLE encoded.
   const char* urtle =
@@ -906,6 +948,8 @@ const Tool* ChooseTool(const string& tool_name) {
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase },
     { "recompact",  "recompacts ninja-internal data structures",
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact },
+    { "restat",  "restats all outputs in the build log",
+      Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRestat },
     { "rules",  "list all rules",
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules },
     { "urtle", NULL,
@@ -1032,17 +1076,21 @@ bool NinjaMain::OpenBuildLog(bool recompact_only) {
     log_path = build_dir_ + "/" + log_path;
 
   string err;
-  if (!build_log_.Load(log_path, &err)) {
+  const LoadStatus status = build_log_.Load(log_path, &err);
+  if (status == LOAD_ERROR) {
     Error("loading build log %s: %s", log_path.c_str(), err.c_str());
     return false;
   }
   if (!err.empty()) {
-    // Hack: Load() can return a warning via err by returning true.
+    // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
     Warning("%s", err.c_str());
     err.clear();
   }
 
   if (recompact_only) {
+    if (status == LOAD_NOT_FOUND) {
+      return true;
+    }
     bool success = build_log_.Recompact(log_path, *this, &err);
     if (!success)
       Error("failed recompaction: %s", err.c_str());
@@ -1067,17 +1115,21 @@ bool NinjaMain::OpenDepsLog(bool recompact_only) {
     path = build_dir_ + "/" + path;
 
   string err;
-  if (!deps_log_.Load(path, &state_, &err)) {
+  const LoadStatus status = deps_log_.Load(path, &state_, &err);
+  if (status == LOAD_ERROR) {
     Error("loading deps log %s: %s", path.c_str(), err.c_str());
     return false;
   }
   if (!err.empty()) {
-    // Hack: Load() can return a warning via err by returning true.
+    // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
     Warning("%s", err.c_str());
     err.clear();
   }
 
   if (recompact_only) {
+    if (status == LOAD_NOT_FOUND) {
+      return true;
+    }
     bool success = deps_log_.Recompact(path, &err);
     if (!success)
       Error("failed recompaction: %s", err.c_str());