Teach DependencyScan to load a dyndep file

Add a LoadDyndeps method to load a dyndep file and update
the edges that name it in their dyndep binding.

6 files changed, 355 insertions, 2 deletions

diff --git a/configure.py b/configure.py
index b56ef89..850bb98 100755
--- a/configure.py
+++ b/configure.py
@@ -496,6 +496,7 @@ for name in ['build',
              'depfile_parser',
              'deps_log',
              'disk_interface',
+             'dyndep',
              'dyndep_parser',
              'edit_distance',
              'eval_env',
diff --git a/src/dyndep.cc b/src/dyndep.cc
new file mode 100644
index 0000000..2aee601
--- /dev/null
+++ b/src/dyndep.cc
@@ -0,0 +1,124 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dyndep.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "debug_flags.h"
+#include "disk_interface.h"
+#include "dyndep_parser.h"
+#include "graph.h"
+#include "state.h"
+#include "util.h"
+
+bool DyndepLoader::LoadDyndeps(Node* node, std::string* err) const {
+  DyndepFile ddf;
+  return LoadDyndeps(node, &ddf, err);
+}
+
+bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf,
+                               std::string* err) const {
+  // We are loading the dyndep file now so it is no longer pending.
+  node->set_dyndep_pending(false);
+
+  // Load the dyndep information from the file.
+  EXPLAIN("loading dyndep file '%s'", node->path().c_str());
+  if (!LoadDyndepFile(node, ddf, err))
+    return false;
+
+  // Update each edge that specified this node as its dyndep binding.
+  std::vector<Edge*> const& out_edges = node->out_edges();
+  for (std::vector<Edge*>::const_iterator oe = out_edges.begin();
+       oe != out_edges.end(); ++oe) {
+    Edge* const edge = *oe;
+    if (edge->dyndep_ != node)
+      continue;
+
+    DyndepFile::iterator ddi = ddf->find(edge);
+    if (ddi == ddf->end()) {
+      *err = ("'" + edge->outputs_[0]->path() + "' "
+              "not mentioned in its dyndep file "
+              "'" + node->path() + "'");
+      return false;
+    }
+
+    ddi->second.used_ = true;
+    Dyndeps const& dyndeps = ddi->second;
+    if (!UpdateEdge(edge, &dyndeps, err)) {
+      return false;
+    }
+  }
+
+  // Reject extra outputs in dyndep file.
+  for (DyndepFile::const_iterator oe = ddf->begin(); oe != ddf->end();
+       ++oe) {
+    if (!oe->second.used_) {
+      Edge* const edge = oe->first;
+      *err = ("dyndep file '" + node->path() + "' mentions output "
+              "'" + edge->outputs_[0]->path() + "' whose build statement "
+              "does not have a dyndep binding for the file");
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps,
+                              std::string* err) const {
+  // Add dyndep-discovered bindings to the edge.
+  // We know the edge already has its own binding
+  // scope because it has a "dyndep" binding.
+  if (dyndeps->restat_)
+    edge->env_->AddBinding("restat", "1");
+
+  // Add the dyndep-discovered outputs to the edge.
+  edge->outputs_.insert(edge->outputs_.end(),
+                        dyndeps->implicit_outputs_.begin(),
+                        dyndeps->implicit_outputs_.end());
+  edge->implicit_outs_ += dyndeps->implicit_outputs_.size();
+
+  // Add this edge as incoming to each new output.
+  for (std::vector<Node*>::const_iterator i =
+           dyndeps->implicit_outputs_.begin();
+       i != dyndeps->implicit_outputs_.end(); ++i) {
+    if ((*i)->in_edge() != NULL) {
+      *err = "multiple rules generate " + (*i)->path();
+      return false;
+    }
+    (*i)->set_in_edge(edge);
+  }
+
+  // Add the dyndep-discovered inputs to the edge.
+  edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_,
+                       dyndeps->implicit_inputs_.begin(),
+                       dyndeps->implicit_inputs_.end());
+  edge->implicit_deps_ += dyndeps->implicit_inputs_.size();
+
+  // Add this edge as outgoing from each new input.
+  for (std::vector<Node*>::const_iterator i =
+           dyndeps->implicit_inputs_.begin();
+       i != dyndeps->implicit_inputs_.end(); ++i)
+    (*i)->AddOutEdge(edge);
+
+  return true;
+}
+
+bool DyndepLoader::LoadDyndepFile(Node* file, DyndepFile* ddf,
+                                  std::string* err) const {
+  DyndepParser parser(state_, disk_interface_, ddf);
+  return parser.Load(file->path(), err);
+}
diff --git a/src/dyndep.h b/src/dyndep.h
index 80c5d1b..907f921 100644
--- a/src/dyndep.h
+++ b/src/dyndep.h
@@ -16,14 +16,18 @@
 #define NINJA_DYNDEP_LOADER_H_
 
 #include <map>
+#include <string>
 #include <vector>
 
+struct DiskInterface;
 struct Edge;
 struct Node;
+struct State;
 
 /// Store dynamically-discovered dependency information for one edge.
 struct Dyndeps {
-  Dyndeps() : restat_(false) {}
+  Dyndeps() : used_(false), restat_(false) {}
+  bool used_;
   bool restat_;
   std::vector<Node*> implicit_inputs_;
   std::vector<Node*> implicit_outputs_;
@@ -35,4 +39,26 @@ struct Dyndeps {
 /// forward-declare it in other headers.
 struct DyndepFile: public std::map<Edge*, Dyndeps> {};
 
+/// DyndepLoader loads dynamically discovered dependencies, as
+/// referenced via the "dyndep" attribute in build files.
+struct DyndepLoader {
+  DyndepLoader(State* state, DiskInterface* disk_interface)
+      : state_(state), disk_interface_(disk_interface) {}
+
+  /// Load a dyndep file from the given node's path and update the
+  /// build graph with the new information.  One overload accepts
+  /// a caller-owned 'DyndepFile' object in which to store the
+  /// information loaded from the dyndep file.
+  bool LoadDyndeps(Node* node, std::string* err) const;
+  bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const;
+
+ private:
+  bool LoadDyndepFile(Node* file, DyndepFile* ddf, std::string* err) const;
+
+  bool UpdateEdge(Edge* edge, Dyndeps const* dyndeps, std::string* err) const;
+
+  State* state_;
+  DiskInterface* disk_interface_;
+};
+
 #endif  // NINJA_DYNDEP_LOADER_H_
diff --git a/src/graph.cc b/src/graph.cc
index 2fbce84..a464299 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -276,6 +276,15 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge,
   return false;
 }
 
+bool DependencyScan::LoadDyndeps(Node* node, string* err) const {
+  return dyndep_loader_.LoadDyndeps(node, err);
+}
+
+bool DependencyScan::LoadDyndeps(Node* node, DyndepFile* ddf,
+                                 string* err) const {
+  return dyndep_loader_.LoadDyndeps(node, ddf, err);
+}
+
 bool Edge::AllInputsReady() const {
   for (vector<Node*>::const_iterator i = inputs_.begin();
        i != inputs_.end(); ++i) {
diff --git a/src/graph.h b/src/graph.h
index 745297d..75edbc5 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -19,6 +19,7 @@
 #include <vector>
 using namespace std;
 
+#include "dyndep.h"
 #include "eval_env.h"
 #include "timestamp.h"
 #include "util.h"
@@ -270,7 +271,8 @@ struct DependencyScan {
                  DepfileParserOptions const* depfile_parser_options)
       : build_log_(build_log),
         disk_interface_(disk_interface),
-        dep_loader_(state, deps_log, disk_interface, depfile_parser_options) {}
+        dep_loader_(state, deps_log, disk_interface, depfile_parser_options),
+        dyndep_loader_(state, disk_interface) {}
 
   /// Update the |dirty_| state of the given node by inspecting its input edge.
   /// Examine inputs, outputs, and command lines to judge whether an edge
@@ -295,6 +297,13 @@ struct DependencyScan {
     return dep_loader_.deps_log();
   }
 
+  /// Load a dyndep file from the given node's path and update the
+  /// build graph with the new information.  One overload accepts
+  /// a caller-owned 'DyndepFile' object in which to store the
+  /// information loaded from the dyndep file.
+  bool LoadDyndeps(Node* node, string* err) const;
+  bool LoadDyndeps(Node* node, DyndepFile* ddf, string* err) const;
+
  private:
   bool RecomputeDirty(Node* node, vector<Node*>* stack, string* err);
   bool VerifyDAG(Node* node, vector<Node*>* stack, string* err);
@@ -307,6 +316,7 @@ struct DependencyScan {
   BuildLog* build_log_;
   DiskInterface* disk_interface_;
   ImplicitDepLoader dep_loader_;
+  DyndepLoader dyndep_loader_;
 };
 
 #endif  // NINJA_GRAPH_H_
diff --git a/src/graph_test.cc b/src/graph_test.cc
index 4a66831..f53c0e9 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -479,3 +479,186 @@ TEST_F(GraphTest, Decanonicalize) {
   EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo");
 }
 #endif
+
+TEST_F(GraphTest, DyndepLoadTrivial) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out: r in || dd\n"
+"  dyndep = dd\n"
+  );
+  fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("", err);
+  EXPECT_FALSE(GetNode("dd")->dyndep_pending());
+
+  Edge* edge = GetNode("out")->in_edge();
+  ASSERT_EQ(1u, edge->outputs_.size());
+  EXPECT_EQ("out", edge->outputs_[0]->path());
+  ASSERT_EQ(2u, edge->inputs_.size());
+  EXPECT_EQ("in", edge->inputs_[0]->path());
+  EXPECT_EQ("dd", edge->inputs_[1]->path());
+  EXPECT_EQ(0u, edge->implicit_deps_);
+  EXPECT_EQ(1u, edge->order_only_deps_);
+  EXPECT_FALSE(edge->GetBindingBool("restat"));
+}
+
+TEST_F(GraphTest, DyndepLoadMissingFile) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out: r in || dd\n"
+"  dyndep = dd\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("loading 'dd': No such file or directory", err);
+}
+
+TEST_F(GraphTest, DyndepLoadMissingEntry) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out: r in || dd\n"
+"  dyndep = dd\n"
+  );
+  fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("'out' not mentioned in its dyndep file 'dd'", err);
+}
+
+TEST_F(GraphTest, DyndepLoadExtraEntry) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out: r in || dd\n"
+"  dyndep = dd\n"
+"build out2: r in || dd\n"
+  );
+  fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep\n"
+"build out2: dyndep\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("dyndep file 'dd' mentions output 'out2' whose build statement "
+            "does not have a dyndep binding for the file", err);
+}
+
+TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules1) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out1 | out-twice.imp: r in1\n"
+"build out2: r in2 || dd\n"
+"  dyndep = dd\n"
+  );
+  fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out2 | out-twice.imp: dyndep\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("multiple rules generate out-twice.imp", err);
+}
+
+TEST_F(GraphTest, DyndepLoadOutputWithMultipleRules2) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out1: r in1 || dd1\n"
+"  dyndep = dd1\n"
+"build out2: r in2 || dd2\n"
+"  dyndep = dd2\n"
+  );
+  fs_.Create("dd1",
+"ninja_dyndep_version = 1\n"
+"build out1 | out-twice.imp: dyndep\n"
+  );
+  fs_.Create("dd2",
+"ninja_dyndep_version = 1\n"
+"build out2 | out-twice.imp: dyndep\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd1")->dyndep_pending());
+  EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd1"), &err));
+  EXPECT_EQ("", err);
+  ASSERT_TRUE(GetNode("dd2")->dyndep_pending());
+  EXPECT_FALSE(scan_.LoadDyndeps(GetNode("dd2"), &err));
+  EXPECT_EQ("multiple rules generate out-twice.imp", err);
+}
+
+TEST_F(GraphTest, DyndepLoadMultiple) {
+  AssertParse(&state_,
+"rule r\n"
+"  command = unused\n"
+"build out1: r in1 || dd\n"
+"  dyndep = dd\n"
+"build out2: r in2 || dd\n"
+"  dyndep = dd\n"
+"build outNot: r in3 || dd\n"
+  );
+  fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out1 | out1imp: dyndep | in1imp\n"
+"build out2: dyndep | in2imp\n"
+"  restat = 1\n"
+  );
+
+  string err;
+  ASSERT_TRUE(GetNode("dd")->dyndep_pending());
+  EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err));
+  EXPECT_EQ("", err);
+  EXPECT_FALSE(GetNode("dd")->dyndep_pending());
+
+  Edge* edge1 = GetNode("out1")->in_edge();
+  ASSERT_EQ(2u, edge1->outputs_.size());
+  EXPECT_EQ("out1", edge1->outputs_[0]->path());
+  EXPECT_EQ("out1imp", edge1->outputs_[1]->path());
+  EXPECT_EQ(1u, edge1->implicit_outs_);
+  ASSERT_EQ(3u, edge1->inputs_.size());
+  EXPECT_EQ("in1", edge1->inputs_[0]->path());
+  EXPECT_EQ("in1imp", edge1->inputs_[1]->path());
+  EXPECT_EQ("dd", edge1->inputs_[2]->path());
+  EXPECT_EQ(1u, edge1->implicit_deps_);
+  EXPECT_EQ(1u, edge1->order_only_deps_);
+  EXPECT_FALSE(edge1->GetBindingBool("restat"));
+  EXPECT_EQ(edge1, GetNode("out1imp")->in_edge());
+  Node* in1imp = GetNode("in1imp");
+  ASSERT_EQ(1u, in1imp->out_edges().size());
+  EXPECT_EQ(edge1, in1imp->out_edges()[0]);
+
+  Edge* edge2 = GetNode("out2")->in_edge();
+  ASSERT_EQ(1u, edge2->outputs_.size());
+  EXPECT_EQ("out2", edge2->outputs_[0]->path());
+  EXPECT_EQ(0u, edge2->implicit_outs_);
+  ASSERT_EQ(3u, edge2->inputs_.size());
+  EXPECT_EQ("in2", edge2->inputs_[0]->path());
+  EXPECT_EQ("in2imp", edge2->inputs_[1]->path());
+  EXPECT_EQ("dd", edge2->inputs_[2]->path());
+  EXPECT_EQ(1u, edge2->implicit_deps_);
+  EXPECT_EQ(1u, edge2->order_only_deps_);
+  EXPECT_TRUE(edge2->GetBindingBool("restat"));
+  Node* in2imp = GetNode("in2imp");
+  ASSERT_EQ(1u, in2imp->out_edges().size());
+  EXPECT_EQ(edge2, in2imp->out_edges()[0]);
+}