SERVER-25809 Add Executable Dependency Tracking for Dagger Project

4 files changed, 106 insertions, 13 deletions

diff --git a/site_scons/site_tools/dagger/__init__.py b/site_scons/site_tools/dagger/__init__.py
index becf164c151..f05228cfe45 100644
--- a/site_scons/site_tools/dagger/__init__.py
+++ b/site_scons/site_tools/dagger/__init__.py
@@ -1,9 +1,11 @@
 """The initialization for the dagger tool. This file provides the initialization for the tool
 and attaches our custom builders and emitters to the build process"""
+import os
+import logging
 
-import dagger
 import SCons
 
+import dagger
 
 def generate(env, **kwargs):
     """The entry point for our tool. However, the builder for
@@ -11,9 +13,16 @@ def generate(env, **kwargs):
     in the environment. When we generate the tool we attach our emitters
     to the native builders for object/libraries.
     """
-    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
+
     env.Replace(LIBEMITTER=SCons.Builder.ListEmitter([env['LIBEMITTER'],
                                                       dagger.emit_lib_db_entry]))
+    running_os = os.sys.platform
+
+    if not (running_os.startswith('win') or running_os.startswith('sun')):
+        env.Replace(PROGEMITTER=SCons.Builder.ListEmitter([env['PROGEMITTER'],
+        dagger.emit_prog_db_entry]))
+
+    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
     suffixes = ['.c', '.cc', '.cxx', '.cpp']
     obj_builders = [static_obj, shared_obj]
     default_emitters = [SCons.Defaults.StaticObjectEmitter,
@@ -29,6 +38,9 @@ def generate(env, **kwargs):
         action=SCons.Action.Action(dagger.write_obj_db, None))
 
     def Dagger(env, target="library_dependency_graph.json"):
+        if running_os.startswith('win') or running_os.startswith('sun'):
+            logging.error("Dagger is only supported on OSX and Linux")
+            return
         result = env.__OBJ_DATABASE(target=target, source=[])
         env.AlwaysBuild(result)
         env.NoCache(result)
diff --git a/site_scons/site_tools/dagger/dagger.py b/site_scons/site_tools/dagger/dagger.py
index 4a1fee2f33b..1eeefe1ea37 100644
--- a/site_scons/site_tools/dagger/dagger.py
+++ b/site_scons/site_tools/dagger/dagger.py
@@ -43,8 +43,10 @@ import SCons
 import graph
 import graph_consts
 
-LIB_DB = []
-OBJ_DB = []
+
+LIB_DB = [] # Stores every SCons library nodes
+OBJ_DB = [] # Stores every SCons object file node
+EXE_DB = {} # Stores every SCons executable node, with the object files that build into it {Executable: [object files]}
 
 
 class DependencyCycleError(SCons.Errors.UserError):
@@ -120,6 +122,13 @@ def emit_obj_db_entry(target, source, env):
         OBJ_DB.append(t)
     return target, source
 
+def emit_prog_db_entry(target, source, env):
+    for t in target:
+        if str(t) is None:
+            continue
+        EXE_DB[t] = [str(s) for s in source]
+
+    return target, source
 
 def emit_lib_db_entry(target, source, env):
     """Emitter for libraries. We add each library
@@ -213,7 +222,6 @@ def __generate_file_rels(obj, g):
     """Generate all file to file and by extension, file to library and library
     to file relationships
     """
-
     file_node = g.get_node(str(obj))
 
     if file_node is None:
@@ -231,6 +239,18 @@ def __generate_file_rels(obj, g):
         for obj in objs:
             g.add_edge(graph_consts.FIL_FIL, file_node.id, obj)
 
+def __generate_exe_rels(exe, g):
+    """Generates all executable to library relationships, and populates the
+    contained files field in each NodeExe object"""
+    exe_node = g.find_node(str(exe), graph_consts.NODE_EXE)
+    for lib in exe.all_children():
+        lib = lib.get_path()
+        if lib is None or not lib.endswith(".a"):
+            continue
+        lib_node = g.find_node(lib, graph_consts.NODE_LIB)
+        g.add_edge(graph_consts.EXE_LIB, exe_node.id, lib_node.id)
+
+    exe_node.contained_files = set(EXE_DB[exe])
 
 def write_obj_db(target, source, env):
     """The bulk of the tool. This method takes all the objects and libraries
@@ -249,6 +269,9 @@ def write_obj_db(target, source, env):
     for obj in OBJ_DB:
         __generate_file_rels(obj, g)
 
+    for exe in EXE_DB.keys():
+        __generate_exe_rels(exe, g)
+
     # target is given as a list of target SCons nodes - this builder is only responsible for
     # building the json target, so this list is of length 1. export_to_json
     # expects a filename, whereas target is a list of SCons nodes so we cast target[0] to str
diff --git a/site_scons/site_tools/dagger/graph.py b/site_scons/site_tools/dagger/graph.py
index ca3815fc493..5ebe6f45061 100644
--- a/site_scons/site_tools/dagger/graph.py
+++ b/site_scons/site_tools/dagger/graph.py
@@ -9,7 +9,6 @@ import graph_consts
 if sys.version_info >= (3, 0):
     basestring = str
 
-
 class Graph(object):
     """Graph class for storing the build dependency graph. The graph stores the
     directed edges as a nested dict of { RelationshipType: {From_Node: Set of
@@ -46,7 +45,7 @@ class Graph(object):
                     if edge["type"] not in edges:
                         edges[edge["type"]] = {}
 
-                    to_edges = set([e["id"] for e in edge["to_node"]])
+                    to_edges = set([str(e["id"]) for e in edge["to_node"]])
                     edges[edge["type"]][edge["from_node"]["id"]] = to_edges
 
                 self._nodes = nodes
@@ -71,6 +70,20 @@ class Graph(object):
     def edges(self):
         return copy.deepcopy(self._edges)
 
+    @nodes.setter
+    def nodes(self, value):
+        if isinstance(value,dict):
+            self._nodes = value
+        else:
+            raise TypeError("Nodes must be a dict")
+
+    @edges.setter
+    def edges(self, value):
+        if isinstance(value, dict):
+            self._edges = value
+        else:
+            raise TypeError("Edges must be a dict")
+
     def get_node(self, id):
         return self._nodes.get(id)
 
@@ -285,7 +298,7 @@ class NodeLib(NodeInterface):
     def __ne__(self, other):
         return not self.__eq__(other)
 
-    def __str__(self):
+    def __repr__(self):
         return self.id
 
 
@@ -411,7 +424,7 @@ class NodeSymbol(NodeInterface):
     def __ne__(self, other):
         return not self.__eq__(other)
 
-    def __str__(self):
+    def __repr__(self):
         return self.id
 
 
@@ -530,13 +543,47 @@ class NodeFile(NodeInterface):
     def __ne__(self, other):
         return not self.__eq__(other)
 
-    def __str__(self):
+    def __repr__(self):
+        return self.id
+
+
+class NodeExe(NodeInterface):
+    def __init__(self, id, name, input=None):
+        if isinstance(input, dict):
+            should_fail = False
+            for k, v in input.iteritems():
+                try:
+                    if isinstance(v, list):
+                        setattr(self, k, set(v))
+                    else:
+                        setattr(self, k, v)
+                except AttributeError as e:
+                    logging.error("found something bad, {0}, {1}", e, type(e))
+                    should_fail = True
+            if should_fail:
+                raise Exception("Problem setting attribute for NodeExe")
+        else:
+            self._id = id
+            self.type = graph_consts.NODE_EXE
+            self._name = name
+            self.contained_files = set()
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def name(self):
+        return self._name
+
+    def __repr__(self):
         return self.id
 
 
 types = {graph_consts.NODE_LIB: NodeLib,
          graph_consts.NODE_SYM: NodeSymbol,
-         graph_consts.NODE_FILE: NodeFile}
+         graph_consts.NODE_FILE: NodeFile,
+         graph_consts.NODE_EXE: NodeExe,}
 
 
 def node_factory(id, nodetype, dict_source=None):
diff --git a/site_scons/site_tools/dagger/graph_consts.py b/site_scons/site_tools/dagger/graph_consts.py
index 1d31bcbfa90..81fe86d75cd 100644
--- a/site_scons/site_tools/dagger/graph_consts.py
+++ b/site_scons/site_tools/dagger/graph_consts.py
@@ -1,15 +1,26 @@
 """Constants for use in graph.py and dagger.py"""
 
+"""Relationship edge types"""
 LIB_LIB = 1
 LIB_FIL = 2
 FIL_LIB = 3
 FIL_FIL = 4
 FIL_SYM = 5
 LIB_SYM = 6
+IMP_LIB_LIB = 7
+EXE_LIB = 8
 
+
+"""NodeTypes"""
 NODE_LIB = 1
 NODE_SYM = 2
 NODE_FILE = 3
+NODE_EXE = 4
+
+RELATIONSHIP_TYPES = range(1, 9)
+NODE_TYPES = range(1, 5)
+
+
+"""Error/query codes"""
+NODE_NOT_FOUND = 1
 
-RELATIONSHIP_TYPES = range(1, 7)
-NODE_TYPES = range(1, 4)