From 6db3c739f65bcd5ee591930d7c7aa8ad6f47e1fd Mon Sep 17 00:00:00 2001 From: Juggls Date: Thu, 25 Aug 2016 17:24:15 -0400 Subject: SERVER-25809 Add Executable Dependency Tracking for Dagger Project --- site_scons/site_tools/dagger/__init__.py | 16 +++++++- site_scons/site_tools/dagger/dagger.py | 29 ++++++++++++-- site_scons/site_tools/dagger/graph.py | 59 +++++++++++++++++++++++++--- site_scons/site_tools/dagger/graph_consts.py | 15 ++++++- 4 files changed, 106 insertions(+), 13 deletions(-) (limited to 'site_scons') diff --git a/site_scons/site_tools/dagger/__init__.py b/site_scons/site_tools/dagger/__init__.py index becf164c151..f05228cfe45 100644 --- a/site_scons/site_tools/dagger/__init__.py +++ b/site_scons/site_tools/dagger/__init__.py @@ -1,9 +1,11 @@ """The initialization for the dagger tool. This file provides the initialization for the tool and attaches our custom builders and emitters to the build process""" +import os +import logging -import dagger import SCons +import dagger def generate(env, **kwargs): """The entry point for our tool. However, the builder for @@ -11,9 +13,16 @@ def generate(env, **kwargs): in the environment. When we generate the tool we attach our emitters to the native builders for object/libraries. """ - static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + env.Replace(LIBEMITTER=SCons.Builder.ListEmitter([env['LIBEMITTER'], dagger.emit_lib_db_entry])) + running_os = os.sys.platform + + if not (running_os.startswith('win') or running_os.startswith('sun')): + env.Replace(PROGEMITTER=SCons.Builder.ListEmitter([env['PROGEMITTER'], + dagger.emit_prog_db_entry])) + + static_obj, shared_obj = SCons.Tool.createObjBuilders(env) suffixes = ['.c', '.cc', '.cxx', '.cpp'] obj_builders = [static_obj, shared_obj] default_emitters = [SCons.Defaults.StaticObjectEmitter, @@ -29,6 +38,9 @@ def generate(env, **kwargs): action=SCons.Action.Action(dagger.write_obj_db, None)) def Dagger(env, target="library_dependency_graph.json"): + if running_os.startswith('win') or running_os.startswith('sun'): + logging.error("Dagger is only supported on OSX and Linux") + return result = env.__OBJ_DATABASE(target=target, source=[]) env.AlwaysBuild(result) env.NoCache(result) diff --git a/site_scons/site_tools/dagger/dagger.py b/site_scons/site_tools/dagger/dagger.py index 4a1fee2f33b..1eeefe1ea37 100644 --- a/site_scons/site_tools/dagger/dagger.py +++ b/site_scons/site_tools/dagger/dagger.py @@ -43,8 +43,10 @@ import SCons import graph import graph_consts -LIB_DB = [] -OBJ_DB = [] + +LIB_DB = [] # Stores every SCons library nodes +OBJ_DB = [] # Stores every SCons object file node +EXE_DB = {} # Stores every SCons executable node, with the object files that build into it {Executable: [object files]} class DependencyCycleError(SCons.Errors.UserError): @@ -120,6 +122,13 @@ def emit_obj_db_entry(target, source, env): OBJ_DB.append(t) return target, source +def emit_prog_db_entry(target, source, env): + for t in target: + if str(t) is None: + continue + EXE_DB[t] = [str(s) for s in source] + + return target, source def emit_lib_db_entry(target, source, env): """Emitter for libraries. We add each library @@ -213,7 +222,6 @@ def __generate_file_rels(obj, g): """Generate all file to file and by extension, file to library and library to file relationships """ - file_node = g.get_node(str(obj)) if file_node is None: @@ -231,6 +239,18 @@ def __generate_file_rels(obj, g): for obj in objs: g.add_edge(graph_consts.FIL_FIL, file_node.id, obj) +def __generate_exe_rels(exe, g): + """Generates all executable to library relationships, and populates the + contained files field in each NodeExe object""" + exe_node = g.find_node(str(exe), graph_consts.NODE_EXE) + for lib in exe.all_children(): + lib = lib.get_path() + if lib is None or not lib.endswith(".a"): + continue + lib_node = g.find_node(lib, graph_consts.NODE_LIB) + g.add_edge(graph_consts.EXE_LIB, exe_node.id, lib_node.id) + + exe_node.contained_files = set(EXE_DB[exe]) def write_obj_db(target, source, env): """The bulk of the tool. This method takes all the objects and libraries @@ -249,6 +269,9 @@ def write_obj_db(target, source, env): for obj in OBJ_DB: __generate_file_rels(obj, g) + for exe in EXE_DB.keys(): + __generate_exe_rels(exe, g) + # target is given as a list of target SCons nodes - this builder is only responsible for # building the json target, so this list is of length 1. export_to_json # expects a filename, whereas target is a list of SCons nodes so we cast target[0] to str diff --git a/site_scons/site_tools/dagger/graph.py b/site_scons/site_tools/dagger/graph.py index ca3815fc493..5ebe6f45061 100644 --- a/site_scons/site_tools/dagger/graph.py +++ b/site_scons/site_tools/dagger/graph.py @@ -9,7 +9,6 @@ import graph_consts if sys.version_info >= (3, 0): basestring = str - class Graph(object): """Graph class for storing the build dependency graph. The graph stores the directed edges as a nested dict of { RelationshipType: {From_Node: Set of @@ -46,7 +45,7 @@ class Graph(object): if edge["type"] not in edges: edges[edge["type"]] = {} - to_edges = set([e["id"] for e in edge["to_node"]]) + to_edges = set([str(e["id"]) for e in edge["to_node"]]) edges[edge["type"]][edge["from_node"]["id"]] = to_edges self._nodes = nodes @@ -71,6 +70,20 @@ class Graph(object): def edges(self): return copy.deepcopy(self._edges) + @nodes.setter + def nodes(self, value): + if isinstance(value,dict): + self._nodes = value + else: + raise TypeError("Nodes must be a dict") + + @edges.setter + def edges(self, value): + if isinstance(value, dict): + self._edges = value + else: + raise TypeError("Edges must be a dict") + def get_node(self, id): return self._nodes.get(id) @@ -285,7 +298,7 @@ class NodeLib(NodeInterface): def __ne__(self, other): return not self.__eq__(other) - def __str__(self): + def __repr__(self): return self.id @@ -411,7 +424,7 @@ class NodeSymbol(NodeInterface): def __ne__(self, other): return not self.__eq__(other) - def __str__(self): + def __repr__(self): return self.id @@ -530,13 +543,47 @@ class NodeFile(NodeInterface): def __ne__(self, other): return not self.__eq__(other) - def __str__(self): + def __repr__(self): + return self.id + + +class NodeExe(NodeInterface): + def __init__(self, id, name, input=None): + if isinstance(input, dict): + should_fail = False + for k, v in input.iteritems(): + try: + if isinstance(v, list): + setattr(self, k, set(v)) + else: + setattr(self, k, v) + except AttributeError as e: + logging.error("found something bad, {0}, {1}", e, type(e)) + should_fail = True + if should_fail: + raise Exception("Problem setting attribute for NodeExe") + else: + self._id = id + self.type = graph_consts.NODE_EXE + self._name = name + self.contained_files = set() + + @property + def id(self): + return self._id + + @property + def name(self): + return self._name + + def __repr__(self): return self.id types = {graph_consts.NODE_LIB: NodeLib, graph_consts.NODE_SYM: NodeSymbol, - graph_consts.NODE_FILE: NodeFile} + graph_consts.NODE_FILE: NodeFile, + graph_consts.NODE_EXE: NodeExe,} def node_factory(id, nodetype, dict_source=None): diff --git a/site_scons/site_tools/dagger/graph_consts.py b/site_scons/site_tools/dagger/graph_consts.py index 1d31bcbfa90..81fe86d75cd 100644 --- a/site_scons/site_tools/dagger/graph_consts.py +++ b/site_scons/site_tools/dagger/graph_consts.py @@ -1,15 +1,26 @@ """Constants for use in graph.py and dagger.py""" +"""Relationship edge types""" LIB_LIB = 1 LIB_FIL = 2 FIL_LIB = 3 FIL_FIL = 4 FIL_SYM = 5 LIB_SYM = 6 +IMP_LIB_LIB = 7 +EXE_LIB = 8 + +"""NodeTypes""" NODE_LIB = 1 NODE_SYM = 2 NODE_FILE = 3 +NODE_EXE = 4 + +RELATIONSHIP_TYPES = range(1, 9) +NODE_TYPES = range(1, 5) + + +"""Error/query codes""" +NODE_NOT_FOUND = 1 -RELATIONSHIP_TYPES = range(1, 7) -NODE_TYPES = range(1, 4) -- cgit v1.2.1