diff options
author | Mathew Robinson <mathew.robinson@mongodb.com> | 2020-01-06 19:53:40 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2020-01-06 19:53:40 +0000 |
commit | 77fd09b418ce6f782eb6cf30568270aa76485d38 (patch) | |
tree | 5f38855ae25567d818377c360558b8af5ff94887 | |
parent | d0671dc95e8144d5828343f7431715537945425b (diff) | |
download | mongo-77fd09b418ce6f782eb6cf30568270aa76485d38.tar.gz |
Revert "SERVER-44947 Allow test execution selection by test source file name"
34 files changed, 796 insertions, 1120 deletions
diff --git a/SConstruct b/SConstruct index d4ec5f0bdb2..316bdb8c841 100644 --- a/SConstruct +++ b/SConstruct @@ -630,15 +630,11 @@ def decide_platform_tools(): def variable_tools_converter(val): tool_list = shlex.split(val) - # This list is not sorted intentionally, the order of tool loading - # matters as some of the tools have dependencies on other tools. return tool_list + [ "distsrc", "gziptool", - "idl_tool", + 'idl_tool', "jsheader", - "mongo_test_execution", - "mongo_test_list", "mongo_benchmark", "mongo_integrationtest", "mongo_unittest", diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py index e3f05f6aa7b..fc93311a9f1 100644 --- a/site_scons/libdeps.py +++ b/site_scons/libdeps.py @@ -57,10 +57,9 @@ import SCons.Errors import SCons.Scanner import SCons.Util -libdeps_env_var = "LIBDEPS" -syslibdeps_env_var = "SYSLIBDEPS" -missing_syslibdep = "MISSING_LIBDEP_" - +libdeps_env_var = 'LIBDEPS' +syslibdeps_env_var = 'SYSLIBDEPS' +missing_syslibdep = 'MISSING_LIBDEP_' class dependency(object): Public, Private, Interface = list(range(3)) @@ -94,15 +93,13 @@ class DependencyCycleError(SCons.Errors.UserError): self.cycle_nodes = [first_node] def __str__(self): - return "Library dependency cycle detected: " + " => ".join( - str(n) for n in self.cycle_nodes - ) + return "Library dependency cycle detected: " + " => ".join(str(n) for n in self.cycle_nodes) def __get_sorted_direct_libdeps(node): direct_sorted = getattr(node.attributes, "libdeps_direct_sorted", False) if not direct_sorted: - direct = getattr(node.attributes, "libdeps_direct", []) + direct = getattr(node.attributes, 'libdeps_direct', []) direct_sorted = sorted(direct, key=lambda t: str(t.target_node)) setattr(node.attributes, "libdeps_direct_sorted", direct_sorted) return direct_sorted @@ -114,7 +111,7 @@ def __get_libdeps(node): Computes the dependencies if they're not already cached. """ - cached_var_name = libdeps_env_var + "_cached" + cached_var_name = libdeps_env_var + '_cached' if hasattr(node.attributes, cached_var_name): return getattr(node.attributes, cached_var_name) @@ -123,7 +120,7 @@ def __get_libdeps(node): marked = set() def visit(n): - if getattr(n.target_node.attributes, "libdeps_exploring", False): + if getattr(n.target_node.attributes, 'libdeps_exploring', False): raise DependencyCycleError(n.target_node) n.target_node.attributes.libdeps_exploring = True @@ -163,33 +160,25 @@ def __get_syslibdeps(node): These are the depencencies listed with SYSLIBDEPS, and are linked using -l. """ - cached_var_name = syslibdeps_env_var + "_cached" + cached_var_name = syslibdeps_env_var + '_cached' if not hasattr(node.attributes, cached_var_name): syslibdeps = node.get_env().Flatten(node.get_env().get(syslibdeps_env_var, [])) for lib in __get_libdeps(node): - for syslib in node.get_env().Flatten( - lib.get_env().get(syslibdeps_env_var, []) - ): + for syslib in node.get_env().Flatten(lib.get_env().get(syslibdeps_env_var, [])): if syslib: if type(syslib) is str and syslib.startswith(missing_syslibdep): - print( - ( - "Target '%s' depends on the availability of a " - "system provided library for '%s', " - "but no suitable library was found during configuration." - % (str(node), syslib[len(missing_syslibdep) :]) - ) - ) + print(("Target '%s' depends on the availability of a " + "system provided library for '%s', " + "but no suitable library was found during configuration." % + (str(node), syslib[len(missing_syslibdep):]))) node.get_env().Exit(1) syslibdeps.append(syslib) setattr(node.attributes, cached_var_name, syslibdeps) return getattr(node.attributes, cached_var_name) - def __missing_syslib(name): return missing_syslibdep + name - def update_scanner(builder): """Update the scanner for "builder" to also scan library dependencies.""" @@ -202,16 +191,14 @@ def update_scanner(builder): result = old_scanner.function(node, env, path) result.extend(__get_libdeps(node)) return result - else: path_function = None def new_scanner(node, env, path=()): return __get_libdeps(node) - builder.target_scanner = SCons.Scanner.Scanner( - function=new_scanner, path_function=path_function - ) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, + path_function=path_function) def get_libdeps(source, target, env, for_signature): @@ -234,8 +221,8 @@ def get_libdeps_objs(source, target, env, for_signature): def get_syslibdeps(source, target, env, for_signature): deps = __get_syslibdeps(target[0]) - lib_link_prefix = env.subst("$LIBLINKPREFIX") - lib_link_suffix = env.subst("$LIBLINKSUFFIX") + lib_link_prefix = env.subst('$LIBLINKPREFIX') + lib_link_suffix = env.subst('$LIBLINKSUFFIX') result = [] for d in deps: # Elements of syslibdeps are either strings (str or unicode), or they're File objects. @@ -244,26 +231,22 @@ def get_syslibdeps(source, target, env, for_signature): # or the compiler-specific equivalent. I.e., 'm' becomes '-lm', but 'File("m.a") is passed # through whole cloth. if type(d) is str: - result.append("%s%s%s" % (lib_link_prefix, d, lib_link_suffix)) + result.append('%s%s%s' % (lib_link_prefix, d, lib_link_suffix)) else: result.append(d) return result - def __append_direct_libdeps(node, prereq_nodes): # We do not bother to decorate nodes that are not actual Objects if type(node) == str: return - if getattr(node.attributes, "libdeps_direct", None) is None: + if getattr(node.attributes, 'libdeps_direct', None) is None: node.attributes.libdeps_direct = [] node.attributes.libdeps_direct.extend(prereq_nodes) -def make_libdeps_emitter( - dependency_builder, - dependency_map=dependency_visibility_ignored, - ignore_progdeps=False, -): +def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibility_ignored, + ignore_progdeps=False): def libdeps_emitter(target, source, env): """SCons emitter that takes values from the LIBDEPS environment variable and converts them to File node objects, binding correct path information into @@ -282,35 +265,29 @@ def make_libdeps_emitter( of the "target" list is made a prerequisite of the elements of LIBDEPS_DEPENDENTS. """ - lib_builder = env["BUILDERS"][dependency_builder] + lib_builder = env['BUILDERS'][dependency_builder] lib_node_factory = lib_builder.target_factory or env.File - prog_builder = env["BUILDERS"]["Program"] + prog_builder = env['BUILDERS']['Program'] prog_node_factory = prog_builder.target_factory or env.File prereqs = [ - dependency(l, dependency_map[dependency.Public]) - for l in env.get(libdeps_env_var, []) + dependency(l, dependency_map[dependency.Public]) for l in env.get(libdeps_env_var, []) if l ] prereqs.extend( dependency(l, dependency_map[dependency.Interface]) - for l in env.get(libdeps_env_var + "_INTERFACE", []) - if l - ) + for l in env.get(libdeps_env_var + '_INTERFACE', []) if l) prereqs.extend( dependency(l, dependency_map[dependency.Private]) - for l in env.get(libdeps_env_var + "_PRIVATE", []) - if l - ) + for l in env.get(libdeps_env_var + '_PRIVATE', []) if l) lib_builder_prefix = lib_builder.get_prefix(env) lib_builder_suffix = lib_builder.get_suffix(env) for prereq in prereqs: - prereqWithIxes = SCons.Util.adjustixes( - prereq.target_node, lib_builder_prefix, lib_builder_suffix - ) + prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder_prefix, + lib_builder_suffix) prereq.target_node = lib_node_factory(prereqWithIxes) for t in target: @@ -318,7 +295,7 @@ def make_libdeps_emitter( # work properly. __append_direct_libdeps(t, prereqs) - for dependent in env.get("LIBDEPS_DEPENDENTS", []): + for dependent in env.get('LIBDEPS_DEPENDENTS', []): if dependent is None: continue @@ -327,19 +304,17 @@ def make_libdeps_emitter( visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes( - dependent, lib_builder_prefix, lib_builder_suffix - ) + dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder_prefix, + lib_builder_suffix) dependentNode = lib_node_factory(dependentWithIxes) - __append_direct_libdeps( - dependentNode, [dependency(target[0], dependency_map[visibility])] - ) + __append_direct_libdeps(dependentNode, + [dependency(target[0], dependency_map[visibility])]) prog_builder_prefix = prog_builder.get_prefix(env) prog_builder_suffix = prog_builder.get_suffix(env) if not ignore_progdeps: - for dependent in env.get("PROGDEPS_DEPENDENTS", []): + for dependent in env.get('PROGDEPS_DEPENDENTS', []): if dependent is None: continue @@ -349,13 +324,11 @@ def make_libdeps_emitter( visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes( - dependent, prog_builder_prefix, prog_builder_suffix - ) + dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder_prefix, + prog_builder_suffix) dependentNode = prog_node_factory(dependentWithIxes) - __append_direct_libdeps( - dependentNode, [dependency(target[0], dependency_map[visibility])] - ) + __append_direct_libdeps(dependentNode, + [dependency(target[0], dependency_map[visibility])]) return target, source @@ -364,7 +337,7 @@ def make_libdeps_emitter( def expand_libdeps_tags(source, target, env, for_signature): results = [] - for expansion in env.get("LIBDEPS_TAG_EXPANSIONS", []): + for expansion in env.get('LIBDEPS_TAG_EXPANSIONS', []): results.append(expansion(source, target, env, for_signature)) return results @@ -403,14 +376,14 @@ def setup_environment(env, emitting_shared=False): """Set up the given build environment to do LIBDEPS tracking.""" try: - env["_LIBDEPS"] + env['_LIBDEPS'] except KeyError: - env["_LIBDEPS"] = "$_LIBDEPS_LIBS" + env['_LIBDEPS'] = '$_LIBDEPS_LIBS' - env["_LIBDEPS_TAGS"] = expand_libdeps_tags - env["_LIBDEPS_GET_LIBS"] = get_libdeps - env["_LIBDEPS_OBJS"] = get_libdeps_objs - env["_SYSLIBDEPS"] = get_syslibdeps + env['_LIBDEPS_TAGS'] = expand_libdeps_tags + env['_LIBDEPS_GET_LIBS'] = get_libdeps + env['_LIBDEPS_OBJS'] = get_libdeps_objs + env['_SYSLIBDEPS'] = get_syslibdeps env[libdeps_env_var] = SCons.Util.CLVar() env[syslibdeps_env_var] = SCons.Util.CLVar() @@ -428,34 +401,29 @@ def setup_environment(env, emitting_shared=False): return indirect_emitter env.Append( - LIBDEPS_LIBEMITTER=make_libdeps_emitter("StaticLibrary"), - LIBEMITTER=make_indirect_emitter("LIBDEPS_LIBEMITTER"), - LIBDEPS_SHAREMITTER=make_libdeps_emitter("SharedArchive", ignore_progdeps=True), - SHAREMITTER=make_indirect_emitter("LIBDEPS_SHAREMITTER"), - LIBDEPS_SHLIBEMITTER=make_libdeps_emitter( - "SharedLibrary", dependency_visibility_honored - ), - SHLIBEMITTER=make_indirect_emitter("LIBDEPS_SHLIBEMITTER"), + LIBDEPS_LIBEMITTER=make_libdeps_emitter('StaticLibrary'), + LIBEMITTER=make_indirect_emitter('LIBDEPS_LIBEMITTER'), + LIBDEPS_SHAREMITTER=make_libdeps_emitter('SharedArchive', ignore_progdeps=True), + SHAREMITTER=make_indirect_emitter('LIBDEPS_SHAREMITTER'), + LIBDEPS_SHLIBEMITTER=make_libdeps_emitter('SharedLibrary', dependency_visibility_honored), + SHLIBEMITTER=make_indirect_emitter('LIBDEPS_SHLIBEMITTER'), LIBDEPS_PROGEMITTER=make_libdeps_emitter( - "SharedLibrary" if emitting_shared else "StaticLibrary" - ), - PROGEMITTER=make_indirect_emitter("LIBDEPS_PROGEMITTER"), + 'SharedLibrary' if emitting_shared else 'StaticLibrary'), + PROGEMITTER=make_indirect_emitter('LIBDEPS_PROGEMITTER'), ) env["_LIBDEPS_LIBS_WITH_TAGS"] = expand_libdeps_with_extraction_flags - env["_LIBDEPS_LIBS"] = ( - "$LINK_WHOLE_ARCHIVE_START " - "$LINK_LIBGROUP_START " - "$_LIBDEPS_LIBS_WITH_TAGS " - "$LINK_LIBGROUP_END " - "$LINK_WHOLE_ARCHIVE_END" - ) + env['_LIBDEPS_LIBS'] = ('$LINK_WHOLE_ARCHIVE_START ' + '$LINK_LIBGROUP_START ' + '$_LIBDEPS_LIBS_WITH_TAGS ' + '$LINK_LIBGROUP_END ' + '$LINK_WHOLE_ARCHIVE_END') - env.Prepend(_LIBFLAGS="$_LIBDEPS_TAGS $_LIBDEPS $_SYSLIBDEPS ") - for builder_name in ("Program", "SharedLibrary", "LoadableModule", "SharedArchive"): + env.Prepend(_LIBFLAGS='$_LIBDEPS_TAGS $_LIBDEPS $_SYSLIBDEPS ') + for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'SharedArchive'): try: - update_scanner(env["BUILDERS"][builder_name]) + update_scanner(env['BUILDERS'][builder_name]) except KeyError: pass @@ -463,7 +431,7 @@ def setup_environment(env, emitting_shared=False): def setup_conftests(conf): def FindSysLibDep(context, name, libs, **kwargs): var = "LIBDEPS_" + name.upper() + "_SYSLIBDEP" - kwargs["autoadd"] = False + kwargs['autoadd'] = False for lib in libs: result = context.sconf.CheckLib(lib, **kwargs) context.did_show_result = 1 @@ -473,4 +441,4 @@ def setup_conftests(conf): context.env[var] = __missing_syslib(name) return context.Result(result) - conf.AddTest("FindSysLibDep", FindSysLibDep) + conf.AddTest('FindSysLibDep', FindSysLibDep) diff --git a/site_scons/site_tools/abilink.py b/site_scons/site_tools/abilink.py index 0dde1e5a262..00f5e71a0a3 100644 --- a/site_scons/site_tools/abilink.py +++ b/site_scons/site_tools/abilink.py @@ -23,14 +23,14 @@ import subprocess def _detect(env): try: - abidw = env["ABIDW"] + abidw = env['ABIDW'] if not abidw: return None return abidw except KeyError: pass - return env.WhereIs("abidw") + return env.WhereIs('abidw') def _add_emitter(builder): @@ -62,16 +62,14 @@ def _add_scanner(builder): new_results.append(abidw if abidw else base) return new_results - builder.target_scanner = SCons.Scanner.Scanner( - function=new_scanner, path_function=path_function - ) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, + path_function=path_function) def _add_action(builder): actions = builder.action builder.action = actions + SCons.Action.Action( - "$ABIDW --no-show-locs $TARGET | md5sum > ${TARGET}.abidw" - ) + "$ABIDW --no-show-locs $TARGET | md5sum > ${TARGET}.abidw") def exists(env): @@ -84,9 +82,9 @@ def generate(env): if not exists(env): return - builder = env["BUILDERS"]["SharedLibrary"] + builder = env['BUILDERS']['SharedLibrary'] _add_emitter(builder) _add_action(builder) _add_scanner(builder) - _add_scanner(env["BUILDERS"]["Program"]) - _add_scanner(env["BUILDERS"]["LoadableModule"]) + _add_scanner(env['BUILDERS']['Program']) + _add_scanner(env['BUILDERS']['LoadableModule']) diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index 0ff74a32144..6328f0ec0a4 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -102,14 +102,26 @@ if __name__ == "__main__": """ RoleInfo = namedtuple( - "RoleInfo", - ["alias_name", "alias", "components", "roles", "actions", "dependencies"], + 'RoleInfo', + [ + 'alias_name', + 'alias', + 'components', + 'roles', + 'actions', + 'dependencies' + ], ) -SuffixMap = namedtuple("SuffixMap", ["directory", "default_roles",],) - +SuffixMap = namedtuple( + 'SuffixMap', + [ + 'directory', + 'default_roles', + ], +) -class DeclaredRole: +class DeclaredRole(): def __init__(self, name, dependencies=None, transitive=False, silent=False): self.name = name @@ -121,18 +133,18 @@ class DeclaredRole: self.transitive = transitive self.silent = silent - def declare_role(env, **kwargs): """Construct a new role declaration""" return DeclaredRole(**kwargs) - def declare_roles(env, roles, base_role=None, meta_role=None): """Given a list of role declarations, validate them and store them in the environment""" role_names = [role.name for role in roles] if len(role_names) != len(set(role_names)): - raise Exception("Cannot declare duplicate roles") + raise Exception( + "Cannot declare duplicate roles" + ) # Ensure that all roles named in dependency lists actually were # passed in as a role. @@ -181,7 +193,9 @@ def declare_roles(env, roles, base_role=None, meta_role=None): silents = [role for role in roles if role.silent] if len(silents) > 1: - raise Exception("No more than one role can be declared as silent") + raise Exception( + "No more than one role can be declared as silent" + ) # If a base role was given, then add it as a dependency of every # role that isn't the base role (which would be circular). @@ -191,7 +205,7 @@ def declare_roles(env, roles, base_role=None, meta_role=None): role.dependencies.add(base_role) # Become a dictionary, so we can look up roles easily. - roles = {role.name: role for role in roles} + roles = { role.name : role for role in roles } # If a meta role was given, then add every role which isn't the # meta role as one of its dependencies. @@ -214,7 +228,6 @@ def generate_alias(env, component, role, target="install"): role="" if env[ROLE_DECLARATIONS][role].silent else "-" + role, ) - def get_alias_map_entry(env, component, role): c_entry = env[ALIAS_MAP][component] try: @@ -227,23 +240,28 @@ def get_alias_map_entry(env, component, role): components=set(), roles=set(), actions=[], - dependencies=[], + dependencies=[] ) c_entry[role] = r_entry return r_entry - def get_package_name(env, component, role): """Return the package file name for the component and role combination.""" basename = env[PACKAGE_ALIAS_MAP].get( - (component, role), "{component}-{role}".format(component=component, role=role) + (component, role), + "{component}-{role}".format(component=component, role=role) ) return basename def get_dependent_actions( - env, components, roles, non_transitive_roles, node, cb=None, + env, + components, + roles, + non_transitive_roles, + node, + cb=None, ): """ Check if node is a transitive dependency of components and roles @@ -267,13 +285,16 @@ def get_dependent_actions( # If they are overlapping then that means we can't transition to a # new role during scanning. if env[BASE_ROLE] not in roles: - can_transfer = non_transitive_roles and roles.isdisjoint(non_transitive_roles) + can_transfer = ( + non_transitive_roles + and roles.isdisjoint(non_transitive_roles) + ) else: can_transfer = True node_roles = { - role - for role in getattr(node.attributes, ROLES, set()) + role for role + in getattr(node.attributes, ROLES, set()) if role != env[META_ROLE] } if ( @@ -287,7 +308,14 @@ def get_dependent_actions( return [] if cb is not None and callable(cb): - return cb(components, roles, non_transitive_roles, node, node_roles, actions,) + return cb( + components, + roles, + non_transitive_roles, + node, + node_roles, + actions, + ) return actions @@ -297,19 +325,17 @@ def scan_for_transitive_install(node, env, cb=None): install_sources = node.sources # Filter out all components = { - component - for component in getattr(node.sources[0].attributes, COMPONENTS, set()) + component for component + in getattr(node.sources[0].attributes, COMPONENTS, set()) if component != "all" } roles = { - role - for role in getattr(node.sources[0].attributes, ROLES, set()) + role for role + in getattr(node.sources[0].attributes, ROLES, set()) if role != env[META_ROLE] } - non_transitive_roles = { - role for role in roles if env[ROLE_DECLARATIONS][role].transitive - } + non_transitive_roles = {role for role in roles if env[ROLE_DECLARATIONS][role].transitive} for install_source in install_sources: install_executor = install_source.get_executor() if not install_executor: @@ -322,7 +348,12 @@ def scan_for_transitive_install(node, env, cb=None): for grandchild in grandchildren: results.extend( get_dependent_actions( - env, components, roles, non_transitive_roles, grandchild, cb=cb, + env, + components, + roles, + non_transitive_roles, + grandchild, + cb=cb, ) ) @@ -357,13 +388,10 @@ def collect_transitive_files(env, source, installed, cache=None): children_to_collect.append(child) if children_to_collect: - files.extend( - collect_transitive_files(env, children_to_collect, installed, cache) - ) + files.extend(collect_transitive_files(env, children_to_collect, installed, cache)) return files - def archive_builder(source, target, env, for_signature): """Build archives of the AutoInstall'd sources.""" if not source: @@ -391,7 +419,7 @@ def archive_builder(source, target, env, for_signature): # In python slicing a string with [:-0] gives an empty string. So # make sure we have a prefix to slice off before trying it. if prefix_elems: - common_ancestor = dest_dir_elems[: -len(prefix_elems)] + common_ancestor = dest_dir_elems[:-len(prefix_elems)] else: common_ancestor = dest_dir_elems @@ -415,7 +443,7 @@ def archive_builder(source, target, env, for_signature): # set of known installed files along to the transitive dependency # walk so we can filter out files that aren't in the install # directory. - installed = env.get("__AIB_INSTALLED_SET", set()) + installed = env.get('__AIB_INSTALLED_SET', set()) transitive_files = collect_transitive_files(env, aliases, installed) paths = {file.get_abspath() for file in transitive_files} @@ -427,9 +455,10 @@ def archive_builder(source, target, env, for_signature): # TODO: relpath is costly, and we do it for every file in the archive here. We should # find a way to avoid the repeated relpath invocation, probably by bucketing by directory. - relative_files = " ".join( - [escape_func(os.path.relpath(path, common_ancestor)) for path in paths] - ) + relative_files = " ".join([ + escape_func(os.path.relpath(path, common_ancestor)) + for path in paths + ]) return " ".join([command_prefix, relative_files]) @@ -448,8 +477,14 @@ def auto_install(env, target, source, **kwargs): roles = roles.union(set(kwargs[ROLES])) component = kwargs.get(PRIMARY_COMPONENT) - if component is not None and (not isinstance(component, str) or " " in component): - raise Exception("AIB_COMPONENT must be a string and contain no whitespace.") + if ( + component is not None + and (not isinstance(component, str) + or " " in component) + ): + raise Exception( + "AIB_COMPONENT must be a string and contain no whitespace." + ) components = { component, @@ -482,12 +517,15 @@ def auto_install(env, target, source, **kwargs): # TODO: Find a way to not need this early subst. target = env.Dir(env.subst(target, source=source)) - action = env.Install(target=target, source=s,) + action = env.Install( + target=target, + source=s, + ) setattr( s.attributes, INSTALL_ACTIONS, - action if isinstance(action, (list, set)) else [action], + action if isinstance(action, (list, set)) else [action] ) actions.append(action) @@ -529,7 +567,8 @@ def finalize_install_dependencies(env): # TODO: $BUILD_ROOT should be $VARIANT_DIR after we fix our dir # setup later on. make_archive_script = env.Textfile( - target="$BUILD_ROOT/aib_make_archive.py", source=[AIB_MAKE_ARCHIVE_CONTENT], + target="$BUILD_ROOT/aib_make_archive.py", + source=[AIB_MAKE_ARCHIVE_CONTENT], ) for component, rolemap in env[ALIAS_MAP].items(): @@ -584,7 +623,7 @@ def auto_install_emitter(target, source, env): # way available to us. # # We're working with upstream to expose this information. - if "conftest" in str(entry): + if 'conftest' in str(entry): continue auto_install_mapping = env[SUFFIX_MAP].get(suffix) @@ -631,20 +670,19 @@ def add_package_name_alias(env, component, role, name): """Add a package name mapping for the combination of component and role.""" # Verify we didn't get a None or empty string for any argument if not name: - raise Exception( - "when setting a package name alias must provide a name parameter" - ) + raise Exception("when setting a package name alias must provide a name parameter") if not component: raise Exception("No component provided for package name alias") if not role: raise Exception("No role provided for package name alias") env[PACKAGE_ALIAS_MAP][(component, role)] = name - def suffix_mapping(env, directory=False, default_roles=False): """Generate a SuffixMap object from source and target.""" - return SuffixMap(directory=directory, default_roles=default_roles,) - + return SuffixMap( + directory=directory, + default_roles=default_roles, + ) def dest_dir_generator(initial_value=None): """Memoized dest_dir_generator""" @@ -658,7 +696,7 @@ def dest_dir_generator(initial_value=None): # absolute path here because if it is the sub Dir call will # not expand correctly. prefix = env.subst("$PREFIX") - if prefix and prefix[0] == "/": + if prefix and prefix[0] == '/': prefix = prefix[1:] if dd[1] is not None and dd[0] == prefix: @@ -713,7 +751,7 @@ def generate(env): # pylint: disable=too-many-statements bld = SCons.Builder.Builder( action=SCons.Action.CommandGeneratorAction( archive_builder, - {"cmdstr": "Building package ${TARGETS[0]} from ${SOURCES[1:]}"}, + {"cmdstr": "Building package ${TARGETS[0]} from ${SOURCES[1:]}"}, ) ) env.Append(BUILDERS={"__AibArchive": bld}) @@ -725,7 +763,7 @@ def generate(env): # pylint: disable=too-many-statements env["DESTDIR"] = dest_dir_generator(env.get("DESTDIR", None)) env["PREFIX_BINDIR"] = env.get("PREFIX_BINDIR", "$DESTDIR/bin") env["PREFIX_LIBDIR"] = env.get("PREFIX_LIBDIR", "$DESTDIR/lib") - env["PREFIX_SHAREDIR"] = env.get("PREFIX_SHAREDIR", "$DESTDIR/share") + env["PREFIX_SHAREDIR"] = env.get("PREFIX_SHAREDIR", "$DESTDIR/share") env["PREFIX_DOCDIR"] = env.get("PREFIX_DOCDIR", "$PREFIX_SHAREDIR/doc") env["PREFIX_INCLUDEDIR"] = env.get("PREFIX_INCLUDEDIR", "$DESTDIR/include") env["PREFIX_DEBUGDIR"] = env.get("PREFIX_DEBUGDIR", _aib_debugdir) @@ -755,7 +793,10 @@ def generate(env): # pylint: disable=too-many-statements base_emitter = builder.emitter # TODO: investigate if using a ListEmitter here can cause # problems if AIB is not loaded last - new_emitter = SCons.Builder.ListEmitter([base_emitter, auto_install_emitter,]) + new_emitter = SCons.Builder.ListEmitter([ + base_emitter, + auto_install_emitter, + ]) builder.emitter = new_emitter base_install_builder = install.BaseInstallBuilder diff --git a/site_scons/site_tools/ccache.py b/site_scons/site_tools/ccache.py index cabb2d316b5..d5ff49c06cf 100644 --- a/site_scons/site_tools/ccache.py +++ b/site_scons/site_tools/ccache.py @@ -21,18 +21,17 @@ import SCons from pkg_resources import parse_version # This is the oldest version of ccache that offers support for -gsplit-dwarf -_ccache_version_min = parse_version("3.2.3") +_ccache_version_min = parse_version('3.2.3') _ccache_version_found = None - def exists(env): """Look for a viable ccache implementation that meets our version requirements.""" # If we already generated, we definitely exist - if "CCACHE_VERSION" in env: + if 'CCACHE_VERSION' in env: return True - ccache = env.get("CCACHE", False) + ccache = env.get('CCACHE', False) if not ccache: return False @@ -40,26 +39,22 @@ def exists(env): if not ccache: return False - pipe = SCons.Action._subproc( - env, - SCons.Util.CLVar(ccache) + ["--version"], - stdin="devnull", - stderr="devnull", - stdout=subprocess.PIPE, - ) + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(ccache) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE) if pipe.wait() != 0: return False validated = False for line in pipe.stdout: - line = line.decode("utf-8") + line = line.decode('utf-8') if validated: continue # consume all data - version_banner = re.search(r"^ccache version", line) + version_banner = re.search(r'^ccache version', line) if not version_banner: continue - ccache_version = re.split("ccache version (.+)", line) + ccache_version = re.split('ccache version (.+)', line) if len(ccache_version) < 2: continue global _ccache_version_found @@ -69,12 +64,11 @@ def exists(env): return validated - def generate(env): """Add ccache support.""" # If we have already generated the tool, don't generate it again. - if "CCACHE_VERSION" in env: + if 'CCACHE_VERSION' in env: return # If we can't find ccache, or it is too old a version, don't @@ -87,18 +81,18 @@ def generate(env): # if ccache is active. Looking at the CCACHE variable in the # environment is not sufficient, since the user may have set it, # but it doesn't work or is out of date. - env["CCACHE_VERSION"] = _ccache_version_found + env['CCACHE_VERSION'] = _ccache_version_found # ccache does not support response files so force scons to always # use the full command # # Note: This only works for Python versions >= 3.5 - env["MAXLINELENGTH"] = math.inf + env['MAXLINELENGTH'] = math.inf # Add ccache to the relevant command lines. Wrap the reference to # ccache in the $( $) pattern so that turning ccache on or off # doesn't invalidate your build. - env["CCCOM"] = "$( $CCACHE $)" + env["CCCOM"] - env["CXXCOM"] = "$( $CCACHE $)" + env["CXXCOM"] - env["SHCCCOM"] = "$( $CCACHE $)" + env["SHCCCOM"] - env["SHCXXCOM"] = "$( $CCACHE $)" + env["SHCXXCOM"] + env['CCCOM'] = '$( $CCACHE $)' + env['CCCOM'] + env['CXXCOM'] = '$( $CCACHE $)' + env['CXXCOM'] + env['SHCCCOM'] = '$( $CCACHE $)' + env['SHCCCOM'] + env['SHCXXCOM'] = '$( $CCACHE $)' + env['SHCXXCOM'] diff --git a/site_scons/site_tools/compilation_db.py b/site_scons/site_tools/compilation_db.py index c3769275e26..03c5e70b42d 100644 --- a/site_scons/site_tools/compilation_db.py +++ b/site_scons/site_tools/compilation_db.py @@ -32,13 +32,13 @@ __COMPILATION_DB_ENTRIES = [] # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if # we could obtain this from SCons. -_CSuffixes = [".c"] -if not SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CSuffixes.append(".C") +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') -_CXXSuffixes = [".cpp", ".cc", ".cxx", ".c++", ".C++"] -if SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CXXSuffixes.append(".C") +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') # We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even @@ -78,13 +78,9 @@ def makeEmitCompilationDbEntry(comstr): dbtarget = __CompilationDbNode(source) entry = env.__COMPILATIONDB_Entry( - target=dbtarget, - source=[], - __COMPILATIONDB_UTARGET=target, - __COMPILATIONDB_USOURCE=source, - __COMPILATIONDB_UACTION=user_action, - __COMPILATIONDB_ENV=env, - ) + target=dbtarget, source=[], __COMPILATIONDB_UTARGET=target, + __COMPILATIONDB_USOURCE=source, __COMPILATIONDB_UACTION=user_action, + __COMPILATIONDB_ENV=env) # TODO: Technically, these next two lines should not be required: it should be fine to # cache the entries. However, they don't seem to update properly. Since they are quick @@ -111,16 +107,15 @@ def CompilationDbEntryAction(target, source, env, **kw): :return: None """ - command = env["__COMPILATIONDB_UACTION"].strfunction( - target=env["__COMPILATIONDB_UTARGET"], - source=env["__COMPILATIONDB_USOURCE"], - env=env["__COMPILATIONDB_ENV"], + command = env['__COMPILATIONDB_UACTION'].strfunction( + target=env['__COMPILATIONDB_UTARGET'], + source=env['__COMPILATIONDB_USOURCE'], + env=env['__COMPILATIONDB_ENV'], ) entry = { - "directory": env.Dir("#").abspath, - "command": command, - "file": str(env["__COMPILATIONDB_USOURCE"][0]), + "directory": env.Dir('#').abspath, "command": command, + "file": str(env['__COMPILATIONDB_USOURCE'][0]) } target[0].write(entry) @@ -132,10 +127,8 @@ def WriteCompilationDb(target, source, env): for s in __COMPILATION_DB_ENTRIES: entries.append(s.read()) - with open(str(target[0]), "w") as target_file: - json.dump( - entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") - ) + with open(str(target[0]), 'w') as target_file: + json.dump(entries, target_file, sort_keys=True, indent=4, separators=(',', ': ')) def ScanCompilationDb(node, env, path): @@ -146,25 +139,18 @@ def generate(env, **kwargs): static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - env["COMPILATIONDB_COMSTR"] = kwargs.get( - "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" - ) + env['COMPILATIONDB_COMSTR'] = kwargs.get('COMPILATIONDB_COMSTR', + 'Building compilation database $TARGET') components_by_suffix = itertools.chain( - itertools.product( - _CSuffixes, - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), - ], - ), - itertools.product( - _CXXSuffixes, - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), - ], - ), + itertools.product(_CSuffixes, [ + (static_obj, SCons.Defaults.StaticObjectEmitter, '$CCCOM'), + (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCCCOM'), + ]), + itertools.product(_CXXSuffixes, [ + (static_obj, SCons.Defaults.StaticObjectEmitter, '$CXXCOM'), + (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCXXCOM'), + ]), ) for entry in components_by_suffix: @@ -173,20 +159,17 @@ def generate(env, **kwargs): # Assumes a dictionary emitter emitter = builder.emitter[suffix] - builder.emitter[suffix] = SCons.Builder.ListEmitter( - [emitter, makeEmitCompilationDbEntry(command),] - ) + builder.emitter[suffix] = SCons.Builder.ListEmitter([ + emitter, + makeEmitCompilationDbEntry(command), + ]) - env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder( - action=SCons.Action.Action(CompilationDbEntryAction, None), - ) + env['BUILDERS']['__COMPILATIONDB_Entry'] = SCons.Builder.Builder( + action=SCons.Action.Action(CompilationDbEntryAction, None), ) - env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder( + env['BUILDERS']['__COMPILATIONDB_Database'] = SCons.Builder.Builder( action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), - target_scanner=SCons.Scanner.Scanner( - function=ScanCompilationDb, node_class=None - ), - ) + target_scanner=SCons.Scanner.Scanner(function=ScanCompilationDb, node_class=None)) def CompilationDatabase(env, target): result = env.__COMPILATIONDB_Database(target=target, source=[]) @@ -196,7 +179,7 @@ def generate(env, **kwargs): return result - env.AddMethod(CompilationDatabase, "CompilationDatabase") + env.AddMethod(CompilationDatabase, 'CompilationDatabase') def exists(env): diff --git a/site_scons/site_tools/dagger/__init__.py b/site_scons/site_tools/dagger/__init__.py index 2e82ebeb058..c63bfc6967e 100644 --- a/site_scons/site_tools/dagger/__init__.py +++ b/site_scons/site_tools/dagger/__init__.py @@ -7,7 +7,6 @@ import SCons from . import dagger - def generate(env, **kwargs): """The entry point for our tool. However, the builder for the JSON file is not actually run until the Dagger method is called @@ -15,43 +14,28 @@ def generate(env, **kwargs): to the native builders for object/libraries. """ - env.Replace( - LIBEMITTER=SCons.Builder.ListEmitter( - [env["LIBEMITTER"], dagger.emit_lib_db_entry] - ) - ) + env.Replace(LIBEMITTER=SCons.Builder.ListEmitter([env['LIBEMITTER'], dagger.emit_lib_db_entry])) running_os = os.sys.platform - if not (running_os.startswith("win") or running_os.startswith("sun")): + if not (running_os.startswith('win') or running_os.startswith('sun')): env.Replace( - PROGEMITTER=SCons.Builder.ListEmitter( - [env["PROGEMITTER"], dagger.emit_prog_db_entry] - ) - ) + PROGEMITTER=SCons.Builder.ListEmitter([env['PROGEMITTER'], dagger.emit_prog_db_entry])) static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - suffixes = [".c", ".cc", ".cxx", ".cpp"] + suffixes = ['.c', '.cc', '.cxx', '.cpp'] obj_builders = [static_obj, shared_obj] - default_emitters = [ - SCons.Defaults.StaticObjectEmitter, - SCons.Defaults.SharedObjectEmitter, - ] + default_emitters = [SCons.Defaults.StaticObjectEmitter, SCons.Defaults.SharedObjectEmitter] for suffix in suffixes: for i in range(len(obj_builders)): obj_builders[i].add_emitter( - suffix, - SCons.Builder.ListEmitter( - [dagger.emit_obj_db_entry, default_emitters[i]] - ), - ) + suffix, SCons.Builder.ListEmitter([dagger.emit_obj_db_entry, default_emitters[i]])) - env["BUILDERS"]["__OBJ_DATABASE"] = SCons.Builder.Builder( - action=SCons.Action.Action(dagger.write_obj_db, None) - ) + env['BUILDERS']['__OBJ_DATABASE'] = SCons.Builder.Builder( + action=SCons.Action.Action(dagger.write_obj_db, None)) def Dagger(env, target="library_dependency_graph.json"): - if running_os.startswith("win") or running_os.startswith("sun"): + if running_os.startswith('win') or running_os.startswith('sun'): logging.error("Dagger is only supported on OSX and Linux") return result = env.__OBJ_DATABASE(target=target, source=[]) @@ -60,7 +44,7 @@ def generate(env, **kwargs): return result - env.AddMethod(Dagger, "Dagger") + env.AddMethod(Dagger, 'Dagger') def exists(env): diff --git a/site_scons/site_tools/dagger/dagger.py b/site_scons/site_tools/dagger/dagger.py index 061401e9bd7..cc208dd23c2 100644 --- a/site_scons/site_tools/dagger/dagger.py +++ b/site_scons/site_tools/dagger/dagger.py @@ -44,12 +44,9 @@ from . import graph from . import graph_consts -LIB_DB = [] # Stores every SCons library nodes -OBJ_DB = [] # Stores every SCons object file node -EXE_DB = ( - {} -) # Stores every SCons executable node, with the object files that build into it {Executable: [object files]} - +LIB_DB = [] # Stores every SCons library nodes +OBJ_DB = [] # Stores every SCons object file node +EXE_DB = {} # Stores every SCons executable node, with the object files that build into it {Executable: [object files]} def list_process(items): """From WIL, converts lists generated from an NM command with unicode strings to lists @@ -60,12 +57,12 @@ def list_process(items): for l in items: if isinstance(l, list): for i in l: - if i.startswith(".L"): + if i.startswith('.L'): continue else: r.append(str(i)) else: - if l.startswith(".L"): + if l.startswith('.L'): continue else: r.append(str(l)) @@ -78,26 +75,26 @@ def get_symbol_worker(object_file, task): """From WIL, launches a worker subprocess which collects either symbols defined or symbols required by an object file""" - platform = "linux" if sys.platform.startswith("linux") else "darwin" + platform = 'linux' if sys.platform.startswith('linux') else 'darwin' - if platform == "linux": - if task == "used": + if platform == 'linux': + if task == 'used': cmd = r'nm "' + object_file + r'" | grep -e "U " | c++filt' - elif task == "defined": + elif task == 'defined': cmd = r'nm "' + object_file + r'" | grep -v -e "U " | c++filt' - elif platform == "darwin": - if task == "used": + elif platform == 'darwin': + if task == 'used': cmd = "nm -u " + object_file + " | c++filt" - elif task == "defined": + elif task == 'defined': cmd = "nm -jU " + object_file + " | c++filt" p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) uses = p.communicate()[0].decode() - if platform == "linux": - return list_process([use[19:] for use in uses.split("\n") if use != ""]) - elif platform == "darwin": - return list_process([use.strip() for use in uses.split("\n") if use != ""]) + if platform == 'linux': + return list_process([use[19:] for use in uses.split('\n') if use != '']) + elif platform == 'darwin': + return list_process([use.strip() for use in uses.split('\n') if use != '']) def emit_obj_db_entry(target, source, env): @@ -138,7 +135,7 @@ def __compute_libdeps(node): env = node.get_env() deps = set() - for child in env.Flatten(getattr(node.attributes, "libdeps_direct", [])): + for child in env.Flatten(getattr(node.attributes, 'libdeps_direct', [])): if not child: continue deps.add(child) diff --git a/site_scons/site_tools/dagger/graph.py b/site_scons/site_tools/dagger/graph.py index fdbc4dc98ba..d8d1e6938ee 100644 --- a/site_scons/site_tools/dagger/graph.py +++ b/site_scons/site_tools/dagger/graph.py @@ -6,7 +6,6 @@ import copy from . import graph_consts - class Graph(object): """Graph class for storing the build dependency graph. The graph stores the directed edges as a nested dict of { RelationshipType: {From_Node: Set of @@ -19,8 +18,8 @@ class Graph(object): A graph can be initialized with a .json file, graph object, or with no args """ if isinstance(input, str): - if input.endswith(".json"): - with open(input, "r") as f: + if input.endswith('.json'): + with open(input, 'r') as f: data = json.load(f, encoding="ascii") nodes = {} should_fail = False @@ -28,9 +27,8 @@ class Graph(object): for node in data["nodes"]: id = str(node["id"]) try: - nodes[id] = node_factory( - id, int(node["node"]["type"]), dict_source=node["node"] - ) + nodes[id] = node_factory(id, int(node["node"]["type"]), + dict_source=node["node"]) except Exception as e: logging.warning("Malformed Data: " + id) should_fail = True @@ -118,9 +116,7 @@ class Graph(object): if from_node not in self._edges[relationship]: self._edges[relationship][from_node] = set() - if any( - item is None for item in (from_node, to_node, from_node_obj, to_node_obj) - ): + if any(item is None for item in (from_node, to_node, from_node_obj, to_node_obj)): raise ValueError self._edges[relationship][from_node].add(to_node) @@ -154,28 +150,21 @@ class Graph(object): edges_dict = self._edges[edge_type] for node in list(edges_dict.keys()): to_nodes = list(self._edges[edge_type][node]) - to_nodes_dicts = [ - {"index": node_index[to_node], "id": to_node} - for to_node in to_nodes - ] - - data["edges"].append( - { - "type": edge_type, - "from_node": {"id": node, "index": node_index[node]}, - "to_node": to_nodes_dicts, - } - ) - - with open(filename, "w", encoding="ascii") as outfile: + to_nodes_dicts = [{"index": node_index[to_node], "id": to_node} + for to_node in to_nodes] + + data["edges"].append({"type": edge_type, + "from_node": {"id": node, + "index": node_index[node]}, + "to_node": to_nodes_dicts}) + + with open(filename, 'w', encoding="ascii") as outfile: json.dump(data, outfile, indent=4) def __str__(self): - return ("<Number of Nodes : {0}, Number of Edges : {1}, " "Hash: {2}>").format( - len(list(self._nodes.keys())), - sum(len(x) for x in list(self._edges.values())), - hash(self), - ) + return ("<Number of Nodes : {0}, Number of Edges : {1}, " + "Hash: {2}>").format(len(list(self._nodes.keys())), + sum(len(x) for x in list(self._edges.values())), hash(self)) class NodeInterface(object, metaclass=abc.ABCMeta): @@ -194,7 +183,6 @@ class NodeInterface(object, metaclass=abc.ABCMeta): class NodeLib(NodeInterface): """NodeLib class which represents a library within the graph """ - def __init__(self, id, name, input=None): if isinstance(input, dict): should_fail = False @@ -295,13 +283,10 @@ class NodeLib(NodeInterface): def __eq__(self, other): if isinstance(other, NodeLib): - return ( - self._id == other._id - and self._defined_symbols == other._defined_symbols - and self._defined_files == other._defined_files - and self._dependent_libs == other._dependent_libs - and self._dependent_files == other._dependent_files - ) + return (self._id == other._id and self._defined_symbols == other._defined_symbols + and self._defined_files == other._defined_files + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files) else: return False @@ -424,13 +409,10 @@ class NodeSymbol(NodeInterface): def __eq__(self, other): if isinstance(other, NodeSymbol): - return ( - self.id == other.id - and self._libs == other._libs - and self._files == other._files - and self._dependent_libs == other._dependent_libs - and self._dependent_files == other._dependent_files - ) + return (self.id == other.id and self._libs == other._libs + and self._files == other._files + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files) else: return False @@ -545,13 +527,10 @@ class NodeFile(NodeInterface): def __eq__(self, other): if isinstance(other, NodeSymbol): - return ( - self.id == other.id - and self._lib == other._lib - and self._dependent_libs == other._dependent_libs - and self._dependent_files == other._dependent_files - and self._defined_symbols == other._defined_symbols - ) + return (self.id == other.id and self._lib == other._lib + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files + and self._defined_symbols == other._defined_symbols) else: return False diff --git a/site_scons/site_tools/dagger/graph_consts.py b/site_scons/site_tools/dagger/graph_consts.py index a2077675463..eae5db9b6c6 100644 --- a/site_scons/site_tools/dagger/graph_consts.py +++ b/site_scons/site_tools/dagger/graph_consts.py @@ -22,3 +22,4 @@ NODE_TYPES = list(range(1, 5)) """Error/query codes""" NODE_NOT_FOUND = 1 + diff --git a/site_scons/site_tools/dagger/graph_test.py b/site_scons/site_tools/dagger/graph_test.py index e532386f852..39bdf77ab7c 100644 --- a/site_scons/site_tools/dagger/graph_test.py +++ b/site_scons/site_tools/dagger/graph_test.py @@ -73,33 +73,22 @@ class CustomAssertions: raise AssertionError("Nodes not of same type") if node1.type == graph_consts.NODE_LIB: - if ( - node1._defined_symbols != node2._defined_symbols - or node1._defined_files != node2._defined_files - or node1._dependent_libs != node2._dependent_libs - or node1._dependent_files != node2._dependent_files - or node1._id != node2._id - ): + if (node1._defined_symbols != node2._defined_symbols + or node1._defined_files != node2._defined_files + or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1._id != node2._id): raise AssertionError("Nodes not equal") elif node1.type == graph_consts.NODE_SYM: - if ( - node1._libs != node2._libs - or node1._files != node2._files - or node1._dependent_libs != node2._dependent_libs - or node1._dependent_files != node2._dependent_files - or node1.id != node2.id - ): + if (node1._libs != node2._libs or node1._files != node2._files + or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1.id != node2.id): raise AssertionError("Nodes not equal") else: - if ( - node1._lib != node2._lib - or node1._dependent_libs != node2._dependent_libs - or node1._dependent_files != node2._dependent_files - or node1.id != node2.id - or node1._defined_symbols != node2._defined_symbols - ): + if (node1._lib != node2._lib or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1.id != node2.id + or node1._defined_symbols != node2._defined_symbols): raise AssertionError("Nodes not equal") @@ -142,80 +131,51 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): self.assertRaises(TypeError, self.g.add_node, "not a node") def test_add_edge_exceptions(self): - self.assertRaises( - TypeError, - self.g.add_edge, - "NOT A RELATIONSHIP", - self.from_node_lib.id, - self.to_node_lib.id, - ) - - self.assertRaises( - ValueError, - self.g.add_edge, - graph_consts.LIB_LIB, - "not a node", - "not a node", - ) + self.assertRaises(TypeError, self.g.add_edge, "NOT A RELATIONSHIP", + self.from_node_lib.id, self.to_node_lib.id) + + self.assertRaises(ValueError, self.g.add_edge, + graph_consts.LIB_LIB, "not a node", "not a node") def test_add_edge_libs(self): - self.g.add_edge( - graph_consts.LIB_LIB, self.from_node_lib.id, self.to_node_lib.id - ) - self.g.add_edge( - graph_consts.LIB_LIB, self.from_node_lib.id, self.to_node_lib.id - ) - self.g.add_edge( - graph_consts.LIB_SYM, self.from_node_lib.id, self.to_node_sym.id - ) - self.g.add_edge( - graph_consts.LIB_FIL, self.from_node_lib.id, self.to_node_file.id - ) - - self.assertEqual( - self.g.edges[graph_consts.LIB_LIB][self.from_node_lib.id], - set([self.to_node_lib.id]), - ) - - self.assertEqual( - self.g.edges[graph_consts.LIB_SYM][self.from_node_lib.id], - set([self.to_node_sym.id]), - ) - - self.assertEqual( - self.g.edges[graph_consts.LIB_FIL][self.from_node_lib.id], - set([self.to_node_file.id]), - ) - - self.assertEqual(self.to_node_lib.dependent_libs, set([self.from_node_lib.id])) + self.g.add_edge(graph_consts.LIB_LIB, self.from_node_lib.id, + self.to_node_lib.id) + self.g.add_edge(graph_consts.LIB_LIB, self.from_node_lib.id, + self.to_node_lib.id) + self.g.add_edge(graph_consts.LIB_SYM, self.from_node_lib.id, + self.to_node_sym.id) + self.g.add_edge(graph_consts.LIB_FIL, self.from_node_lib.id, + self.to_node_file.id) + + self.assertEqual(self.g.edges[graph_consts.LIB_LIB][ + self.from_node_lib.id], set([self.to_node_lib.id])) + + self.assertEqual(self.g.edges[graph_consts.LIB_SYM][ + self.from_node_lib.id], set([self.to_node_sym.id])) + + self.assertEqual(self.g.edges[graph_consts.LIB_FIL][ + self.from_node_lib.id], set([self.to_node_file.id])) + + self.assertEqual(self.to_node_lib.dependent_libs, + set([self.from_node_lib.id])) def test_add_edge_files(self): - self.g.add_edge( - graph_consts.FIL_FIL, self.from_node_file.id, self.to_node_file.id - ) - self.g.add_edge( - graph_consts.FIL_SYM, self.from_node_file.id, self.to_node_sym.id - ) - self.g.add_edge( - graph_consts.FIL_LIB, self.from_node_file.id, self.to_node_lib.id - ) - - self.assertEqual( - self.g.edges[graph_consts.FIL_FIL][self.from_node_file.id], - set([self.to_node_file.id]), - ) - self.assertEqual( - self.g.edges[graph_consts.FIL_SYM][self.from_node_file.id], - set([self.to_node_sym.id]), - ) - self.assertEqual( - self.g.edges[graph_consts.FIL_LIB][self.from_node_file.id], - set([self.to_node_lib.id]), - ) - - self.assertEqual( - self.to_node_file.dependent_files, set([self.from_node_file.id]) - ) + self.g.add_edge(graph_consts.FIL_FIL, self.from_node_file.id, + self.to_node_file.id) + self.g.add_edge(graph_consts.FIL_SYM, self.from_node_file.id, + self.to_node_sym.id) + self.g.add_edge(graph_consts.FIL_LIB, self.from_node_file.id, + self.to_node_lib.id) + + self.assertEqual(self.g.edges[graph_consts.FIL_FIL][ + self.from_node_file.id], set([self.to_node_file.id])) + self.assertEqual(self.g.edges[graph_consts.FIL_SYM][ + self.from_node_file.id], set([self.to_node_sym.id])) + self.assertEqual(self.g.edges[graph_consts.FIL_LIB][ + self.from_node_file.id], set([self.to_node_lib.id])) + + self.assertEqual(self.to_node_file.dependent_files, + set([self.from_node_file.id])) def test_export_to_json(self): generated_graph = generate_graph() @@ -237,11 +197,10 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): # the need for a custom assertion self.assertNodeEquals( - graph_fromJSON.get_node(id), correct_graph.get_node(id) - ) + graph_fromJSON.get_node(id), correct_graph.get_node(id)) self.assertEqual(graph_fromJSON.edges, correct_graph.edges) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/site_scons/site_tools/distsrc.py b/site_scons/site_tools/distsrc.py index 6fab5559521..cc72c0655f7 100644 --- a/site_scons/site_tools/distsrc.py +++ b/site_scons/site_tools/distsrc.py @@ -26,15 +26,13 @@ from distutils.spawn import find_executable __distsrc_callbacks = [] - class DistSrcFile: def __init__(self, **kwargs): - [setattr(self, key, val) for (key, val) in list(kwargs.items())] + [ setattr(self, key, val) for (key, val) in list(kwargs.items()) ] def __str__(self): return self.name - class DistSrcArchive: def __init__(self, archive_type, archive_file, filename, mode): self.archive_type = archive_type @@ -46,14 +44,17 @@ class DistSrcArchive: def Open(filename): if filename.endswith("tar"): return DistSrcTarArchive( - "tar", - tarfile.open(filename, "r", format=tarfile.PAX_FORMAT), + 'tar', + tarfile.open(filename, 'r', format=tarfile.PAX_FORMAT), filename, - "r", + 'r', ) elif filename.endswith("zip"): return DistSrcZipArchive( - "zip", zipfile.ZipFile(filename, "a"), filename, "a", + 'zip', + zipfile.ZipFile(filename, 'a'), + filename, + 'a', ) def close(self): @@ -77,31 +78,29 @@ class DistSrcTarArchive(DistSrcArchive): uid=item_data.uid, gid=item_data.gid, uname=item_data.uname, - gname=item_data.uname, + gname=item_data.uname ) - def append_file_contents( - self, - filename, - file_contents, - mtime=time.time(), - mode=0o644, - uname="root", - gname="root", - ): + def append_file_contents(self, filename, file_contents, + mtime=time.time(), + mode=0o644, + uname="root", + gname="root"): file_metadata = tarfile.TarInfo(name=filename) file_metadata.mtime = mtime file_metadata.mode = mode file_metadata.uname = uname file_metadata.gname = gname file_metadata.size = len(file_contents) - file_buf = io.BytesIO(file_contents.encode("utf-8")) - if self.archive_mode == "r": + file_buf = io.BytesIO(file_contents.encode('utf-8')) + if self.archive_mode == 'r': self.archive_file.close() self.archive_file = tarfile.open( - self.archive_name, "a", format=tarfile.PAX_FORMAT, + self.archive_name, + 'a', + format=tarfile.PAX_FORMAT, ) - self.archive_mode = "a" + self.archive_mode = 'a' self.archive_file.addfile(file_metadata, fileobj=file_buf) def append_file(self, filename, localfile): @@ -127,31 +126,24 @@ class DistSrcZipArchive(DistSrcArchive): uid=0, gid=0, uname="root", - gname="root", + gname="root" ) - def append_file_contents( - self, - filename, - file_contents, - mtime=time.time(), - mode=0o644, - uname="root", - gname="root", - ): + def append_file_contents(self, filename, file_contents, + mtime=time.time(), + mode=0o644, + uname="root", + gname="root"): self.archive_file.writestr(filename, file_contents) def append_file(self, filename, localfile): self.archive_file.write(localfile, arcname=filename) - def build_error_action(msg): def error_stub(target=None, source=None, env=None): print(msg) env.Exit(1) - - return [error_stub] - + return [ error_stub ] def distsrc_action_generator(source, target, env, for_signature): # This is done in two stages because env.WhereIs doesn't seem to work @@ -171,20 +163,16 @@ def distsrc_action_generator(source, target, env, for_signature): archive_wrapper.close() target_ext = str(target[0])[-3:] - if not target_ext in ["zip", "tar"]: + if not target_ext in [ 'zip', 'tar' ]: print("Invalid file format for distsrc. Must be tar or zip file") env.Exit(1) - git_cmd = ( - '"%s" archive --format %s --output %s --prefix ${MONGO_DIST_SRC_PREFIX} HEAD' - % (git_path, target_ext, target[0]) - ) + git_cmd = "\"%s\" archive --format %s --output %s --prefix ${MONGO_DIST_SRC_PREFIX} HEAD" % ( + git_path, target_ext, target[0]) return [ SCons.Action.Action(git_cmd, "Running git archive for $TARGET"), - SCons.Action.Action( - run_distsrc_callbacks, "Running distsrc callbacks for $TARGET" - ), + SCons.Action.Action(run_distsrc_callbacks, "Running distsrc callbacks for $TARGET") ] @@ -193,10 +181,8 @@ def add_callback(env, fn): def generate(env, **kwargs): - env.AddMethod(add_callback, "AddDistSrcCallback") - env["BUILDERS"]["__DISTSRC"] = SCons.Builder.Builder( - generator=distsrc_action_generator, - ) + env.AddMethod(add_callback, 'AddDistSrcCallback') + env['BUILDERS']['__DISTSRC'] = SCons.Builder.Builder(generator=distsrc_action_generator, ) def DistSrc(env, target): result = env.__DISTSRC(target=target, source=[]) @@ -204,7 +190,7 @@ def generate(env, **kwargs): env.NoCache(result) return result - env.AddMethod(DistSrc, "DistSrc") + env.AddMethod(DistSrc, 'DistSrc') def exists(env): diff --git a/site_scons/site_tools/git_decider.py b/site_scons/site_tools/git_decider.py index 1d56a302e17..fb0c9c7657b 100644 --- a/site_scons/site_tools/git_decider.py +++ b/site_scons/site_tools/git_decider.py @@ -20,22 +20,21 @@ def generate(env, **kwargs): # Grab the existing decider functions out of the environment # so we can invoke them when we can't use Git. base_decider = env.decide_target - if base_decider != env.decide_source: + if (base_decider != env.decide_source): raise Exception("Decider environment seems broken") from git import Git - - thisRepo = Git(env.Dir("#").abspath) - currentGitState = thisRepo.ls_files("--stage") - lines = currentGitState.split("\n") + thisRepo = Git(env.Dir('#').abspath) + currentGitState = thisRepo.ls_files('--stage') + lines = currentGitState.split('\n') file_sha1_map = {} for line in lines: line_content = line.split() file_sha1_map[env.File(line_content[3]).path] = line_content[1] - for m in thisRepo.ls_files("-m").split("\n"): - if m: + for m in thisRepo.ls_files('-m').split('\n'): + if (m): del file_sha1_map[env.File(m).path] def is_known_to_git(dependency): @@ -48,7 +47,7 @@ def generate(env, **kwargs): dependency.get_ninfo().csig = gitInfoForDep return False - if not (hasattr(prev_ni, "csig")): + if not (hasattr(prev_ni, 'csig')): prev_ni.csig = gitInfoForDep result = gitInfoForDep == prev_ni.csig @@ -65,8 +64,7 @@ def generate(env, **kwargs): def exists(env): try: from git import Git - - Git(env.Dir("#").abspath).ls_files("--stage") + Git(env.Dir('#').abspath).ls_files('--stage') return True except: return False diff --git a/site_scons/site_tools/gziptool.py b/site_scons/site_tools/gziptool.py index dce6062c993..3142d46cae9 100644 --- a/site_scons/site_tools/gziptool.py +++ b/site_scons/site_tools/gziptool.py @@ -16,20 +16,19 @@ import SCons import gzip import shutil - def GZipAction(target, source, env, **kw): - dst_gzip = gzip.GzipFile(str(target[0]), "wb") - with open(str(source[0]), "rb") as src_file: + dst_gzip = gzip.GzipFile(str(target[0]), 'wb') + with open(str(source[0]), 'rb') as src_file: shutil.copyfileobj(src_file, dst_gzip) dst_gzip.close() - def generate(env, **kwargs): - env["BUILDERS"]["__GZIPTOOL"] = SCons.Builder.Builder( + env['BUILDERS']['__GZIPTOOL'] = SCons.Builder.Builder( action=SCons.Action.Action(GZipAction, "$GZIPTOOL_COMSTR") ) - env["GZIPTOOL_COMSTR"] = kwargs.get( - "GZIPTOOL_COMSTR", "Compressing $TARGET with gzip" + env['GZIPTOOL_COMSTR'] = kwargs.get( + "GZIPTOOL_COMSTR", + "Compressing $TARGET with gzip" ) def GZipTool(env, target, source): @@ -37,7 +36,7 @@ def generate(env, **kwargs): env.AlwaysBuild(result) return result - env.AddMethod(GZipTool, "GZip") + env.AddMethod(GZipTool, 'GZip') def exists(env): diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py index 2a7e72b8f5a..dfa7aa32748 100644 --- a/site_scons/site_tools/icecream.py +++ b/site_scons/site_tools/icecream.py @@ -20,8 +20,8 @@ import subprocess from pkg_resources import parse_version -_icecream_version_min = parse_version("1.1rc2") -_ccache_nocpp2_version = parse_version("3.4.1") +_icecream_version_min = parse_version('1.1rc2') +_ccache_nocpp2_version = parse_version('3.4.1') # I'd prefer to use value here, but amazingly, its __str__ returns the @@ -39,7 +39,6 @@ class _BoundSubstitution: self.result = self.env.subst(self.expression) return self.result - def generate(env): if not exists(env): @@ -51,33 +50,31 @@ def generate(env): # little differently if ccache is in play. If you don't use the # TOOLS variable to configure your tools, you should explicitly # load the ccache tool before you load icecream. - if "ccache" in env["TOOLS"] and not "CCACHE_VERSION" in env: - env.Tool("ccache") - ccache_enabled = "CCACHE_VERSION" in env + if 'ccache' in env['TOOLS'] and not 'CCACHE_VERSION' in env: + env.Tool('ccache') + ccache_enabled = ('CCACHE_VERSION' in env) # Absoluteify, so we can derive ICERUN - env["ICECC"] = env.WhereIs("$ICECC") + env['ICECC'] = env.WhereIs('$ICECC') - if not "ICERUN" in env: - env["ICERUN"] = env.File("$ICECC").File("icerun") + if not 'ICERUN' in env: + env['ICERUN'] = env.File('$ICECC').File('icerun') # Absoluteify, for parity with ICECC - env["ICERUN"] = env.WhereIs("$ICERUN") + env['ICERUN'] = env.WhereIs('$ICERUN') # We can't handle sanitizer blacklist files, so disable icecc then, and just flow through # icerun to prevent slamming the local system with a huge -j value. if any( - f.startswith("-fsanitize-blacklist=") - for fs in ["CCFLAGS", "CFLAGS", "CXXFLAGS"] - for f in env[fs] - ): - env["ICECC"] = "$ICERUN" + f.startswith("-fsanitize-blacklist=") for fs in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS'] + for f in env[fs]): + env['ICECC'] = '$ICERUN' # Make CC and CXX absolute paths too. It is better for icecc. - env["CC"] = env.WhereIs("$CC") - env["CXX"] = env.WhereIs("$CXX") + env['CC'] = env.WhereIs('$CC') + env['CXX'] = env.WhereIs('$CXX') - if "ICECC_VERSION" in env: + if 'ICECC_VERSION' in env: # TODO: # # If ICECC_VERSION is a file, we are done. If it is a file @@ -89,8 +86,8 @@ def generate(env): pass else: # Make a predictable name for the toolchain - icecc_version_target_filename = env.subst("$CC$CXX").replace("/", "_") - icecc_version_dir = env.Dir("$BUILD_ROOT/scons/icecc") + icecc_version_target_filename = env.subst('$CC$CXX').replace('/', '_') + icecc_version_dir = env.Dir('$BUILD_ROOT/scons/icecc') icecc_version = icecc_version_dir.File(icecc_version_target_filename) # There is a weird ordering problem that occurs when the ninja generator @@ -107,10 +104,14 @@ def generate(env): # Make an isolated environment so that our setting of ICECC_VERSION in the environment # doesn't appear when executing icecc_create_env toolchain_env = env.Clone() - if toolchain_env.ToolchainIs("clang"): + if toolchain_env.ToolchainIs('clang'): toolchain = toolchain_env.Command( target=icecc_version, - source=["$ICECC_CREATE_ENV", "$CC", "$CXX"], + source=[ + '$ICECC_CREATE_ENV', + '$CC', + '$CXX' + ], action=[ "${SOURCES[0]} --clang ${SOURCES[1].abspath} /bin/true $TARGET", ], @@ -118,7 +119,11 @@ def generate(env): else: toolchain = toolchain_env.Command( target=icecc_version, - source=["$ICECC_CREATE_ENV", "$CC", "$CXX"], + source=[ + '$ICECC_CREATE_ENV', + '$CC', + '$CXX' + ], action=[ "${SOURCES[0]} --gcc ${SOURCES[1].abspath} ${SOURCES[2].abspath} $TARGET", ], @@ -133,13 +138,13 @@ def generate(env): # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if # we could obtain this from SCons. - _CSuffixes = [".c"] - if not SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CSuffixes.append(".C") + _CSuffixes = ['.c'] + if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') - _CXXSuffixes = [".cpp", ".cc", ".cxx", ".c++", ".C++"] - if SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CXXSuffixes.append(".C") + _CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] + if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') suffixes = _CSuffixes + _CXXSuffixes for object_builder in SCons.Tool.createObjBuilders(env): @@ -148,35 +153,44 @@ def generate(env): if not suffix in suffixes: continue base = emitterdict[suffix] - emitterdict[suffix] = SCons.Builder.ListEmitter( - [base, icecc_toolchain_dependency_emitter] - ) + emitterdict[suffix] = SCons.Builder.ListEmitter([ + base, + icecc_toolchain_dependency_emitter + ]) # Add ICECC_VERSION to the environment, pointed at the generated # file so that we can expand it in the realpath expressions for # CXXCOM and friends below. - env["ICECC_VERSION"] = icecc_version + env['ICECC_VERSION'] = icecc_version - if env.ToolchainIs("clang"): - env["ENV"]["ICECC_CLANG_REMOTE_CPP"] = 1 + if env.ToolchainIs('clang'): + env['ENV']['ICECC_CLANG_REMOTE_CPP'] = 1 - if ccache_enabled and env["CCACHE_VERSION"] >= _ccache_nocpp2_version: - env.AppendUnique(CCFLAGS=["-frewrite-includes"]) - env["ENV"]["CCACHE_NOCPP2"] = 1 + if ccache_enabled and env['CCACHE_VERSION'] >= _ccache_nocpp2_version: + env.AppendUnique( + CCFLAGS=[ + '-frewrite-includes' + ] + ) + env['ENV']['CCACHE_NOCPP2'] = 1 else: - env.AppendUnique(CCFLAGS=["-fdirectives-only"]) + env.AppendUnique( + CCFLAGS=[ + '-fdirectives-only' + ] + ) if ccache_enabled: - env["ENV"]["CCACHE_NOCPP2"] = 1 + env['ENV']['CCACHE_NOCPP2'] = 1 - if "ICECC_SCHEDULER" in env: - env["ENV"]["USE_SCHEDULER"] = env["ICECC_SCHEDULER"] + if 'ICECC_SCHEDULER' in env: + env['ENV']['USE_SCHEDULER'] = env['ICECC_SCHEDULER'] # Make sure it is a file node so that we can call `.abspath` on it # below. We must defer the abspath and realpath calls until after # the tool has completed and we have begun building, since we need # the real toolchain tarball to get created first on disk as part # of the DAG walk. - env["ICECC_VERSION"] = env.File("$ICECC_VERSION") + env['ICECC_VERSION'] = env.File('$ICECC_VERSION') # Not all platforms have the readlink utility, so create our own # generator for that. @@ -191,21 +205,20 @@ def generate(env): # nice to be able to memoize away this call, but we should # think carefully about where to store the result of such # memoization. - return os.path.realpath(env["ICECC_VERSION"].abspath) - - env["ICECC_VERSION_GEN"] = icecc_version_gen + return os.path.realpath(env['ICECC_VERSION'].abspath) + env['ICECC_VERSION_GEN'] = icecc_version_gen # Build up the string we will set in the environment to tell icecream # about the compiler package. - icecc_version_string = "${ICECC_VERSION_GEN}" - if "ICECC_VERSION_ARCH" in env: - icecc_version_string = "${ICECC_VERSION_ARCH}:" + icecc_version_string + icecc_version_string = '${ICECC_VERSION_GEN}' + if 'ICECC_VERSION_ARCH' in env: + icecc_version_string = '${ICECC_VERSION_ARCH}:' + icecc_version_string # Use our BoundSubstitition class to put ICECC_VERSION into # env['ENV'] with substitution in play. This lets us defer doing # the realpath in the generator above until after we have made the # tarball. - env["ENV"]["ICECC_VERSION"] = _BoundSubstitution(env, icecc_version_string) + env['ENV']['ICECC_VERSION'] = _BoundSubstitution(env, icecc_version_string) # If ccache is in play we actually want the icecc binary in the # CCACHE_PREFIX environment variable, not on the command line, per @@ -217,13 +230,13 @@ def generate(env): # compiler flags (things like -fdirectives-only), but we still try # to do the right thing. if ccache_enabled: - env["ENV"]["CCACHE_PREFIX"] = _BoundSubstitution(env, "$ICECC") + env['ENV']['CCACHE_PREFIX'] = _BoundSubstitution(env, '$ICECC') else: - icecc_string = "$( $ICECC $)" - env["CCCOM"] = " ".join([icecc_string, env["CCCOM"]]) - env["CXXCOM"] = " ".join([icecc_string, env["CXXCOM"]]) - env["SHCCCOM"] = " ".join([icecc_string, env["SHCCCOM"]]) - env["SHCXXCOM"] = " ".join([icecc_string, env["SHCXXCOM"]]) + icecc_string = '$( $ICECC $)' + env['CCCOM'] = ' '.join([icecc_string, env['CCCOM']]) + env['CXXCOM'] = ' '.join([icecc_string, env['CXXCOM']]) + env['SHCCCOM'] = ' '.join([icecc_string, env['SHCCCOM']]) + env['SHCXXCOM'] = ' '.join([icecc_string, env['SHCXXCOM']]) # Make link like jobs flow through icerun so we don't kill the # local machine. @@ -231,9 +244,9 @@ def generate(env): # TODO: Should we somehow flow SPAWN or other universal shell launch through # ICERUN to avoid saturating the local machine, and build something like # ninja pools? - env["ARCOM"] = "$( $ICERUN $) " + env["ARCOM"] - env["LINKCOM"] = "$( $ICERUN $) " + env["LINKCOM"] - env["SHLINKCOM"] = "$( $ICERUN $) " + env["SHLINKCOM"] + env['ARCOM'] = '$( $ICERUN $) ' + env['ARCOM'] + env['LINKCOM'] = '$( $ICERUN $) ' + env['LINKCOM'] + env['SHLINKCOM'] = '$( $ICERUN $) ' + env['SHLINKCOM'] # Uncomment these to debug your icecc integration # env['ENV']['ICECC_DEBUG'] = 'debug' @@ -242,33 +255,29 @@ def generate(env): def exists(env): - icecc = env.get("ICECC", False) + icecc = env.get('ICECC', False) if not icecc: return False icecc = env.WhereIs(icecc) if not icecc: return False - pipe = SCons.Action._subproc( - env, - SCons.Util.CLVar(icecc) + ["--version"], - stdin="devnull", - stderr="devnull", - stdout=subprocess.PIPE, - ) + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(icecc) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE) if pipe.wait() != 0: return False validated = False for line in pipe.stdout: - line = line.decode("utf-8") + line = line.decode('utf-8') if validated: continue # consume all data - version_banner = re.search(r"^ICECC ", line) + version_banner = re.search(r'^ICECC ', line) if not version_banner: continue - icecc_version = re.split("ICECC (.+)", line) + icecc_version = re.split('ICECC (.+)', line) if len(icecc_version) < 2: continue icecc_version = parse_version(icecc_version[1]) diff --git a/site_scons/site_tools/idl_tool.py b/site_scons/site_tools/idl_tool.py index b041dc8cb7e..3edfc55a9b4 100755 --- a/site_scons/site_tools/idl_tool.py +++ b/site_scons/site_tools/idl_tool.py @@ -32,20 +32,18 @@ def idlc_emitter(target, source, env): first_source = str(source[0]) if not first_source.endswith(".idl"): - raise ValueError( - "Bad idl file name '%s', it must end with '.idl' " % (first_source) - ) + raise ValueError("Bad idl file name '%s', it must end with '.idl' " % (first_source)) base_file_name, _ = SCons.Util.splitext(str(target[0])) target_source = base_file_name + "_gen.cpp" target_header = base_file_name + "_gen.h" - env.Alias("generated-sources", [target_source, target_header]) + env.Alias('generated-sources', [target_source, target_header]) return [target_source, target_header], source -IDLCAction = SCons.Action.Action("$IDLCCOM", "$IDLCCOMSTR") +IDLCAction = SCons.Action.Action('$IDLCCOM', '$IDLCCOMSTR') def idl_scanner(node, env, path): @@ -55,30 +53,24 @@ def idl_scanner(node, env, path): nodes_deps_list = IDL_GLOBAL_DEPS[:] - with open(str(node), encoding="utf-8") as file_stream: - parsed_doc = idlc.parser.parse( - file_stream, str(node), idlc.CompilerImportResolver(["src"]) - ) + with open(str(node), encoding='utf-8') as file_stream: + parsed_doc = idlc.parser.parse(file_stream, str(node), + idlc.CompilerImportResolver(['src'])) if not parsed_doc.errors and parsed_doc.spec.imports is not None: - nodes_deps_list.extend( - [env.File(d) for d in sorted(parsed_doc.spec.imports.dependencies)] - ) + nodes_deps_list.extend([ + env.File(d) for d in sorted(parsed_doc.spec.imports.dependencies) + ]) setattr(node.attributes, "IDL_NODE_DEPS", nodes_deps_list) return nodes_deps_list -idl_scanner = SCons.Scanner.Scanner(function=idl_scanner, skeys=[".idl"]) +idl_scanner = SCons.Scanner.Scanner(function=idl_scanner, skeys=['.idl']) # TODO: create a scanner for imports when imports are implemented -IDLCBuilder = SCons.Builder.Builder( - action=IDLCAction, - emitter=idlc_emitter, - srcsuffx=".idl", - suffix=".cpp", - source_scanner=idl_scanner, -) +IDLCBuilder = SCons.Builder.Builder(action=IDLCAction, emitter=idlc_emitter, srcsuffx=".idl", + suffix=".cpp", source_scanner=idl_scanner) def generate(env): @@ -86,27 +78,22 @@ def generate(env): env.Append(SCANNERS=idl_scanner) - env["BUILDERS"]["Idlc"] = bld + env['BUILDERS']['Idlc'] = bld sys.path.append(env.Dir("#buildscripts").get_abspath()) import buildscripts.idl.idl.compiler as idlc_mod - global idlc idlc = idlc_mod - env["IDLC"] = sys.executable + " buildscripts/idl/idlc.py" - env["IDLCFLAGS"] = "" - base_dir = env.subst("$BUILD_ROOT/$VARIANT_DIR").replace("#", "") - env["IDLCCOM"] = ( - "$IDLC --include src --base_dir %s --target_arch $TARGET_ARCH --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES " - % (base_dir) - ) - env["IDLCSUFFIX"] = ".idl" - - IDL_GLOBAL_DEPS = env.Glob("#buildscripts/idl/*.py") + env.Glob( - "#buildscripts/idl/idl/*.py" - ) - env["IDL_HAS_INLINE_DEPENDENCIES"] = True + env['IDLC'] = sys.executable + " buildscripts/idl/idlc.py" + env['IDLCFLAGS'] = '' + base_dir = env.subst('$BUILD_ROOT/$VARIANT_DIR').replace("#", "") + env['IDLCCOM'] = '$IDLC --include src --base_dir %s --target_arch $TARGET_ARCH --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % ( + base_dir) + env['IDLCSUFFIX'] = '.idl' + + IDL_GLOBAL_DEPS = env.Glob('#buildscripts/idl/*.py') + env.Glob('#buildscripts/idl/idl/*.py') + env['IDL_HAS_INLINE_DEPENDENCIES'] = True def exists(env): diff --git a/site_scons/site_tools/incremental_link.py b/site_scons/site_tools/incremental_link.py index dddd8b770f6..31f16a482da 100644 --- a/site_scons/site_tools/incremental_link.py +++ b/site_scons/site_tools/incremental_link.py @@ -21,29 +21,28 @@ def _tag_as_precious(target, source, env): def generate(env): - builders = env["BUILDERS"] - for builder in ("Program", "SharedLibrary", "LoadableModule"): + builders = env['BUILDERS'] + for builder in ('Program', 'SharedLibrary', 'LoadableModule'): emitter = builders[builder].emitter - builders[builder].emitter = SCons.Builder.ListEmitter( - [emitter, _tag_as_precious,] - ) + builders[builder].emitter = SCons.Builder.ListEmitter([ + emitter, + _tag_as_precious, + ]) def exists(env): # By default, the windows linker is incremental, so unless # overridden in the environment with /INCREMENTAL:NO, the tool is # in play. - if env.TargetOSIs("windows") and not "/INCREMENTAL:NO" in env["LINKFLAGS"]: + if env.TargetOSIs('windows') and not "/INCREMENTAL:NO" in env['LINKFLAGS']: return True # On posix platforms, excluding darwin, we may have enabled # incremental linking. Check for the relevant flags. - if ( - env.TargetOSIs("posix") - and not env.TargetOSIs("darwin") - and "-fuse-ld=gold" in env["LINKFLAGS"] - and "-Wl,--incremental" in env["LINKFLAGS"] - ): + if env.TargetOSIs('posix') and \ + not env.TargetOSIs('darwin') and \ + "-fuse-ld=gold" in env['LINKFLAGS'] and \ + "-Wl,--incremental" in env['LINKFLAGS']: return True return False diff --git a/site_scons/site_tools/jsheader.py b/site_scons/site_tools/jsheader.py index f842a8672e4..4c2765b7108 100644 --- a/site_scons/site_tools/jsheader.py +++ b/site_scons/site_tools/jsheader.py @@ -2,15 +2,12 @@ from SCons.Script import Action def jsToH(env, target, source): - return env.Command( - target=target, - source=["#site_scons/site_tools/jstoh.py"] + source, - action=Action("$PYTHON ${SOURCES[0]} $TARGET ${SOURCES[1:]}"), - ) + return env.Command(target=target, source=['#site_scons/site_tools/jstoh.py'] + source, + action=Action('$PYTHON ${SOURCES[0]} $TARGET ${SOURCES[1:]}')) def generate(env, **kw): - env.AddMethod(jsToH, "JSHeader") + env.AddMethod(jsToH, 'JSHeader') def exists(env): diff --git a/site_scons/site_tools/jstoh.py b/site_scons/site_tools/jstoh.py index 5f88043d3bb..50c0b66cf32 100644 --- a/site_scons/site_tools/jstoh.py +++ b/site_scons/site_tools/jstoh.py @@ -9,39 +9,37 @@ def jsToHeader(target, source): h = [ '#include "mongo/base/string_data.h"', '#include "mongo/scripting/engine.h"', - "namespace mongo {", - "namespace JSFiles{", + 'namespace mongo {', + 'namespace JSFiles{', ] def lineToChars(s): - return ",".join(str(ord(c)) for c in (s.rstrip() + "\n")) + "," + return ','.join(str(ord(c)) for c in (s.rstrip() + '\n')) + ',' for s in source: filename = str(s) - objname = os.path.split(filename)[1].split(".")[0] - stringname = "_jscode_raw_" + objname + objname = os.path.split(filename)[1].split('.')[0] + stringname = '_jscode_raw_' + objname - h.append("constexpr char " + stringname + "[] = {") + h.append('constexpr char ' + stringname + "[] = {") - with open(filename, "r") as f: + with open(filename, 'r') as f: for line in f: h.append(lineToChars(line)) h.append("0};") # symbols aren't exported w/o this - h.append("extern const JSFile %s;" % objname) - h.append( - 'const JSFile %s = { "%s", StringData(%s, sizeof(%s) - 1) };' - % (objname, filename.replace("\\", "/"), stringname, stringname) - ) + h.append('extern const JSFile %s;' % objname) + h.append('const JSFile %s = { "%s", StringData(%s, sizeof(%s) - 1) };' % + (objname, filename.replace('\\', '/'), stringname, stringname)) h.append("} // namespace JSFiles") h.append("} // namespace mongo") h.append("") - text = "\n".join(h) + text = '\n'.join(h) - with open(outFile, "w") as out: + with open(outFile, 'w') as out: try: out.write(text) finally: diff --git a/site_scons/site_tools/libtool.py b/site_scons/site_tools/libtool.py index 2ef2415627e..07527bb7f50 100644 --- a/site_scons/site_tools/libtool.py +++ b/site_scons/site_tools/libtool.py @@ -3,14 +3,14 @@ import SCons def generate(env): - env["AR"] = "libtool" - env["ARCOM"] = "$AR -static -o $TARGET $ARFLAGS $SOURCES" - env["ARFLAGS"] = ["-s", "-no_warning_for_no_symbols"] + env['AR'] = 'libtool' + env['ARCOM'] = '$AR -static -o $TARGET $ARFLAGS $SOURCES' + env['ARFLAGS'] = ["-s", "-no_warning_for_no_symbols"] # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" - env["RANLIBCOMSTR"] = "Skipping ranlib for libtool generated target $TARGET" + env['RANLIBCOM'] = '' + env['RANLIBCOMSTR'] = 'Skipping ranlib for libtool generated target $TARGET' def exists(env): - return env.detect("libtool") + return env.detect('libtool') diff --git a/site_scons/site_tools/mongo_benchmark.py b/site_scons/site_tools/mongo_benchmark.py index 400512e738a..c765adba744 100644 --- a/site_scons/site_tools/mongo_benchmark.py +++ b/site_scons/site_tools/mongo_benchmark.py @@ -1,60 +1,75 @@ -# Copyright 2019 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Pseudo-builders for building and registering benchmarks. +"""Pseudo-builders for building and registering benchmarks. """ +import os from SCons.Script import Action - def exists(env): return True +_benchmarks = [] +def register_benchmark(env, test): + _benchmarks.append(test.path) + env.Alias('$BENCHMARK_ALIAS', test) + +def benchmark_list_builder_action(env, target, source): + ofile = open(str(target[0]), 'w') + try: + for s in _benchmarks: + print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() def build_benchmark(env, target, source, **kwargs): bmEnv = env.Clone() - bmEnv.InjectThirdParty(libraries=["benchmark"]) + bmEnv.InjectThirdParty(libraries=['benchmark']) - if bmEnv.TargetOSIs("windows"): + if bmEnv.TargetOSIs('windows'): bmEnv.Append(LIBS=["ShLwApi.lib"]) - libdeps = kwargs.get("LIBDEPS", []) - libdeps.append("$BUILD_DIR/mongo/unittest/benchmark_main") + libdeps = kwargs.get('LIBDEPS', []) + libdeps.append('$BUILD_DIR/mongo/unittest/benchmark_main') - kwargs["LIBDEPS"] = libdeps - kwargs["INSTALL_ALIAS"] = ["benchmarks"] + kwargs['LIBDEPS'] = libdeps + kwargs['INSTALL_ALIAS'] = ['benchmarks'] - benchmark_test_components = {"tests", "benchmarks"} - if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENT"].endswith("-benchmark"): - kwargs["AIB_COMPONENT"] += "-benchmark" + benchmark_test_components = {'tests', 'benchmarks'} + if ( + 'AIB_COMPONENT' in kwargs + and not kwargs['AIB_COMPONENT'].endswith('-benchmark') + ): + kwargs['AIB_COMPONENT'] += '-benchmark' - if "AIB_COMPONENTS_EXTRA" in kwargs: - benchmark_test_components = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( - benchmark_test_components - ) + if 'AIB_COMPONENTS_EXTRA' in kwargs: + benchmark_test_components = set(kwargs['AIB_COMPONENTS_EXTRA']).union(benchmark_test_components) - kwargs["AIB_COMPONENTS_EXTRA"] = benchmark_test_components + kwargs['AIB_COMPONENTS_EXTRA'] = benchmark_test_components result = bmEnv.Program(target, source, **kwargs) - bmEnv.RegisterTest("$BENCHMARK_LIST", result[0]) - bmEnv.Alias("$BENCHMARK_ALIAS", result) + bmEnv.RegisterBenchmark(result[0]) + hygienic = bmEnv.GetOption('install-mode') == 'hygienic' + if not hygienic: + installed_test = bmEnv.Install("#/build/benchmark/", result[0]) + env.Command( + target="#@{}".format(os.path.basename(installed_test[0].path)), + source=installed_test, + action="${SOURCES[0]}" + ) + else: + test_bin_name = os.path.basename(result[0].path) + env.Command( + target="#@{}".format(test_bin_name), + source=["$PREFIX_BINDIR/{}".format(test_bin_name)], + action="${SOURCES[0]}" + ) return result def generate(env): - env.TestList("$BENCHMARK_LIST", source=[]) - env.AddMethod(build_benchmark, "Benchmark") - env.Alias("$BENCHMARK_ALIAS", "$BENCHMARK_LIST") + env.Command('$BENCHMARK_LIST', env.Value(_benchmarks), + Action(benchmark_list_builder_action, "Generating $TARGET")) + env.AddMethod(register_benchmark, 'RegisterBenchmark') + env.AddMethod(build_benchmark, 'Benchmark') + env.Alias('$BENCHMARK_ALIAS', '$BENCHMARK_LIST') diff --git a/site_scons/site_tools/mongo_integrationtest.py b/site_scons/site_tools/mongo_integrationtest.py index 4cc89fc57bd..95e3b3b8d77 100644 --- a/site_scons/site_tools/mongo_integrationtest.py +++ b/site_scons/site_tools/mongo_integrationtest.py @@ -1,38 +1,51 @@ -""" -Pseudo-builders for building and registering integration tests. -""" +'''Pseudo-builders for building and registering integration tests. +''' from SCons.Script import Action - def exists(env): return True +_integration_tests = [] +def register_integration_test(env, test): + installed_test = env.Install('#/build/integration_tests/', test) + _integration_tests.append(installed_test[0].path) + env.Alias('$INTEGRATION_TEST_ALIAS', installed_test) + +def integration_test_list_builder_action(env, target, source): + ofile = open(str(target[0]), 'w') + try: + for s in _integration_tests: + print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() def build_cpp_integration_test(env, target, source, **kwargs): - libdeps = kwargs.get("LIBDEPS", []) - libdeps.append("$BUILD_DIR/mongo/unittest/integration_test_main") + libdeps = kwargs.get('LIBDEPS', []) + libdeps.append( '$BUILD_DIR/mongo/unittest/integration_test_main' ) - kwargs["LIBDEPS"] = libdeps - integration_test_components = {"tests", "integration-tests"} + kwargs['LIBDEPS'] = libdeps + integration_test_components = {'tests', 'integration-tests'} - if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENT"].endswith("-test"): - kwargs["AIB_COMPONENT"] += "-test" + if ( + 'AIB_COMPONENT' in kwargs + and not kwargs['AIB_COMPONENT'].endswith('-test') + ): + kwargs['AIB_COMPONENT'] += '-test' - if "AIB_COMPONENTS_EXTRA" in kwargs: - kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( - integration_test_components - ) + if 'AIB_COMPONENTS_EXTRA' in kwargs: + kwargs['AIB_COMPONENTS_EXTRA'] = set(kwargs['AIB_COMPONENTS_EXTRA']).union(integration_test_components) else: - kwargs["AIB_COMPONENTS_EXTRA"] = integration_test_components + kwargs['AIB_COMPONENTS_EXTRA'] = integration_test_components result = env.Program(target, source, **kwargs) - env.RegisterTest("$INTEGRATION_TEST_LIST", result[0]) - env.Alias("$INTEGRATION_TEST_ALIAS", result[0]) - + env.RegisterIntegrationTest(result[0]) return result def generate(env): - env.TestList("$INTEGRATION_TEST_LIST", source=[]) - env.AddMethod(build_cpp_integration_test, "CppIntegrationTest") - env.Alias("$INTEGRATION_TEST_ALIAS", "$INTEGRATION_TEST_LIST") + env.Command('$INTEGRATION_TEST_LIST', env.Value(_integration_tests), + Action(integration_test_list_builder_action, 'Generating $TARGET')) + env.AddMethod(register_integration_test, 'RegisterIntegrationTest') + env.AddMethod(build_cpp_integration_test, 'CppIntegrationTest') + env.Alias('$INTEGRATION_TEST_ALIAS', '$INTEGRATION_TEST_LIST') diff --git a/site_scons/site_tools/mongo_libfuzzer.py b/site_scons/site_tools/mongo_libfuzzer.py index 85041d6ae21..c81f80dccf2 100644 --- a/site_scons/site_tools/mongo_libfuzzer.py +++ b/site_scons/site_tools/mongo_libfuzzer.py @@ -7,50 +7,56 @@ def exists(env): return True +_libfuzzer_tests = [] + + +def register_libfuzzer_test(env, test): + _libfuzzer_tests.append(test.path) + env.Alias('$LIBFUZZER_TEST_ALIAS', test) + + def libfuzzer_test_list_builder_action(env, target, source): - with open(str(target[0]), "w") as ofile: + with open(str(target[0]), 'w') as ofile: for s in _libfuzzer_tests: - print("\t" + str(s)) - ofile.write("%s\n" % s) + print('\t' + str(s)) + ofile.write('%s\n' % s) + def build_cpp_libfuzzer_test(env, target, source, **kwargs): myenv = env.Clone() - if not myenv.IsSanitizerEnabled("fuzzer"): + if not myenv.IsSanitizerEnabled('fuzzer'): return [] - libdeps = kwargs.get("LIBDEPS", []) - kwargs["LIBDEPS"] = libdeps - kwargs["INSTALL_ALIAS"] = ["tests"] - sanitizer_option = "-fsanitize=fuzzer" + libdeps = kwargs.get('LIBDEPS', []) + kwargs['LIBDEPS'] = libdeps + kwargs['INSTALL_ALIAS'] = ['tests'] + sanitizer_option = '-fsanitize=fuzzer' myenv.Prepend(LINKFLAGS=[sanitizer_option]) - libfuzzer_test_components = {"tests", "fuzzertests"} - if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENTS"].endswith( - "-fuzzertest" + libfuzzer_test_components = {'tests', 'fuzzertests'} + if ( + 'AIB_COMPONENT' in kwargs + and not kwargs['AIB_COMPONENTS'].endswith('-fuzzertest') ): - kwargs["AIB_COMPONENT"] += "-fuzzertest" + kwargs['AIB_COMPONENT'] += '-fuzzertest' - if "AIB_COMPONENTS_EXTRA" in kwargs: - libfuzzer_test_components = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( - libfuzzer_test_components - ) + if 'AIB_COMPONENTS_EXTRA' in kwargs: + libfuzzer_test_components = set(kwargs['AIB_COMPONENTS_EXTRA']).union(libfuzzer_test_components) - kwargs["AIB_COMPONENTS_EXTRA"] = libfuzzer_test_components + kwargs['AIB_COMPONENTS_EXTRA'] = libfuzzer_test_components result = myenv.Program(target, source, **kwargs) - myenv.RegisterTest("$LIBFUZZER_TEST_LIST", result[0]) - myenv.Alias("$LIBFUZZER_TEST_ALIAS", result) - - # TODO: remove when hygienic is default - hygienic = myenv.GetOption("install-mode") == "hygienic" + myenv.RegisterLibfuzzerTest(result[0]) + hygienic = myenv.GetOption('install-mode') == 'hygienic' if not hygienic: myenv.Install("#/build/libfuzzer_tests/", result[0]) - return result def generate(env): - env.TestList("$LIBFUZZER_TEST_LIST", source=[]) - env.AddMethod(build_cpp_libfuzzer_test, "CppLibfuzzerTest") - env.Alias("$LIBFUZZER_TEST_ALIAS", "$LIBFUZZER_TEST_LIST") + env.Command('$LIBFUZZER_TEST_LIST', env.Value(_libfuzzer_tests), + Action(libfuzzer_test_list_builder_action, "Generating $TARGET")) + env.AddMethod(register_libfuzzer_test, 'RegisterLibfuzzerTest') + env.AddMethod(build_cpp_libfuzzer_test, 'CppLibfuzzerTest') + env.Alias('$LIBFUZZER_TEST_ALIAS', '$LIBFUZZER_TEST_LIST') diff --git a/site_scons/site_tools/mongo_test_execution.py b/site_scons/site_tools/mongo_test_execution.py deleted file mode 100644 index 7424f726aed..00000000000 --- a/site_scons/site_tools/mongo_test_execution.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2019 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - - -def generate_test_execution_aliases(env, test): - hygienic = env.GetOption("install-mode") == "hygienic" - if hygienic and getattr(test.attributes, "AIB_INSTALL_ACTIONS", []): - installed = getattr(test.attributes, "AIB_INSTALL_ACTIONS") - else: - installed = [test] - - target_name = os.path.basename(installed[0].get_path()) - command = env.Command( - target="#+{}".format(target_name), - source=installed, - action="${SOURCES[0]} $UNITTEST_FLAGS", - NINJA_POOL="console", - ) - - env.Alias("test-execution-aliases", command) - for source in test.sources: - source_base_name = os.path.basename(source.get_path()) - # Strip suffix - dot_idx = source_base_name.rfind(".") - suffix = source_base_name[dot_idx:] - if suffix in env["TEST_EXECUTION_SUFFIX_BLACKLIST"]: - continue - - source_name = source_base_name[:dot_idx] - if target_name == source_name: - continue - - source_command = env.Command( - target="#+{}".format(source_name), - source=installed, - action="${SOURCES[0]} -fileNameFilter $TEST_SOURCE_FILE_NAME $UNITTEST_FLAGS", - TEST_SOURCE_FILE_NAME=source_name, - NINJA_POOL="console", - ) - - env.Alias("test-execution-aliases", source_command) - - -def exists(env): - return True - - -def generate(env): - # Wire up dependency for Ninja generator to collect the test - # execution aliases - test_execution_aliases = env.Alias("test-execution-aliases") - env.Alias("all", test_execution_aliases) - env.Alias("install-all-meta", test_execution_aliases) - env.AddMethod(generate_test_execution_aliases, "GenerateTestExecutionAliases") - - env["TEST_EXECUTION_SUFFIX_BLACKLIST"] = env.get( - "TEST_EXECUTION_SUFFIX_BLACKLIST", [".in"] - ) - - # TODO: Remove when the new ninja generator is the only supported generator - env["_NINJA_NO_TEST_EXECUTION"] = True diff --git a/site_scons/site_tools/mongo_test_list.py b/site_scons/site_tools/mongo_test_list.py deleted file mode 100644 index 8ea4a427354..00000000000 --- a/site_scons/site_tools/mongo_test_list.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2019 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Pseudo-builders for building test lists for Resmoke""" - -import SCons -from collections import defaultdict - -TEST_REGISTRY = defaultdict(list) - - -def register_test(env, file_name, test): - """Register test into the dictionary of tests for file_name""" - test_path = test.path - if getattr(test.attributes, "AIB_INSTALL_ACTIONS", []): - test_path = getattr(test.attributes, "AIB_INSTALL_ACTIONS")[0].path - - if SCons.Util.is_String(file_name): - file_name = env.File(file_name).path - else: - file_name = file_name.path - - TEST_REGISTRY[file_name].append(test_path) - env.GenerateTestExecutionAliases(test) - - -def test_list_builder_action(env, target, source): - """Build a test list used by resmoke.py to execute binary tests.""" - if SCons.Util.is_String(target[0]): - filename = env.subst(target[0]) - else: - filename = target[0].path - - source = [env.subst(s) if SCons.Util.is_String(s) else s.path for s in source] - - with open(filename, "w") as ofile: - tests = TEST_REGISTRY[filename] - if source: - tests.extend(source) - - for s in tests: - ofile.write("{}\n".format(str(s))) - - -TEST_LIST_BUILDER = SCons.Builder.Builder( - action=SCons.Action.FunctionAction( - test_list_builder_action, {"cmdstr": "Generating $TARGETS"}, - ) -) - - -def exists(env): - return True - - -def generate(env): - env.Append(BUILDERS={"TestList": TEST_LIST_BUILDER}) - env.AddMethod(register_test, "RegisterTest") diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py index 84fc256aca6..0ddd1b069ad 100644 --- a/site_scons/site_tools/mongo_unittest.py +++ b/site_scons/site_tools/mongo_unittest.py @@ -1,59 +1,66 @@ -# Copyright 2019 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pseudo-builders for building and registering unit tests.""" +'''Pseudo-builders for building and registering unit tests.''' +import os + from SCons.Script import Action +def exists(env): + return True + + +_unittests = [] def register_unit_test(env, test): - """ - Kept around for compatibility. + _unittests.append(test.path) - Some SConscripts called RegisterUnitTest directly. - """ - env.RegisterTest("$UNITTEST_LIST", test) + hygienic = env.GetOption('install-mode') == 'hygienic' + if hygienic and getattr(test.attributes, "AIB_INSTALL_ACTIONS", []): + installed = getattr(test.attributes, "AIB_INSTALL_ACTIONS") + else: + installed = [test] + env.Command( + target="#@{}".format(os.path.basename(installed[0].get_path())), + source=installed, + action="${SOURCES[0]}" + ) -def exists(env): - return True + env.Alias('$UNITTEST_ALIAS', test) +def unit_test_list_builder_action(env, target, source): + ofile = open(str(target[0]), 'w') + try: + for s in _unittests: + print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() def build_cpp_unit_test(env, target, source, **kwargs): - libdeps = kwargs.get("LIBDEPS", []) - libdeps.append("$BUILD_DIR/mongo/unittest/unittest_main") - - kwargs["LIBDEPS"] = libdeps - unit_test_components = {"tests", "unittests"} - if "AIB_COMPONENT" in kwargs and not kwargs["AIB_COMPONENT"].endswith("-test"): - kwargs["AIB_COMPONENT"] += "-test" - - if "AIB_COMPONENTS_EXTRA" in kwargs: - kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( - unit_test_components - ) + libdeps = kwargs.get('LIBDEPS', []) + libdeps.append( '$BUILD_DIR/mongo/unittest/unittest_main' ) + + kwargs['LIBDEPS'] = libdeps + unit_test_components = {'tests', 'unittests'} + if ( + 'AIB_COMPONENT' in kwargs + and not kwargs['AIB_COMPONENT'].endswith('-test') + ): + kwargs['AIB_COMPONENT'] += '-test' + + if 'AIB_COMPONENTS_EXTRA' in kwargs: + kwargs['AIB_COMPONENTS_EXTRA'] = set(kwargs['AIB_COMPONENTS_EXTRA']).union(unit_test_components) else: - kwargs["AIB_COMPONENTS_EXTRA"] = unit_test_components + kwargs['AIB_COMPONENTS_EXTRA'] = unit_test_components result = env.Program(target, source, **kwargs) - env.RegisterTest("$UNITTEST_LIST", result[0]) - env.Alias("$UNITTEST_ALIAS", result[0]) + env.RegisterUnitTest(result[0]) return result def generate(env): - env.TestList("$UNITTEST_LIST", source=[]) - env.AddMethod(build_cpp_unit_test, "CppUnitTest") - env.AddMethod(register_unit_test, "RegisterUnitTest") - env.Alias("$UNITTEST_ALIAS", "$UNITTEST_LIST") + env.Command('$UNITTEST_LIST', env.Value(_unittests), + Action(unit_test_list_builder_action, 'Generating $TARGET')) + env.AddMethod(register_unit_test, 'RegisterUnitTest') + env.AddMethod(build_cpp_unit_test, 'CppUnitTest') + env.Alias('$UNITTEST_ALIAS', '$UNITTEST_LIST') diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index e2b50bfd6d2..52b567e2b7c 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -29,7 +29,6 @@ from SCons.Script import COMMAND_LINE_TARGETS NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" -NINJA_POOLS = "__NINJA_CUSTOM_POOLS" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" NINJA_BUILD = "NINJA_BUILD" NINJA_OUTPUTS = "__NINJA_OUTPUTS" @@ -148,15 +147,19 @@ class SConsToNinjaTranslator: def __init__(self, env): self.env = env self.func_handlers = { + # Skip conftest builders "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the # target isn't static. We let the commands that ninja runs do # this check for us. "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. "installFunc": _install_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, + } self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) @@ -302,18 +305,13 @@ class SConsToNinjaTranslator: # Make sure we didn't generate an empty cmdline if cmdline: - ninja_build = { + return { "outputs": all_outputs, "rule": "CMD", "variables": {"cmd": cmdline}, "implicit": dependencies, } - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["pool"] - - return ninja_build - elif results[0]["rule"] == "phony": return { "outputs": all_outputs, @@ -367,8 +365,10 @@ class NinjaState: ), ), "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - sys.executable, " ".join([escape(arg) for arg in sys.argv]) + sys.executable, + " ".join([escape(arg) for arg in sys.argv]) ), + # This must be set to a global default per: # https://ninja-build.org/manual.html # @@ -381,11 +381,13 @@ class NinjaState: "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", "description": "Building $out", }, + # We add the deps processing variables to this below. "CMD_W_DEPS": { "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", "description": "Building $out", }, + "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -451,10 +453,6 @@ class NinjaState: }, } - self.pools = { - "install_pool": self.env.GetOption("num_jobs") / 2, - } - if env["PLATFORM"] == "win32": self.rules["CMD_W_DEPS"]["deps"] = "msvc" else: @@ -462,7 +460,6 @@ class NinjaState: self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) def generate_builds(self, node): """Generate a ninja build rule for node and it's children.""" @@ -522,8 +519,7 @@ class NinjaState: ninja.comment("Generated by scons. DO NOT EDIT.") - for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) + ninja.pool("install_pool", self.env.GetOption("num_jobs") / 2) for var, val in self.variables.items(): ninja.variable(var, val) @@ -533,10 +529,12 @@ class NinjaState: generated_source_files = { output + # First find builds which have header files in their outputs. for build in self.builds if is_generated_source(build["outputs"]) for output in build["outputs"] + # Collect only the header files from the builds with them # in their output. We do this because is_generated_source # returns True if it finds a header in any of the outputs, @@ -621,8 +619,7 @@ class NinjaState: ninja.build( ninja_file, rule="REGENERATE", - implicit=[self.env.File("#SConstruct").get_abspath()] - + glob("src/**/SConscript", recursive=True), + implicit=[self.env.File("#SConstruct").get_abspath()] + glob("src/**/SConscript", recursive=True), ) ninja.build( @@ -764,9 +761,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Ninja but for now, and partially because the existing Ninja # generator does so, we just disable it all together. cmd = cmd.replace("\n", " && ").strip() - if env["PLATFORM"] == "win32" and ( - "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd - ): + if env["PLATFORM"] == "win32" and ("embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd): cmd = " && ".join(cmd.split(" && ")[0:-1]) if cmd.endswith("&&"): @@ -814,33 +809,13 @@ def get_command(env, node, action): # pylint: disable=too-many-branches setattr(node.attributes, "NINJA_ENV_ENV", command_env) - ninja_build = { + return { "outputs": outputs, "implicit": implicit, "rule": rule, "variables": {"cmd": command_env + cmd}, } - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" @@ -932,17 +907,12 @@ def register_custom_rule(env, rule, command, description=""): } -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - - def ninja_csig(original): """Return a dummy csig""" def wrapper(self): name = str(self) - if "SConscript" in name or "SConstruct" in name: + if 'SConscript' in name or 'SConstruct' in name: return original(self) return "dummy_ninja_csig" @@ -954,7 +924,7 @@ def ninja_contents(original): def wrapper(self): name = str(self) - if "SConscript" in name or "SConstruct" in name: + if 'SConscript' in name or 'SConstruct' in name: return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") @@ -1035,7 +1005,7 @@ class NinjaEternalTempFile(SCons.Platform.TempFileMunge): node = target[0] if SCons.Util.is_List(target) else target if node is not None: - cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) + cmdlist = getattr(node.attributes, 'tempfile_cmdlist', None) if cmdlist is not None: return cmdlist @@ -1044,7 +1014,7 @@ class NinjaEternalTempFile(SCons.Platform.TempFileMunge): # If TempFileMunge.__call__ returns a string it means that no # response file was needed. No processing required so just # return the command. - if isinstance(cmd, str): + if isinstance(cmd, str): return cmd # Strip the removal commands from the command list. @@ -1084,14 +1054,15 @@ class NinjaEternalTempFile(SCons.Platform.TempFileMunge): # Note the weird newline and rm command in the middle # element and the lack of TEMPFILEPREFIX on the last # element. - prefix = env.subst("$TEMPFILEPREFIX") + prefix = env.subst('$TEMPFILEPREFIX') if not prefix: - prefix = "@" + prefix = '@' new_cmdlist = [cmd[0], prefix + cmd[-1]] - setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) + setattr(node.attributes, 'tempfile_cmdlist', new_cmdlist) return new_cmdlist + def _print_cmd_str(*_args, **_kwargs): """Disable this method""" pass @@ -1137,12 +1108,6 @@ def generate(env): env[NINJA_RULES] = {} env.AddMethod(register_custom_rule, "NinjaRule") - # Provides a mechanism for inject custom Ninja pools which can - # be used by providing the NINJA_POOL="name" as an - # OverrideEnvironment variable in a builder call. - env[NINJA_POOLS] = {} - env.AddMethod(register_custom_pool, "NinjaPool") - # Add the ability to register custom NinjaRuleMappings for Command # builders. We don't store this dictionary in the env to prevent # accidental deletion of the CC/XXCOM mappings. You can still @@ -1192,12 +1157,8 @@ def generate(env): # slows down the build significantly and we don't need contents or # content signatures calculated when generating a ninja file since # we're not doing any SCons caching or building. - SCons.Executor.Executor.get_contents = ninja_contents( - SCons.Executor.Executor.get_contents - ) - SCons.Node.Alias.Alias.get_contents = ninja_contents( - SCons.Node.Alias.Alias.get_contents - ) + SCons.Executor.Executor.get_contents = ninja_contents(SCons.Executor.Executor.get_contents) + SCons.Node.Alias.Alias.get_contents = ninja_contents(SCons.Node.Alias.Alias.get_contents) SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) @@ -1240,7 +1201,7 @@ def generate(env): if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaEternalTempFile + env['TEMPFILE'] = NinjaEternalTempFile # Force the SConsign to be written, we benefit from SCons caching of # implicit dependencies and conftests. Unfortunately, we have to do this diff --git a/site_scons/site_tools/separate_debug.py b/site_scons/site_tools/separate_debug.py index 4347f3031ab..211a31db84a 100644 --- a/site_scons/site_tools/separate_debug.py +++ b/site_scons/site_tools/separate_debug.py @@ -26,16 +26,16 @@ def _update_builder(env, builder, bitcode): if origin: origin_results = old_scanner(origin, env, path) for origin_result in origin_results: - origin_result_debug_file = getattr( - origin_result.attributes, "separate_debug_file", None - ) + origin_result_debug_file = getattr(origin_result.attributes, "separate_debug_file", + None) if origin_result_debug_file: results.append(origin_result_debug_file) # TODO: Do we need to do the same sort of drag along for bcsymbolmap files? return results builder.target_scanner = SCons.Scanner.Scanner( - function=new_scanner, path_function=old_path_function, + function=new_scanner, + path_function=old_path_function, ) base_action = builder.action @@ -49,38 +49,26 @@ def _update_builder(env, builder, bitcode): # us. We could then also remove a lot of the compiler and sysroot # setup from the etc/scons/xcode_*.vars files, which would be a # win as well. - if env.TargetOSIs("darwin"): + if env.TargetOSIs('darwin'): if bitcode: base_action.list.append( SCons.Action.Action( "dsymutil -num-threads=1 $TARGET --symbol-map=${TARGET}.bcsymbolmap -o ${TARGET}.dSYM", - "Generating debug info for $TARGET into ${TARGET}.dSYM", - ) - ) + "Generating debug info for $TARGET into ${TARGET}.dSYM")) else: base_action.list.append( - SCons.Action.Action( - "dsymutil -num-threads=1 $TARGET -o ${TARGET}.dSYM", - "Generating debug info for $TARGET into ${TARGET}.dSYM", - ) - ) - base_action.list.append( - SCons.Action.Action("strip -Sx ${TARGET}", "Stripping ${TARGET}") - ) - elif env.TargetOSIs("posix"): - base_action.list.extend( - [ - SCons.Action.Action( - "${OBJCOPY} --only-keep-debug $TARGET ${TARGET}.debug", - "Generating debug info for $TARGET into ${TARGET}.debug", - ), - SCons.Action.Action( - "${OBJCOPY} --strip-debug --add-gnu-debuglink ${TARGET}.debug ${TARGET}", - "Stripping debug info from ${TARGET} and adding .gnu.debuglink to ${TARGET}.debug", - ), - ] - ) + SCons.Action.Action("dsymutil -num-threads=1 $TARGET -o ${TARGET}.dSYM", + "Generating debug info for $TARGET into ${TARGET}.dSYM")) + base_action.list.append(SCons.Action.Action("strip -Sx ${TARGET}", "Stripping ${TARGET}")) + elif env.TargetOSIs('posix'): + base_action.list.extend([ + SCons.Action.Action("${OBJCOPY} --only-keep-debug $TARGET ${TARGET}.debug", + "Generating debug info for $TARGET into ${TARGET}.debug"), + SCons.Action.Action( + "${OBJCOPY} --strip-debug --add-gnu-debuglink ${TARGET}.debug ${TARGET}", + "Stripping debug info from ${TARGET} and adding .gnu.debuglink to ${TARGET}.debug"), + ]) else: pass @@ -91,15 +79,15 @@ def _update_builder(env, builder, bitcode): def new_emitter(target, source, env): bitcode_file = None - if env.TargetOSIs("darwin"): + if env.TargetOSIs('darwin'): debug_file = env.Entry(str(target[0]) + ".dSYM") env.Precious(debug_file) if bitcode: bitcode_file = env.File(str(target[0]) + ".bcsymbolmap") - elif env.TargetOSIs("posix"): + elif env.TargetOSIs('posix'): debug_file = env.File(str(target[0]) + ".debug") - elif env.TargetOSIs("windows"): - debug_file = env.File(env.subst("${PDB}", target=target)) + elif env.TargetOSIs('windows'): + debug_file = env.File(env.subst('${PDB}', target=target)) else: pass @@ -133,22 +121,18 @@ def generate(env): # later was a better time to address this. We should also consider # moving all bitcode setup into a separate tool. bitcode = False - if env.TargetOSIs("darwin") and any( - flag == "-fembed-bitcode" for flag in env["LINKFLAGS"] - ): + if env.TargetOSIs('darwin') and any(flag == "-fembed-bitcode" for flag in env['LINKFLAGS']): bitcode = True - env.AppendUnique( - LINKFLAGS=[ - "-Wl,-bitcode_hide_symbols", - "-Wl,-bitcode_symbol_map,${TARGET}.bcsymbolmap", - ] - ) + env.AppendUnique(LINKFLAGS=[ + "-Wl,-bitcode_hide_symbols", + "-Wl,-bitcode_symbol_map,${TARGET}.bcsymbolmap", + ]) # TODO: For now, not doing this for programs. Need to update # auto_install_binaries to understand to install the debug symbol # for target X to the same target location as X. - for builder in ["Program", "SharedLibrary", "LoadableModule"]: - _update_builder(env, env["BUILDERS"][builder], bitcode) + for builder in ['Program', 'SharedLibrary', 'LoadableModule']: + _update_builder(env, env['BUILDERS'][builder], bitcode) def exists(env): diff --git a/site_scons/site_tools/split_dwarf.py b/site_scons/site_tools/split_dwarf.py index 9fe960d1b70..c57b9e96822 100644 --- a/site_scons/site_tools/split_dwarf.py +++ b/site_scons/site_tools/split_dwarf.py @@ -14,24 +14,24 @@ import SCons -_splitDwarfFlag = "-gsplit-dwarf" +_splitDwarfFlag = '-gsplit-dwarf' # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if # we could obtain this from SCons. -_CSuffixes = [".c"] -if not SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CSuffixes.append(".C") +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') -_CXXSuffixes = [".cpp", ".cc", ".cxx", ".c++", ".C++"] -if SCons.Util.case_sensitive_suffixes(".c", ".C"): - _CXXSuffixes.append(".C") +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') def _dwo_emitter(target, source, env): new_targets = [] for t in target: base, ext = SCons.Util.splitext(str(t)) - if not any(ext == env[osuffix] for osuffix in ["OBJSUFFIX", "SHOBJSUFFIX"]): + if not any(ext == env[osuffix] for osuffix in ['OBJSUFFIX', 'SHOBJSUFFIX']): continue # TODO: Move 'dwo' into DWOSUFFIX so it can be customized? For # now, GCC doesn't let you control the output filename, so it @@ -44,12 +44,12 @@ def _dwo_emitter(target, source, env): def generate(env): suffixes = [] - if _splitDwarfFlag in env["CCFLAGS"]: + if _splitDwarfFlag in env['CCFLAGS']: suffixes = _CSuffixes + _CXXSuffixes else: - if _splitDwarfFlag in env["CFLAGS"]: + if _splitDwarfFlag in env['CFLAGS']: suffixes.extend(_CSuffixes) - if _splitDwarfFlag in env["CXXFLAGS"]: + if _splitDwarfFlag in env['CXXFLAGS']: suffixes.extend(_CXXSuffixes) for object_builder in SCons.Tool.createObjBuilders(env): @@ -58,8 +58,11 @@ def generate(env): if not suffix in suffixes: continue base = emitterdict[suffix] - emitterdict[suffix] = SCons.Builder.ListEmitter([base, _dwo_emitter,]) + emitterdict[suffix] = SCons.Builder.ListEmitter([ + base, + _dwo_emitter, + ]) def exists(env): - return any(_splitDwarfFlag in env[f] for f in ["CCFLAGS", "CFLAGS", "CXXFLAGS"]) + return any(_splitDwarfFlag in env[f] for f in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS']) diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py index cbc0cd03890..145589c7996 100644 --- a/site_scons/site_tools/thin_archive.py +++ b/site_scons/site_tools/thin_archive.py @@ -19,25 +19,21 @@ import subprocess def exists(env): - if not "AR" in env: + if not 'AR' in env: return False - ar = env.subst(env["AR"]) + ar = env.subst(env['AR']) if not ar: return False # If the user has done anything confusing with ARFLAGS, bail out. We want to find # an item in ARFLAGS of the exact form 'rc'. - if not "rc" in env["ARFLAGS"]: + if not "rc" in env['ARFLAGS']: return False - pipe = SCons.Action._subproc( - env, - SCons.Util.CLVar(ar) + ["--version"], - stdin="devnull", - stderr="devnull", - stdout=subprocess.PIPE, - ) + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(ar) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE) if pipe.wait() != 0: return False @@ -45,11 +41,10 @@ def exists(env): for line in pipe.stdout: if found: continue # consume all data - found = re.search(r"^GNU ar|^LLVM", line.decode("utf-8")) + found = re.search(r'^GNU ar|^LLVM', line.decode('utf-8')) return bool(found) - def _add_emitter(builder): base_emitter = builder.emitter @@ -75,25 +70,23 @@ def _add_scanner(builder): new_results.extend(base.children()) return new_results - builder.target_scanner = SCons.Scanner.Scanner( - function=new_scanner, path_function=path_function - ) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, + path_function=path_function) def generate(env): if not exists(env): return - env["ARFLAGS"] = SCons.Util.CLVar( - [arflag if arflag != "rc" else "rcsTD" for arflag in env["ARFLAGS"]] - ) + env['ARFLAGS'] = SCons.Util.CLVar( + [arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']]) # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" - env["RANLIBCOMSTR"] = "Skipping ranlib for thin archive $TARGET" + env['RANLIBCOM'] = '' + env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET' - for builder in ["StaticLibrary", "SharedArchive"]: - _add_emitter(env["BUILDERS"][builder]) + for builder in ['StaticLibrary', 'SharedArchive']: + _add_emitter(env['BUILDERS'][builder]) - for builder in ["SharedLibrary", "LoadableModule", "Program"]: - _add_scanner(env["BUILDERS"][builder]) + for builder in ['SharedLibrary', 'LoadableModule', 'Program']: + _add_scanner(env['BUILDERS'][builder]) diff --git a/site_scons/site_tools/vcredist.py b/site_scons/site_tools/vcredist.py index 67f34e69724..62e14010329 100644 --- a/site_scons/site_tools/vcredist.py +++ b/site_scons/site_tools/vcredist.py @@ -19,12 +19,10 @@ import winreg import SCons - def exists(env): - result = "msvc" in env["TOOLS"] + result = 'msvc' in env['TOOLS'] return result - # How to locate the Merge Modules path is described in: # # - VS2019: https://docs.microsoft.com/en-us/visualstudio/releases/2019/redistribution#visual-c-runtime-files @@ -44,18 +42,17 @@ def exists(env): # # TODO: Expand this map as needed. target_arch_expansion_map = { - "amd64": "x64", - "arm": None, - "arm64": "arm64", - "emt64": "x64", - "i386": "x86", - "x86": "x86", - "x86_64": "x64", + 'amd64' : 'x64', + 'arm' : None, + 'arm64' : 'arm64', + 'emt64' : 'x64', + 'i386' : 'x86', + 'x86' : 'x86', + 'x86_64' : 'x64', } - def _get_programfiles(): - result = os.getenv("ProgramFiles(x86)") + result = os.getenv('ProgramFiles(x86)') # When we run this under cygwin, the environment is broken, fall # back to hard coded C:\Program Files (x86) if result is None: @@ -64,22 +61,20 @@ def _get_programfiles(): return None return result - def _get_merge_module_name_for_feature(env, feature): - version_components = env["MSVC_VERSION"].split(".") + version_components = env['MSVC_VERSION'].split('.') return "Microsoft_VC{msvc_major}{msvc_minor}_{feature}_{target_arch}.msm".format( msvc_major=version_components[0], msvc_minor=version_components[1], feature=feature, - target_arch=target_arch_expansion_map[env.subst("$TARGET_ARCH")], + target_arch=target_arch_expansion_map[env.subst('$TARGET_ARCH')] ) - def generate(env): if not exists(env): return - env.Tool("msvc") + env.Tool('msvc') env.AddMethod(_get_merge_module_name_for_feature, "GetMergeModuleNameForFeature") @@ -90,7 +85,7 @@ def generate(env): # https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering # for details on the various version numbers in play for # the Microsoft toolchain. - msvc_major, msvc_minor = env["MSVC_VERSION"].split(".") + msvc_major, msvc_minor = env['MSVC_VERSION'].split('.') if msvc_major != "14": return @@ -109,7 +104,7 @@ def generate(env): # On VS2015 the merge modules are in the program files directory, # not under the VS install dir. - if msvc_minor == "0": + if msvc_minor == '0': if not programfilesx86: programfilesx86 = _get_programfiles() @@ -118,9 +113,9 @@ def generate(env): mergemodulepath = os.path.join(programfilesx86, "Common Files", "Merge Modules") if os.path.isdir(mergemodulepath): - env["MSVS"]["VCREDISTMERGEMODULEPATH"] = mergemodulepath + env['MSVS']['VCREDISTMERGEMODULEPATH'] = mergemodulepath - if not "VSINSTALLDIR" in env["MSVS"]: + if not 'VSINSTALLDIR' in env['MSVS']: # Compute a VS version based on the VC version. VC 14.0 is VS 2015, VC # 14.1 is VS 2017. Also compute the next theoretical version by @@ -128,9 +123,7 @@ def generate(env): # that we can use as an argument to the -version flag to vswhere. vs_version = int(msvc_major) + int(msvc_minor) vs_version_next = vs_version + 1 - vs_version_range = "[{vs_version}.0, {vs_version_next}.0)".format( - vs_version=vs_version, vs_version_next=vs_version_next - ) + vs_version_range = '[{vs_version}.0, {vs_version_next}.0)'.format(vs_version=vs_version, vs_version_next=vs_version_next) if not programfilesx86: programfilesx86 = _get_programfiles() @@ -138,52 +131,28 @@ def generate(env): return # Use vswhere (it has a fixed stable path) to query where Visual Studio is installed. - env["MSVS"]["VSINSTALLDIR"] = ( - subprocess.check_output( - [ - os.path.join( - programfilesx86, - "Microsoft Visual Studio", - "Installer", - "vswhere.exe", - ), - "-version", - vs_version_range, - "-property", - "installationPath", - "-nologo", - ] - ) - .decode("utf-8") - .strip() - ) - - vsinstall_dir = env["MSVS"]["VSINSTALLDIR"] + env['MSVS']['VSINSTALLDIR'] = subprocess.check_output([os.path.join(programfilesx86, "Microsoft Visual Studio", "Installer", "vswhere.exe"), "-version", vs_version_range, "-property", "installationPath", "-nologo"]).decode('utf-8').strip() + + vsinstall_dir = env['MSVS']['VSINSTALLDIR'] # Combine and set the full merge module path redist_root = os.path.join(vsinstall_dir, "VC", "Redist", "MSVC") if not os.path.isdir(redist_root): return - env["MSVS"]["VCREDISTROOT"] = redist_root + env['MSVS']['VCREDISTROOT'] = redist_root # Check the registry key that has the runtime lib version try: # TOOO: This x64 needs to be abstracted away. Is it the host # arch, or the target arch? My guess is host. - vsruntime_key_name = "SOFTWARE\\Microsoft\\VisualStudio\\{msvc_major}.0\\VC\\Runtimes\\x64".format( - msvc_major=msvc_major - ) + vsruntime_key_name = "SOFTWARE\\Microsoft\\VisualStudio\\{msvc_major}.0\\VC\\Runtimes\\x64".format(msvc_major=msvc_major) vsruntime_key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, vsruntime_key_name) - vslib_version, vslib_version_type = winreg.QueryValueEx( - vsruntime_key, "Version" - ) + vslib_version, vslib_version_type = winreg.QueryValueEx(vsruntime_key, "Version") except WindowsError: return # Fallback to directory search if we don't find the expected version - redist_path = os.path.join( - redist_root, re.match("v(\d+\.\d+\.\d+)\.\d+", vslib_version).group(1) - ) + redist_path = os.path.join(redist_root, re.match("v(\d+\.\d+\.\d+)\.\d+", vslib_version).group(1)) if not os.path.isdir(redist_path): redist_path = None dirs = os.listdir(redist_root) @@ -195,12 +164,12 @@ def generate(env): break else: return - env["MSVS"]["VCREDISTPATH"] = redist_path + env['MSVS']['VCREDISTPATH'] = redist_path if mergemodulepath is None and msvc_minor != "0": mergemodulepath = os.path.join(redist_path, "MergeModules") if os.path.isdir(mergemodulepath): - env["MSVS"]["VCREDISTMERGEMODULEPATH"] = mergemodulepath + env['MSVS']['VCREDISTMERGEMODULEPATH'] = mergemodulepath # Keep these in preference order. The way with the {} in between # the dots appears to be the more modern form, but we select the @@ -216,17 +185,15 @@ def generate(env): "vc_redist.{}.exe", ] - expansion = target_arch_expansion_map.get(env.subst("$TARGET_ARCH"), None) + expansion = target_arch_expansion_map.get(env.subst('$TARGET_ARCH'), None) if not expansion: return - vcredist_candidates = [ - c.format(expansion) for c in vcredist_search_template_sequence - ] + vcredist_candidates = [c.format(expansion) for c in vcredist_search_template_sequence] for candidate in vcredist_candidates: candidate = os.path.join(redist_path, candidate) if os.path.isfile(candidate): break else: return - env["MSVS"]["VCREDISTEXE"] = candidate + env['MSVS']['VCREDISTEXE'] = candidate diff --git a/site_scons/site_tools/xcode.py b/site_scons/site_tools/xcode.py index d4dbc23c8ef..5ddebb2e003 100644 --- a/site_scons/site_tools/xcode.py +++ b/site_scons/site_tools/xcode.py @@ -1,16 +1,12 @@ import os - def exists(env): - return env.Detect("xcrun") - + return env.Detect('xcrun') def generate(env): if not exists(env): return - if "DEVELOPER_DIR" in os.environ: - env["ENV"]["DEVELOPER_DIR"] = os.environ["DEVELOPER_DIR"] - print( - "NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands" - ) + if 'DEVELOPER_DIR' in os.environ: + env['ENV']['DEVELOPER_DIR'] = os.environ['DEVELOPER_DIR'] + print("NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands") diff --git a/src/mongo/db/ftdc/SConscript b/src/mongo/db/ftdc/SConscript index 0e165ac38c1..fbe7aa01fea 100644 --- a/src/mongo/db/ftdc/SConscript +++ b/src/mongo/db/ftdc/SConscript @@ -99,7 +99,7 @@ env.CppUnitTest( 'file_manager_test.cpp', 'file_writer_test.cpp', 'ftdc_test.cpp', - 'ftdc_util_test.cpp', + 'util_test.cpp', 'varint_test.cpp', ], LIBDEPS=[ diff --git a/src/mongo/db/ftdc/ftdc_util_test.cpp b/src/mongo/db/ftdc/util_test.cpp index b50d7fa8dc1..b50d7fa8dc1 100644 --- a/src/mongo/db/ftdc/ftdc_util_test.cpp +++ b/src/mongo/db/ftdc/util_test.cpp |