diff options
Diffstat (limited to 'site_scons')
29 files changed, 353 insertions, 288 deletions
diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py index d2844a8d68b..e08104b0042 100644 --- a/site_scons/libdeps.py +++ b/site_scons/libdeps.py @@ -61,7 +61,7 @@ syslibdeps_env_var = 'SYSLIBDEPS' missing_syslibdep = 'MISSING_LIBDEP_' class dependency(object): - Public, Private, Interface = range(3) + Public, Private, Interface = list(range(3)) def __init__(self, value, deptype): self.target_node = value @@ -70,28 +70,31 @@ class dependency(object): def __str__(self): return str(self.target_node) + dependency_visibility_ignored = { - dependency.Public : dependency.Public, - dependency.Private : dependency.Public, - dependency.Interface : dependency.Public, + dependency.Public: dependency.Public, + dependency.Private: dependency.Public, + dependency.Interface: dependency.Public, } dependency_visibility_honored = { - dependency.Public : dependency.Public, - dependency.Private : dependency.Private, - dependency.Interface : dependency.Interface, + dependency.Public: dependency.Public, + dependency.Private: dependency.Private, + dependency.Interface: dependency.Interface, } + class DependencyCycleError(SCons.Errors.UserError): """Exception representing a cycle discovered in library dependencies.""" - def __init__(self, first_node ): + def __init__(self, first_node): super(DependencyCycleError, self).__init__() self.cycle_nodes = [first_node] def __str__(self): return "Library dependency cycle detected: " + " => ".join(str(n) for n in self.cycle_nodes) + def __get_sorted_direct_libdeps(node): direct_sorted = getattr(node.attributes, "libdeps_direct_sorted", False) if not direct_sorted: @@ -100,8 +103,8 @@ def __get_sorted_direct_libdeps(node): setattr(node.attributes, "libdeps_direct_sorted", direct_sorted) return direct_sorted -def __get_libdeps(node): +def __get_libdeps(node): """Given a SCons Node, return its library dependencies, topologically sorted. Computes the dependencies if they're not already cached. @@ -133,7 +136,7 @@ def __get_libdeps(node): marked.add(n.target_node) tsorted.append(n.target_node) - except DependencyCycleError, e: + except DependencyCycleError as e: if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]: e.cycle_nodes.insert(0, n.target_node) raise @@ -150,6 +153,7 @@ def __get_libdeps(node): return tsorted + def __get_syslibdeps(node): """ Given a SCons Node, return its system library dependencies. @@ -161,11 +165,11 @@ def __get_syslibdeps(node): for lib in __get_libdeps(node): for syslib in node.get_env().Flatten(lib.get_env().get(syslibdeps_env_var, [])): if syslib: - if type(syslib) in (str, unicode) and syslib.startswith(missing_syslibdep): - print("Target '%s' depends on the availability of a " + if type(syslib) is str and syslib.startswith(missing_syslibdep): + print(("Target '%s' depends on the availability of a " "system provided library for '%s', " "but no suitable library was found during configuration." % - (str(node), syslib[len(missing_syslibdep):])) + (str(node), syslib[len(missing_syslibdep):]))) node.get_env().Exit(1) syslibdeps.append(syslib) setattr(node.attributes, cached_var_name, syslibdeps) @@ -181,17 +185,20 @@ def update_scanner(builder): if old_scanner: path_function = old_scanner.path_function + def new_scanner(node, env, path=()): result = old_scanner.function(node, env, path) result.extend(__get_libdeps(node)) return result else: path_function = None + def new_scanner(node, env, path=()): return __get_libdeps(node) builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, - path_function=path_function) + path_function=path_function) + def get_libdeps(source, target, env, for_signature): """Implementation of the special _LIBDEPS environment variable. @@ -202,6 +209,7 @@ def get_libdeps(source, target, env, for_signature): target = env.Flatten([target]) return __get_libdeps(target[0]) + def get_libdeps_objs(source, target, env, for_signature): objs = [] for lib in get_libdeps(source, target, env, for_signature): @@ -209,6 +217,7 @@ def get_libdeps_objs(source, target, env, for_signature): objs.extend(lib.sources) return objs + def get_syslibdeps(source, target, env, for_signature): deps = __get_syslibdeps(target[0]) lib_link_prefix = env.subst('$LIBLINKPREFIX') @@ -220,7 +229,7 @@ def get_syslibdeps(source, target, env, for_signature): # they're believed to represent library short names, that should be prefixed with -l # or the compiler-specific equivalent. I.e., 'm' becomes '-lm', but 'File("m.a") is passed # through whole cloth. - if type(d) in (str, unicode): + if type(d) is str: result.append('%s%s%s' % (lib_link_prefix, d, lib_link_suffix)) else: result.append(d) @@ -234,8 +243,9 @@ def __append_direct_libdeps(node, prereq_nodes): node.attributes.libdeps_direct = [] node.attributes.libdeps_direct.extend(prereq_nodes) -def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibility_ignored, ignore_progdeps=False): +def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibility_ignored, + ignore_progdeps=False): def libdeps_emitter(target, source, env): """SCons emitter that takes values from the LIBDEPS environment variable and converts them to File node objects, binding correct path information into @@ -260,13 +270,20 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit prog_builder = env['BUILDERS']['Program'] prog_node_factory = prog_builder.target_factory or env.File - prereqs = [dependency(l, dependency_map[dependency.Public]) for l in env.get(libdeps_env_var, []) if l] - prereqs.extend(dependency(l, dependency_map[dependency.Interface]) for l in env.get(libdeps_env_var + '_INTERFACE', []) if l) - prereqs.extend(dependency(l, dependency_map[dependency.Private]) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l) + prereqs = [ + dependency(l, dependency_map[dependency.Public]) for l in env.get(libdeps_env_var, []) + if l + ] + prereqs.extend( + dependency(l, dependency_map[dependency.Interface]) + for l in env.get(libdeps_env_var + '_INTERFACE', []) if l) + prereqs.extend( + dependency(l, dependency_map[dependency.Private]) + for l in env.get(libdeps_env_var + '_PRIVATE', []) if l) for prereq in prereqs: - prereqWithIxes = SCons.Util.adjustixes( - prereq.target_node, lib_builder.get_prefix(env), lib_builder.get_suffix(env)) + prereqWithIxes = SCons.Util.adjustixes(prereq.target_node, lib_builder.get_prefix(env), + lib_builder.get_suffix(env)) prereq.target_node = lib_node_factory(prereqWithIxes) for t in target: @@ -283,10 +300,11 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes( - dependent, lib_builder.get_prefix(env), lib_builder.get_suffix(env)) + dependentWithIxes = SCons.Util.adjustixes(dependent, lib_builder.get_prefix(env), + lib_builder.get_suffix(env)) dependentNode = lib_node_factory(dependentWithIxes) - __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) + __append_direct_libdeps(dependentNode, + [dependency(target[0], dependency_map[visibility])]) if not ignore_progdeps: for dependent in env.get('PROGDEPS_DEPENDENTS', []): @@ -299,21 +317,24 @@ def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibilit visibility = dependent[1] dependent = dependent[0] - dependentWithIxes = SCons.Util.adjustixes( - dependent, prog_builder.get_prefix(env), prog_builder.get_suffix(env)) + dependentWithIxes = SCons.Util.adjustixes(dependent, prog_builder.get_prefix(env), + prog_builder.get_suffix(env)) dependentNode = prog_node_factory(dependentWithIxes) - __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) + __append_direct_libdeps(dependentNode, + [dependency(target[0], dependency_map[visibility])]) return target, source return libdeps_emitter + def expand_libdeps_tags(source, target, env, for_signature): results = [] for expansion in env.get('LIBDEPS_TAG_EXPANSIONS', []): results.append(expansion(source, target, env, for_signature)) return results + def setup_environment(env, emitting_shared=False): """Set up the given build environment to do LIBDEPS tracking.""" @@ -339,19 +360,18 @@ def setup_environment(env, emitting_shared=False): def make_indirect_emitter(variable): def indirect_emitter(target, source, env): return env[variable](target, source, env) + return indirect_emitter env.Append( LIBDEPS_LIBEMITTER=make_libdeps_emitter('StaticLibrary'), LIBEMITTER=make_indirect_emitter('LIBDEPS_LIBEMITTER'), - LIBDEPS_SHAREMITTER=make_libdeps_emitter('SharedArchive', ignore_progdeps=True), SHAREMITTER=make_indirect_emitter('LIBDEPS_SHAREMITTER'), - LIBDEPS_SHLIBEMITTER=make_libdeps_emitter('SharedLibrary', dependency_visibility_honored), SHLIBEMITTER=make_indirect_emitter('LIBDEPS_SHLIBEMITTER'), - - LIBDEPS_PROGEMITTER=make_libdeps_emitter('SharedLibrary' if emitting_shared else 'StaticLibrary'), + LIBDEPS_PROGEMITTER=make_libdeps_emitter( + 'SharedLibrary' if emitting_shared else 'StaticLibrary'), PROGEMITTER=make_indirect_emitter('LIBDEPS_PROGEMITTER'), ) @@ -362,9 +382,11 @@ def setup_environment(env, emitting_shared=False): if 'init-no-global-side-effects' in env.Entry(lib).get_env().get('LIBDEPS_TAGS', []): result.append(str(lib)) else: - result.extend(env.subst('$LINK_WHOLE_ARCHIVE_LIB_START' - '$TARGET' - '$LINK_WHOLE_ARCHIVE_LIB_END', target=lib).split()) + result.extend( + env.subst( + '$LINK_WHOLE_ARCHIVE_LIB_START' + '$TARGET' + '$LINK_WHOLE_ARCHIVE_LIB_END', target=lib).split()) return result env['_LIBDEPS_LIBS_WITH_TAGS'] = expand_libdeps_with_extraction_flags @@ -382,6 +404,7 @@ def setup_environment(env, emitting_shared=False): except KeyError: pass + def setup_conftests(conf): def FindSysLibDep(context, name, libs, **kwargs): var = "LIBDEPS_" + name.upper() + "_SYSLIBDEP" @@ -394,4 +417,5 @@ def setup_conftests(conf): return context.Result(result) context.env[var] = __missing_syslib(name) return context.Result(result) + conf.AddTest('FindSysLibDep', FindSysLibDep) diff --git a/site_scons/mongo/__init__.py b/site_scons/mongo/__init__.py index 510bd7bcc2f..f77478092b9 100644 --- a/site_scons/mongo/__init__.py +++ b/site_scons/mongo/__init__.py @@ -5,4 +5,4 @@ def print_build_failures(): from SCons.Script import GetBuildFailures for bf in GetBuildFailures(): - print "%s failed: %s" % (bf.node, bf.errstr) + print("%s failed: %s" % (bf.node, bf.errstr)) diff --git a/site_scons/mongo/generators.py b/site_scons/mongo/generators.py index c07e86a4d14..052f18d3a0a 100644 --- a/site_scons/mongo/generators.py +++ b/site_scons/mongo/generators.py @@ -1,6 +1,6 @@ # -*- mode: python; -*- -import md5 +import hashlib # Default and alternative generator definitions go here. @@ -15,22 +15,69 @@ import md5 # want to define them. def default_buildinfo_environment_data(): return ( - ('distmod', '$MONGO_DISTMOD', True, True,), - ('distarch', '$MONGO_DISTARCH', True, True,), - ('cc', '$CC_VERSION', True, False,), - ('ccflags', '$CCFLAGS', True, False,), - ('cxx', '$CXX_VERSION', True, False,), - ('cxxflags', '$CXXFLAGS', True, False,), - ('linkflags', '$LINKFLAGS', True, False,), - ('target_arch', '$TARGET_ARCH', True, True,), - ('target_os', '$TARGET_OS', True, False,), + ( + 'distmod', + '$MONGO_DISTMOD', + True, + True, + ), + ( + 'distarch', + '$MONGO_DISTARCH', + True, + True, + ), + ( + 'cc', + '$CC_VERSION', + True, + False, + ), + ( + 'ccflags', + '$CCFLAGS', + True, + False, + ), + ( + 'cxx', + '$CXX_VERSION', + True, + False, + ), + ( + 'cxxflags', + '$CXXFLAGS', + True, + False, + ), + ( + 'linkflags', + '$LINKFLAGS', + True, + False, + ), + ( + 'target_arch', + '$TARGET_ARCH', + True, + True, + ), + ( + 'target_os', + '$TARGET_OS', + True, + False, + ), ) + # If you want buildInfo and --version to be relatively empty, set # MONGO_BUILDINFO_ENVIRONMENT_DATA = empty_buildinfo_environment_data() def empty_buildinfo_environment_data(): return () + def default_variant_dir_generator(target, source, env, for_signature): if env.GetOption('cache') != None: @@ -44,11 +91,11 @@ def default_variant_dir_generator(target, source, env, for_signature): # Hash the named options and their values, and take the first 8 characters of the hash as # the variant name - hasher = md5.md5() + hasher = hashlib.md5() for option in variant_options: - hasher.update(option) - hasher.update(str(env.GetOption(option))) - variant_dir = hasher.hexdigest()[0:8] + hasher.update(option.encode('utf-8')) + hasher.update(str(env.GetOption(option)).encode('utf-8')) + variant_dir = str(hasher.hexdigest()[0:8]) # If our option hash yields a well known hash, replace it with its name. known_variant_hashes = { diff --git a/site_scons/mongo/platform.py b/site_scons/mongo/platform.py index 05df9017ac4..f5d15b4924c 100644 --- a/site_scons/mongo/platform.py +++ b/site_scons/mongo/platform.py @@ -1,5 +1,4 @@ # -*- mode: python; -*- - """ Support code related to OS detection in general. System specific facilities or customization hooks live in mongo_platform_<PLATFORM>.py files. @@ -12,6 +11,7 @@ import os # This needs to precede the options section so that we can only offer some options on certain # operating systems. + # This function gets the running OS as identified by Python # It should only be used to set up defaults for options/variables, because # its value could potentially be overridden by setting TARGET_OS on the @@ -34,25 +34,28 @@ def get_running_os_name(): running_os = 'unknown' return running_os + def env_get_os_name_wrapper(self): return self['TARGET_OS'] + def is_os_raw(target_os, os_list_to_check): - darwin_os_list = [ 'macOS', 'tvOS', 'tvOS-sim', 'iOS', 'iOS-sim', 'watchOS', 'watchOS-sim' ] - linux_os_list = [ 'android', 'linux' ] - posix_os_list = [ 'openbsd', 'freebsd', 'solaris', 'emscripten' ] + darwin_os_list + linux_os_list + darwin_os_list = ['macOS', 'tvOS', 'tvOS-sim', 'iOS', 'iOS-sim', 'watchOS', 'watchOS-sim'] + linux_os_list = ['android', 'linux'] + posix_os_list = ['openbsd', 'freebsd', 'solaris', 'emscripten'] + darwin_os_list + linux_os_list os_families = { - "darwin": darwin_os_list, - "posix": posix_os_list, - "linux": linux_os_list, + "darwin": darwin_os_list, + "posix": posix_os_list, + "linux": linux_os_list, } for os in os_list_to_check: - if os == target_os or ( os in os_families and target_os in os_families[os] ): + if os == target_os or (os in os_families and target_os in os_families[os]): return True return False + # This function tests the running OS as identified by Python # It should only be used to set up defaults for options/variables, because # its value could potentially be overridden by setting TARGET_OS on the @@ -60,5 +63,6 @@ def is_os_raw(target_os, os_list_to_check): def is_running_os(*os_list): return is_os_raw(get_running_os_name(), os_list) + def env_os_is_wrapper(self, *os_list): return is_os_raw(self['TARGET_OS'], os_list) diff --git a/site_scons/mongo/toolchain.py b/site_scons/mongo/toolchain.py index ffa74f56f6f..9bab92a68b7 100644 --- a/site_scons/mongo/toolchain.py +++ b/site_scons/mongo/toolchain.py @@ -6,31 +6,22 @@ import SCons # Helper functions for generic toolchain things go here + def get_toolchain_ver(env, tool): # By default we don't know the version of each tool, and only report what # command gets executed (gcc vs /opt/mongodbtoolchain/bin/gcc). verstr = "version unknown" proc = None if env.ToolchainIs('clang', 'gcc'): - proc = SCons.Action._subproc(env, - env.subst("${%s} --version" % tool), - stdout=subprocess.PIPE, - stderr='devnull', - stdin='devnull', - universal_newlines=True, - error='raise', - shell=True) + proc = SCons.Action._subproc(env, env.subst("${%s} --version" % tool), + stdout=subprocess.PIPE, stderr='devnull', stdin='devnull', + universal_newlines=True, error='raise', shell=True) verstr = proc.stdout.readline() elif env.ToolchainIs('msvc') and env.TargetOSIs('windows'): - proc = SCons.Action._subproc(env, - env.subst("${%s}" % tool), - stdout='devnull', - stderr=subprocess.PIPE, - stdin='devnull', - universal_newlines=True, - error='raise', - shell=True) + proc = SCons.Action._subproc(env, env.subst("${%s}" % tool), stdout='devnull', + stderr=subprocess.PIPE, stdin='devnull', + universal_newlines=True, error='raise', shell=True) verstr = proc.stderr.readline() # If we started a process, we should drain its stdout/stderr and wait for diff --git a/site_scons/site_tools/abilink.py b/site_scons/site_tools/abilink.py index 65f2a995719..00f5e71a0a3 100644 --- a/site_scons/site_tools/abilink.py +++ b/site_scons/site_tools/abilink.py @@ -20,6 +20,7 @@ import subprocess # TODO: Make a variable for the md5sum utility (allow any hasher) # TODO: Add an ABILINKCOM variable to the Action, so it can be silenced. + def _detect(env): try: abidw = env['ABIDW'] @@ -31,6 +32,7 @@ def _detect(env): return env.WhereIs('abidw') + def _add_emitter(builder): base_emitter = builder.emitter @@ -47,6 +49,7 @@ def _add_emitter(builder): new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) builder.emitter = new_emitter + def _add_scanner(builder): old_scanner = builder.target_scanner path_function = old_scanner.path_function @@ -59,16 +62,21 @@ def _add_scanner(builder): new_results.append(abidw if abidw else base) return new_results - builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, + path_function=path_function) + def _add_action(builder): actions = builder.action - builder.action = actions + SCons.Action.Action("$ABIDW --no-show-locs $TARGET | md5sum > ${TARGET}.abidw") + builder.action = actions + SCons.Action.Action( + "$ABIDW --no-show-locs $TARGET | md5sum > ${TARGET}.abidw") + def exists(env): result = _detect(env) != None return result + def generate(env): if not exists(env): diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py index 9a71b4919b7..6261058c268 100644 --- a/site_scons/site_tools/auto_install_binaries.py +++ b/site_scons/site_tools/auto_install_binaries.py @@ -1,24 +1,26 @@ import SCons + def exists(env): return True + def generate(env): env.Tool('install') suffix_map = { - env.subst('$PROGSUFFIX') : 'bin', - '.dylib' : 'lib', + env.subst('$PROGSUFFIX'): 'bin', + '.dylib': 'lib', # TODO: These 'lib' answers are incorrect. The location for the debug info # should be the same as the target itself, which might be bin or lib. We need # a solution for that. When that is fixed, add 'Program' back into the list # of separate debug targets in the separate_debug.py tool. - '.dSYM' : 'lib', - '.debug' : 'lib', - '.so' : 'lib', - '.dll' : 'bin', - '.lib' : 'lib', + '.dSYM': 'lib', + '.debug': 'lib', + '.so': 'lib', + '.dll': 'bin', + '.lib': 'lib', } def auto_install(env, target, source, **kwargs): @@ -67,7 +69,8 @@ def generate(env): if auto_install_location: tentry_install_tags = env.get('INSTALL_ALIAS', []) setattr(tentry.attributes, 'INSTALL_ALIAS', tentry_install_tags) - install = env.AutoInstall(auto_install_location, tentry, INSTALL_ALIAS=tentry_install_tags) + install = env.AutoInstall(auto_install_location, tentry, + INSTALL_ALIAS=tentry_install_tags) return (target, source) def add_emitter(builder): @@ -99,7 +102,7 @@ def generate(env): from SCons.Tool import install base_install_builder = install.BaseInstallBuilder - assert(base_install_builder.target_scanner == None) + assert (base_install_builder.target_scanner == None) base_install_builder.target_scanner = SCons.Scanner.Scanner( function=scan_for_transitive_install, diff --git a/site_scons/site_tools/compilation_db.py b/site_scons/site_tools/compilation_db.py index 161ac6061ea..07aa2d96e58 100644 --- a/site_scons/site_tools/compilation_db.py +++ b/site_scons/site_tools/compilation_db.py @@ -28,7 +28,7 @@ import itertools # compilation database can access the complete list, and also so that the writer has easy # access to write all of the files. But it seems clunky. How can the emitter and the scanner # communicate more gracefully? -__COMPILATION_DB_ENTRIES=[] +__COMPILATION_DB_ENTRIES = [] # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if # we could obtain this from SCons. @@ -40,6 +40,7 @@ _CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] if SCons.Util.case_sensitive_suffixes('.c', '.C'): _CXXSuffixes.append('.C') + # We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even # integrate with the cache, but there doesn't seem to be much call for it. class __CompilationDbNode(SCons.Node.Python.Value): @@ -77,11 +78,8 @@ def makeEmitCompilationDbEntry(comstr): dbtarget = __CompilationDbNode(source) entry = env.__COMPILATIONDB_Entry( - target=dbtarget, - source=[], - __COMPILATIONDB_UTARGET=target, - __COMPILATIONDB_USOURCE=source, - __COMPILATIONDB_UACTION=user_action, + target=dbtarget, source=[], __COMPILATIONDB_UTARGET=target, + __COMPILATIONDB_USOURCE=source, __COMPILATIONDB_UACTION=user_action, __COMPILATIONDB_ENV=env) # TODO: Technically, these next two lines should not be required: it should be fine to @@ -112,11 +110,11 @@ def CompilationDbEntryAction(target, source, env, **kw): command = env['__COMPILATIONDB_UACTION'].strfunction( target=env['__COMPILATIONDB_UTARGET'], source=env['__COMPILATIONDB_USOURCE'], - env=env['__COMPILATIONDB_ENV'],) + env=env['__COMPILATIONDB_ENV'], + ) entry = { - "directory": env.Dir('#').abspath, - "command": command, + "directory": env.Dir('#').abspath, "command": command, "file": str(env['__COMPILATIONDB_USOURCE'][0]) } @@ -130,21 +128,19 @@ def WriteCompilationDb(target, source, env): entries.append(s.read()) with open(str(target[0]), 'w') as target_file: - json.dump(entries, target_file, - sort_keys=True, - indent=4, - separators=(',', ': ')) + json.dump(entries, target_file, sort_keys=True, indent=4, separators=(',', ': ')) def ScanCompilationDb(node, env, path): return __COMPILATION_DB_ENTRIES + def generate(env, **kwargs): static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - env['COMPILATIONDB_COMSTR'] = kwargs.get( - 'COMPILATIONDB_COMSTR', 'Building compilation database $TARGET') + env['COMPILATIONDB_COMSTR'] = kwargs.get('COMPILATIONDB_COMSTR', + 'Building compilation database $TARGET') components_by_suffix = itertools.chain( itertools.product(_CSuffixes, [ @@ -163,28 +159,20 @@ def generate(env, **kwargs): # Assumes a dictionary emitter emitter = builder.emitter[suffix] - builder.emitter[suffix] = SCons.Builder.ListEmitter( - [ - emitter, - makeEmitCompilationDbEntry(command), - ] - ) + builder.emitter[suffix] = SCons.Builder.ListEmitter([ + emitter, + makeEmitCompilationDbEntry(command), + ]) env['BUILDERS']['__COMPILATIONDB_Entry'] = SCons.Builder.Builder( - action=SCons.Action.Action(CompilationDbEntryAction, None), - ) + action=SCons.Action.Action(CompilationDbEntryAction, None), ) env['BUILDERS']['__COMPILATIONDB_Database'] = SCons.Builder.Builder( action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), - target_scanner=SCons.Scanner.Scanner( - function=ScanCompilationDb, - node_class=None) - ) + target_scanner=SCons.Scanner.Scanner(function=ScanCompilationDb, node_class=None)) def CompilationDatabase(env, target): - result = env.__COMPILATIONDB_Database( - target=target, - source=[]) + result = env.__COMPILATIONDB_Database(target=target, source=[]) env.AlwaysBuild(result) env.NoCache(result) @@ -193,5 +181,6 @@ def generate(env, **kwargs): env.AddMethod(CompilationDatabase, 'CompilationDatabase') + def exists(env): return True diff --git a/site_scons/site_tools/dagger/__init__.py b/site_scons/site_tools/dagger/__init__.py index f05228cfe45..c63bfc6967e 100644 --- a/site_scons/site_tools/dagger/__init__.py +++ b/site_scons/site_tools/dagger/__init__.py @@ -5,7 +5,7 @@ import logging import SCons -import dagger +from . import dagger def generate(env, **kwargs): """The entry point for our tool. However, the builder for @@ -14,25 +14,22 @@ def generate(env, **kwargs): to the native builders for object/libraries. """ - env.Replace(LIBEMITTER=SCons.Builder.ListEmitter([env['LIBEMITTER'], - dagger.emit_lib_db_entry])) + env.Replace(LIBEMITTER=SCons.Builder.ListEmitter([env['LIBEMITTER'], dagger.emit_lib_db_entry])) running_os = os.sys.platform if not (running_os.startswith('win') or running_os.startswith('sun')): - env.Replace(PROGEMITTER=SCons.Builder.ListEmitter([env['PROGEMITTER'], - dagger.emit_prog_db_entry])) + env.Replace( + PROGEMITTER=SCons.Builder.ListEmitter([env['PROGEMITTER'], dagger.emit_prog_db_entry])) static_obj, shared_obj = SCons.Tool.createObjBuilders(env) suffixes = ['.c', '.cc', '.cxx', '.cpp'] obj_builders = [static_obj, shared_obj] - default_emitters = [SCons.Defaults.StaticObjectEmitter, - SCons.Defaults.SharedObjectEmitter] + default_emitters = [SCons.Defaults.StaticObjectEmitter, SCons.Defaults.SharedObjectEmitter] for suffix in suffixes: for i in range(len(obj_builders)): - obj_builders[i].add_emitter(suffix, SCons.Builder.ListEmitter([ - dagger.emit_obj_db_entry, default_emitters[i] - ])) + obj_builders[i].add_emitter( + suffix, SCons.Builder.ListEmitter([dagger.emit_obj_db_entry, default_emitters[i]])) env['BUILDERS']['__OBJ_DATABASE'] = SCons.Builder.Builder( action=SCons.Action.Action(dagger.write_obj_db, None)) diff --git a/site_scons/site_tools/dagger/dagger.py b/site_scons/site_tools/dagger/dagger.py index bace834783b..cc208dd23c2 100644 --- a/site_scons/site_tools/dagger/dagger.py +++ b/site_scons/site_tools/dagger/dagger.py @@ -40,8 +40,8 @@ import sys import SCons -import graph -import graph_consts +from . import graph +from . import graph_consts LIB_DB = [] # Stores every SCons library nodes @@ -92,11 +92,9 @@ def get_symbol_worker(object_file, task): uses = p.communicate()[0].decode() if platform == 'linux': - return list_process([use[19:] for use in uses.split('\n') - if use != '']) + return list_process([use[19:] for use in uses.split('\n') if use != '']) elif platform == 'darwin': - return list_process([use.strip() for use in uses.split('\n') - if use != '']) + return list_process([use.strip() for use in uses.split('\n') if use != '']) def emit_obj_db_entry(target, source, env): @@ -109,6 +107,7 @@ def emit_obj_db_entry(target, source, env): OBJ_DB.append(t) return target, source + def emit_prog_db_entry(target, source, env): for t in target: if str(t) is None: @@ -117,6 +116,7 @@ def emit_prog_db_entry(target, source, env): return target, source + def emit_lib_db_entry(target, source, env): """Emitter for libraries. We add each library into our global variable""" @@ -210,6 +210,7 @@ def __generate_file_rels(obj, g): for obj in objs: g.add_edge(graph_consts.FIL_FIL, file_node.id, obj) + def __generate_exe_rels(exe, g): """Generates all executable to library relationships, and populates the contained files field in each NodeExe object""" @@ -223,6 +224,7 @@ def __generate_exe_rels(exe, g): exe_node.contained_files = set(EXE_DB[exe]) + def write_obj_db(target, source, env): """The bulk of the tool. This method takes all the objects and libraries which we have stored in the global LIB_DB and OBJ_DB variables and @@ -240,7 +242,7 @@ def write_obj_db(target, source, env): for obj in OBJ_DB: __generate_file_rels(obj, g) - for exe in EXE_DB.keys(): + for exe in list(EXE_DB.keys()): __generate_exe_rels(exe, g) # target is given as a list of target SCons nodes - this builder is only responsible for diff --git a/site_scons/site_tools/dagger/graph.py b/site_scons/site_tools/dagger/graph.py index 5ebe6f45061..40c7fd9b2d9 100644 --- a/site_scons/site_tools/dagger/graph.py +++ b/site_scons/site_tools/dagger/graph.py @@ -4,10 +4,7 @@ import abc import json import copy -import graph_consts - -if sys.version_info >= (3, 0): - basestring = str +from . import graph_consts class Graph(object): """Graph class for storing the build dependency graph. The graph stores the @@ -20,7 +17,7 @@ class Graph(object): """ A graph can be initialized with a .json file, graph object, or with no args """ - if isinstance(input, basestring): + if isinstance(input, str): if input.endswith('.json'): with open(input, 'r') as f: data = json.load(f, encoding="ascii") @@ -72,7 +69,7 @@ class Graph(object): @nodes.setter def nodes(self, value): - if isinstance(value,dict): + if isinstance(value, dict): self._nodes = value else: raise TypeError("Nodes must be a dict") @@ -141,7 +138,7 @@ class Graph(object): node_dict["id"] = id node_dict["node"] = {} - for property, value in vars(node).iteritems(): + for property, value in vars(node).items(): if isinstance(value, set): node_dict["node"][property] = list(value) else: @@ -151,7 +148,7 @@ class Graph(object): for edge_type in graph_consts.RELATIONSHIP_TYPES: edges_dict = self._edges[edge_type] - for node in edges_dict.keys(): + for node in list(edges_dict.keys()): to_nodes = list(self._edges[edge_type][node]) to_nodes_dicts = [{"index": node_index[to_node], "id": to_node} for to_node in to_nodes] @@ -166,14 +163,13 @@ class Graph(object): def __str__(self): return ("<Number of Nodes : {0}, Number of Edges : {1}, " - "Hash: {2}>").format(len(self._nodes.keys()), - sum(len(x) for x in self._edges.values()), hash(self)) + "Hash: {2}>").format(len(list(self._nodes.keys())), + sum(len(x) for x in list(self._edges.values())), hash(self)) -class NodeInterface(object): +class NodeInterface(object, metaclass=abc.ABCMeta): """Abstract base class for all Node Objects - All nodes must have an id and name """ - __metaclass__ = abc.ABCMeta @abc.abstractproperty def id(self): @@ -190,7 +186,7 @@ class NodeLib(NodeInterface): def __init__(self, id, name, input=None): if isinstance(input, dict): should_fail = False - for k, v in input.iteritems(): + for k, v in input.items(): try: if isinstance(v, list): setattr(self, k, set(v)) @@ -287,10 +283,10 @@ class NodeLib(NodeInterface): def __eq__(self, other): if isinstance(other, NodeLib): - return (self._id == other._id and self._defined_symbols == other._defined_symbols and - self._defined_files == other._defined_files and - self._dependent_libs == other._dependent_libs and - self._dependent_files == other._dependent_files) + return (self._id == other._id and self._defined_symbols == other._defined_symbols + and self._defined_files == other._defined_files + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files) else: return False @@ -310,7 +306,7 @@ class NodeSymbol(NodeInterface): if isinstance(input, dict): should_fail = False - for k, v in input.iteritems(): + for k, v in input.items(): try: if isinstance(v, list): setattr(self, k, set(v)) @@ -413,11 +409,10 @@ class NodeSymbol(NodeInterface): def __eq__(self, other): if isinstance(other, NodeSymbol): - return (self.id == other.id and self._libs == other._libs and - self._files == other._files and - self._dependent_libs == other._dependent_libs and - self._dependent_files == other._dependent_files - ) + return (self.id == other.id and self._libs == other._libs + and self._files == other._files + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files) else: return False @@ -435,7 +430,7 @@ class NodeFile(NodeInterface): def __init__(self, id, name, input=None): if isinstance(input, dict): should_fail = False - for k, v in input.iteritems(): + for k, v in input.items(): try: if isinstance(v, list): setattr(self, k, set(v)) @@ -526,16 +521,16 @@ class NodeFile(NodeInterface): self.add_dependent_lib(from_node.library) g.add_edge(graph_consts.LIB_FIL, from_node.library, self.id) if lib_node is not None: - lib_node.add_dependent_file(from_node.id) - lib_node.add_dependent_lib(from_node.library) - g.add_edge(graph_consts.FIL_LIB, from_node.id, lib_node.id) + lib_node.add_dependent_file(from_node.id) + lib_node.add_dependent_lib(from_node.library) + g.add_edge(graph_consts.FIL_LIB, from_node.id, lib_node.id) def __eq__(self, other): if isinstance(other, NodeSymbol): - return (self.id == other.id and self._lib == other._lib and - self._dependent_libs == other._dependent_libs and - self._dependent_files == other._dependent_files and - self._defined_symbols == other._defined_symbols) + return (self.id == other.id and self._lib == other._lib + and self._dependent_libs == other._dependent_libs + and self._dependent_files == other._dependent_files + and self._defined_symbols == other._defined_symbols) else: return False @@ -551,7 +546,7 @@ class NodeExe(NodeInterface): def __init__(self, id, name, input=None): if isinstance(input, dict): should_fail = False - for k, v in input.iteritems(): + for k, v in input.items(): try: if isinstance(v, list): setattr(self, k, set(v)) @@ -580,10 +575,12 @@ class NodeExe(NodeInterface): return self.id -types = {graph_consts.NODE_LIB: NodeLib, - graph_consts.NODE_SYM: NodeSymbol, - graph_consts.NODE_FILE: NodeFile, - graph_consts.NODE_EXE: NodeExe,} +types = { + graph_consts.NODE_LIB: NodeLib, + graph_consts.NODE_SYM: NodeSymbol, + graph_consts.NODE_FILE: NodeFile, + graph_consts.NODE_EXE: NodeExe, +} def node_factory(id, nodetype, dict_source=None): diff --git a/site_scons/site_tools/dagger/graph_consts.py b/site_scons/site_tools/dagger/graph_consts.py index 81fe86d75cd..eae5db9b6c6 100644 --- a/site_scons/site_tools/dagger/graph_consts.py +++ b/site_scons/site_tools/dagger/graph_consts.py @@ -1,5 +1,4 @@ """Constants for use in graph.py and dagger.py""" - """Relationship edge types""" LIB_LIB = 1 LIB_FIL = 2 @@ -17,8 +16,8 @@ NODE_SYM = 2 NODE_FILE = 3 NODE_EXE = 4 -RELATIONSHIP_TYPES = range(1, 9) -NODE_TYPES = range(1, 5) +RELATIONSHIP_TYPES = list(range(1, 9)) +NODE_TYPES = list(range(1, 5)) """Error/query codes""" diff --git a/site_scons/site_tools/dagger/graph_test.py b/site_scons/site_tools/dagger/graph_test.py index bc84f5868c7..39bdf77ab7c 100644 --- a/site_scons/site_tools/dagger/graph_test.py +++ b/site_scons/site_tools/dagger/graph_test.py @@ -5,8 +5,8 @@ from JSON import json import unittest -import graph -import graph_consts +from . import graph +from . import graph_consts def generate_graph(): @@ -73,26 +73,22 @@ class CustomAssertions: raise AssertionError("Nodes not of same type") if node1.type == graph_consts.NODE_LIB: - if (node1._defined_symbols != node2._defined_symbols or - node1._defined_files != node2._defined_files or - node1._dependent_libs != node2._dependent_libs or - node1._dependent_files != node2._dependent_files or - node1._id != node2._id): + if (node1._defined_symbols != node2._defined_symbols + or node1._defined_files != node2._defined_files + or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1._id != node2._id): raise AssertionError("Nodes not equal") elif node1.type == graph_consts.NODE_SYM: - if (node1._libs != node2._libs or node1._files != node2._files or - node1._dependent_libs != node2._dependent_libs or - node1._dependent_files != node2._dependent_files or - node1.id != node2.id): + if (node1._libs != node2._libs or node1._files != node2._files + or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1.id != node2.id): raise AssertionError("Nodes not equal") else: - if (node1._lib != node2._lib or - node1._dependent_libs != node2._dependent_libs or - node1._dependent_files != node2._dependent_files or - node1.id != node2.id or - node1._defined_symbols != node2._defined_symbols): + if (node1._lib != node2._lib or node1._dependent_libs != node2._dependent_libs + or node1._dependent_files != node2._dependent_files or node1.id != node2.id + or node1._defined_symbols != node2._defined_symbols): raise AssertionError("Nodes not equal") @@ -104,11 +100,9 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): self.from_node_lib = graph.NodeLib("from_node_lib", "from_node_lib") self.to_node_lib = graph.NodeLib("to_node_lib", "to_node_lib") - self.from_node_file = graph.NodeFile( - "from_node_file", "from_node_file") + self.from_node_file = graph.NodeFile("from_node_file", "from_node_file") self.to_node_file = graph.NodeFile("to_node_file", "to_node_file") - self.from_node_sym = graph.NodeSymbol( - "from_node_symbol", "from_node_symbol") + self.from_node_sym = graph.NodeSymbol("from_node_symbol", "from_node_symbol") self.to_node_sym = graph.NodeSymbol("to_node_symbol", "to_node_symbol") self.g.add_node(self.from_node_lib) @@ -122,15 +116,15 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): node = graph.NodeLib("test_node", "test_node") self.g._nodes = {"test_node": node} - self.assertEquals(self.g.get_node("test_node"), node) + self.assertEqual(self.g.get_node("test_node"), node) - self.assertEquals(self.g.get_node("missing_node"), None) + self.assertEqual(self.g.get_node("missing_node"), None) def test_add_node(self): node = graph.NodeLib("test_node", "test_node") self.g.add_node(node) - self.assertEquals(self.g.get_node("test_node"), node) + self.assertEqual(self.g.get_node("test_node"), node) self.assertRaises(ValueError, self.g.add_node, node) @@ -153,16 +147,16 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): self.g.add_edge(graph_consts.LIB_FIL, self.from_node_lib.id, self.to_node_file.id) - self.assertEquals(self.g.edges[graph_consts.LIB_LIB][ + self.assertEqual(self.g.edges[graph_consts.LIB_LIB][ self.from_node_lib.id], set([self.to_node_lib.id])) - self.assertEquals(self.g.edges[graph_consts.LIB_SYM][ + self.assertEqual(self.g.edges[graph_consts.LIB_SYM][ self.from_node_lib.id], set([self.to_node_sym.id])) - self.assertEquals(self.g.edges[graph_consts.LIB_FIL][ + self.assertEqual(self.g.edges[graph_consts.LIB_FIL][ self.from_node_lib.id], set([self.to_node_file.id])) - self.assertEquals(self.to_node_lib.dependent_libs, + self.assertEqual(self.to_node_lib.dependent_libs, set([self.from_node_lib.id])) def test_add_edge_files(self): @@ -173,14 +167,14 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): self.g.add_edge(graph_consts.FIL_LIB, self.from_node_file.id, self.to_node_lib.id) - self.assertEquals(self.g.edges[graph_consts.FIL_FIL][ + self.assertEqual(self.g.edges[graph_consts.FIL_FIL][ self.from_node_file.id], set([self.to_node_file.id])) - self.assertEquals(self.g.edges[graph_consts.FIL_SYM][ + self.assertEqual(self.g.edges[graph_consts.FIL_SYM][ self.from_node_file.id], set([self.to_node_sym.id])) - self.assertEquals(self.g.edges[graph_consts.FIL_LIB][ + self.assertEqual(self.g.edges[graph_consts.FIL_LIB][ self.from_node_file.id], set([self.to_node_lib.id])) - self.assertEquals(self.to_node_file.dependent_files, + self.assertEqual(self.to_node_file.dependent_files, set([self.from_node_file.id])) def test_export_to_json(self): @@ -188,7 +182,7 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): generated_graph.export_to_json("export_test.json") generated = open("export_test.json", "r") correct = open("test_graph.json", "r") - self.assertEquals(json.load(generated), json.load(correct)) + self.assertEqual(json.load(generated), json.load(correct)) generated.close() correct.close() @@ -196,7 +190,7 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): graph_fromJSON = graph.Graph("test_graph.json") correct_graph = generate_graph() - for id in graph_fromJSON.nodes.keys(): + for id in list(graph_fromJSON.nodes.keys()): # for some reason, neither # assertTrue(graph_fromJSON.get_node(id) == correct_graph.get_node(str(id))) # nor assertEquals() seem to call the correct eq method here, hence @@ -205,7 +199,7 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): self.assertNodeEquals( graph_fromJSON.get_node(id), correct_graph.get_node(id)) - self.assertEquals(graph_fromJSON.edges, correct_graph.edges) + self.assertEqual(graph_fromJSON.edges, correct_graph.edges) if __name__ == '__main__': diff --git a/site_scons/site_tools/distsrc.py b/site_scons/site_tools/distsrc.py index 861f5d9e2e2..cc72c0655f7 100644 --- a/site_scons/site_tools/distsrc.py +++ b/site_scons/site_tools/distsrc.py @@ -20,7 +20,7 @@ import shutil import tarfile import time import zipfile -import StringIO +import io from distutils.spawn import find_executable @@ -28,7 +28,7 @@ __distsrc_callbacks = [] class DistSrcFile: def __init__(self, **kwargs): - [ setattr(self, key, val) for (key, val) in kwargs.items() ] + [ setattr(self, key, val) for (key, val) in list(kwargs.items()) ] def __str__(self): return self.name @@ -60,6 +60,7 @@ class DistSrcArchive: def close(self): self.archive_file.close() + class DistSrcTarArchive(DistSrcArchive): def __iter__(self): file_list = self.archive_file.getnames() @@ -82,7 +83,7 @@ class DistSrcTarArchive(DistSrcArchive): def append_file_contents(self, filename, file_contents, mtime=time.time(), - mode=0644, + mode=0o644, uname="root", gname="root"): file_metadata = tarfile.TarInfo(name=filename) @@ -91,7 +92,7 @@ class DistSrcTarArchive(DistSrcArchive): file_metadata.uname = uname file_metadata.gname = gname file_metadata.size = len(file_contents) - file_buf = StringIO.StringIO(file_contents) + file_buf = io.BytesIO(file_contents.encode('utf-8')) if self.archive_mode == 'r': self.archive_file.close() self.archive_file = tarfile.open( @@ -105,6 +106,7 @@ class DistSrcTarArchive(DistSrcArchive): def append_file(self, filename, localfile): self.archive_file.add(localfile, arcname=filename) + class DistSrcZipArchive(DistSrcArchive): def __iter__(self): file_list = self.archive_file.namelist() @@ -119,7 +121,7 @@ class DistSrcZipArchive(DistSrcArchive): name=key, size=item_data.file_size, mtime=time.mktime(fixed_time), - mode=0775 if is_dir else 0664, + mode=0o775 if is_dir else 0o664, type=tarfile.DIRTYPE if is_dir else tarfile.REGTYPE, uid=0, gid=0, @@ -129,7 +131,7 @@ class DistSrcZipArchive(DistSrcArchive): def append_file_contents(self, filename, file_contents, mtime=time.time(), - mode=0644, + mode=0o644, uname="root", gname="root"): self.archive_file.writestr(filename, file_contents) @@ -139,7 +141,7 @@ class DistSrcZipArchive(DistSrcArchive): def build_error_action(msg): def error_stub(target=None, source=None, env=None): - print msg + print(msg) env.Exit(1) return [ error_stub ] @@ -162,7 +164,7 @@ def distsrc_action_generator(source, target, env, for_signature): target_ext = str(target[0])[-3:] if not target_ext in [ 'zip', 'tar' ]: - print "Invalid file format for distsrc. Must be tar or zip file" + print("Invalid file format for distsrc. Must be tar or zip file") env.Exit(1) git_cmd = "\"%s\" archive --format %s --output %s --prefix ${MONGO_DIST_SRC_PREFIX} HEAD" % ( @@ -173,14 +175,14 @@ def distsrc_action_generator(source, target, env, for_signature): SCons.Action.Action(run_distsrc_callbacks, "Running distsrc callbacks for $TARGET") ] + def add_callback(env, fn): __distsrc_callbacks.append(fn) + def generate(env, **kwargs): env.AddMethod(add_callback, 'AddDistSrcCallback') - env['BUILDERS']['__DISTSRC'] = SCons.Builder.Builder( - generator=distsrc_action_generator, - ) + env['BUILDERS']['__DISTSRC'] = SCons.Builder.Builder(generator=distsrc_action_generator, ) def DistSrc(env, target): result = env.__DISTSRC(target=target, source=[]) @@ -190,5 +192,6 @@ def generate(env, **kwargs): env.AddMethod(DistSrc, 'DistSrc') + def exists(env): return True diff --git a/site_scons/site_tools/git_decider.py b/site_scons/site_tools/git_decider.py index cd69de95085..51e6cd54b26 100644 --- a/site_scons/site_tools/git_decider.py +++ b/site_scons/site_tools/git_decider.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. - # If available, uses Git metadata to decide whether files are out of date. + def generate(env, **kwargs): # Grab the existing decider functions out of the environment @@ -47,7 +47,7 @@ def generate(env, **kwargs): dependency.get_ninfo().csig = gitInfoForDep return False - if not(hasattr(prev_ni, 'csig')): + if not (hasattr(prev_ni, 'csig')): prev_ni.csig = gitInfoForDep result = gitInfoForDep == prev_ni.csig @@ -60,6 +60,7 @@ def generate(env, **kwargs): env.Decider(MongoGitDecider) + def exists(env): try: from git import Git diff --git a/site_scons/site_tools/gziptool.py b/site_scons/site_tools/gziptool.py index 8419352b481..3142d46cae9 100644 --- a/site_scons/site_tools/gziptool.py +++ b/site_scons/site_tools/gziptool.py @@ -18,7 +18,7 @@ import shutil def GZipAction(target, source, env, **kw): dst_gzip = gzip.GzipFile(str(target[0]), 'wb') - with open(str(source[0]), 'r') as src_file: + with open(str(source[0]), 'rb') as src_file: shutil.copyfileobj(src_file, dst_gzip) dst_gzip.close() @@ -38,5 +38,6 @@ def generate(env, **kwargs): env.AddMethod(GZipTool, 'GZip') + def exists(env): return True diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py index 9838b633490..003594a7a87 100644 --- a/site_scons/site_tools/icecream.py +++ b/site_scons/site_tools/icecream.py @@ -22,6 +22,7 @@ from pkg_resources import parse_version icecream_version_min = '1.1rc2' + def generate(env): if not exists(env): @@ -31,14 +32,16 @@ def generate(env): env['ICECC'] = env.WhereIs('$ICECC') if not 'ICERUN' in env: - env['ICERUN'] = env.File('$ICECC').File('icerun') + env['ICERUN'] = env.File('$ICECC').File('icerun') # Absoluteify, for parity with ICECC env['ICERUN'] = env.WhereIs('$ICERUN') # We can't handle sanitizer blacklist files, so disable icecc then, and just flow through # icerun to prevent slamming the local system with a huge -j value. - if any(f.startswith("-fsanitize-blacklist=") for fs in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS'] for f in env[fs]): + if any( + f.startswith("-fsanitize-blacklist=") for fs in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS'] + for f in env[fs]): env['ICECC'] = '$ICERUN' # Make CC and CXX absolute paths too. It is better for icecc. @@ -46,7 +49,7 @@ def generate(env): env['CXX'] = env.WhereIs('$CXX') # Make a predictable name for the toolchain - icecc_version_target_filename=env.subst('$CC$CXX').replace('/', '_') + icecc_version_target_filename = env.subst('$CC$CXX').replace('/', '_') icecc_version = env.Dir('$BUILD_ROOT/scons/icecc').File(icecc_version_target_filename) # Make an isolated environment so that our setting of ICECC_VERSION in the environment @@ -63,12 +66,9 @@ def generate(env): env['ENV']['ICECC_CLANG_REMOTE_CPP'] = 1 else: toolchain = toolchain_env.Command( - target=icecc_version, - source=['$ICECC_CREATE_ENV', '$CC', '$CXX'], - action=[ + target=icecc_version, source=['$ICECC_CREATE_ENV', '$CC', '$CXX'], action=[ "${SOURCES[0]} --gcc ${SOURCES[1].abspath} ${SOURCES[2].abspath} $TARGET", - ] - ) + ]) env.AppendUnique(CCFLAGS=['-fdirectives-only']) # Add ICECC_VERSION to the environment, pointed at the generated @@ -99,7 +99,7 @@ def generate(env): suffixes = _CSuffixes + _CXXSuffixes for object_builder in SCons.Tool.createObjBuilders(env): emitterdict = object_builder.builder.emitter - for suffix in emitterdict.iterkeys(): + for suffix in emitterdict.keys(): if not suffix in suffixes: continue base = emitterdict[suffix] @@ -128,6 +128,7 @@ def generate(env): # env['ENV']['ICECC_DEBUG'] = 'debug' # env['ENV']['ICECC_LOGFILE'] = 'icecc.log' + def exists(env): icecc = env.get('ICECC', False) @@ -135,10 +136,9 @@ def exists(env): return False icecc = env.WhereIs(icecc) - pipe = SCons.Action._subproc(env, SCons.Util.CLVar(icecc) + ['--version'], - stdin = 'devnull', - stderr = 'devnull', - stdout = subprocess.PIPE) + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(icecc) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE, text=True) if pipe.wait() != 0: return False diff --git a/site_scons/site_tools/idl_tool.py b/site_scons/site_tools/idl_tool.py index c7f64293f5d..58873b576d0 100755 --- a/site_scons/site_tools/idl_tool.py +++ b/site_scons/site_tools/idl_tool.py @@ -21,6 +21,7 @@ import sys import SCons + def idlc_emitter(target, source, env): """For each input IDL file, the tool produces a .cpp and .h file.""" first_source = str(source[0]) @@ -44,9 +45,9 @@ def idl_scanner(node, env, path): # Use the import scanner mode of the IDL compiler to file imported files cmd = [sys.executable, "buildscripts/idl/idlc.py", '--include','src', str(node), '--write-dependencies'] try: - deps_str = subprocess.check_output(cmd) + deps_str = subprocess.check_output(cmd).decode('utf-8') except subprocess.CalledProcessError as e: - print("IDLC ERROR: %s" % (e.output) ) + print(("IDLC ERROR: %s" % (e.output) )) raise deps_list = deps_str.splitlines() @@ -61,29 +62,26 @@ def idl_scanner(node, env, path): idl_scanner = SCons.Scanner.Scanner(function=idl_scanner, skeys=['.idl']) # TODO: create a scanner for imports when imports are implemented -IDLCBuilder = SCons.Builder.Builder( - action=IDLCAction, - emitter=idlc_emitter, - srcsuffx=".idl", - suffix=".cpp", - source_scanner = idl_scanner - ) +IDLCBuilder = SCons.Builder.Builder(action=IDLCAction, emitter=idlc_emitter, srcsuffx=".idl", + suffix=".cpp", source_scanner=idl_scanner) def generate(env): bld = IDLCBuilder - env.Append(SCANNERS = idl_scanner) + env.Append(SCANNERS=idl_scanner) env['BUILDERS']['Idlc'] = bld env['IDLC'] = sys.executable + " buildscripts/idl/idlc.py" env['IDLCFLAGS'] = '' base_dir = env.subst('$BUILD_ROOT/$VARIANT_DIR').replace("#", "") - env['IDLCCOM'] = '$IDLC --include src --base_dir %s --target_arch $TARGET_ARCH --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % (base_dir) + env['IDLCCOM'] = '$IDLC --include src --base_dir %s --target_arch $TARGET_ARCH --header ${TARGETS[1]} --output ${TARGETS[0]} $SOURCES ' % ( + base_dir) env['IDLCSUFFIX'] = '.idl' env['IDL_HAS_INLINE_DEPENDENCIES'] = True + def exists(env): return True diff --git a/site_scons/site_tools/incremental_link.py b/site_scons/site_tools/incremental_link.py index cf74ef9674c..31f16a482da 100644 --- a/site_scons/site_tools/incremental_link.py +++ b/site_scons/site_tools/incremental_link.py @@ -14,10 +14,12 @@ import SCons + def _tag_as_precious(target, source, env): env.Precious(target) return target, source + def generate(env): builders = env['BUILDERS'] for builder in ('Program', 'SharedLibrary', 'LoadableModule'): @@ -27,6 +29,7 @@ def generate(env): _tag_as_precious, ]) + def exists(env): # By default, the windows linker is incremental, so unless # overridden in the environment with /INCREMENTAL:NO, the tool is diff --git a/site_scons/site_tools/jsheader.py b/site_scons/site_tools/jsheader.py index cb418506200..4c2765b7108 100644 --- a/site_scons/site_tools/jsheader.py +++ b/site_scons/site_tools/jsheader.py @@ -1,13 +1,14 @@ from SCons.Script import Action + def jsToH(env, target, source): - return env.Command( - target=target, - source=['#site_scons/site_tools/jstoh.py'] + source, - action=Action('$PYTHON ${SOURCES[0]} $TARGET ${SOURCES[1:]}')) + return env.Command(target=target, source=['#site_scons/site_tools/jstoh.py'] + source, + action=Action('$PYTHON ${SOURCES[0]} $TARGET ${SOURCES[1:]}')) + def generate(env, **kw): env.AddMethod(jsToH, 'JSHeader') + def exists(env): return True diff --git a/site_scons/site_tools/jstoh.py b/site_scons/site_tools/jstoh.py index 26eb6cbbf24..50c0b66cf32 100644 --- a/site_scons/site_tools/jstoh.py +++ b/site_scons/site_tools/jstoh.py @@ -39,7 +39,7 @@ def jsToHeader(target, source): text = '\n'.join(h) - with open(outFile, 'wb') as out: + with open(outFile, 'w') as out: try: out.write(text) finally: @@ -48,7 +48,7 @@ def jsToHeader(target, source): if __name__ == "__main__": if len(sys.argv) < 3: - print "Must specify [target] [source] " + print("Must specify [target] [source] ") sys.exit(1) jsToHeader(sys.argv[1], sys.argv[2:]) diff --git a/site_scons/site_tools/libtool.py b/site_scons/site_tools/libtool.py index 84a646ed066..a098d722fe8 100644 --- a/site_scons/site_tools/libtool.py +++ b/site_scons/site_tools/libtool.py @@ -1,5 +1,6 @@ import SCons + def generate(env): env['AR'] = 'libtool' @@ -13,5 +14,6 @@ def generate(env): env['RANLIBCOM'] = noop_action env['RANLIBCOMSTR'] = 'Skipping ranlib for libtool generated target $TARGET' + def exists(env): return env.detect('libtool') diff --git a/site_scons/site_tools/mongo_benchmark.py b/site_scons/site_tools/mongo_benchmark.py index 2064516aa1f..d9c6a02f91e 100644 --- a/site_scons/site_tools/mongo_benchmark.py +++ b/site_scons/site_tools/mongo_benchmark.py @@ -11,10 +11,10 @@ def register_benchmark(env, test): env.Alias('$BENCHMARK_ALIAS', test) def benchmark_list_builder_action(env, target, source): - ofile = open(str(target[0]), 'wb') + ofile = open(str(target[0]), 'w') try: for s in _benchmarks: - print '\t' + str(s) + print('\t' + str(s)) ofile.write('%s\n' % s) finally: ofile.close() @@ -40,9 +40,10 @@ def build_benchmark(env, target, source, **kwargs): bmEnv.Install("#/build/benchmark/", result[0]) return result + def generate(env): env.Command('$BENCHMARK_LIST', env.Value(_benchmarks), - Action(benchmark_list_builder_action, "Generating $TARGET")) + Action(benchmark_list_builder_action, "Generating $TARGET")) env.AddMethod(register_benchmark, 'RegisterBenchmark') env.AddMethod(build_benchmark, 'Benchmark') env.Alias('$BENCHMARK_ALIAS', '$BENCHMARK_LIST') diff --git a/site_scons/site_tools/mongo_integrationtest.py b/site_scons/site_tools/mongo_integrationtest.py index 0ced90c9493..8e830958960 100644 --- a/site_scons/site_tools/mongo_integrationtest.py +++ b/site_scons/site_tools/mongo_integrationtest.py @@ -12,10 +12,10 @@ def register_integration_test(env, test): env.Alias('$INTEGRATION_TEST_ALIAS', installed_test) def integration_test_list_builder_action(env, target, source): - ofile = open(str(target[0]), 'wb') + ofile = open(str(target[0]), 'w') try: for s in _integration_tests: - print '\t' + str(s) + print('\t' + str(s)) ofile.write('%s\n' % s) finally: ofile.close() @@ -31,9 +31,10 @@ def build_cpp_integration_test(env, target, source, **kwargs): env.RegisterIntegrationTest(result[0]) return result + def generate(env): env.Command('$INTEGRATION_TEST_LIST', env.Value(_integration_tests), - Action(integration_test_list_builder_action, "Generating $TARGET")) + Action(integration_test_list_builder_action, "Generating $TARGET")) env.AddMethod(register_integration_test, 'RegisterIntegrationTest') env.AddMethod(build_cpp_integration_test, 'CppIntegrationTest') env.Alias('$INTEGRATION_TEST_ALIAS', '$INTEGRATION_TEST_LIST') diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py index 2ad0f51bfd0..938be7f9741 100644 --- a/site_scons/site_tools/mongo_unittest.py +++ b/site_scons/site_tools/mongo_unittest.py @@ -11,10 +11,10 @@ def register_unit_test(env, test): env.Alias('$UNITTEST_ALIAS', test) def unit_test_list_builder_action(env, target, source): - ofile = open(str(target[0]), 'wb') + ofile = open(str(target[0]), 'w') try: for s in _unittests: - print '\t' + str(s) + print('\t' + str(s)) ofile.write('%s\n' % s) finally: ofile.close() @@ -33,9 +33,10 @@ def build_cpp_unit_test(env, target, source, **kwargs): env.Install("#/build/unittests/", result[0]) return result + def generate(env): env.Command('$UNITTEST_LIST', env.Value(_unittests), - Action(unit_test_list_builder_action, "Generating $TARGET")) + Action(unit_test_list_builder_action, "Generating $TARGET")) env.AddMethod(register_unit_test, 'RegisterUnitTest') env.AddMethod(build_cpp_unit_test, 'CppUnitTest') env.Alias('$UNITTEST_ALIAS', '$UNITTEST_LIST') diff --git a/site_scons/site_tools/separate_debug.py b/site_scons/site_tools/separate_debug.py index 3edd50518e1..6e78796244b 100644 --- a/site_scons/site_tools/separate_debug.py +++ b/site_scons/site_tools/separate_debug.py @@ -14,6 +14,7 @@ import SCons + def _update_builder(env, builder, bitcode): old_scanner = builder.target_scanner @@ -25,7 +26,8 @@ def _update_builder(env, builder, bitcode): if origin: origin_results = old_scanner(origin, env, path) for origin_result in origin_results: - origin_result_debug_file = getattr(origin_result.attributes, "separate_debug_file", None) + origin_result_debug_file = getattr(origin_result.attributes, "separate_debug_file", + None) if origin_result_debug_file: results.append(origin_result_debug_file) # TODO: Do we need to do the same sort of drag along for bcsymbolmap files? @@ -52,38 +54,26 @@ def _update_builder(env, builder, bitcode): base_action.list.append( SCons.Action.Action( "dsymutil $TARGET --symbol-map=${TARGET}.bcsymbolmap -o ${TARGET}.dSYM", - "Generating debug info for $TARGET into ${TARGET}.dSYM" - ) - ) + "Generating debug info for $TARGET into ${TARGET}.dSYM")) else: base_action.list.append( - SCons.Action.Action( - "dsymutil $TARGET -o ${TARGET}.dSYM", - "Generating debug info for $TARGET into ${TARGET}.dSYM" - ) - ) - base_action.list.append( - SCons.Action.Action( - "strip -Sx ${TARGET}", - "Stripping ${TARGET}" - ) - ) + SCons.Action.Action("dsymutil $TARGET -o ${TARGET}.dSYM", + "Generating debug info for $TARGET into ${TARGET}.dSYM")) + base_action.list.append(SCons.Action.Action("strip -Sx ${TARGET}", "Stripping ${TARGET}")) elif env.TargetOSIs('posix'): base_action.list.extend([ - SCons.Action.Action( - "${OBJCOPY} --only-keep-debug $TARGET ${TARGET}.debug", - "Generating debug info for $TARGET into ${TARGET}.debug" - ), + SCons.Action.Action("${OBJCOPY} --only-keep-debug $TARGET ${TARGET}.debug", + "Generating debug info for $TARGET into ${TARGET}.debug"), SCons.Action.Action( "${OBJCOPY} --strip-debug --add-gnu-debuglink ${TARGET}.debug ${TARGET}", - "Stripping debug info from ${TARGET} and adding .gnu.debuglink to ${TARGET}.debug" - ), + "Stripping debug info from ${TARGET} and adding .gnu.debuglink to ${TARGET}.debug"), ]) else: pass base_emitter = builder.emitter + def new_emitter(target, source, env): bitcode_file = None @@ -111,6 +101,7 @@ def _update_builder(env, builder, bitcode): new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) builder.emitter = new_emitter + def generate(env): if not exists(env): return @@ -132,12 +123,12 @@ def generate(env): "-Wl,-bitcode_symbol_map,${TARGET}.bcsymbolmap", ]) - # TODO: For now, not doing this for programs. Need to update # auto_install_binaries to understand to install the debug symbol # for target X to the same target location as X. for builder in ['SharedLibrary', 'LoadableModule']: _update_builder(env, env['BUILDERS'][builder], bitcode) + def exists(env): return True diff --git a/site_scons/site_tools/split_dwarf.py b/site_scons/site_tools/split_dwarf.py index 95130c9e9a3..c57b9e96822 100644 --- a/site_scons/site_tools/split_dwarf.py +++ b/site_scons/site_tools/split_dwarf.py @@ -26,6 +26,7 @@ _CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] if SCons.Util.case_sensitive_suffixes('.c', '.C'): _CXXSuffixes.append('.C') + def _dwo_emitter(target, source, env): new_targets = [] for t in target: @@ -40,6 +41,7 @@ def _dwo_emitter(target, source, env): targets = target + new_targets return (targets, source) + def generate(env): suffixes = [] if _splitDwarfFlag in env['CCFLAGS']: @@ -52,7 +54,7 @@ def generate(env): for object_builder in SCons.Tool.createObjBuilders(env): emitterdict = object_builder.builder.emitter - for suffix in emitterdict.iterkeys(): + for suffix in emitterdict.keys(): if not suffix in suffixes: continue base = emitterdict[suffix] @@ -61,5 +63,6 @@ def generate(env): _dwo_emitter, ]) + def exists(env): return any(_splitDwarfFlag in env[f] for f in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS']) diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py index 15357874438..b4b5b91ad5e 100644 --- a/site_scons/site_tools/thin_archive.py +++ b/site_scons/site_tools/thin_archive.py @@ -17,6 +17,7 @@ import SCons import re import subprocess + def exists(env): if not 'AR' in env: return False @@ -30,10 +31,9 @@ def exists(env): if not "rc" in env['ARFLAGS']: return False - pipe = SCons.Action._subproc(env, SCons.Util.CLVar(ar) + ['--version'], - stdin = 'devnull', - stderr = 'devnull', - stdout = subprocess.PIPE) + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(ar) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE) if pipe.wait() != 0: return False @@ -41,7 +41,7 @@ def exists(env): for line in pipe.stdout: if found: continue # consume all data - found = re.search(r'^GNU ar|^LLVM', line) + found = re.search(r'^GNU ar|^LLVM', line.decode('utf-8')) return bool(found) @@ -56,6 +56,7 @@ def _add_emitter(builder): new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) builder.emitter = new_emitter + def _add_scanner(builder): old_scanner = builder.target_scanner path_function = old_scanner.path_function @@ -69,13 +70,16 @@ def _add_scanner(builder): new_results.extend(base.children()) return new_results - builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, + path_function=path_function) + def generate(env): if not exists(env): return - env['ARFLAGS'] = SCons.Util.CLVar([arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']]) + env['ARFLAGS'] = SCons.Util.CLVar( + [arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']]) def noop_action(env, target, source): pass diff --git a/site_scons/site_tools/xcode.py b/site_scons/site_tools/xcode.py index 9ec68c35470..5ddebb2e003 100644 --- a/site_scons/site_tools/xcode.py +++ b/site_scons/site_tools/xcode.py @@ -9,4 +9,4 @@ def generate(env): if 'DEVELOPER_DIR' in os.environ: env['ENV']['DEVELOPER_DIR'] = os.environ['DEVELOPER_DIR'] - print "NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands" + print("NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands") |