diff options
| -rw-r--r-- | SConstruct | 24 | ||||
| -rw-r--r-- | site_scons/site_tools/ccache.py | 78 | ||||
| -rw-r--r-- | site_scons/site_tools/icecream.py | 91 | ||||
| -rw-r--r-- | site_scons/site_tools/ninja.py | 2 |
4 files changed, 151 insertions, 44 deletions
diff --git a/SConstruct b/SConstruct index 5c7095c8565..6451263d080 100644 --- a/SConstruct +++ b/SConstruct @@ -110,15 +110,6 @@ add_option('ninja', help='Enable the build.ninja generator tool', ) -add_option('ccache', - choices=['true', 'false'], - default='false', - nargs='?', - const='true', - type='choice', - help='Enable ccache support', -) - add_option('prefix', help='installation prefix (conficts with DESTDIR, PREFIX, and --install-mode=hygienic)', ) @@ -696,18 +687,18 @@ env_vars.Add('ARFLAGS', converter=variable_shlex_converter) env_vars.Add('CCACHE', - help='Path to ccache used for the --ccache option. Defaults to first ccache in PATH.') + help='Tell SCons where the ccache binary is') env_vars.Add( 'CACHE_SIZE', - help='Maximum size of the cache (in gigabytes)', + help='Maximum size of the SCons cache (in gigabytes)', default=32, converter=lambda x:int(x) ) env_vars.Add( 'CACHE_PRUNE_TARGET', - help='Maximum percent in-use in cache after pruning', + help='Maximum percent in-use in SCons cache after pruning', default=66, converter=lambda x:int(x) ) @@ -3740,7 +3731,10 @@ def doConfigure(myenv): env = doConfigure( env ) env["NINJA_SYNTAX"] = "#site_scons/third_party/ninja_syntax.py" -# Now that we are done with configure checks, enable icecream, if available. +# Now that we are done with configure checks, enable ccache and +# icecream, if available. Per the rules declared in the icecream tool, +# load the ccache tool first. +env.Tool('ccache') env.Tool('icecream') if get_option('ninja') == 'true': @@ -3818,10 +3812,6 @@ if get_option('ninja') == 'true': env.NinjaRegisterFunctionHandler("write_uuid_to_file", fakelib_in_ninja) - # Load ccache after icecream since order matters when we're both changing CCCOM - if get_option('ccache') == 'true': - env.Tool('ccache') - # TODO: Later, this should live somewhere more graceful. if get_option('install-mode') == 'hygienic': diff --git a/site_scons/site_tools/ccache.py b/site_scons/site_tools/ccache.py index 077768dbd3d..d5ff49c06cf 100644 --- a/site_scons/site_tools/ccache.py +++ b/site_scons/site_tools/ccache.py @@ -14,25 +14,85 @@ import math import os +import re +import subprocess + import SCons +from pkg_resources import parse_version + +# This is the oldest version of ccache that offers support for -gsplit-dwarf +_ccache_version_min = parse_version('3.2.3') +_ccache_version_found = None def exists(env): - """Always enable""" - ccache_path = env.get('CCACHE', env.WhereIs('ccache')) - return os.path.exists(ccache_path) + """Look for a viable ccache implementation that meets our version requirements.""" + + # If we already generated, we definitely exist + if 'CCACHE_VERSION' in env: + return True + + ccache = env.get('CCACHE', False) + if not ccache: + return False + + ccache = env.WhereIs(ccache) + if not ccache: + return False + + pipe = SCons.Action._subproc(env, + SCons.Util.CLVar(ccache) + ['--version'], stdin='devnull', + stderr='devnull', stdout=subprocess.PIPE) + + if pipe.wait() != 0: + return False + validated = False + for line in pipe.stdout: + line = line.decode('utf-8') + if validated: + continue # consume all data + version_banner = re.search(r'^ccache version', line) + if not version_banner: + continue + ccache_version = re.split('ccache version (.+)', line) + if len(ccache_version) < 2: + continue + global _ccache_version_found + _ccache_version_found = parse_version(ccache_version[1]) + if _ccache_version_found >= _ccache_version_min: + validated = True + + return validated def generate(env): """Add ccache support.""" + + # If we have already generated the tool, don't generate it again. + if 'CCACHE_VERSION' in env: + return + + # If we can't find ccache, or it is too old a version, don't + # generate. + if not exists(env): + return + + # Record our found CCACHE_VERSION. Other tools that need to know + # about ccache (like iecc) should query this variable to determine + # if ccache is active. Looking at the CCACHE variable in the + # environment is not sufficient, since the user may have set it, + # but it doesn't work or is out of date. + env['CCACHE_VERSION'] = _ccache_version_found + # ccache does not support response files so force scons to always # use the full command # # Note: This only works for Python versions >= 3.5 env['MAXLINELENGTH'] = math.inf - env['CCACHE'] = env.get('CCACHE', env.WhereIs('ccache')) - env['CCCOM'] = '$CCACHE ' + env['CCCOM'] - env['CXXCOM'] = '$CCACHE ' + env['CXXCOM'] - env['SHCCCOM'] = '$CCACHE ' + env['SHCCCOM'] - env['SHCXXCOM'] = '$CCACHE ' + env['SHCXXCOM'] - + # Add ccache to the relevant command lines. Wrap the reference to + # ccache in the $( $) pattern so that turning ccache on or off + # doesn't invalidate your build. + env['CCCOM'] = '$( $CCACHE $)' + env['CCCOM'] + env['CXXCOM'] = '$( $CCACHE $)' + env['CXXCOM'] + env['SHCCCOM'] = '$( $CCACHE $)' + env['SHCCCOM'] + env['SHCXXCOM'] = '$( $CCACHE $)' + env['SHCXXCOM'] diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py index c97b4ecc817..dfa7aa32748 100644 --- a/site_scons/site_tools/icecream.py +++ b/site_scons/site_tools/icecream.py @@ -20,13 +20,40 @@ import subprocess from pkg_resources import parse_version -icecream_version_min = '1.1rc2' +_icecream_version_min = parse_version('1.1rc2') +_ccache_nocpp2_version = parse_version('3.4.1') + + +# I'd prefer to use value here, but amazingly, its __str__ returns the +# *initial* value of the Value and not the built value, if +# available. That seems like a bug. In the meantime, make our own very +# sinmple Substition thing. +class _BoundSubstitution: + def __init__(self, env, expression): + self.env = env + self.expression = expression + self.result = None + + def __str__(self): + if self.result is None: + self.result = self.env.subst(self.expression) + return self.result def generate(env): if not exists(env): return + # If we are going to load the ccache tool, but we haven't done so + # yet, then explicitly do it now. We need the ccache tool to be in + # place before we setup icecream because we need to do things a + # little differently if ccache is in play. If you don't use the + # TOOLS variable to configure your tools, you should explicitly + # load the ccache tool before you load icecream. + if 'ccache' in env['TOOLS'] and not 'CCACHE_VERSION' in env: + env.Tool('ccache') + ccache_enabled = ('CCACHE_VERSION' in env) + # Absoluteify, so we can derive ICERUN env['ICECC'] = env.WhereIs('$ICECC') @@ -78,7 +105,7 @@ def generate(env): # doesn't appear when executing icecc_create_env toolchain_env = env.Clone() if toolchain_env.ToolchainIs('clang'): - toolchain = env.Command( + toolchain = toolchain_env.Command( target=icecc_version, source=[ '$ICECC_CREATE_ENV', @@ -138,12 +165,22 @@ def generate(env): if env.ToolchainIs('clang'): env['ENV']['ICECC_CLANG_REMOTE_CPP'] = 1 + + if ccache_enabled and env['CCACHE_VERSION'] >= _ccache_nocpp2_version: + env.AppendUnique( + CCFLAGS=[ + '-frewrite-includes' + ] + ) + env['ENV']['CCACHE_NOCPP2'] = 1 else: env.AppendUnique( CCFLAGS=[ '-fdirectives-only' ] ) + if ccache_enabled: + env['ENV']['CCACHE_NOCPP2'] = 1 if 'ICECC_SCHEDULER' in env: env['ENV']['USE_SCHEDULER'] = env['ICECC_SCHEDULER'] @@ -161,26 +198,45 @@ def generate(env): # Be careful here. If we are running with the ninja tool, many things # may have been monkey patched away. Rely only on `os`, not things # that may try to stat. The abspath appears to be ok. + # + # TODO: Another idea would be to eternally memoize lstat in + # the ninja module, and then we could return to using a call + # to islink on the ICECC_VERSION file. Similarly, it would be + # nice to be able to memoize away this call, but we should + # think carefully about where to store the result of such + # memoization. return os.path.realpath(env['ICECC_VERSION'].abspath) env['ICECC_VERSION_GEN'] = icecc_version_gen - # Build up the string we will stick at the front of the compile - # line. We wrap it in the magic "don't consider this part of the - # build signature" sigils in the hope that enabling and disabling - # icecream won't cause rebuilds. This is unlikely to really work, - # since above we have maybe changed compiler flags (things like - # -fdirectives-only), but we still try to do the right thing. + # Build up the string we will set in the environment to tell icecream + # about the compiler package. icecc_version_string = '${ICECC_VERSION_GEN}' if 'ICECC_VERSION_ARCH' in env: icecc_version_string = '${ICECC_VERSION_ARCH}:' + icecc_version_string - icecc_version_string = '$( ICECC_VERSION={value} $ICECC $)'.format(value=icecc_version_string) - # Amend the various C compilation command strings to start with - # the icecream prelude. - env['CCCOM'] = ' '.join([icecc_version_string, env['CCCOM']]) - env['CXXCOM'] = ' '.join([icecc_version_string, env['CXXCOM']]) - env['SHCCCOM'] = ' '.join([icecc_version_string, env['SHCCCOM']]) - env['SHCXXCOM'] = ' '.join([icecc_version_string, env['SHCXXCOM']]) + # Use our BoundSubstitition class to put ICECC_VERSION into + # env['ENV'] with substitution in play. This lets us defer doing + # the realpath in the generator above until after we have made the + # tarball. + env['ENV']['ICECC_VERSION'] = _BoundSubstitution(env, icecc_version_string) + + # If ccache is in play we actually want the icecc binary in the + # CCACHE_PREFIX environment variable, not on the command line, per + # the ccache documentation on compiler wrappers. Otherwise, just + # put $ICECC on the command line. We wrap it in the magic "don't + # consider this part of the build signature" sigils in the hope + # that enabling and disabling icecream won't cause rebuilds. This + # is unlikely to really work, since above we have maybe changed + # compiler flags (things like -fdirectives-only), but we still try + # to do the right thing. + if ccache_enabled: + env['ENV']['CCACHE_PREFIX'] = _BoundSubstitution(env, '$ICECC') + else: + icecc_string = '$( $ICECC $)' + env['CCCOM'] = ' '.join([icecc_string, env['CCCOM']]) + env['CXXCOM'] = ' '.join([icecc_string, env['CXXCOM']]) + env['SHCCCOM'] = ' '.join([icecc_string, env['SHCCCOM']]) + env['SHCXXCOM'] = ' '.join([icecc_string, env['SHCXXCOM']]) # Make link like jobs flow through icerun so we don't kill the # local machine. @@ -203,6 +259,8 @@ def exists(env): if not icecc: return False icecc = env.WhereIs(icecc) + if not icecc: + return False pipe = SCons.Action._subproc(env, SCons.Util.CLVar(icecc) + ['--version'], stdin='devnull', @@ -223,8 +281,7 @@ def exists(env): if len(icecc_version) < 2: continue icecc_version = parse_version(icecc_version[1]) - needed_version = parse_version(icecream_version_min) - if icecc_version >= needed_version: + if icecc_version >= _icecream_version_min: validated = True return validated diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 8823121de90..52b567e2b7c 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -805,7 +805,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches if env["PLATFORM"] == "win32": command_env += "set '{}={}' && ".format(key, value) else: - command_env += "{}={}; ".format(key, value) + command_env += "export {}={}; ".format(key, value) setattr(node.attributes, "NINJA_ENV_ENV", command_env) |