diff options
| author | William Deegan <bill@baddogconsulting.com> | 2017-08-07 15:45:39 -0700 |
|---|---|---|
| committer | William Deegan <bill@baddogconsulting.com> | 2017-08-07 15:45:39 -0700 |
| commit | fd2d3d079bd9c88545f7389809e011c9c72ee85d (patch) | |
| tree | 74a1221558c955a1fa6beedb16b295e3e300dde3 /src/engine | |
| parent | 657c8b30f165b4f537b75a090ddbab6ada640d1a (diff) | |
| parent | d49e4ae08c5c32bd8466f176f5c19d34ad858700 (diff) | |
| download | scons-fd2d3d079bd9c88545f7389809e011c9c72ee85d.tar.gz | |
Merged in thosrtanner/trt-scons-sig-suppress (pull request #390)
Diffstat (limited to 'src/engine')
120 files changed, 3305 insertions, 3146 deletions
diff --git a/src/engine/.aeignore b/src/engine/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in index f7d5aa77..32f4965d 100644 --- a/src/engine/MANIFEST.in +++ b/src/engine/MANIFEST.in @@ -47,12 +47,12 @@ SCons/Script/Interactive.py SCons/Script/Main.py SCons/Script/SConscript.py SCons/Script/SConsOptions.py -SCons/Sig.py SCons/Subst.py SCons/Taskmaster.py SCons/Tool/__init__.py SCons/Tool/386asm.py SCons/Tool/aixc++.py +SCons/Tool/aixcxx.py SCons/Tool/aixcc.py SCons/Tool/aixf77.py SCons/Tool/aixlink.py @@ -60,12 +60,11 @@ SCons/Tool/applelink.py SCons/Tool/ar.py SCons/Tool/as.py SCons/Tool/bcc32.py -SCons/Tool/BitKeeper.py SCons/Tool/c++.py +SCons/Tool/cxx.py SCons/Tool/cc.py SCons/Tool/cyglink.py SCons/Tool/cvf.py -SCons/Tool/CVS.py SCons/Tool/DCommon.py SCons/Tool/default.py SCons/Tool/dmd.py @@ -81,6 +80,7 @@ SCons/Tool/filesystem.py SCons/Tool/fortran.py SCons/Tool/FortranCommon.py SCons/Tool/g++.py +SCons/Tool/gxx.py SCons/Tool/g77.py SCons/Tool/gas.py SCons/Tool/gcc.py @@ -89,6 +89,7 @@ SCons/Tool/gfortran.py SCons/Tool/gnulink.py SCons/Tool/gs.py SCons/Tool/hpc++.py +SCons/Tool/hpcxx.py SCons/Tool/hpcc.py SCons/Tool/hplink.py SCons/Tool/icc.py @@ -132,22 +133,20 @@ SCons/Tool/packaging/*.py SCons/Tool/pdf.py SCons/Tool/pdflatex.py SCons/Tool/pdftex.py -SCons/Tool/Perforce.py SCons/Tool/PharLapCommon.py SCons/Tool/qt.py -SCons/Tool/RCS.py SCons/Tool/rmic.py SCons/Tool/rpcgen.py SCons/Tool/rpm.py SCons/Tool/rpmutils.py -SCons/Tool/SCCS.py SCons/Tool/sgiar.py SCons/Tool/sgic++.py +SCons/Tool/sgicxx.py SCons/Tool/sgicc.py SCons/Tool/sgilink.py -SCons/Tool/Subversion.py SCons/Tool/sunar.py SCons/Tool/sunc++.py +SCons/Tool/suncxx.py SCons/Tool/suncc.py SCons/Tool/sunf77.py SCons/Tool/sunf90.py @@ -170,7 +169,7 @@ SCons/Variables/PackageVariable.py SCons/Variables/PathVariable.py SCons/Warnings.py SCons/Tool/GettextCommon.py -SCons/Tool/gettext.py +SCons/Tool/gettext_tool.py SCons/Tool/msgfmt.py SCons/Tool/msginit.py SCons/Tool/msgmerge.py diff --git a/src/engine/SCons/.aeignore b/src/engine/SCons/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index de9bf5ca..d6fe30b4 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -60,6 +60,7 @@ this module: get_presig() Fetches the "contents" of a subclass for signature calculation. The varlist is added to this to produce the Action's contents. + TODO(?): Change this to always return ascii/bytes and not unicode (or py3 strings) strfunction() Returns a substituted string representation of the Action. @@ -99,12 +100,13 @@ way for wrapping up the functions. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -import dis import os import pickle import re import sys import subprocess +import itertools +import inspect import SCons.Debug from SCons.Debug import logInstanceCreation @@ -123,37 +125,25 @@ print_actions = 1 execute_actions = 1 print_actions_presub = 0 +# Use pickle protocol 1 when pickling functions for signature +# otherwise python3 and python2 will yield different pickles +# for the same object. +# This is due to default being 1 for python 2.7, and 3 for 3.x +# TODO: We can roll this forward to 2 (if it has value), but not +# before a deprecation cycle as the sconsigns will change +ACTION_SIGNATURE_PICKLE_PROTOCOL = 1 + + def rfile(n): try: return n.rfile() except AttributeError: return n + def default_exitstatfunc(s): return s -try: - SET_LINENO = dis.SET_LINENO - HAVE_ARGUMENT = dis.HAVE_ARGUMENT -except AttributeError: - remove_set_lineno_codes = lambda x: x -else: - def remove_set_lineno_codes(code): - result = [] - n = len(code) - i = 0 - while i < n: - c = code[i] - op = ord(c) - if op >= HAVE_ARGUMENT: - if op != SET_LINENO: - result.append(code[i:i+3]) - i = i+3 - else: - result.append(c) - i = i+1 - return ''.join(result) - strip_quotes = re.compile('^[\'"](.*)[\'"]$') @@ -175,8 +165,8 @@ def _callable_contents(obj): return _code_contents(obj) except AttributeError: - # Test if obj is a function object. - return _function_contents(obj) + # Test if obj is a function object. + return _function_contents(obj) def _object_contents(obj): @@ -204,20 +194,23 @@ def _object_contents(obj): # Test if obj is a function object. return _function_contents(obj) - except AttributeError: - # Should be a pickable Python object. + except AttributeError as ae: + # Should be a pickle-able Python object. try: - return pickle.dumps(obj) - except (pickle.PicklingError, TypeError): + return _object_instance_content(obj) + # pickling an Action instance or object doesn't yield a stable + # content as instance property may be dumped in different orders + # return pickle.dumps(obj, ACTION_SIGNATURE_PICKLE_PROTOCOL) + except (pickle.PicklingError, TypeError, AttributeError) as ex: # This is weird, but it seems that nested classes # are unpickable. The Python docs say it should # always be a PicklingError, but some Python # versions seem to return TypeError. Just do # the best we can. - return str(obj) + return bytearray(repr(obj), 'utf-8') -def _code_contents(code): +def _code_contents(code, docstring=None): """Return the signature contents of a code object. By providing direct access to the code object of the @@ -227,62 +220,169 @@ def _code_contents(code): number indications in the compiled byte code. Boo! So we remove the line number byte codes to prevent recompilations from moving a Python function. + + See: + * https://docs.python.org/2/library/inspect.html + * http://python-reference.readthedocs.io/en/latest/docs/code/index.html + For info on what each co_ variable provides + + The signature is as follows (should be byte/chars): + co_argcount, len(co_varnames), len(co_cellvars), len(co_freevars), + ( comma separated signature for each object in co_consts ), + ( comma separated signature for each object in co_names ), + ( The bytecode with line number bytecodes removed from co_code ) + + co_argcount - Returns the number of positional arguments (including arguments with default values). + co_varnames - Returns a tuple containing the names of the local variables (starting with the argument names). + co_cellvars - Returns a tuple containing the names of local variables that are referenced by nested functions. + co_freevars - Returns a tuple containing the names of free variables. (?) + co_consts - Returns a tuple containing the literals used by the bytecode. + co_names - Returns a tuple containing the names used by the bytecode. + co_code - Returns a string representing the sequence of bytecode instructions. + """ - contents = [] + # contents = [] # The code contents depends on the number of local variables # but not their actual names. - contents.append(b"{}, {}".format(code.co_argcount, len(code.co_varnames))) - contents.append(b", {}, {}".format(len(code.co_cellvars), len(code.co_freevars))) + contents = bytearray("{}, {}".format(code.co_argcount, len(code.co_varnames)), 'utf-8') + + contents.extend(b", ") + contents.extend(bytearray(str(len(code.co_cellvars)), 'utf-8')) + contents.extend(b", ") + contents.extend(bytearray(str(len(code.co_freevars)), 'utf-8')) # The code contents depends on any constants accessed by the # function. Note that we have to call _object_contents on each # constants because the code object of nested functions can # show-up among the constants. - # - # Note that we also always ignore the first entry of co_consts - # which contains the function doc string. We assume that the - # function does not access its doc string. - contents.append(b',(' + b','.join(map(_object_contents,code.co_consts[1:])) + b')') + + z = [_object_contents(cc) for cc in code.co_consts[1:]] + contents.extend(b',(') + contents.extend(bytearray(',', 'utf-8').join(z)) + contents.extend(b')') # The code contents depends on the variable names used to # accessed global variable, as changing the variable name changes # the variable actually accessed and therefore changes the # function result. - contents.append(b',(' + b','.join(map(_object_contents,code.co_names)) + b')') - + z= [bytearray(_object_contents(cc)) for cc in code.co_names] + contents.extend(b',(') + contents.extend(bytearray(',','utf-8').join(z)) + contents.extend(b')') # The code contents depends on its actual code!!! - contents.append(b',(' + remove_set_lineno_codes(code.co_code) + b')') + contents.extend(b',(') + contents.extend(code.co_code) + contents.extend(b')') - return b''.join(contents) + return contents def _function_contents(func): - """Return the signature contents of a function.""" + """Return the signature contents of a function. - contents = [_code_contents(func.__code__)] + The signature is as follows (should be byte/chars): + < _code_contents (see above) from func.__code__ > + ,( comma separated _object_contents for function argument defaults) + ,( comma separated _object_contents for any closure contents ) + + + See also: https://docs.python.org/3/reference/datamodel.html + func.__code__ - The code object representing the compiled function body. + func.__defaults__ - A tuple containing default argument values for those arguments + that have defaults, or None if no arguments have a default value + func.__closure__ - None or a tuple of cells that contain bindings for the function's free variables. + """ + + contents = [_code_contents(func.__code__, func.__doc__)] # The function contents depends on the value of defaults arguments if func.__defaults__: - contents.append(b',(' + b','.join(map(_object_contents,func.__defaults__)) + b')') + + function_defaults_contents = [_object_contents(cc) for cc in func.__defaults__] + + defaults = bytearray(b',(') + defaults.extend(bytearray(b',').join(function_defaults_contents)) + defaults.extend(b')') + + contents.append(defaults) else: contents.append(b',()') # The function contents depends on the closure captured cell values. closure = func.__closure__ or [] - #xxx = [_object_contents(x.cell_contents) for x in closure] try: - xxx = [_object_contents(x.cell_contents) for x in closure] + closure_contents = [_object_contents(x.cell_contents) for x in closure] except AttributeError: - xxx = [] - contents.append(b',(' + ','.join(xxx).encode('ascii') + b')') + closure_contents = [] + + contents.append(b',(') + contents.append(bytearray(b',').join(closure_contents)) + contents.append(b')') - return b''.join(contents) + retval = bytearray(b'').join(contents) + return retval +def _object_instance_content(obj): + """ + Returns consistant content for a action class or an instance thereof + :param obj: Should be either and action class or an instance thereof + :return: bytearray or bytes representing the obj suitable for generating + a signiture from. + """ + retval = bytearray() + + if obj is None: + return b'N.' + + if isinstance(obj, SCons.Util.BaseStringTypes): + return SCons.Util.to_bytes(obj) + + inst_class = obj.__class__ + inst_class_name = bytearray(obj.__class__.__name__,'utf-8') + inst_class_module = bytearray(obj.__class__.__module__,'utf-8') + inst_class_hierarchy = bytearray(repr(inspect.getclasstree([obj.__class__,])),'utf-8') + # print("ICH:%s : %s"%(inst_class_hierarchy, repr(obj))) + + properties = [(p, getattr(obj, p, "None")) for p in dir(obj) if not (p[:2] == '__' or inspect.ismethod(getattr(obj, p)) or inspect.isbuiltin(getattr(obj,p))) ] + properties.sort() + properties_str = ','.join(["%s=%s"%(p[0],p[1]) for p in properties]) + properties_bytes = bytearray(properties_str,'utf-8') + + methods = [p for p in dir(obj) if inspect.ismethod(getattr(obj, p))] + methods.sort() + + method_contents = [] + for m in methods: + # print("Method:%s"%m) + v = _function_contents(getattr(obj, m)) + # print("[%s->]V:%s [%s]"%(m,v,type(v))) + method_contents.append(v) + + retval = bytearray(b'{') + retval.extend(inst_class_name) + retval.extend(b":") + retval.extend(inst_class_module) + retval.extend(b'}[[') + retval.extend(inst_class_hierarchy) + retval.extend(b']]{{') + retval.extend(bytearray(b",").join(method_contents)) + retval.extend(b"}}{{{") + retval.extend(properties_bytes) + retval.extend(b'}}}') + return retval + + # print("class :%s"%inst_class) + # print("class_name :%s"%inst_class_name) + # print("class_module :%s"%inst_class_module) + # print("Class hier :\n%s"%pp.pformat(inst_class_hierarchy)) + # print("Inst Properties:\n%s"%pp.pformat(properties)) + # print("Inst Methods :\n%s"%pp.pformat(methods)) + def _actionAppend(act1, act2): # This function knows how to slap two actions together. # Mainly, it handles ListActions by concatenating into @@ -304,6 +404,7 @@ def _actionAppend(act1, act2): else: return ListAction([ a1, a2 ]) + def _do_create_keywords(args, kw): """This converts any arguments after the action argument into their equivalent keywords and adds them to the kw argument. @@ -331,6 +432,7 @@ def _do_create_keywords(args, kw): raise SCons.Errors.UserError( 'Cannot have both strfunction and cmdstr args to Action()') + def _do_create_action(act, kw): """This is the actual "implementation" for the Action factory method, below. This handles the @@ -383,6 +485,7 @@ def _do_create_action(act, kw): # Else fail silently (???) return None + def _do_create_list_action(act, kw): """A factory for list actions. Convert the input list into Actions and then wrap them in a ListAction.""" @@ -397,6 +500,7 @@ def _do_create_list_action(act, kw): else: return ListAction(acts) + def Action(act, *args, **kw): """A factory for action objects.""" # Really simple: the _do_create_* routines do the heavy lifting. @@ -405,6 +509,7 @@ def Action(act, *args, **kw): return _do_create_list_action(act, kw) return _do_create_action(act, kw) + class ActionBase(object): """Base class for all types of action objects that can be held by other objects (Builders, Executors, etc.) This provides the @@ -422,8 +527,18 @@ class ActionBase(object): return str(self) def get_contents(self, target, source, env): - result = [ self.get_presig(target, source, env) ] - result = [ SCons.Util.to_bytes(r) for r in result ] + result = self.get_presig(target, source, env) + + if not isinstance(result,(bytes, bytearray)): + result = bytearray("",'utf-8').join([ SCons.Util.to_bytes(r) for r in result ]) + else: + # Make a copy and put in bytearray, without this the contents returned by get_presig + # can be changed by the logic below, appending with each call and causing very + # hard to track down issues... + result = bytearray(result) + + # At this point everything should be a bytearray + # This should never happen, as the Action() factory should wrap # the varlist, but just in case an action is created directly, # we duplicate this check here. @@ -431,8 +546,18 @@ class ActionBase(object): if is_String(vl): vl = (vl,) for v in vl: # do the subst this way to ignore $(...$) parts: - result.append(SCons.Util.to_bytes(env.subst_target_source('${'+v+'}', SCons.Subst.SUBST_SIG, target, source))) - return b''.join(result) + if isinstance(result, bytearray): + result.extend(SCons.Util.to_bytes(env.subst_target_source('${'+v+'}', SCons.Subst.SUBST_SIG, target, source))) + else: + raise Exception("WE SHOULD NEVER GET HERE result should be bytearray not:%s"%type(result)) + # result.append(SCons.Util.to_bytes(env.subst_target_source('${'+v+'}', SCons.Subst.SUBST_SIG, target, source))) + + + if isinstance(result, (bytes,bytearray)): + return result + else: + raise Exception("WE SHOULD NEVER GET HERE - #2 result should be bytearray not:%s" % type(result)) + # return b''.join(result) def __add__(self, other): return _actionAppend(self, other) @@ -462,6 +587,7 @@ class ActionBase(object): """ return self.targets + class _ActionAction(ActionBase): """Base class for actions that create output objects.""" def __init__(self, cmdstr=_null, strfunction=_null, varlist=(), @@ -495,14 +621,16 @@ class _ActionAction(ActionBase): SCons.Util.AddMethod(self, batch_key, 'batch_key') def print_cmd_line(self, s, target, source, env): - # In python 3, and in some of our tests, sys.stdout is - # a String io object, and it takes unicode strings only - # In other cases it's a regular Python 2.x file object - # which takes strings (bytes), and if you pass those a - # unicode object they try to decode with 'ascii' codec - # which fails if the cmd line has any hi-bit-set chars. - # This code assumes s is a regular string, but should - # work if it's unicode too. + """ + In python 3, and in some of our tests, sys.stdout is + a String io object, and it takes unicode strings only + In other cases it's a regular Python 2.x file object + which takes strings (bytes), and if you pass those a + unicode object they try to decode with 'ascii' codec + which fails if the cmd line has any hi-bit-set chars. + This code assumes s is a regular string, but should + work if it's unicode too. + """ try: sys.stdout.write(s + u"\n") except UnicodeDecodeError: @@ -601,13 +729,17 @@ def _string_from_cmd_list(cmd_list): cl.append(arg) return ' '.join(cl) -# A fiddlin' little function that has an 'import SCons.Environment' which -# can't be moved to the top level without creating an import loop. Since -# this import creates a local variable named 'SCons', it blocks access to -# the global variable, so we move it here to prevent complaints about local -# variables being used uninitialized. default_ENV = None + + def get_default_ENV(env): + """ + A fiddlin' little function that has an 'import SCons.Environment' which + can't be moved to the top level without creating an import loop. Since + this import creates a local variable named 'SCons', it blocks access to + the global variable, so we move it here to prevent complaints about local + variables being used uninitialized. + """ global default_ENV try: return env['ENV'] @@ -622,12 +754,15 @@ def get_default_ENV(env): default_ENV = SCons.Environment.Environment()['ENV'] return default_ENV -# This function is still in draft mode. We're going to need something like -# it in the long run as more and more places use subprocess, but I'm sure -# it'll have to be tweaked to get the full desired functionality. -# one special arg (so far?), 'error', to tell what to do with exceptions. + def _subproc(scons_env, cmd, error = 'ignore', **kw): - """Do common setup for a subprocess.Popen() call""" + """Do common setup for a subprocess.Popen() call + + This function is still in draft mode. We're going to need something like + it in the long run as more and more places use subprocess, but I'm sure + it'll have to be tweaked to get the full desired functionality. + one special arg (so far?), 'error', to tell what to do with exceptions. + """ # allow std{in,out,err} to be "'devnull'" io = kw.get('stdin') if is_String(io) and io == 'devnull': @@ -645,7 +780,7 @@ def _subproc(scons_env, cmd, error = 'ignore', **kw): # Ensure that the ENV values are all strings: new_env = {} - for key, value in list(ENV.items()): + for key, value in ENV.items(): if is_List(value): # If the value is a list, then we assume it is a path list, # because that's a pretty common list-like value to stick @@ -669,7 +804,7 @@ def _subproc(scons_env, cmd, error = 'ignore', **kw): # return a dummy Popen instance that only returns error class dummyPopen(object): def __init__(self, e): self.exception = e - def communicate(self,input=None): return ('','') + def communicate(self, input=None): return ('', '') def wait(self): return -self.exception.errno stdin = None class f(object): @@ -679,6 +814,7 @@ def _subproc(scons_env, cmd, error = 'ignore', **kw): stdout = stderr = f() return dummyPopen(e) + class CommandAction(_ActionAction): """Class for command-execution actions.""" def __init__(self, cmd, **kw): @@ -695,7 +831,7 @@ class CommandAction(_ActionAction): _ActionAction.__init__(self, **kw) if is_List(cmd): - if list(filter(is_List, cmd)): + if [c for c in cmd if is_List(c)]: raise TypeError("CommandAction should be given only " \ "a single command") self.cmd_list = cmd @@ -772,7 +908,7 @@ class CommandAction(_ActionAction): ENV = get_default_ENV(env) # Ensure that the ENV values are all strings: - for key, value in list(ENV.items()): + for key, value in ENV.items(): if not is_String(value): if is_List(value): # If the value is a list, then we assume it is a @@ -845,6 +981,7 @@ class CommandAction(_ActionAction): res.append(env.fs.File(d)) return res + class CommandGeneratorAction(ActionBase): """Class for command-generator actions.""" def __init__(self, generator, kw): @@ -916,25 +1053,25 @@ class CommandGeneratorAction(ActionBase): return self._generate(None, None, env, 1, executor).get_targets(env, executor) - -# A LazyAction is a kind of hybrid generator and command action for -# strings of the form "$VAR". These strings normally expand to other -# strings (think "$CCCOM" to "$CC -c -o $TARGET $SOURCE"), but we also -# want to be able to replace them with functions in the construction -# environment. Consequently, we want lazy evaluation and creation of -# an Action in the case of the function, but that's overkill in the more -# normal case of expansion to other strings. -# -# So we do this with a subclass that's both a generator *and* -# a command action. The overridden methods all do a quick check -# of the construction variable, and if it's a string we just call -# the corresponding CommandAction method to do the heavy lifting. -# If not, then we call the same-named CommandGeneratorAction method. -# The CommandGeneratorAction methods work by using the overridden -# _generate() method, that is, our own way of handling "generation" of -# an action based on what's in the construction variable. - class LazyAction(CommandGeneratorAction, CommandAction): + """ + A LazyAction is a kind of hybrid generator and command action for + strings of the form "$VAR". These strings normally expand to other + strings (think "$CCCOM" to "$CC -c -o $TARGET $SOURCE"), but we also + want to be able to replace them with functions in the construction + environment. Consequently, we want lazy evaluation and creation of + an Action in the case of the function, but that's overkill in the more + normal case of expansion to other strings. + + So we do this with a subclass that's both a generator *and* + a command action. The overridden methods all do a quick check + of the construction variable, and if it's a string we just call + the corresponding CommandAction method to do the heavy lifting. + If not, then we call the same-named CommandGeneratorAction method. + The CommandGeneratorAction methods work by using the overridden + _generate() method, that is, our own way of handling "generation" of + an action based on what's in the construction variable. + """ def __init__(self, var, kw): if SCons.Debug.track_instances: logInstanceCreation(self, 'Action.LazyAction') @@ -1013,6 +1150,7 @@ class FunctionAction(_ActionAction): c = env.subst(self.cmdstr, SUBST_RAW, target, source) if c: return c + def array(a): def quote(s): try: @@ -1086,7 +1224,6 @@ class FunctionAction(_ActionAction): # more information about this issue. del exc_info - def get_presig(self, target, source, env): """Return the signature contents of this callable action.""" try: @@ -1126,7 +1263,7 @@ class ListAction(ActionBase): Simple concatenation of the signatures of the elements. """ - return b"".join([x.get_contents(target, source, env) for x in self.list]) + return b"".join([bytes(x.get_contents(target, source, env)) for x in self.list]) def __call__(self, target, source, env, exitstatfunc=_null, presub=_null, show=_null, execute=_null, chdir=_null, executor=None): @@ -1153,6 +1290,7 @@ class ListAction(ActionBase): result[var] = True return list(result.keys()) + class ActionCaller(object): """A class for delaying calling an Action function with specific (positional and keyword) arguments until the Action is actually @@ -1171,16 +1309,16 @@ class ActionCaller(object): actfunc = self.parent.actfunc try: # "self.actfunc" is a function. - contents = str(actfunc.__code__.co_code) + contents = actfunc.__code__.co_code except AttributeError: # "self.actfunc" is a callable object. try: - contents = str(actfunc.__call__.__func__.__code__.co_code) + contents = actfunc.__call__.__func__.__code__.co_code except AttributeError: # No __call__() method, so it might be a builtin # or something like that. Do the best we can. - contents = str(actfunc) - contents = remove_set_lineno_codes(contents) + contents = repr(actfunc) + return contents def subst(self, s, target, source, env): @@ -1223,6 +1361,7 @@ class ActionCaller(object): def __str__(self): return self.parent.strfunc(*self.args, **self.kw) + class ActionFactory(object): """A factory class that will wrap up an arbitrary function as an SCons-executable Action object. diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index f0c48ea9..2398c106 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -132,7 +132,7 @@ class Environment(object): self.d['SPAWN'] = scons_env['SPAWN'] self.d['PSPAWN'] = scons_env['PSPAWN'] self.d['ESCAPE'] = scons_env['ESCAPE'] - for k, v in list(kw.items()): + for k, v in kw.items(): self.d[k] = v # Just use the underlying scons_subst*() utility methods. def subst(self, strSubst, raw=0, target=[], source=[], conv=None): @@ -157,12 +157,12 @@ class Environment(object): def Clone(self, **kw): res = Environment() res.d = SCons.Util.semi_deepcopy(self.d) - for k, v in list(kw.items()): + for k, v in kw.items(): res.d[k] = v return res def sig_dict(self): d = {} - for k,v in list(self.items()): d[k] = v + for k,v in self.items(): d[k] = v d['TARGETS'] = ['__t1__', '__t2__', '__t3__', '__t4__', '__t5__', '__t6__'] d['TARGET'] = d['TARGETS'][0] d['SOURCES'] = ['__s1__', '__s2__', '__s3__', '__s4__', '__s5__', '__s6__'] @@ -1428,10 +1428,12 @@ class CommandGeneratorActionTestCase(unittest.TestCase): def LocalFunc(): pass - func_matches = [ - b"0, 0, 0, 0,(),(),(d\000\000S),(),()", - b"0, 0, 0, 0,(),(),(d\x00\x00S),(),()", - ] + # Since the python bytecode has per version differences, we need different expected results per version + func_matches = { + (2,7) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,5) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,6) : bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + } meth_matches = [ b"1, 1, 0, 0,(),(),(d\000\000S),(),()", @@ -1448,11 +1450,11 @@ class CommandGeneratorActionTestCase(unittest.TestCase): a = self.factory(f_global) c = a.get_contents(target=[], source=[], env=env) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected \n"+repr(func_matches[sys.version_info[:2]]) a = self.factory(f_local) c = a.get_contents(target=[], source=[], env=env) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected \n"+repr(func_matches[sys.version_info[:2]]) def f_global(target, source, env, for_signature): return SCons.Action.Action(GlobalFunc, varlist=['XYZ']) @@ -1460,7 +1462,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): def f_local(target, source, env, for_signature): return SCons.Action.Action(LocalFunc, varlist=['XYZ']) - matches_foo = [x + b"foo" for x in func_matches] + matches_foo = func_matches[sys.version_info[:2]] + b'foo' a = self.factory(f_global) c = a.get_contents(target=[], source=[], env=env) @@ -1590,40 +1592,48 @@ class FunctionActionTestCase(unittest.TestCase): def LocalFunc(): pass - func_matches = [ - b"0, 0, 0, 0,(),(),(d\000\000S),(),()", - b"0, 0, 0, 0,(),(),(d\x00\x00S),(),()", - ] + func_matches = { + (2,7) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,5) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,6) : bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + } - meth_matches = [ - b"1, 1, 0, 0,(),(),(d\000\000S),(),()", - b"1, 1, 0, 0,(),(),(d\x00\x00S),(),()", - ] + meth_matches = { + (2,7) : bytearray(b'1, 1, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,5) : bytearray(b'1, 1, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,6) : bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), + } def factory(act, **kw): return SCons.Action.FunctionAction(act, kw) a = factory(GlobalFunc) c = a.get_contents(target=[], source=[], env=Environment()) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) + a = factory(LocalFunc) c = a.get_contents(target=[], source=[], env=Environment()) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) - matches_foo = [x + b"foo" for x in func_matches] + matches_foo = func_matches[sys.version_info[:2]] + b'foo' a = factory(GlobalFunc, varlist=['XYZ']) c = a.get_contents(target=[], source=[], env=Environment()) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) + # assert c in func_matches, repr(c) + c = a.get_contents(target=[], source=[], env=Environment(XYZ='foo')) - assert c in matches_foo, repr(c) + assert c == matches_foo, repr(c) ##TODO: is this set of tests still needed? # Make sure a bare string varlist works a = factory(GlobalFunc, varlist='XYZ') c = a.get_contents(target=[], source=[], env=Environment()) - assert c in func_matches, repr(c) + # assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) + + c = a.get_contents(target=[], source=[], env=Environment(XYZ='foo')) assert c in matches_foo, repr(c) @@ -1640,7 +1650,7 @@ class FunctionActionTestCase(unittest.TestCase): lc = LocalClass() a = factory(lc.LocalMethod) c = a.get_contents(target=[], source=[], env=Environment()) - assert c in meth_matches, repr(c) + assert c == meth_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(meth_matches[sys.version_info[:2]]) def test_strfunction(self): """Test the FunctionAction.strfunction() method @@ -1806,10 +1816,12 @@ class LazyActionTestCase(unittest.TestCase): def LocalFunc(): pass - func_matches = [ - b"0, 0, 0, 0,(),(),(d\000\000S),(),()", - b"0, 0, 0, 0,(),(),(d\x00\x00S),(),()", - ] + + func_matches = { + (2,7) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,5) : bytearray(b'0, 0, 0, 0,(),(),(d\x00\x00S),(),()'), + (3,6) : bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + } meth_matches = [ b"1, 1, 0, 0,(),(),(d\000\000S),(),()", @@ -1824,22 +1836,28 @@ class LazyActionTestCase(unittest.TestCase): env = Environment(FOO = factory(GlobalFunc)) c = a.get_contents(target=[], source=[], env=env) - assert c in func_matches, repr(c) + # assert c in func_matches, "Got\n"+repr(c)+"\nExpected one of \n"+"\n".join([repr(f) for f in func_matches]) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) + + env = Environment(FOO = factory(LocalFunc)) c = a.get_contents(target=[], source=[], env=env) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) + + # matches_foo = [x + b"foo" for x in func_matches] + matches_foo = func_matches[sys.version_info[:2]] + b'foo' - matches_foo = [x + b"foo" for x in func_matches] env = Environment(FOO = factory(GlobalFunc, varlist=['XYZ'])) c = a.get_contents(target=[], source=[], env=env) - assert c in func_matches, repr(c) + assert c == func_matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(func_matches[sys.version_info[:2]]) env['XYZ'] = 'foo' c = a.get_contents(target=[], source=[], env=env) assert c in matches_foo, repr(c) + class ActionCallerTestCase(unittest.TestCase): def test___init__(self): """Test creation of an ActionCaller""" @@ -1856,25 +1874,23 @@ class ActionCallerTestCase(unittest.TestCase): def LocalFunc(): pass - matches = [ - b"d\000\000S", - b"d\x00\x00S" - ] + + matches = { + (2,7) : b'd\x00\x00S', + (3,5) : b'd\x00\x00S', + (3,6) : b'd\x00S\x00', + } + af = SCons.Action.ActionFactory(GlobalFunc, strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c in matches, repr(c) + assert c == matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(matches[sys.version_info[:2]]) af = SCons.Action.ActionFactory(LocalFunc, strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c in matches, repr(c) - - matches = [ - b'd\000\000S', - b"d\x00\x00S" - ] + assert c == matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(matches[sys.version_info[:2]]) class LocalActFunc(object): def __call__(self): @@ -1883,12 +1899,12 @@ class ActionCallerTestCase(unittest.TestCase): af = SCons.Action.ActionFactory(GlobalActFunc(), strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c in matches, repr(c) + assert c == matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(matches[sys.version_info[:2]]) af = SCons.Action.ActionFactory(LocalActFunc(), strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c in matches, repr(c) + assert c == matches[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected one of \n"+repr(matches[sys.version_info[:2]]) matches = [ b"<built-in function str>", @@ -1899,7 +1915,9 @@ class ActionCallerTestCase(unittest.TestCase): ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) assert c == "<built-in function str>" or \ - c == "<type 'str'>", repr(c) + c == "<type 'str'>" or \ + c == "<class 'str'>", repr(c) + # ^^ class str for python3 def test___call__(self): """Test calling an ActionCaller""" @@ -2020,6 +2038,70 @@ class ActionCompareTestCase(unittest.TestCase): assert dog.get_name(env) == 'DOG', dog.get_name(env) +class TestClass(object): + """A test class used by ObjectContentsTestCase.test_object_contents""" + def __init__(self): + self.a = "a" + self.b = "b" + def method(self, arg): + pass + + +class ObjectContentsTestCase(unittest.TestCase): + + def test_function_contents(self): + """Test that Action._function_contents works""" + + def func1(a, b, c): + """A test function""" + return a + + # Since the python bytecode has per version differences, we need different expected results per version + expected = { + (2,7) : bytearray(b'3, 3, 0, 0,(),(),(|\x00\x00S),(),()'), + (3,5) : bytearray(b'3, 3, 0, 0,(),(),(|\x00\x00S),(),()'), + (3,6) : bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()'), + } + + c = SCons.Action._function_contents(func1) + assert c == expected[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected \n"+"\n"+repr(expected[sys.version_info[:2]]) + + + def test_object_contents(self): + """Test that Action._object_contents works""" + + # See definition above + o = TestClass() + c = SCons.Action._object_contents(o) + + # c = SCons.Action._object_instance_content(o) + + # Since the python bytecode has per version differences, we need different expected results per version + expected = { + (2,7): bytearray(b"{TestClass:__main__}[[[(<type \'object\'>, ()), [(<class \'__main__.TestClass\'>, (<type \'object\'>,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01\x00|\x00\x00_\x00\x00d\x02\x00|\x00\x00_\x01\x00d\x00\x00S),(),(),2, 2, 0, 0,(),(),(d\x00\x00S),(),()}}{{{a=a,b=b}}}"), + (3,5): bytearray(b"{TestClass:__main__}[[[(<class \'object\'>, ()), [(<class \'__main__.TestClass\'>, (<class \'object\'>,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01\x00|\x00\x00_\x00\x00d\x02\x00|\x00\x00_\x01\x00d\x00\x00S),(),(),2, 2, 0, 0,(),(),(d\x00\x00S),(),()}}{{{a=a,b=b}}}"), + (3,6): bytearray(b"{TestClass:__main__}[[[(<class \'object\'>, ()), [(<class \'__main__.TestClass\'>, (<class \'object\'>,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + } + + assert c == expected[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected \n"+"\n"+repr(expected[sys.version_info[:2]]) + + def test_code_contents(self): + """Test that Action._code_contents works""" + + code = compile("print('Hello, World!')", '<string>', 'exec') + c = SCons.Action._code_contents(code) + + # Since the python bytecode has per version differences, we need different expected results per version + expected = { + (2,7) : bytearray(b'0, 0, 0, 0,(N.),(),(d\x00\x00GHd\x01\x00S)'), + (3,5) : bytearray(b'0, 0, 0, 0,(N.),(print),(e\x00\x00d\x00\x00\x83\x01\x00\x01d\x01\x00S)'), + (3,6) : bytearray(b'0, 0, 0, 0,(N.),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), + } + + assert c == expected[sys.version_info[:2]], "Got\n"+repr(c)+"\nExpected \n"+"\n"+expected[sys.version_info[:2]] + + + if __name__ == "__main__": suite = unittest.TestSuite() tclasses = [ _ActionActionTestCase, @@ -2031,13 +2113,17 @@ if __name__ == "__main__": LazyActionTestCase, ActionCallerTestCase, ActionFactoryTestCase, - ActionCompareTestCase ] + ActionCompareTestCase, + ObjectContentsTestCase ] for tclass in tclasses: names = unittest.getTestCaseNames(tclass, 'test_') suite.addTests(list(map(tclass, names))) TestUnit.run(suite) + # Swap this for above to debug otherwise you can't run individual tests as TestUnit is swallowing arguments + # unittest.main() + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index c7bce3a3..21f1b4ae 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -299,7 +299,7 @@ def _node_errors(builder, env, tlist, slist): msg = "Two different environments were specified for target %s,\n\tbut they appear to have the same action: %s" % (t, action.genstring(tlist, slist, t.env)) SCons.Warnings.warn(SCons.Warnings.DuplicateEnvironmentWarning, msg) else: - msg = "Two environments with different actions were specified for the same target: %s\n(action 1: %s)\n(action 2: %s)" % (t,t_contents,contents) + msg = "Two environments with different actions were specified for the same target: %s\n(action 1: %s)\n(action 2: %s)" % (t,t_contents.decode('utf-8'),contents.decode('utf-8')) raise UserError(msg) if builder.multi: if t.builder != builder: @@ -610,6 +610,8 @@ class BuilderBase(object): else: ekw = self.executor_kw.copy() ekw['chdir'] = chdir + if 'chdir' in ekw and SCons.Util.is_String(ekw['chdir']): + ekw['chdir'] = env.subst(ekw['chdir']) if kw: if 'srcdir' in kw: def prependDirIfRelative(f, srcdir=kw['srcdir']): diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py index ca35abc0..1e544a13 100644 --- a/src/engine/SCons/BuilderTests.py +++ b/src/engine/SCons/BuilderTests.py @@ -79,7 +79,7 @@ class Environment(object): self.d['SHELL'] = scons_env['SHELL'] self.d['SPAWN'] = scons_env['SPAWN'] self.d['ESCAPE'] = scons_env['ESCAPE'] - for k, v in list(kw.items()): + for k, v in kw.items(): self.d[k] = v global env_arg2nodes_called env_arg2nodes_called = None @@ -140,7 +140,7 @@ class Environment(object): return list(self.d.items()) def sig_dict(self): d = {} - for k,v in list(self.items()): d[k] = v + for k,v in self.items(): d[k] = v d['TARGETS'] = ['__t1__', '__t2__', '__t3__', '__t4__', '__t5__', '__t6__'] d['TARGET'] = d['TARGETS'][0] d['SOURCES'] = ['__s1__', '__s2__', '__s3__', '__s4__', '__s5__', '__s6__'] diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py index 8a52928d..494db98f 100644 --- a/src/engine/SCons/CacheDirTests.py +++ b/src/engine/SCons/CacheDirTests.py @@ -44,7 +44,7 @@ class Action(object): def genstring(self, target, source, env): return str(self) def get_contents(self, target, source, env): - return '' + return bytearray('','utf-8') class Builder(object): def __init__(self, environment, action): diff --git a/src/engine/SCons/Debug.py b/src/engine/SCons/Debug.py index 6ac5f27d..706b4c45 100644 --- a/src/engine/SCons/Debug.py +++ b/src/engine/SCons/Debug.py @@ -89,7 +89,7 @@ def dumpLoggedInstances(classes, file=sys.stdout): obj = ref() if obj is not None: file.write(' %s:\n' % obj) - for key, value in list(obj.__dict__.items()): + for key, value in obj.__dict__.items(): file.write(' %20s : %s\n' % (key, value)) @@ -97,7 +97,8 @@ def dumpLoggedInstances(classes, file=sys.stdout): if sys.platform[:5] == "linux": # Linux doesn't actually support memory usage stats from getrusage(). def memory(): - mstr = open('/proc/self/stat').read() + with open('/proc/self/stat') as f: + mstr = f.read() mstr = mstr.split()[22] return int(mstr) elif sys.platform[:6] == 'darwin': @@ -163,7 +164,7 @@ def caller_trace(back=0): # print a single caller and its callers, if any def _dump_one_caller(key, file, level=0): leader = ' '*level - for v,c in sorted([(-v,c) for c,v in list(caller_dicts[key].items())]): + for v,c in sorted([(-v,c) for c,v in caller_dicts[key].items()]): file.write("%s %6d %s:%d(%s)\n" % ((leader,-v) + func_shorten(c[-3:]))) if c in caller_dicts: _dump_one_caller(c, file, level+1) @@ -233,6 +234,7 @@ def Trace(msg, file=None, mode='w', tstamp=None): PreviousTime = now fp.write(msg) fp.flush() + fp.close() # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index f0959829..69d5c943 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -264,7 +264,10 @@ def copy_func(dest, src, symlinks=True): shutil.copy2(src, dest) return 0 else: - return shutil.copytree(src, dest, symlinks) + shutil.copytree(src, dest, symlinks) + # copytree returns None in python2 and destination string in python3 + # A error is raised in both cases, so we can just return 0 for success + return 0 Copy = ActionFactory( copy_func, @@ -459,7 +462,7 @@ def processDefines(defs): else: l.append(str(d[0])) elif SCons.Util.is_Dict(d): - for macro,value in list(d.items()): + for macro,value in d.items(): if value is not None: l.append(str(macro) + '=' + str(value)) else: @@ -485,6 +488,7 @@ def processDefines(defs): l = [str(defs)] return l + def _defines(prefix, defs, suffix, env, c=_concat_ixes): """A wrapper around _concat_ixes that turns a list or string into a list of C preprocessor command-line definitions. @@ -492,6 +496,7 @@ def _defines(prefix, defs, suffix, env, c=_concat_ixes): return c(prefix, env.subst_path(processDefines(defs)), suffix, env) + class NullCmdGenerator(object): """This is a callable class that can be used in place of other command generators if you don't want them to do anything. @@ -510,6 +515,7 @@ class NullCmdGenerator(object): def __call__(self, target, source, env, for_signature=None): return self.cmd + class Variable_Method_Caller(object): """A class for finding a construction variable on the stack and calling one of its methods. diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index ed8ef786..480a1d68 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -152,7 +152,7 @@ def _set_BUILDERS(env, key, value): except KeyError: bd = BuilderDict(kwbd, env) env._dict[key] = bd - for k, v in list(value.items()): + for k, v in value.items(): if not SCons.Builder.is_a_Builder(v): raise SCons.Errors.UserError('%s is not a Builder.' % repr(v)) bd.update(value) @@ -324,7 +324,7 @@ class BuilderDict(UserDict): delattr(self.env, item) def update(self, dict): - for i, v in list(dict.items()): + for i, v in dict.items(): self.__setitem__(i, v) @@ -515,7 +515,7 @@ class SubstitutionEnvironment(object): def subst_kw(self, kw, raw=0, target=None, source=None): nkw = {} - for k, v in list(kw.items()): + for k, v in kw.items(): k = self.subst(k, raw, target, source) if SCons.Util.is_String(v): v = self.subst(v, raw, target, source) @@ -627,7 +627,7 @@ class SubstitutionEnvironment(object): if not o: return self overrides = {} merges = None - for key, value in list(o.items()): + for key, value in o.items(): if key == 'parse_flags': merges = value else: @@ -815,7 +815,7 @@ class SubstitutionEnvironment(object): if not unique: self.Append(**args) return self - for key, value in list(args.items()): + for key, value in args.items(): if not value: continue try: @@ -984,7 +984,7 @@ class Base(SubstitutionEnvironment): # Now restore the passed-in and customized variables # to the environment, since the values the user set explicitly # should override any values set by the tools. - for key, val in list(save.items()): + for key, val in save.items(): self._dict[key] = val # Finally, apply any flags to be merged in @@ -1131,7 +1131,7 @@ class Base(SubstitutionEnvironment): in an Environment. """ kw = copy_non_reserved_keywords(kw) - for key, val in list(kw.items()): + for key, val in kw.items(): # It would be easier on the eyes to write this using # "continue" statements whenever we finish processing an item, # but Python 1.5.2 apparently doesn't let you use "continue" @@ -1201,7 +1201,7 @@ class Base(SubstitutionEnvironment): update_dict(val) except (AttributeError, TypeError, ValueError): if SCons.Util.is_Dict(val): - for k, v in list(val.items()): + for k, v in val.items(): orig[k] = v else: orig[val] = None @@ -1247,7 +1247,7 @@ class Base(SubstitutionEnvironment): values move to end. """ kw = copy_non_reserved_keywords(kw) - for key, val in list(kw.items()): + for key, val in kw.items(): if SCons.Util.is_List(val): val = _delete_duplicates(val, delete_existing) if key not in self._dict or self._dict[key] in ('', None): @@ -1363,7 +1363,7 @@ class Base(SubstitutionEnvironment): val = [val] elif SCons.Util.is_Dict(val): tmp = [] - for i,j in list(val.items()): + for i,j in val.items(): if j is not None: tmp.append((i,j)) else: @@ -1405,7 +1405,7 @@ class Base(SubstitutionEnvironment): # so the tools can use the new variables kw = copy_non_reserved_keywords(kw) new = {} - for key, value in list(kw.items()): + for key, value in kw.items(): new[key] = SCons.Subst.scons_subst_once(value, self, key) clone.Replace(**new) @@ -1605,7 +1605,7 @@ class Base(SubstitutionEnvironment): in an Environment. """ kw = copy_non_reserved_keywords(kw) - for key, val in list(kw.items()): + for key, val in kw.items(): # It would be easier on the eyes to write this using # "continue" statements whenever we finish processing an item, # but Python 1.5.2 apparently doesn't let you use "continue" @@ -1659,7 +1659,7 @@ class Base(SubstitutionEnvironment): update_dict(val) except (AttributeError, TypeError, ValueError): if SCons.Util.is_Dict(val): - for k, v in list(val.items()): + for k, v in val.items(): orig[k] = v else: orig[val] = None @@ -1696,7 +1696,7 @@ class Base(SubstitutionEnvironment): values move to front. """ kw = copy_non_reserved_keywords(kw) - for key, val in list(kw.items()): + for key, val in kw.items(): if SCons.Util.is_List(val): val = _delete_duplicates(val, not delete_existing) if key not in self._dict or self._dict[key] in ('', None): @@ -1983,6 +1983,15 @@ class Base(SubstitutionEnvironment): return result return self.fs.Dir(s, *args, **kw) + def PyPackageDir(self, modulename): + s = self.subst(modulename) + if SCons.Util.is_Sequence(s): + result=[] + for e in s: + result.append(self.fs.PyPackageDir(e)) + return result + return self.fs.PyPackageDir(s) + def NoClean(self, *targets): """Tags a target so that it will not be cleaned by -c""" tlist = [] @@ -2368,19 +2377,21 @@ class OverrideEnvironment(Base): Environment = Base -# An entry point for returning a proxy subclass instance that overrides -# the subst*() methods so they don't actually perform construction -# variable substitution. This is specifically intended to be the shim -# layer in between global function calls (which don't want construction -# variable substitution) and the DefaultEnvironment() (which would -# substitute variables if left to its own devices).""" -# -# We have to wrap this in a function that allows us to delay definition of -# the class until it's necessary, so that when it subclasses Environment -# it will pick up whatever Environment subclass the wrapper interface -# might have assigned to SCons.Environment.Environment. def NoSubstitutionProxy(subject): + """ + An entry point for returning a proxy subclass instance that overrides + the subst*() methods so they don't actually perform construction + variable substitution. This is specifically intended to be the shim + layer in between global function calls (which don't want construction + variable substitution) and the DefaultEnvironment() (which would + substitute variables if left to its own devices). + + We have to wrap this in a function that allows us to delay definition of + the class until it's necessary, so that when it subclasses Environment + it will pick up whatever Environment subclass the wrapper interface + might have assigned to SCons.Environment.Environment. + """ class _NoSubstitutionProxy(Environment): def __init__(self, subject): self.__dict__['__subject'] = subject diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml index 65d71ffe..ccee68d7 100644 --- a/src/engine/SCons/Environment.xml +++ b/src/engine/SCons/Environment.xml @@ -327,7 +327,7 @@ Examples: # which the method will be called; the Python # convention is to call it 'self'. def my_method(self, arg): - print "my_method() got", arg + print("my_method() got", arg) # Use the global AddMethod() function to add a method # to the Environment class. This @@ -2504,6 +2504,29 @@ env.PrependUnique(CCFLAGS = '-g', FOO = ['foo.yyy']) </summary> </scons_function> +<scons_function name="PyPackageDir"> +<arguments> +(modulename) +</arguments> +<summary> +<para> +This returns a Directory Node similar to Dir. +The python module / package is looked up and if located +the directory is returned for the location. +<varname>modulename</varname> +Is a named python package / module to +lookup the directory for it's location. +</para> +<para> +If +<varname>modulename</varname> +is a list, SCons returns a list of Dir nodes. +Construction variables are expanded in +<varname>modulename</varname>. +</para> +</summary> +</scons_function> + <scons_function name="Replace"> <arguments signature="env"> (key=val, [...]) diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index e3259d42..229858fc 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -164,7 +164,7 @@ class TestEnvironmentFixture(object): default_keys = { 'CC' : 'cc', 'CCFLAGS' : '-DNDEBUG', 'ENV' : { 'TMP' : '/tmp' } } - for key, value in list(default_keys.items()): + for key, value in default_keys.items(): if key not in kw: kw[key] = value if 'BUILDERS' not in kw: @@ -247,7 +247,9 @@ class SubstitutionTestCase(unittest.TestCase): """ env = SubstitutionEnvironment(XXX = 'x', YYY = 'y') items = list(env.items()) - assert items == [('XXX','x'), ('YYY','y')], items + assert len(items) == 2 and ('XXX','x') in items and ('YYY','y') in items, items + # Was. This fails under py3 as order changes + # assert items == [('XXX','x'), ('YYY','y')], items def test_arg2nodes(self): """Test the arg2nodes method @@ -1459,8 +1461,6 @@ def exists(env): assert env['SOURCE'] == 's', env['SOURCE'] assert env['SOURCES'] == 'sss', env['SOURCES'] - - def test_Append(self): """Test appending to construction variables in an Environment """ diff --git a/src/engine/SCons/EnvironmentValues.py b/src/engine/SCons/EnvironmentValues.py new file mode 100644 index 00000000..e2efccbe --- /dev/null +++ b/src/engine/SCons/EnvironmentValues.py @@ -0,0 +1,97 @@ +import re + +_is_valid_var = re.compile(r'[_a-zA-Z]\w*$') + +_rm = re.compile(r'\$[()]') +_remove = re.compile(r'\$\([^\$]*(\$[^\)][^\$]*)*\$\)') + +# Regular expressions for splitting strings and handling substitutions, +# for use by the scons_subst() and scons_subst_list() functions: +# +# The first expression compiled matches all of the $-introduced tokens +# that we need to process in some way, and is used for substitutions. +# The expressions it matches are: +# +# "$$" +# "$(" +# "$)" +# "$variable" [must begin with alphabetic or underscore] +# "${any stuff}" +# +# The second expression compiled is used for splitting strings into tokens +# to be processed, and it matches all of the tokens listed above, plus +# the following that affect how arguments do or don't get joined together: +# +# " " [white space] +# "non-white-space" [without any dollar signs] +# "$" [single dollar sign] +# +_dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}' +_dollar_exps = re.compile(r'(%s)' % _dollar_exps_str) +_separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str) + +# This regular expression is used to replace strings of multiple white +# space characters in the string result from the scons_subst() function. +_space_sep = re.compile(r'[\t ]+(?![^{]*})') + +class ValueTypes(object): + """ + Enum to store what type of value the variable holds. + """ + UNKNOWN = 0 + STRING = 1 + CALLABLE = 2 + VARIABLE = 3 + + +class EnvironmentValue(object): + """ + Hold a single value. We're going to cache parsed version of the file + We're going to keep track of variables which feed into this values evaluation + """ + def __init__(self, value): + self.value = value + self.var_type = ValueTypes.UNKNOWN + + if callable(self.value): + self.var_type = ValueTypes.CALLABLE + else: + self.parse_value() + + + def parse_value(self): + """ + Scan the string and break into component values + """ + + try: + if '$' not in self.value: + self._parsed = self.value + self.var_type = ValueTypes.STRING + else: + # Now we need to parse the specified string + result = _dollar_exps.sub(sub_match, args) + print(result) + pass + except TypeError: + # likely callable? either way we don't parse + self._parsed = self.value + + def parse_trial(self): + """ + Try alternate parsing methods. + :return: + """ + parts = [] + for c in self.value: + + + +class EnvironmentValues(object): + """ + A class to hold all the environment variables + """ + def __init__(self, **kw): + self._dict = {} + for k in kw: + self._dict[k] = EnvironmentValue(kw[k]) diff --git a/src/engine/SCons/EnvironmentValuesTest.py b/src/engine/SCons/EnvironmentValuesTest.py new file mode 100644 index 00000000..58ee9cf4 --- /dev/null +++ b/src/engine/SCons/EnvironmentValuesTest.py @@ -0,0 +1,16 @@ +import unittest + +from SCons.EnvironmentValues import EnvironmentValues + +class MyTestCase(unittest.TestCase): + def test_simple_environmentValues(self): + """Test comparing SubstitutionEnvironments + """ + + env1 = EnvironmentValues(XXX='x') + env2 = EnvironmentValues(XXX='x',XX="$X", X1="${X}", X2="$($X$)") + + + +if __name__ == '__main__': + unittest.main() diff --git a/src/engine/SCons/Errors.py b/src/engine/SCons/Errors.py index 3cc9c6dd..1df8cb91 100644 --- a/src/engine/SCons/Errors.py +++ b/src/engine/SCons/Errors.py @@ -30,10 +30,12 @@ and user errors in SCons. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import shutil import SCons.Util + class BuildError(Exception): - """ Errors occuring while building. + """ Errors occurring while building. BuildError have the following attributes: @@ -86,12 +88,15 @@ class BuildError(Exception): is not due to the an action failure) """ - def __init__(self, + def __init__(self, node=None, errstr="Unknown error", status=2, exitstatus=2, filename=None, executor=None, action=None, command=None, exc_info=(None, None, None)): - - self.errstr = errstr + + # py3: errstr should be string and not bytes. + # import pdb; pdb.set_trace() + + self.errstr = SCons.Util.to_str(errstr) self.status = status self.exitstatus = exitstatus self.filename = filename @@ -102,7 +107,7 @@ class BuildError(Exception): self.action = action self.command = command - Exception.__init__(self, node, errstr, status, exitstatus, filename, + Exception.__init__(self, node, errstr, status, exitstatus, filename, executor, action, command, exc_info) def __str__(self): @@ -141,6 +146,9 @@ def convert_to_BuildError(status, exc_info=None): The buildError.status we set here will normally be used as the exit status of the "scons" process. """ + + # import pdb; pdb.set_trace() + if not exc_info and isinstance(status, Exception): exc_info = (status.__class__, status, None) @@ -162,6 +170,21 @@ def convert_to_BuildError(status, exc_info=None): status=2, exitstatus=2, exc_info=exc_info) + elif isinstance(status, shutil.SameFileError): + # PY3 has a exception for when copying file to itself + # It's object provides info differently than below + try: + filename = status.filename + except AttributeError: + filename = None + + buildError = BuildError( + errstr=status.args[0], + status=status.errno, + exitstatus=2, + filename=filename, + exc_info=exc_info) + elif isinstance(status, (EnvironmentError, OSError, IOError)): # If an IOError/OSError happens, raise a BuildError. # Report the name of the file or directory that caused the @@ -197,7 +220,7 @@ def convert_to_BuildError(status, exc_info=None): exitstatus=2) #import sys - #sys.stderr.write("convert_to_BuildError: status %s => (errstr %s, status %s)"%(status,buildError.errstr, buildError.status)) + #sys.stderr.write("convert_to_BuildError: status %s => (errstr %s, status %s)\n"%(status,buildError.errstr, buildError.status)) return buildError # Local Variables: diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index dd5088d6..bce1549c 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -26,6 +26,7 @@ Nodes. # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +from __future__ import print_function __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" @@ -455,10 +456,16 @@ class Executor(object, with_metaclass(NoSlotsPyPy)): except KeyError: pass env = self.get_build_env() - result = b"".join([action.get_contents(self.get_all_targets(), - self.get_all_sources(), - env) - for action in self.get_action_list()]) + + action_list = self.get_action_list() + all_targets = self.get_all_targets() + all_sources = self.get_all_sources() + + result = bytearray("",'utf-8').join([action.get_contents(all_targets, + all_sources, + env) + for action in action_list]) + self._memo['get_contents'] = result return result diff --git a/src/engine/SCons/Node/.aeignore b/src/engine/SCons/Node/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/Node/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Node/Alias.py b/src/engine/SCons/Node/Alias.py index f229a9fa..a035816c 100644 --- a/src/engine/SCons/Node/Alias.py +++ b/src/engine/SCons/Node/Alias.py @@ -89,7 +89,7 @@ class AliasNodeInfo(SCons.Node.NodeInfoBase): """ # TODO check or discard version del state['_version_id'] - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('__weakref__',): setattr(self, key, value) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 0de39893..8c1161d0 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -220,7 +220,12 @@ needs_normpath_match = needs_normpath_check.match # there should be *no* changes to the external file system(s)... # -if hasattr(os, 'link'): +# For Now disable hard & softlinks for win32 +# PY3 supports them, but the rest of SCons is not ready for this +# in some cases user permissions may be required. +# TODO: See if theres a reasonable way to enable using links on win32/64 + +if hasattr(os, 'link') and sys.platform != 'win32': def _hardlink_func(fs, src, dst): # If the source is a symlink, we can't just hard-link to it # because a relative symlink may point somewhere completely @@ -236,7 +241,7 @@ if hasattr(os, 'link'): else: _hardlink_func = None -if hasattr(os, 'symlink'): +if hasattr(os, 'symlink') and sys.platform != 'win32': def _softlink_func(fs, src, dst): fs.symlink(src, dst) else: @@ -351,33 +356,6 @@ class _Null(object): _null = _Null() -DefaultSCCSBuilder = None -DefaultRCSBuilder = None - -def get_DefaultSCCSBuilder(): - global DefaultSCCSBuilder - if DefaultSCCSBuilder is None: - import SCons.Builder - # "env" will get filled in by Executor.get_build_env() - # calling SCons.Defaults.DefaultEnvironment() when necessary. - act = SCons.Action.Action('$SCCSCOM', '$SCCSCOMSTR') - DefaultSCCSBuilder = SCons.Builder.Builder(action = act, - env = None, - name = "DefaultSCCSBuilder") - return DefaultSCCSBuilder - -def get_DefaultRCSBuilder(): - global DefaultRCSBuilder - if DefaultRCSBuilder is None: - import SCons.Builder - # "env" will get filled in by Executor.get_build_env() - # calling SCons.Defaults.DefaultEnvironment() when necessary. - act = SCons.Action.Action('$RCS_COCOM', '$RCS_COCOMSTR') - DefaultRCSBuilder = SCons.Builder.Builder(action = act, - env = None, - name = "DefaultRCSBuilder") - return DefaultRCSBuilder - # Cygwin's os.path.normcase pretends it's on a case-sensitive filesystem. _is_cygwin = sys.platform == "cygwin" if os.path.normcase("TeSt") == os.path.normpath("TeSt") and not _is_cygwin: @@ -422,46 +400,12 @@ def do_diskcheck_match(node, predicate, errorfmt): def ignore_diskcheck_match(node, predicate, errorfmt): pass -def do_diskcheck_rcs(node, name): - try: - rcs_dir = node.rcs_dir - except AttributeError: - if node.entry_exists_on_disk('RCS'): - rcs_dir = node.Dir('RCS') - else: - rcs_dir = None - node.rcs_dir = rcs_dir - if rcs_dir: - return rcs_dir.entry_exists_on_disk(name+',v') - return None - -def ignore_diskcheck_rcs(node, name): - return None - -def do_diskcheck_sccs(node, name): - try: - sccs_dir = node.sccs_dir - except AttributeError: - if node.entry_exists_on_disk('SCCS'): - sccs_dir = node.Dir('SCCS') - else: - sccs_dir = None - node.sccs_dir = sccs_dir - if sccs_dir: - return sccs_dir.entry_exists_on_disk('s.'+name) - return None -def ignore_diskcheck_sccs(node, name): - return None diskcheck_match = DiskChecker('match', do_diskcheck_match, ignore_diskcheck_match) -diskcheck_rcs = DiskChecker('rcs', do_diskcheck_rcs, ignore_diskcheck_rcs) -diskcheck_sccs = DiskChecker('sccs', do_diskcheck_sccs, ignore_diskcheck_sccs) diskcheckers = [ diskcheck_match, - diskcheck_rcs, - diskcheck_sccs, ] def set_diskcheck(list): @@ -477,6 +421,11 @@ class EntryProxy(SCons.Util.Proxy): __str__ = SCons.Util.Delegate('__str__') + # In PY3 if a class defines __eq__, then it must explicitly provide + # __hash__. Since SCons.Util.Proxy provides __eq__ we need the following + # see: https://docs.python.org/3.1/reference/datamodel.html#object.__hash__ + __hash__ = SCons.Util.Delegate('__hash__') + def __get_abspath(self): entry = self.get() return SCons.Subst.SpecialAttrWrapper(entry.get_abspath(), @@ -574,6 +523,7 @@ class EntryProxy(SCons.Util.Proxy): else: return attr_function(self) + class Base(SCons.Node.Node): """A generic class for file system entries. This class is for when we don't know yet whether the entry being looked up is a file @@ -690,6 +640,10 @@ class Base(SCons.Node.Node): return self._save_str() return self._get_str() + def __lt__(self, other): + """ less than operator used by sorting on py3""" + return str(self) < str(other) + @SCons.Memoize.CountMethodCall def _save_str(self): try: @@ -972,8 +926,6 @@ class Entry(Base): 'root', 'dirname', 'on_disk_entries', - 'sccs_dir', - 'rcs_dir', 'released_target_info', 'contentsig'] @@ -1438,6 +1390,35 @@ class FS(LocalFS): if not isinstance(d, SCons.Node.Node): d = self.Dir(d) self.Top.addRepository(d) + + def PyPackageDir(self, modulename): + """Locate the directory of a given python module name + + For example scons might resolve to + Windows: C:\Python27\Lib\site-packages\scons-2.5.1 + Linux: /usr/lib/scons + + This can be useful when we want to determine a toolpath based on a python module name""" + + dirpath = '' + if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] in (0,1,2,3,4)): + # Python2 Code + import imp + splitname = modulename.split('.') + srchpths = sys.path + for item in splitname: + file, path, desc = imp.find_module(item, srchpths) + if file is not None: + path = os.path.dirname(path) + srchpths = [path] + dirpath = path + else: + # Python3 Code + import importlib.util + modspec = importlib.util.find_spec(modulename) + dirpath = os.path.dirname(modspec.origin) + return self._lookup(dirpath, None, Dir, True) + def variant_dir_target_climb(self, orig, dir, tail): """Create targets in corresponding variant directories @@ -1519,8 +1500,6 @@ class Dir(Base): 'root', 'dirname', 'on_disk_entries', - 'sccs_dir', - 'rcs_dir', 'released_target_info', 'contentsig'] @@ -2086,9 +2065,7 @@ class Dir(Base): return node def file_on_disk(self, name): - if self.entry_exists_on_disk(name) or \ - diskcheck_rcs(self, name) or \ - diskcheck_sccs(self, name): + if self.entry_exists_on_disk(name): try: return self.File(name) except TypeError: pass node = self.srcdir_duplicate(name) @@ -2203,7 +2180,7 @@ class Dir(Base): # We use the .name attribute from the Node because the keys of # the dir.entries dictionary are normalized (that is, all upper # case) on case-insensitive systems like Windows. - node_names = [ v.name for k, v in list(dir.entries.items()) + node_names = [ v.name for k, v in dir.entries.items() if k not in ('.', '..') ] names.extend(node_names) if not strings: @@ -2434,6 +2411,7 @@ class RootDir(Dir): def src_builder(self): return _null + class FileNodeInfo(SCons.Node.NodeInfoBase): __slots__ = ('csig', 'timestamp', 'size') current_version_id = 2 @@ -2481,10 +2459,11 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): """ # TODO check or discard version del state['_version_id'] - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('__weakref__',): setattr(self, key, value) + class FileBuildInfo(SCons.Node.BuildInfoBase): __slots__ = () current_version_id = 2 @@ -2515,6 +2494,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): pass else: setattr(self, attr, list(map(node_to_str, val))) + def convert_from_sconsign(self, dir, name): """ Converts a newly-read FileBuildInfo object for in-SCons use @@ -2523,6 +2503,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): perform--but we're leaving this method here to make that clear. """ pass + def prepare_dependencies(self): """ Prepares a FileBuildInfo object for explaining what changed @@ -2551,6 +2532,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): s = ni.str_to_node(s) nodes.append(s) setattr(self, nattr, nodes) + def format(self, names=0): result = [] bkids = self.bsources + self.bdepends + self.bimplicit @@ -2563,6 +2545,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): result.append('%s [%s]' % (self.bactsig, self.bact)) return '\n'.join(result) + class File(Base): """A class for files in a file system. """ @@ -2579,8 +2562,6 @@ class File(Base): 'root', 'dirname', 'on_disk_entries', - 'sccs_dir', - 'rcs_dir', 'released_target_info', 'contentsig'] @@ -2653,10 +2634,12 @@ class File(Base): def get_contents(self): return SCons.Node._get_contents_map[self._func_get_contents](self) - # This attempts to figure out what the encoding of the text is - # based upon the BOM bytes, and then decodes the contents so that - # it's a valid python string. def get_text_contents(self): + """ + This attempts to figure out what the encoding of the text is + based upon the BOM bytes, and then decodes the contents so that + it's a valid python string. + """ contents = self.get_contents() # The behavior of various decode() methods and functions # w.r.t. the initial BOM bytes is different for different @@ -2664,15 +2647,15 @@ class File(Base): # them, but has a 'utf-8-sig' which does; 'utf-16' seems to # strip them; etc.) Just sidestep all the complication by # explicitly stripping the BOM before we decode(). - if contents.startswith(codecs.BOM_UTF8): + if contents[:len(codecs.BOM_UTF8)] == codecs.BOM_UTF8: return contents[len(codecs.BOM_UTF8):].decode('utf-8') - if contents.startswith(codecs.BOM_UTF16_LE): + if contents[:len(codecs.BOM_UTF16_LE)] == codecs.BOM_UTF16_LE: return contents[len(codecs.BOM_UTF16_LE):].decode('utf-16-le') - if contents.startswith(codecs.BOM_UTF16_BE): + if contents[:len(codecs.BOM_UTF16_BE)] == codecs.BOM_UTF16_BE: return contents[len(codecs.BOM_UTF16_BE):].decode('utf-16-be') try: return contents.decode() - except UnicodeDecodeError: + except (UnicodeDecodeError, AttributeError) as e: return contents @@ -3019,12 +3002,7 @@ class File(Base): return None scb = self.dir.src_builder() if scb is _null: - if diskcheck_sccs(self.dir, self.name): - scb = get_DefaultSCCSBuilder() - elif diskcheck_rcs(self.dir, self.name): - scb = get_DefaultRCSBuilder() - else: - scb = None + scb = None if scb is not None: try: b = self.builder diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index b9c19bc8..399ac06a 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -86,7 +86,7 @@ class Action(object): def show(self, string): pass def get_contents(self, target, source, env): - return "" + return bytearray("",'utf-8') def genstring(self, target, source, env): return "" def strfunction(self, targets, sources, env): @@ -231,7 +231,7 @@ class VariantDirTestCase(unittest.TestCase): # Build path exists assert f2.exists() # ...and exists() should copy the file from src to build path - assert test.read(['work', 'build', 'var2', 'test.in']) == 'test.in',\ + assert test.read(['work', 'build', 'var2', 'test.in']) == bytearray('test.in','utf-8'),\ test.read(['work', 'build', 'var2', 'test.in']) # Since exists() is true, so should rexists() be assert f2.rexists() @@ -260,7 +260,7 @@ class VariantDirTestCase(unittest.TestCase): # Build path should exist assert f4.exists() # ...and copy over the file into the local build path - assert test.read(['work', 'build', 'var2', 'test2.in']) == 'test2.in' + assert test.read(['work', 'build', 'var2', 'test2.in']) == bytearray('test2.in','utf-8') # should exist in repository, since exists() is true assert f4.rexists() # rfile() should point to ourselves @@ -273,12 +273,12 @@ class VariantDirTestCase(unittest.TestCase): assert f5.exists() # We should not copy the file from the source dir, since this is # a derived file. - assert test.read(['work', 'build', 'var1', 'test.out']) == 'test.old' + assert test.read(['work', 'build', 'var1', 'test.out']) == bytearray('test.old','utf-8') assert f6.exists() # We should not copy the file from the source dir, since this is # a derived file. - assert test.read(['work', 'build', 'var2', 'test.out']) == 'test.old' + assert test.read(['work', 'build', 'var2', 'test.out']) == bytearray('test.old','utf-8') f7 = fs.File('build/var1/test2.out') f8 = fs.File('build/var2/test2.out') @@ -334,7 +334,7 @@ class VariantDirTestCase(unittest.TestCase): test.write([ 'work', 'build', 'var1', 'asourcefile' ], 'stuff') f10 = fs.File('build/var1/asourcefile') assert f10.exists() - assert f10.get_contents() == 'stuff', f10.get_contents() + assert f10.get_contents() == bytearray('stuff','utf-8'), f10.get_contents() f11 = fs.File('src/file11') t, m = f11.alter_targets() @@ -485,6 +485,14 @@ class VariantDirTestCase(unittest.TestCase): real_symlink = os.symlink except AttributeError: real_symlink = None + + # Disable symlink and link for now in win32. + # We don't have a consistant plan to make these work as yet + # They are only supported with PY3 + if sys.platform == 'win32': + real_symlink = None + real_link = None + real_copy = shutil.copy2 simulator = LinkSimulator(duplicate, real_link, real_symlink, real_copy) @@ -673,7 +681,7 @@ class BaseTestCase(_tempdirTestCase): nonexistent = fs.Entry('nonexistent') assert not nonexistent.isfile() - if hasattr(os, 'symlink'): + if sys.platform != 'win32' and hasattr(os, 'symlink'): def test_islink(self): """Test the Base.islink() method""" test = self.test @@ -1298,7 +1306,7 @@ class FSTestCase(_tempdirTestCase): # get_contents() returns the binary contents. test.write("binary_file", "Foo\x1aBar") f1 = fs.File(test.workpath("binary_file")) - assert f1.get_contents() == "Foo\x1aBar", f1.get_contents() + assert f1.get_contents() == bytearray("Foo\x1aBar",'utf-8'), f1.get_contents() # This tests to make sure we can decode UTF-8 text files. test_string = u"Foo\x1aBar" @@ -1367,7 +1375,7 @@ class FSTestCase(_tempdirTestCase): try: e = fs.Entry('file') c = e.get_contents() - assert c == "file\n", c + assert c == bytearray("file\n",'utf-8'), c assert e.__class__ == SCons.Node.FS.File finally: test.unlink("file") @@ -1399,7 +1407,7 @@ class FSTestCase(_tempdirTestCase): except SyntaxError: assert c == "" - if hasattr(os, 'symlink'): + if sys.platform != 'win32' and hasattr(os, 'symlink'): os.symlink('nonexistent', test.workpath('dangling_symlink')) e = fs.Entry('dangling_symlink') c = e.get_contents() @@ -1494,16 +1502,14 @@ class FSTestCase(_tempdirTestCase): assert r, r assert not os.path.exists(test.workpath('exists')), "exists was not removed" - symlink = test.workpath('symlink') - try: + if sys.platform != 'win32' and hasattr(os, 'symlink'): + symlink = test.workpath('symlink') os.symlink(test.workpath('does_not_exist'), symlink) assert os.path.islink(symlink) f = fs.File('symlink') r = f.remove() assert r, r assert not os.path.islink(symlink), "symlink was not removed" - except AttributeError: - pass test.write('can_not_remove', "can_not_remove\n") test.writable(test.workpath('.'), 0) @@ -3099,7 +3105,7 @@ class RepositoryTestCase(_tempdirTestCase): test.write(["rep3", "contents"], "Con\x1aTents\n") try: c = fs.File("contents").get_contents() - assert c == "Con\x1aTents\n", "got '%s'" % c + assert c == bytearray("Con\x1aTents\n",'utf-8'), "got '%s'" % c finally: test.unlink(["rep3", "contents"]) @@ -3271,18 +3277,11 @@ class has_src_builderTestCase(unittest.TestCase): fs = SCons.Node.FS.FS(test.workpath('')) os.chdir(test.workpath('')) test.subdir('sub1') - test.subdir('sub2', ['sub2', 'SCCS'], ['sub2', 'RCS']) sub1 = fs.Dir('sub1', '.') f1 = fs.File('f1', sub1) f2 = fs.File('f2', sub1) f3 = fs.File('f3', sub1) - sub2 = fs.Dir('sub2', '.') - f4 = fs.File('f4', sub2) - f5 = fs.File('f5', sub2) - f6 = fs.File('f6', sub2) - f7 = fs.File('f7', sub2) - f8 = fs.File('f8', sub2) h = f1.has_src_builder() assert not h, h @@ -3307,32 +3306,6 @@ class has_src_builderTestCase(unittest.TestCase): assert h, h assert f3.builder is b1, f3.builder - f7.set_src_builder(b1) - f8.builder_set(b1) - - test.write(['sub2', 'SCCS', 's.f5'], "sub2/SCCS/s.f5\n") - test.write(['sub2', 'RCS', 'f6,v'], "sub2/RCS/f6,v\n") - h = f4.has_src_builder() - assert not h, h - h = f4.has_builder() - assert not h, h - h = f5.has_src_builder() - assert h, h - h = f5.has_builder() - assert h, h - h = f6.has_src_builder() - assert h, h - h = f6.has_builder() - assert h, h - h = f7.has_src_builder() - assert h, h - h = f7.has_builder() - assert h, h - h = f8.has_src_builder() - assert not h, h - h = f8.has_builder() - assert h, h - class prepareTestCase(unittest.TestCase): def runTest(self): """Test the prepare() method""" diff --git a/src/engine/SCons/Node/Python.py b/src/engine/SCons/Node/Python.py index 92cc3204..8c47c97f 100644 --- a/src/engine/SCons/Node/Python.py +++ b/src/engine/SCons/Node/Python.py @@ -58,7 +58,7 @@ class ValueNodeInfo(SCons.Node.NodeInfoBase): del state['__weakref__'] except KeyError: pass - + return state def __setstate__(self, state): @@ -67,7 +67,7 @@ class ValueNodeInfo(SCons.Node.NodeInfoBase): """ # TODO check or discard version del state['_version_id'] - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('__weakref__',): setattr(self, key, value) @@ -77,7 +77,7 @@ class ValueBuildInfo(SCons.Node.BuildInfoBase): current_version_id = 2 class Value(SCons.Node.Node): - """A class for Python variables, typically passed on the command line + """A class for Python variables, typically passed on the command line or generated by a script, but not from a file or some other source. """ @@ -108,7 +108,7 @@ class Value(SCons.Node.Node): is_up_to_date = SCons.Node.Node.children_are_up_to_date def is_under(self, dir): - # Make Value nodes get built regardless of + # Make Value nodes get built regardless of # what directory scons was run from. Value nodes # are outside the filesystem: return 1 @@ -133,10 +133,17 @@ class Value(SCons.Node.Node): ###TODO: something reasonable about universal newlines contents = str(self.value) for kid in self.children(None): - contents = contents + kid.get_contents() + contents = contents + kid.get_contents().decode() return contents - get_contents = get_text_contents ###TODO should return 'bytes' value + def get_contents(self): + text_contents = self.get_text_contents() + try: + return text_contents.encode() + except UnicodeDecodeError: + # Already encoded as python2 str are bytes + return text_contents + def changed_since_last_build(self, target, prev_ni): cur_csig = self.get_csig() diff --git a/src/engine/SCons/Node/PythonTests.py b/src/engine/SCons/Node/PythonTests.py index e2e36bf9..346542b0 100644 --- a/src/engine/SCons/Node/PythonTests.py +++ b/src/engine/SCons/Node/PythonTests.py @@ -90,15 +90,15 @@ class ValueTestCase(unittest.TestCase): """ v1 = SCons.Node.Python.Value('aaa') csig = v1.get_csig(None) - assert csig == 'aaa', csig + assert csig.decode() == 'aaa', csig v2 = SCons.Node.Python.Value(7) csig = v2.get_csig(None) - assert csig == '7', csig + assert csig.decode() == '7', csig v3 = SCons.Node.Python.Value(None) csig = v3.get_csig(None) - assert csig == 'None', csig + assert csig.decode() == 'None', csig class ValueNodeInfoTestCase(unittest.TestCase): def test___init__(self): diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 2bf38c2f..0409d3b2 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -350,6 +350,7 @@ class NodeInfoBase(object): """ __slots__ = ('__weakref__',) current_version_id = 2 + def update(self, node): try: field_list = self.field_list @@ -366,8 +367,10 @@ class NodeInfoBase(object): pass else: setattr(self, f, func()) + def convert(self, node, val): pass + def merge(self, other): """ Merge the fields of another object into this object. Already existing @@ -427,7 +430,7 @@ class NodeInfoBase(object): # TODO check or discard version del state['_version_id'] - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('__weakref__',): setattr(self, key, value) @@ -445,6 +448,7 @@ class BuildInfoBase(object): __slots__ = ("bsourcesigs", "bdependsigs", "bimplicitsigs", "bactsig", "bsources", "bdepends", "bact", "bimplicit", "__weakref__") current_version_id = 2 + def __init__(self): # Create an object attribute from the class attribute so it ends up # in the pickled data in the .sconsign file. @@ -452,6 +456,7 @@ class BuildInfoBase(object): self.bdependsigs = [] self.bimplicitsigs = [] self.bactsig = None + def merge(self, other): """ Merge the fields of another object into this object. Already existing @@ -488,7 +493,7 @@ class BuildInfoBase(object): """ # TODO check or discard version del state['_version_id'] - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('__weakref__',): setattr(self, key, value) @@ -1131,38 +1136,22 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bactsig = SCons.Util.MD5signature(executor.get_contents()) if self._specific_sources: - sources = [] - for s in self.sources: - if s not in ignore_set: - sources.append(s) + sources = [ s for s in self.sources if not s in ignore_set] + else: sources = executor.get_unignored_sources(self, self.ignore) + seen = set() - bsources = [] - bsourcesigs = [] - for s in sources: - if not s in seen: - seen.add(s) - bsources.append(s) - bsourcesigs.append(s.get_ninfo()) - binfo.bsources = bsources - binfo.bsourcesigs = bsourcesigs - - depends = self.depends - dependsigs = [] - for d in depends: - if d not in ignore_set: - dependsigs.append(d.get_ninfo()) - binfo.bdepends = depends - binfo.bdependsigs = dependsigs - - implicit = self.implicit or [] - implicitsigs = [] - for i in implicit: - if i not in ignore_set: - implicitsigs.append(i.get_ninfo()) - binfo.bimplicit = implicit - binfo.bimplicitsigs = implicitsigs + binfo.bsources = [s for s in sources if s not in seen and not seen.add(s)] + binfo.bsourcesigs = [s.get_ninfo() for s in binfo.bsources] + + + binfo.bdepends = self.depends + binfo.bdependsigs = [d.get_ninfo() for d in self.depends if d not in ignore_set] + + binfo.bimplicit = self.implicit or [] + binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit if i not in ignore_set] + return binfo @@ -1651,6 +1640,9 @@ class Node(object, with_metaclass(NoSlotsPyPy)): if old.bact == new.bact: lines.append("the contents of the build action changed\n" + fmt_with_title('action: ', new.bact)) + + # lines.append("the contents of the build action changed [%s] [%s]\n"%(old.bactsig,new.bactsig) + + # fmt_with_title('action: ', new.bact)) else: lines.append("the build action changed:\n" + fmt_with_title('old: ', old.bact) + diff --git a/src/engine/SCons/PathList.py b/src/engine/SCons/PathList.py index 77e30c40..76cbeab8 100644 --- a/src/engine/SCons/PathList.py +++ b/src/engine/SCons/PathList.py @@ -104,11 +104,11 @@ class _PathList(object): pl = [] for p in pathlist: try: - index = p.find('$') + found = '$' in p except (AttributeError, TypeError): type = TYPE_OBJECT else: - if index == -1: + if not found: type = TYPE_STRING_NO_SUBST else: type = TYPE_STRING_SUBST diff --git a/src/engine/SCons/Platform/.aeignore b/src/engine/SCons/Platform/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/Platform/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Platform/__init__.py b/src/engine/SCons/Platform/__init__.py index ebdbb4c7..1654d0ae 100644 --- a/src/engine/SCons/Platform/__init__.py +++ b/src/engine/SCons/Platform/__init__.py @@ -217,7 +217,7 @@ class TempFileMunge(object): prefix = '@' args = list(map(SCons.Subst.quote_spaces, cmd[1:])) - os.write(fd, " ".join(args) + "\n") + os.write(fd, bytearray(" ".join(args) + "\n",'utf-8')) os.close(fd) # XXX Using the SCons.Action.print_actions value directly # like this is bogus, but expedient. This class should diff --git a/src/engine/SCons/Platform/darwin.py b/src/engine/SCons/Platform/darwin.py index 1cf4aeb5..c85b75b5 100644 --- a/src/engine/SCons/Platform/darwin.py +++ b/src/engine/SCons/Platform/darwin.py @@ -63,6 +63,10 @@ def generate(env): env.AppendENVPath('PATHOSX', line.strip('\n')) f.close() + # Not sure why this wasn't the case all along? + if env['ENV'].get('PATHOSX', False) and os.environ.get('SCONS_USE_MAC_PATHS', False): + env.AppendENVPath('PATH',env['ENV']['PATHOSX']) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Platform/posix.py b/src/engine/SCons/Platform/posix.py index 190a2a65..8db08db1 100644 --- a/src/engine/SCons/Platform/posix.py +++ b/src/engine/SCons/Platform/posix.py @@ -56,7 +56,7 @@ def escape(arg): for c in special: arg = arg.replace(c, slash+c) - # print "ESCAPE RESULT: %s"%arg + # print("ESCAPE RESULT: %s" % arg) return '"' + arg + '"' diff --git a/src/engine/SCons/Platform/win32.py b/src/engine/SCons/Platform/win32.py index b9defbbe..0e9e01f9 100644 --- a/src/engine/SCons/Platform/win32.py +++ b/src/engine/SCons/Platform/win32.py @@ -77,7 +77,7 @@ else: def __init__(self, *args, **kw): _builtin_file.__init__(self, *args, **kw) win32api.SetHandleInformation(msvcrt.get_osfhandle(self.fileno()), - win32con.HANDLE_FLAG_INHERIT, 0) + win32con.HANDLE_FLAG_INHERIT, 0) file = _scons_file else: import io @@ -92,6 +92,37 @@ else: setattr(io, io_class, _scons_file) + +if False: + # Now swap out shutil.filecopy and filecopy2 for win32 api native CopyFile + try: + from ctypes import windll + import shutil + + CopyFile = windll.kernel32.CopyFileA + SetFileTime = windll.kernel32.SetFileTime + + _shutil_copy = shutil.copy + _shutil_copy2 = shutil.copy2 + + shutil.copy2 = CopyFile + + def win_api_copyfile(src,dst): + CopyFile(src,dst) + os.utime(dst) + + shutil.copy = win_api_copyfile + + except AttributeError: + parallel_msg = \ + "Couldn't override shutil.copy or shutil.copy2 falling back to shutil defaults" + + + + + + + try: import threading spawn_lock = threading.Lock() @@ -195,10 +226,10 @@ def piped_spawn(sh, escape, cmd, args, env, stdout, stderr): def exec_spawn(l, env): try: result = spawnve(os.P_WAIT, l[0], l, env) - except OSError as e: + except (OSError, EnvironmentError) as e: try: - result = exitvalmap[e[0]] - sys.stderr.write("scons: %s: %s\n" % (l[0], e[1])) + result = exitvalmap[e.errno] + sys.stderr.write("scons: %s: %s\n" % (l[0], e.strerror)) except KeyError: result = 127 if len(l) > 2: @@ -208,7 +239,7 @@ def exec_spawn(l, env): command = l[0] else: command = l[0] - sys.stderr.write("scons: unknown OSError exception code %d - '%s': %s\n" % (e[0], command, e[1])) + sys.stderr.write("scons: unknown OSError exception code %d - '%s': %s\n" % (e.errno, command, e.strerror)) return result @@ -256,6 +287,10 @@ def get_system_root(): raise except: pass + + # Ensure system root is a string and not unicode + # (This only matters for py27 were unicode in env passed to POpen fails) + val = str(val) _system_root = val return val diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index c68d1c6f..889af4a0 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -110,7 +110,7 @@ def _createConfigH(target, source, env): #define %(DEFNAME)s_SEEN """ % {'DEFNAME' : defname}) - t.write(source[0].get_contents()) + t.write(source[0].get_contents().decode()) t.write(""" #endif /* %(DEFNAME)s_SEEN */ """ % {'DEFNAME' : defname}) @@ -164,11 +164,11 @@ class ConfigureCacheError(SConfError): # define actions for building text files def _createSource( target, source, env ): fd = open(str(target[0]), "w") - fd.write(source[0].get_contents()) + fd.write(source[0].get_contents().decode()) fd.close() def _stringSource( target, source, env ): return (str(target[0]) + ' <-\n |' + - source[0].get_contents().replace( '\n', "\n |" ) ) + source[0].get_contents().decode().replace( '\n', "\n |" ) ) class SConfBuildInfo(SCons.Node.FS.FileBuildInfo): """ @@ -609,7 +609,7 @@ class SConfBase(object): ok = self.TryBuild(self.env.SConfActionBuilder, text, extension) del self.env['BUILDERS']['SConfActionBuilder'] if ok: - outputStr = self.lastTarget.get_contents() + outputStr = self.lastTarget.get_contents().decode() return (1, outputStr) return (0, "") @@ -643,7 +643,7 @@ class SConfBase(object): node = self.env.Command(output, prog, [ [ pname, ">", "${TARGET}"] ]) ok = self.BuildNodes(node) if ok: - outputStr = output.get_contents() + outputStr = SCons.Util.to_str(output.get_contents()) return( 1, outputStr) return (0, "") diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index 15499ed9..f4c6f899 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -48,7 +48,7 @@ class SConfTestCase(unittest.TestCase): def setUp(self): # we always want to start with a clean directory self.save_cwd = os.getcwd() - self.test = TestCmd.TestCmd(workdir = '') + self.test = TestCmd.TestCmd(workdir = '') os.chdir(self.test.workpath('')) def tearDown(self): @@ -102,14 +102,17 @@ class SConfTestCase(unittest.TestCase): import SCons.Platform.win32 - file = SCons.Platform.win32._builtin_file - open = SCons.Platform.win32._builtin_open + try: + file = SCons.Platform.win32._builtin_file + open = SCons.Platform.win32._builtin_open + except AttributeError: + pass def _baseTryXXX(self, TryFunc): # TryCompile and TryLink are much the same, so we can test them # in one method, we pass the function as a string ('TryCompile', # 'TryLink'), so we are aware of reloading modules. - + def checks(self, sconf, TryFuncString): TryFunc = self.SConf.SConfBase.__dict__[TryFuncString] res1 = TryFunc( sconf, "int main() { return 0; }\n", ".c" ) @@ -128,7 +131,7 @@ class SConfTestCase(unittest.TestCase): assert res[0] and not res[1], res finally: sconf.Finish() - + # 2.1 test the error caching mechanism (no dependencies have changed) self._resetSConfState() sconf = self.SConf.SConf(self.scons_env, @@ -139,9 +142,9 @@ class SConfTestCase(unittest.TestCase): assert res[0] and not res[1], res finally: sconf.Finish() - # we should have exactly one one error cached - log = self.test.read( self.test.workpath('config.log') ) - expr = re.compile( ".*failed in a previous run and all", re.DOTALL ) + # we should have exactly one one error cached + log = str(self.test.read( self.test.workpath('config.log') )) + expr = re.compile( ".*failed in a previous run and all", re.DOTALL ) firstOcc = expr.match( log ) assert firstOcc is not None, log secondOcc = expr.match( log, firstOcc.end(0) ) @@ -171,6 +174,7 @@ class SConfTestCase(unittest.TestCase): conf_dir=self.test.workpath('config.tests'), log_file=self.test.workpath('config.log')) import SCons.Builder + import SCons.Node class MyBuilder(SCons.Builder.BuilderBase): def __init__(self): self.prefix = '' @@ -179,7 +183,7 @@ class SConfTestCase(unittest.TestCase): class MyNode(object): def __init__(self, name): self.name = name - self.state = None + self.state = SCons.Node.no_state self.waiting_parents = set() self.side_effects = [] self.builder = None @@ -238,11 +242,11 @@ class SConfTestCase(unittest.TestCase): """Test SConf.TryCompile """ self._baseTryXXX( "TryCompile" ) #self.SConf.SConf.TryCompile ) - + def test_TryLink(self): """Test SConf.TryLink """ - self._baseTryXXX( "TryLink" ) #self.SConf.SConf.TryLink ) + self._baseTryXXX( "TryLink" ) #self.SConf.SConf.TryLink ) def test_TryRun(self): """Test SConf.TryRun @@ -255,10 +259,10 @@ int main() { return 0; } """ - res1 = sconf.TryRun( prog, ".c" ) + res1 = sconf.TryRun( prog, ".c" ) res2 = sconf.TryRun( "not a c program\n", ".c" ) return (res1, res2) - + self._resetSConfState() sconf = self.SConf.SConf(self.scons_env, conf_dir=self.test.workpath('config.tests'), @@ -282,8 +286,9 @@ int main() { assert not res[1][0] and res[1][1] == "", res finally: sconf.Finish() - # we should have exactly one error cached - log = self.test.read( self.test.workpath('config.log') ) + # we should have exactly one error cached + # creating string here because it's bytes by default on py3 + log = str(self.test.read( self.test.workpath('config.log') )) expr = re.compile( ".*failed in a previous run and all", re.DOTALL ) firstOcc = expr.match( log ) assert firstOcc is not None, log @@ -305,7 +310,7 @@ int main() { log_file=self.test.workpath('config.log')) try: (ret, output) = sconf.TryAction(action=actionOK) - assert ret and output == "RUN OK" + os.linesep, (ret, output) + assert ret and output.encode('utf-8') == bytearray("RUN OK"+os.linesep,'utf-8'), (ret, output) (ret, output) = sconf.TryAction(action=actionFAIL) assert not ret and output == "", (ret, output) finally: @@ -352,7 +357,7 @@ int main() { try: self._test_check_compilers('CC', sconf.CheckCC, 'CheckCC') except AssertionError: - sys.stderr.write(self.test.read('config.log')) + sys.stderr.write(self.test.read('config.log', mode='r')) raise finally: sconf.Finish() @@ -368,7 +373,7 @@ int main() { try: self._test_check_compilers('SHCC', sconf.CheckSHCC, 'CheckSHCC') except AssertionError: - sys.stderr.write(self.test.read('config.log')) + sys.stderr.write(self.test.read('config.log', mode='r')) raise finally: sconf.Finish() @@ -384,7 +389,7 @@ int main() { try: self._test_check_compilers('CXX', sconf.CheckCXX, 'CheckCXX') except AssertionError: - sys.stderr.write(self.test.read('config.log')) + sys.stderr.write(self.test.read('config.log', mode='r')) raise finally: sconf.Finish() @@ -400,7 +405,7 @@ int main() { try: self._test_check_compilers('SHCXX', sconf.CheckSHCXX, 'CheckSHCXX') except AssertionError: - sys.stderr.write(self.test.read('config.log')) + sys.stderr.write(self.test.read('config.log', mode='r')) raise finally: sconf.Finish() @@ -625,8 +630,8 @@ int main() { else: r = sconf.CheckProg('cmd.exe') self.assertIn('cmd.exe',r) - - + + r = sconf.CheckProg('hopefully-not-a-program') assert r is None @@ -715,7 +720,7 @@ int main() { # In ANSI C, malloc should be available in stdlib r = sconf.CheckDeclaration('malloc', includes = "#include <stdlib.h>") assert r, "malloc not declared ??" - # For C++, __cplusplus should be declared + # For C++, __cplusplus should be declared r = sconf.CheckDeclaration('__cplusplus', language = 'C++') assert r, "__cplusplus not declared in C++ ??" r = sconf.CheckDeclaration('__cplusplus', language = 'C') @@ -759,7 +764,7 @@ int main() { test.Result( ret ) assert ret and output == "Hello", (ret, output) return ret - + self._resetSConfState() sconf = self.SConf.SConf(self.scons_env, @@ -771,7 +776,7 @@ int main() { assert ret, ret finally: sconf.Finish() - + if __name__ == "__main__": suite = unittest.makeSuite(SConfTestCase, 'test_') diff --git a/src/engine/SCons/SConsign.py b/src/engine/SCons/SConsign.py index 75d2c419..5042aa1d 100644 --- a/src/engine/SCons/SConsign.py +++ b/src/engine/SCons/SConsign.py @@ -41,6 +41,7 @@ import SCons.Warnings from SCons.compat import PICKLE_PROTOCOL + def corrupt_dblite_warning(filename): SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, "Ignoring corrupt .sconsign file: %s"%filename) @@ -48,7 +49,7 @@ def corrupt_dblite_warning(filename): SCons.dblite.ignore_corrupt_dbfiles = 1 SCons.dblite.corruption_warning = corrupt_dblite_warning -#XXX Get rid of the global array so this becomes re-entrant. +# XXX Get rid of the global array so this becomes re-entrant. sig_files = [] # Info for the database SConsign implementation (now the default): @@ -62,6 +63,7 @@ DB_Module = SCons.dblite DB_Name = ".sconsign" DB_sync_list = [] + def Get_DataBase(dir): global DataBase, DB_Module, DB_Name top = dir.fs.Top @@ -90,6 +92,7 @@ def Get_DataBase(dir): print("DataBase =", DataBase) raise + def Reset(): """Reset global state. Used by unit tests that end up using SConsign multiple times to get a clean slate for each test.""" @@ -99,6 +102,7 @@ def Reset(): normcase = os.path.normcase + def write(): global sig_files for sig_file in sig_files: @@ -117,6 +121,7 @@ def write(): else: closemethod() + class SConsignEntry(object): """ Wrapper class for the generic entry in a .sconsign file. @@ -155,10 +160,11 @@ class SConsignEntry(object): return state def __setstate__(self, state): - for key, value in list(state.items()): + for key, value in state.items(): if key not in ('_version_id','__weakref__'): setattr(self, key, value) + class Base(object): """ This is the controlling class for the signatures for the collection of @@ -199,7 +205,7 @@ class Base(object): pass def merge(self): - for key, node in list(self.to_be_merged.items()): + for key, node in self.to_be_merged.items(): entry = node.get_stored_info() try: ninfo = entry.ninfo @@ -213,6 +219,7 @@ class Base(object): self.entries[key] = entry self.to_be_merged = {} + class DB(Base): """ A Base subclass that reads and writes signature information @@ -245,7 +252,7 @@ class DB(Base): except Exception as e: SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e)) - for key, entry in list(self.entries.items()): + for key, entry in self.entries.items(): entry.convert_from_sconsign(dir, key) if mode == "r": @@ -272,7 +279,7 @@ class DB(Base): # the Repository; we only write to our own .sconsign file, # not to .sconsign files in Repositories. path = normcase(self.dir.get_internal_path()) - for key, entry in list(self.entries.items()): + for key, entry in self.entries.items(): entry.convert_to_sconsign() db[path] = pickle.dumps(self.entries, PICKLE_PROTOCOL) @@ -285,6 +292,7 @@ class DB(Base): else: syncmethod() + class Dir(Base): def __init__(self, fp=None, dir=None): """ @@ -301,9 +309,10 @@ class Dir(Base): raise TypeError if dir: - for key, entry in list(self.entries.items()): + for key, entry in self.entries.items(): entry.convert_from_sconsign(dir, key) + class DirFile(Dir): """ Encapsulates reading and writing a per-directory .sconsign file. @@ -360,7 +369,7 @@ class DirFile(Dir): fname = self.sconsign except IOError: return - for key, entry in list(self.entries.items()): + for key, entry in self.entries.items(): entry.convert_to_sconsign() pickle.dump(self.entries, file, PICKLE_PROTOCOL) file.close() @@ -394,6 +403,7 @@ class DirFile(Dir): ForDirectory = DB + def File(name, dbm_module=None): """ Arrange for all signatures to be stored in a global .sconsign.db* diff --git a/src/engine/SCons/Scanner/.aeignore b/src/engine/SCons/Scanner/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/Scanner/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Scanner/Dir.py b/src/engine/SCons/Scanner/Dir.py index cbfb6fb0..3b33fe5a 100644 --- a/src/engine/SCons/Scanner/Dir.py +++ b/src/engine/SCons/Scanner/Dir.py @@ -27,7 +27,7 @@ import SCons.Scanner def only_dirs(nodes): is_Dir = lambda n: isinstance(n.disambiguate(), SCons.Node.FS.Dir) - return list(filter(is_Dir, nodes)) + return [node for node in nodes if is_Dir(node)] def DirScanner(**kw): """Return a prototype Scanner instance for scanning diff --git a/src/engine/SCons/Scanner/LaTeX.py b/src/engine/SCons/Scanner/LaTeX.py index cb89cf0c..e5abb0c2 100644 --- a/src/engine/SCons/Scanner/LaTeX.py +++ b/src/engine/SCons/Scanner/LaTeX.py @@ -37,7 +37,9 @@ import SCons.Util # list of graphics file extensions for TeX and LaTeX TexGraphics = ['.eps', '.ps'] -LatexGraphics = ['.pdf', '.png', '.jpg', '.gif', '.tif'] +#LatexGraphics = ['.pdf', '.png', '.jpg', '.gif', '.tif'] +LatexGraphics = [ '.png', '.jpg', '.gif', '.tif'] + # Used as a return value of modify_env_var if the variable is not set. class _Null(object): @@ -224,14 +226,14 @@ class LaTeX(SCons.Scanner.Base): """ def __init__(self, dictionary): self.dictionary = {} - for k,n in list(dictionary.items()): + for k,n in dictionary.items(): self.dictionary[k] = ( SCons.Scanner.FindPathDirs(n), FindENVPathDirs(n) ) def __call__(self, env, dir=None, target=None, source=None, argument=None): di = {} - for k,(c,cENV) in list(self.dictionary.items()): + for k,(c,cENV) in self.dictionary.items(): di[k] = ( c(env, dir=None, target=None, source=None, argument=None) , cENV(env, dir=None, target=None, source=None, @@ -397,6 +399,7 @@ class LaTeX(SCons.Scanner.Base): include = queue.pop() inc_type, inc_subdir, inc_filename = include + try: if seen[inc_filename] == 1: continue diff --git a/src/engine/SCons/Scanner/RC.py b/src/engine/SCons/Scanner/RC.py index 61393ae9..82f8ddc9 100644 --- a/src/engine/SCons/Scanner/RC.py +++ b/src/engine/SCons/Scanner/RC.py @@ -30,21 +30,32 @@ Definition Language) files. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import re + import SCons.Node.FS import SCons.Scanner -import re + + +def no_tlb(nodes): + """ + Filter out .tlb files as they are binary and shouldn't be scanned + """ + # print("Nodes:%s"%[str(n) for n in nodes]) + return [n for n in nodes if str(n)[-4:] != '.tlb'] + def RCScan(): """Return a prototype Scanner instance for scanning RC source files""" - + res_re= r'^(?:\s*#\s*(?:include)|' \ '.*?\s+(?:ICON|BITMAP|CURSOR|HTML|FONT|MESSAGETABLE|TYPELIB|REGISTRY|D3DFX)' \ '\s*.*?)' \ '\s*(<|"| )([^>"\s]+)(?:[>"\s])*$' - resScanner = SCons.Scanner.ClassicCPP( "ResourceScanner", - "$RCSUFFIXES", - "CPPPATH", - res_re ) + resScanner = SCons.Scanner.ClassicCPP("ResourceScanner", + "$RCSUFFIXES", + "CPPPATH", + res_re, + recursive=no_tlb) return resScanner diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py index e34dcada..8050216c 100644 --- a/src/engine/SCons/Scanner/ScannerTests.py +++ b/src/engine/SCons/Scanner/ScannerTests.py @@ -105,7 +105,7 @@ class ScannerTestCase(unittest.TestCase): assert str(s) == 'fooscan', str(s) assert s.argument == 888, s.argument - + class BaseTestCase(unittest.TestCase): class skey_node(object): @@ -447,6 +447,17 @@ class CurrentTestCase(unittest.TestCase): self.failUnless(ic.func_called, "did not call func()") class ClassicTestCase(unittest.TestCase): + + def func(self, filename, env, target, *args): + self.filename = filename + self.env = env + self.target = target + + if len(args) > 0: + self.arg = args[0] + + return self.deps + def test_find_include(self): """Test the Scanner.Classic find_include() method""" env = DummyEnvironment() @@ -548,6 +559,25 @@ class ClassicTestCase(unittest.TestCase): ret = s.function(n, env, ('foo5',)) assert ret == ['jkl', 'mno'], ret + def test_recursive(self): + """Test the Scanner.Classic class recursive flag""" + nodes = [1, 2, 3, 4] + + + s = SCons.Scanner.Classic("Test", [], None, "", function=self.func, recursive=1) + n = s.recurse_nodes(nodes) + self.failUnless(n == n, + "recursive = 1 didn't return all nodes: %s" % n) + + def odd_only(nodes): + return [n for n in nodes if n % 2] + + s = SCons.Scanner.Classic("Test", [], None, "", function=self.func, recursive=odd_only) + n = s.recurse_nodes(nodes) + self.failUnless(n == [1, 3], + "recursive = 1 didn't return all nodes: %s" % n) + + class ClassicCPPTestCase(unittest.TestCase): diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py index 28be6426..c0f1a933 100644 --- a/src/engine/SCons/Scanner/__init__.py +++ b/src/engine/SCons/Scanner/__init__.py @@ -188,12 +188,12 @@ class Base(object): def path(self, env, dir=None, target=None, source=None): if not self.path_function: return () - if not self.argument is _null: + if self.argument is not _null: return self.path_function(env, dir, target, source, self.argument) else: return self.path_function(env, dir, target, source) - def __call__(self, node, env, path = ()): + def __call__(self, node, env, path=()): """ This method scans a single object. 'node' is the node that will be passed to the scanner function, and 'env' is the @@ -206,16 +206,16 @@ class Base(object): self = self.select(node) if not self.argument is _null: - list = self.function(node, env, path, self.argument) + node_list = self.function(node, env, path, self.argument) else: - list = self.function(node, env, path) + node_list = self.function(node, env, path) kw = {} if hasattr(node, 'dir'): kw['directory'] = node.dir node_factory = env.get_factory(self.node_factory) nodes = [] - for l in list: + for l in node_list: if self.node_class and not isinstance(l, self.node_class): l = node_factory(l, **kw) nodes.append(l) @@ -259,7 +259,7 @@ class Base(object): def _recurse_no_nodes(self, nodes): return [] - recurse_nodes = _recurse_no_nodes + # recurse_nodes = _recurse_no_nodes def add_scanner(self, skey, scanner): self.function[skey] = scanner @@ -283,7 +283,7 @@ class Selector(Base): self.dict = dict self.skeys = list(dict.keys()) - def __call__(self, node, env, path = ()): + def __call__(self, node, env, path=()): return self.select(node)(node, env, path) def select(self, node): @@ -326,7 +326,7 @@ class Classic(Current): self.cre = re.compile(regex, re.M) - def _scan(node, env, path=(), self=self): + def _scan(node, _, path=(), self=self): node = node.rfile() if not node.exists(): return [] @@ -334,7 +334,12 @@ class Classic(Current): kw['function'] = _scan kw['path_function'] = FindPathDirs(path_variable) - kw['recursive'] = 1 + + # Allow recursive to propagate if child class specifies. + # In this case resource scanner needs to specify a filter on which files + # get recursively processed. Previously was hardcoded to 1 instead of + # defaulted to 1. + kw['recursive'] = kw.get('recursive', 1) kw['skeys'] = suffixes kw['name'] = name @@ -356,7 +361,7 @@ class Classic(Current): if node.includes is not None: includes = node.includes else: - includes = self.find_include_names (node) + includes = self.find_include_names(node) # Intern the names of the include files. Saves some memory # if the same header is included many times. node.includes = list(map(SCons.Util.silent_intern, includes)) @@ -393,7 +398,7 @@ class ClassicCPP(Classic): the contained filename in group 1. """ def find_include(self, include, source_dir, path): - include = list (map (SCons.Util.to_str, include)) + include = list(map(SCons.Util.to_str, include)) if include[0] == '"': paths = (source_dir,) + tuple(path) else: diff --git a/src/engine/SCons/Script/.aeignore b/src/engine/SCons/Script/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/Script/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Script/Interactive.py b/src/engine/SCons/Script/Interactive.py index 3c3d23a0..6be7b270 100644 --- a/src/engine/SCons/Script/Interactive.py +++ b/src/engine/SCons/Script/Interactive.py @@ -121,7 +121,7 @@ class SConsInteractiveCmd(cmd.Cmd): def __init__(self, **kw): cmd.Cmd.__init__(self) - for key, val in list(kw.items()): + for key, val in kw.items(): setattr(self, key, val) if sys.platform == 'win32': @@ -222,7 +222,7 @@ class SConsInteractiveCmd(cmd.Cmd): def get_unseen_children(node, parent, seen_nodes=seen_nodes): def is_unseen(node, seen_nodes=seen_nodes): return node not in seen_nodes - return list(filter(is_unseen, node.children(scan=1))) + return [child for child in node.children(scan=1) if is_unseen(child)] def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): seen_nodes[node] = 1 diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index f8cb24c1..c810634b 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -47,6 +47,7 @@ import os import sys import time import traceback +import sysconfig import SCons.CacheDir import SCons.Debug @@ -65,6 +66,7 @@ import SCons.Warnings import SCons.Script.Interactive + def fetch_win32_parallel_msg(): # A subsidiary function that exists solely to isolate this import # so we don't have to pull it in on all platforms, and so that an @@ -75,6 +77,7 @@ def fetch_win32_parallel_msg(): import SCons.Platform.win32 return SCons.Platform.win32.parallel_msg + def revert_io(): # This call is added to revert stderr and stdout to the original # ones just in case some build rule or something else in the system @@ -91,6 +94,7 @@ progress_display = SCons.Util.DisplayEngine() first_command_start = None last_command_end = None + class Progressor(object): prev = '' count = 0 @@ -154,9 +158,11 @@ def Progress(*args, **kw): _BuildFailures = [] + def GetBuildFailures(): return _BuildFailures + class BuildTask(SCons.Taskmaster.OutOfDateTask): """An SCons build task.""" progress = ProgressObject @@ -306,6 +312,7 @@ class BuildTask(SCons.Taskmaster.OutOfDateTask): if explanation: sys.stdout.write("scons: " + explanation) + class CleanTask(SCons.Taskmaster.AlwaysTask): """An SCons clean task.""" def fs_delete(self, path, pathstr, remove=True): @@ -1162,7 +1169,7 @@ def _build_targets(fs, options, targets, target_top): # or not a file, so go ahead and keep it as a default # target and let the engine sort it out: return 1 - d = list(filter(check_dir, SCons.Script.DEFAULT_TARGETS)) + d = [tgt for tgt in SCons.Script.DEFAULT_TARGETS if check_dir(tgt)] SCons.Script.DEFAULT_TARGETS[:] = d target_top = None lookup_top = None @@ -1236,7 +1243,7 @@ def _build_targets(fs, options, targets, target_top): if options.taskmastertrace_file == '-': tmtrace = sys.stdout elif options.taskmastertrace_file: - tmtrace = open(options.taskmastertrace_file, 'wb') + tmtrace = open(options.taskmastertrace_file, 'w') else: tmtrace = None taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) @@ -1245,16 +1252,19 @@ def _build_targets(fs, options, targets, target_top): # various print_* settings, tree_printer list, etc. BuildTask.options = options + + python_has_threads = sysconfig.get_config_var('WITH_THREAD') + # to check if python configured with threads. global num_jobs num_jobs = options.num_jobs jobs = SCons.Job.Jobs(num_jobs, taskmaster) if num_jobs > 1: msg = None - if jobs.num_jobs == 1: + if sys.platform == 'win32': + msg = fetch_win32_parallel_msg() + elif jobs.num_jobs == 1 or not python_has_threads: msg = "parallel builds are unsupported by this version of Python;\n" + \ "\tignoring -j or num_jobs option.\n" - elif sys.platform == 'win32': - msg = fetch_win32_parallel_msg() if msg: SCons.Warnings.warn(SCons.Warnings.NoParallelSupportWarning, msg) diff --git a/src/engine/SCons/Script/Main.xml b/src/engine/SCons/Script/Main.xml index 07dce9a4..98db5bf3 100644 --- a/src/engine/SCons/Script/Main.xml +++ b/src/engine/SCons/Script/Main.xml @@ -256,7 +256,7 @@ import atexit def print_build_failures(): from SCons.Script import GetBuildFailures for bf in GetBuildFailures(): - print "%s failed: %s" % (bf.node, bf.errstr) + print("%s failed: %s" % (bf.node, bf.errstr)) atexit.register(print_build_failures) </example_commands> @@ -577,7 +577,7 @@ every 10 Nodes: <example_commands> def my_progress_function(node, *args, **kw): - print 'Evaluating node %s!' % node + print('Evaluating node %s!' % node) Progress(my_progress_function, interval=10) </example_commands> diff --git a/src/engine/SCons/Script/SConsOptions.py b/src/engine/SCons/Script/SConsOptions.py index 501e4cef..288a7a5a 100644 --- a/src/engine/SCons/Script/SConsOptions.py +++ b/src/engine/SCons/Script/SConsOptions.py @@ -63,6 +63,7 @@ def diskcheck_convert(value): raise ValueError(v) return result + class SConsValues(optparse.Values): """ Holder class for uniform access to SCons options, regardless @@ -112,7 +113,18 @@ class SConsValues(optparse.Values): try: return self.__dict__['__SConscript_settings__'][attr] except KeyError: - return getattr(self.__dict__['__defaults__'], attr) + try: + return getattr(self.__dict__['__defaults__'], attr) + except KeyError: + # Added because with py3 this is a new class, + # not a classic class, and due to the way + # In that case it will create an object without + # __defaults__, and then query for __setstate__ + # which will throw an exception of KeyError + # deepcopy() is expecting AttributeError if __setstate__ + # is not available. + raise AttributeError(attr) + settable = [ 'clean', @@ -186,6 +198,7 @@ class SConsValues(optparse.Values): self.__SConscript_settings__[name] = value + class SConsOption(optparse.Option): def convert_value(self, opt, value): if value is not None: diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py index a7c8a370..558e28f9 100644 --- a/src/engine/SCons/Script/SConscript.py +++ b/src/engine/SCons/Script/SConscript.py @@ -165,7 +165,7 @@ def _SConscript(fs, *files, **kw): try: SCons.Script.sconscript_reading = SCons.Script.sconscript_reading + 1 if fn == "-": - exec(sys.stdin, call_stack[-1].globals) + exec(sys.stdin.read(), call_stack[-1].globals) else: if isinstance(fn, SCons.Node.Node): f = fn @@ -179,10 +179,10 @@ def _SConscript(fs, *files, **kw): fs.chdir(top, change_os_dir=1) if f.rexists(): actual = f.rfile() - _file_ = open(actual.get_abspath(), "r") + _file_ = open(actual.get_abspath(), "rb") elif f.srcnode().rexists(): actual = f.srcnode().rfile() - _file_ = open(actual.get_abspath(), "r") + _file_ = open(actual.get_abspath(), "rb") elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -192,7 +192,7 @@ def _SConscript(fs, *files, **kw): f.built() f.builder_set(None) if f.exists(): - _file_ = open(f.get_abspath(), "r") + _file_ = open(f.get_abspath(), "rb") if _file_: # Chdir to the SConscript directory. Use a path # name relative to the SConstruct file so that if @@ -248,6 +248,7 @@ def _SConscript(fs, *files, **kw): pass try: try: +# _file_ = SCons.Util.to_str(_file_) exec(compile(_file_.read(), _file_.name, 'exec'), call_stack[-1].globals) except SConscriptReturn: @@ -526,7 +527,7 @@ class SConsEnvironment(SCons.Environment.Base): return x ls = list(map(subst_element, ls)) subst_kw = {} - for key, val in list(kw.items()): + for key, val in kw.items(): if SCons.Util.is_String(val): val = self.subst(val) elif SCons.Util.is_List(val): diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 3fa3a480..5bdd63e1 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -337,6 +337,7 @@ GlobalDefaultEnvironmentFunctions = [ 'Local', 'ParseDepends', 'Precious', + 'PyPackageDir', 'Repository', 'Requires', 'SConsignFile', diff --git a/src/engine/SCons/Sig.py b/src/engine/SCons/Sig.py deleted file mode 100644 index 35fffc14..00000000 --- a/src/engine/SCons/Sig.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__doc__ = """Place-holder for the old SCons.Sig module hierarchy - -This is no longer used, but code out there (such as the NSIS module on -the SCons wiki) may try to import SCons.Sig. If so, we generate a warning -that points them to the line that caused the import, and don't die. - -If someone actually tried to use the sub-modules or functions within -the package (for example, SCons.Sig.MD5.signature()), then they'll still -get an AttributeError, but at least they'll know where to start looking. -""" - -import SCons.Util -import SCons.Warnings - -msg = 'The SCons.Sig module no longer exists.\n' \ - ' Remove the following "import SCons.Sig" line to eliminate this warning:' - -SCons.Warnings.warn(SCons.Warnings.DeprecatedSigModuleWarning, msg) - -default_calc = None -default_module = None - -class MD5Null(SCons.Util.Null): - def __repr__(self): - return "MD5Null()" - -class TimeStampNull(SCons.Util.Null): - def __repr__(self): - return "TimeStampNull()" - -MD5 = MD5Null() -TimeStamp = TimeStampNull() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Subst.py b/src/engine/SCons/Subst.py index b81d79c8..a68b54db 100644 --- a/src/engine/SCons/Subst.py +++ b/src/engine/SCons/Subst.py @@ -442,7 +442,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ return s else: key = s[1:] - if key[0] == '{' or key.find('.') >= 0: + if key[0] == '{' or '.' in key: if key[0] == '{': key = key[1:-1] try: diff --git a/src/engine/SCons/SubstTests.py b/src/engine/SCons/SubstTests.py index f2cbc3f9..66041280 100644 --- a/src/engine/SCons/SubstTests.py +++ b/src/engine/SCons/SubstTests.py @@ -254,7 +254,7 @@ class SubstTestCase(unittest.TestCase): else: if result != expect: if failed == 0: print() - print(" input %s => %s did not match %s" % (repr(input), repr(result), repr(expect))) + print(" input %s => \n%s did not match \n%s" % (repr(input), repr(result), repr(expect))) failed = failed + 1 del cases[:2] fmt = "%d %s() cases failed" @@ -596,7 +596,10 @@ class scons_subst_TestCase(SubstTestCase): except SCons.Errors.UserError as e: expect = [ # Python 2.3, 2.4, 2.5 - "TypeError `func() takes exactly 3 arguments (1 given)' trying to evaluate `${func(1)}'" + "TypeError `func() takes exactly 3 arguments (1 given)' trying to evaluate `${func(1)}'", + + # Python 3.5 (and 3.x?) + "TypeError `func() missing 2 required positional arguments: 'b' and 'c'' trying to evaluate `${func(1)}'" ] assert str(e) in expect, repr(str(e)) else: diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index a620ba14..9bd95975 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -479,7 +479,7 @@ class Task(object): if p.ref_count == 0: self.tm.candidates.append(p) - for p, subtract in list(parents.items()): + for p, subtract in parents.items(): p.ref_count = p.ref_count - subtract if T: T.write(self.trace_message(u'Task.postprocess()', p, @@ -542,12 +542,21 @@ class Task(object): exc_type, exc_value = exc exc_traceback = None + # raise exc_type(exc_value).with_traceback(exc_traceback) if sys.version_info[0] == 2: exec("raise exc_type, exc_value, exc_traceback") else: # sys.version_info[0] == 3: - exec("raise exc_type(*exc_value.args).with_traceback(exc_traceback)") + if isinstance(exc_value, Exception): #hasattr(exc_value, 'with_traceback'): + # If exc_value is an exception, then just reraise + exec("raise exc_value.with_traceback(exc_traceback)") + else: + # else we'll create an exception using the value and raise that + exec("raise exc_type(exc_value).with_traceback(exc_traceback)") + # raise e.__class__, e.__class__(e), sys.exc_info()[2] + # exec("raise exc_type(exc_value).with_traceback(exc_traceback)") + class AlwaysTask(Task): @@ -772,7 +781,7 @@ class Taskmaster(object): self.ready_exc = None T = self.trace - if T: T.write(u'\n' + self.trace_message('Looking for a node to evaluate')) + if T: T.write(SCons.Util.UnicodeType('\n') + self.trace_message('Looking for a node to evaluate')) while True: node = self.next_candidate() @@ -851,13 +860,13 @@ class Taskmaster(object): if childstate <= NODE_EXECUTING: children_not_ready.append(child) - # These nodes have not even been visited yet. Add # them to the list so that on some next pass we can # take a stab at evaluating them (or their children). children_not_visited.reverse() self.candidates.extend(self.order(children_not_visited)) - #if T and children_not_visited: + + # if T and children_not_visited: # T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) # T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) @@ -960,7 +969,7 @@ class Taskmaster(object): task = self.tasker(self, tlist, node in self.original_top, node) try: task.make_ready() - except: + except Exception as e : # We had a problem just trying to get this task ready (like # a child couldn't be linked to a VariantDir when deciding # whether this node is current). Arrange to raise the diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index e795f5c1..d237d60a 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -849,14 +849,18 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() t.exception_set((MyException, "exception value")) exc_caught = None + exc_actually_caught = None + exc_value = None try: t.prepare() except MyException as e: exc_caught = 1 - except: + exc_value = e + except Exception as e: + exc_actually_caught = e pass - assert exc_caught, "did not catch expected MyException" - assert str(e) == "exception value", e + assert exc_caught, "did not catch expected MyException: %s"%exc_actually_caught + assert str(exc_value) == "exception value", exc_value assert built_text is None, built_text # Regression test, make sure we prepare not only @@ -1065,10 +1069,20 @@ class TaskmasterTestCase(unittest.TestCase): assert t.exception == 3 try: 1//0 - except: pass - t.exception_set(None) + except: + # Moved from below + t.exception_set(None) + #pass + +# import pdb; pdb.set_trace() + + # Having this here works for python 2.x, + # but it is a tuple (None, None, None) when called outside + # an except statement + # t.exception_set(None) + exc_type, exc_value, exc_tb = t.exception - assert exc_type is ZeroDivisionError, exc_type + assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s"%exc_type exception_values = [ "integer division or modulo", "integer division or modulo by zero", @@ -1078,13 +1092,14 @@ class TaskmasterTestCase(unittest.TestCase): class Exception1(Exception): pass - t.exception_set((Exception1, None)) + # Previously value was None, but while PY2 None = "", in Py3 None != "", so set to "" + t.exception_set((Exception1, "")) try: t.exception_raise() except: exc_type, exc_value = sys.exc_info()[:2] assert exc_type == Exception1, exc_type - assert str(exc_value) == '', exc_value + assert str(exc_value) == '', "Expecting empty string got:%s (type %s)"%(exc_value,type(exc_value)) else: assert 0, "did not catch expected exception" diff --git a/src/engine/SCons/Tool/.aeignore b/src/engine/SCons/Tool/.aeignore deleted file mode 100644 index 22ebd62b..00000000 --- a/src/engine/SCons/Tool/.aeignore +++ /dev/null @@ -1,5 +0,0 @@ -*,D -*.pyc -.*.swp -.consign -.sconsign diff --git a/src/engine/SCons/Tool/BitKeeper.py b/src/engine/SCons/Tool/BitKeeper.py deleted file mode 100644 index 44632d78..00000000 --- a/src/engine/SCons/Tool/BitKeeper.py +++ /dev/null @@ -1,66 +0,0 @@ -"""SCons.Tool.BitKeeper.py - -Tool-specific initialization for the BitKeeper source code control -system. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import SCons.Action -import SCons.Builder -import SCons.Util - -def generate(env): - """Add a Builder factory function and construction variables for - BitKeeper to an Environment.""" - - def BitKeeperFactory(env=env): - """ """ - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The BitKeeper() factory is deprecated and there is no replacement.""") - act = SCons.Action.Action("$BITKEEPERCOM", "$BITKEEPERCOMSTR") - return SCons.Builder.Builder(action = act, env = env) - - env.BitKeeper = BitKeeperFactory - - env['BITKEEPER'] = 'bk' - env['BITKEEPERGET'] = '$BITKEEPER get' - env['BITKEEPERGETFLAGS'] = SCons.Util.CLVar('') - env['BITKEEPERCOM'] = '$BITKEEPERGET $BITKEEPERGETFLAGS $TARGET' - -def exists(env): - return env.Detect('bk') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/BitKeeper.xml b/src/engine/SCons/Tool/BitKeeper.xml deleted file mode 100644 index 30a5e779..00000000 --- a/src/engine/SCons/Tool/BitKeeper.xml +++ /dev/null @@ -1,123 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<tool name="BitKeeper"> -<summary> -<para> -Sets construction variables for the BitKeeper -source code control system. -</para> -</summary> -<sets> -<item>BITKEEPER</item> -<item>BITKEEPERGET</item> -<item>BITKEEPERGETFLAGS</item> -<item>BITKEEPERCOM</item> -</sets> -<uses> -<item>BITKEEPERCOMSTR</item> -</uses> -</tool> - -<cvar name="BITKEEPER"> -<summary> -<para> -The BitKeeper executable. -</para> -</summary> -</cvar> - -<cvar name="BITKEEPERCOM"> -<summary> -<para> -The command line for -fetching source files using BitKeeper. -</para> -</summary> -</cvar> - -<cvar name="BITKEEPERCOMSTR"> -<summary> -<para> -The string displayed when fetching -a source file using BitKeeper. -If this is not set, then &cv-link-BITKEEPERCOM; -(the command line) is displayed. -</para> -</summary> -</cvar> - -<cvar name="BITKEEPERGET"> -<summary> -<para> -The command (&cv-link-BITKEEPER;) and subcommand -for fetching source files using BitKeeper. -</para> -</summary> -</cvar> - -<cvar name="BITKEEPERGETFLAGS"> -<summary> -<para> -Options that are passed to the BitKeeper -<command>get</command> -subcommand. -</para> -</summary> -</cvar> - -<scons_function name="BitKeeper"> -<arguments signature="env"> -() -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -using BitKeeper. -The returned Builder -is intended to be passed to the -&f-SourceCode; -function. -</para> - -<para> -This function is deprecated. For details, see the entry for the -&f-SourceCode; -function. -</para> - -<para> -Example: -</para> - -<example_commands> -env.SourceCode('.', env.BitKeeper()) -</example_commands> -</summary> -</scons_function> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/CVS.py b/src/engine/SCons/Tool/CVS.py deleted file mode 100644 index 08cf04ca..00000000 --- a/src/engine/SCons/Tool/CVS.py +++ /dev/null @@ -1,72 +0,0 @@ -"""SCons.Tool.CVS.py - -Tool-specific initialization for CVS. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import SCons.Action -import SCons.Builder -import SCons.Util - -def generate(env): - """Add a Builder factory function and construction variables for - CVS to an Environment.""" - - def CVSFactory(repos, module='', env=env): - """ """ - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The CVS() factory is deprecated and there is no replacement.""") - # fail if repos is not an absolute path name? - if module != '': - # Don't use os.path.join() because the name we fetch might - # be across a network and must use POSIX slashes as separators. - module = module + '/' - env['CVSCOM'] = '$CVS $CVSFLAGS co $CVSCOFLAGS -d ${TARGET.dir} $CVSMODULE${TARGET.posix}' - act = SCons.Action.Action('$CVSCOM', '$CVSCOMSTR') - return SCons.Builder.Builder(action = act, - env = env, - CVSREPOSITORY = repos, - CVSMODULE = module) - - env.CVS = CVSFactory - - env['CVS'] = 'cvs' - env['CVSFLAGS'] = SCons.Util.CLVar('-d $CVSREPOSITORY') - env['CVSCOFLAGS'] = SCons.Util.CLVar('') - env['CVSCOM'] = '$CVS $CVSFLAGS co $CVSCOFLAGS ${TARGET.posix}' - -def exists(env): - return env.Detect('cvs') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/CVS.xml b/src/engine/SCons/Tool/CVS.xml deleted file mode 100644 index 1e695c54..00000000 --- a/src/engine/SCons/Tool/CVS.xml +++ /dev/null @@ -1,159 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<tool name="CVS"> -<summary> -<para> -Sets construction variables for the CVS source code -management system. -</para> -</summary> -<sets> -<item>CVS</item> -<item>CVSCOM</item> -<item>CVSFLAGS</item> -<item>CVSCOFLAGS</item> -</sets> -<uses> -<item>CVSCOMSTR</item> -</uses> -</tool> - -<cvar name="CVS"> -<summary> -<para> -The CVS executable. -</para> -</summary> -</cvar> - -<cvar name="CVSCOFLAGS"> -<summary> -<para> -Options that are passed to the CVS checkout subcommand. -</para> -</summary> -</cvar> - -<cvar name="CVSCOM"> -<summary> -<para> -The command line used to -fetch source files from a CVS repository. -</para> -</summary> -</cvar> - -<cvar name="CVSCOMSTR"> -<summary> -<para> -The string displayed when fetching -a source file from a CVS repository. -If this is not set, then &cv-link-CVSCOM; -(the command line) is displayed. -</para> -</summary> -</cvar> - -<cvar name="CVSFLAGS"> -<summary> -<para> -General options that are passed to CVS. -By default, this is set to -<literal>-d $CVSREPOSITORY</literal> -to specify from where the files must be fetched. -</para> -</summary> -</cvar> - -<cvar name="CVSREPOSITORY"> -<summary> -<para> -The path to the CVS repository. -This is referenced in the default -&cv-link-CVSFLAGS; value. -</para> -</summary> -</cvar> - -<scons_function name="CVS"> -<arguments signature="env"> -(repository, module) -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -from the specified -CVS -<varname>repository</varname>. -The returned Builder -is intended to be passed to the -&f-link-SourceCode; -function. -</para> - -<para> -This function is deprecated. For details, see the entry for the -&f-SourceCode; -function. -</para> - -<para> -The optional specified -<varname>module</varname> -will be added to the beginning -of all repository path names; -this can be used, in essence, -to strip initial directory names -from the repository path names, -so that you only have to -replicate part of the repository -directory hierarchy in your -local build directory. -</para> - -<para> -Examples: -</para> - -<example_commands> -# Will fetch foo/bar/src.c -# from /usr/local/CVSROOT/foo/bar/src.c. -env.SourceCode('.', env.CVS('/usr/local/CVSROOT')) - -# Will fetch bar/src.c -# from /usr/local/CVSROOT/foo/bar/src.c. -env.SourceCode('.', env.CVS('/usr/local/CVSROOT', 'foo')) - -# Will fetch src.c -# from /usr/local/CVSROOT/foo/bar/src.c. -env.SourceCode('.', env.CVS('/usr/local/CVSROOT', 'foo/bar')) -</example_commands> -</summary> -</scons_function> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/DCommon.py b/src/engine/SCons/Tool/DCommon.py index 02a5e732..71c4d8d1 100644 --- a/src/engine/SCons/Tool/DCommon.py +++ b/src/engine/SCons/Tool/DCommon.py @@ -1,3 +1,5 @@ +from __future__ import print_function + """SCons.Tool.DCommon Common code for the various D tools. @@ -5,6 +7,7 @@ Common code for the various D tools. Coded by Russel Winder (russel@winder.org.uk) 2012-09-06 """ + # # __COPYRIGHT__ # @@ -32,6 +35,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path + def isD(env, source): if not source: return 0 @@ -42,6 +46,7 @@ def isD(env, source): return 1 return 0 + def addDPATHToEnv(env, executable): dPath = env.WhereIs(executable) if dPath: @@ -49,6 +54,14 @@ def addDPATHToEnv(env, executable): if os.path.isdir(phobosDir): env.Append(DPATH=[phobosDir]) + +def allAtOnceEmitter(target, source, env): + if env['DC'] in ('ldc2', 'dmd'): + env.SideEffect(str(target[0]) + '.o', target[0]) + env.Clean(target[0], str(target[0]) + '.o') + return target, source + + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Tool/FortranCommon.py b/src/engine/SCons/Tool/FortranCommon.py index e450730b..ec409c03 100644 --- a/src/engine/SCons/Tool/FortranCommon.py +++ b/src/engine/SCons/Tool/FortranCommon.py @@ -207,7 +207,7 @@ def add_f90_to_env(env): except KeyError: F90Suffixes = ['.f90'] - #print "Adding %s to f90 suffixes" % F90Suffixes + #print("Adding %s to f90 suffixes" % F90Suffixes) try: F90PPSuffixes = env['F90PPFILESUFFIXES'] except KeyError: @@ -223,7 +223,7 @@ def add_f95_to_env(env): except KeyError: F95Suffixes = ['.f95'] - #print "Adding %s to f95 suffixes" % F95Suffixes + #print("Adding %s to f95 suffixes" % F95Suffixes) try: F95PPSuffixes = env['F95PPFILESUFFIXES'] except KeyError: @@ -239,7 +239,7 @@ def add_f03_to_env(env): except KeyError: F03Suffixes = ['.f03'] - #print "Adding %s to f95 suffixes" % F95Suffixes + #print("Adding %s to f95 suffixes" % F95Suffixes) try: F03PPSuffixes = env['F03PPFILESUFFIXES'] except KeyError: diff --git a/src/engine/SCons/Tool/GettextCommon.py b/src/engine/SCons/Tool/GettextCommon.py index e23a2d75..3b840a67 100644 --- a/src/engine/SCons/Tool/GettextCommon.py +++ b/src/engine/SCons/Tool/GettextCommon.py @@ -29,15 +29,32 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.Warnings import re + ############################################################################# class XgettextToolWarning(SCons.Warnings.Warning): pass + + class XgettextNotFound(XgettextToolWarning): pass + + class MsginitToolWarning(SCons.Warnings.Warning): pass + + class MsginitNotFound(MsginitToolWarning): pass + + class MsgmergeToolWarning(SCons.Warnings.Warning): pass + + class MsgmergeNotFound(MsgmergeToolWarning): pass + + class MsgfmtToolWarning(SCons.Warnings.Warning): pass + + class MsgfmtNotFound(MsgfmtToolWarning): pass + + ############################################################################# SCons.Warnings.enableWarningClass(XgettextToolWarning) SCons.Warnings.enableWarningClass(XgettextNotFound) @@ -47,367 +64,406 @@ SCons.Warnings.enableWarningClass(MsgmergeToolWarning) SCons.Warnings.enableWarningClass(MsgmergeNotFound) SCons.Warnings.enableWarningClass(MsgfmtToolWarning) SCons.Warnings.enableWarningClass(MsgfmtNotFound) + + ############################################################################# ############################################################################# class _POTargetFactory(object): - """ A factory of `PO` target files. - - Factory defaults differ from these of `SCons.Node.FS.FS`. We set `precious` - (this is required by builders and actions gettext) and `noclean` flags by - default for all produced nodes. - """ - def __init__( self, env, nodefault = True, alias = None, precious = True - , noclean = True ): - """ Object constructor. - - **Arguments** - - - *env* (`SCons.Environment.Environment`) - - *nodefault* (`boolean`) - if `True`, produced nodes will be ignored - from default target `'.'` - - *alias* (`string`) - if provided, produced nodes will be automatically - added to this alias, and alias will be set as `AlwaysBuild` - - *precious* (`boolean`) - if `True`, the produced nodes will be set as - `Precious`. - - *noclen* (`boolean`) - if `True`, the produced nodes will be excluded - from `Clean`. + """ A factory of `PO` target files. + + Factory defaults differ from these of `SCons.Node.FS.FS`. We set `precious` + (this is required by builders and actions gettext) and `noclean` flags by + default for all produced nodes. """ - self.env = env - self.alias = alias - self.precious = precious - self.noclean = noclean - self.nodefault = nodefault - - def _create_node(self, name, factory, directory = None, create = 1): - """ Create node, and set it up to factory settings. """ - import SCons.Util - node = factory(name, directory, create) - node.set_noclean(self.noclean) - node.set_precious(self.precious) - if self.nodefault: - self.env.Ignore('.', node) - if self.alias: - self.env.AlwaysBuild(self.env.Alias(self.alias, node)) - return node - - def Entry(self, name, directory = None, create = 1): - """ Create `SCons.Node.FS.Entry` """ - return self._create_node(name, self.env.fs.Entry, directory, create) - - def File(self, name, directory = None, create = 1): - """ Create `SCons.Node.FS.File` """ - return self._create_node(name, self.env.fs.File, directory, create) + + def __init__(self, env, nodefault=True, alias=None, precious=True + , noclean=True): + """ Object constructor. + + **Arguments** + + - *env* (`SCons.Environment.Environment`) + - *nodefault* (`boolean`) - if `True`, produced nodes will be ignored + from default target `'.'` + - *alias* (`string`) - if provided, produced nodes will be automatically + added to this alias, and alias will be set as `AlwaysBuild` + - *precious* (`boolean`) - if `True`, the produced nodes will be set as + `Precious`. + - *noclen* (`boolean`) - if `True`, the produced nodes will be excluded + from `Clean`. + """ + self.env = env + self.alias = alias + self.precious = precious + self.noclean = noclean + self.nodefault = nodefault + + def _create_node(self, name, factory, directory=None, create=1): + """ Create node, and set it up to factory settings. """ + import SCons.Util + node = factory(name, directory, create) + node.set_noclean(self.noclean) + node.set_precious(self.precious) + if self.nodefault: + self.env.Ignore('.', node) + if self.alias: + self.env.AlwaysBuild(self.env.Alias(self.alias, node)) + return node + + def Entry(self, name, directory=None, create=1): + """ Create `SCons.Node.FS.Entry` """ + return self._create_node(name, self.env.fs.Entry, directory, create) + + def File(self, name, directory=None, create=1): + """ Create `SCons.Node.FS.File` """ + return self._create_node(name, self.env.fs.File, directory, create) + + ############################################################################# ############################################################################# _re_comment = re.compile(r'(#[^\n\r]+)$', re.M) _re_lang = re.compile(r'([a-zA-Z0-9_]+)', re.M) + + ############################################################################# -def _read_linguas_from_files(env, linguas_files = None): - """ Parse `LINGUAS` file and return list of extracted languages """ - import SCons.Util - import SCons.Environment - global _re_comment - global _re_lang - if not SCons.Util.is_List(linguas_files) \ - and not SCons.Util.is_String(linguas_files) \ - and not isinstance(linguas_files, SCons.Node.FS.Base) \ - and linguas_files: - # If, linguas_files==True or such, then read 'LINGUAS' file. - linguas_files = [ 'LINGUAS' ] - if linguas_files is None: - return [] - fnodes = env.arg2nodes(linguas_files) - linguas = [] - for fnode in fnodes: - contents = _re_comment.sub("", fnode.get_text_contents()) - ls = [ l for l in _re_lang.findall(contents) if l ] - linguas.extend(ls) - return linguas +def _read_linguas_from_files(env, linguas_files=None): + """ Parse `LINGUAS` file and return list of extracted languages """ + import SCons.Util + import SCons.Environment + global _re_comment + global _re_lang + if not SCons.Util.is_List(linguas_files) \ + and not SCons.Util.is_String(linguas_files) \ + and not isinstance(linguas_files, SCons.Node.FS.Base) \ + and linguas_files: + # If, linguas_files==True or such, then read 'LINGUAS' file. + linguas_files = ['LINGUAS'] + if linguas_files is None: + return [] + fnodes = env.arg2nodes(linguas_files) + linguas = [] + for fnode in fnodes: + contents = _re_comment.sub("", fnode.get_text_contents()) + ls = [l for l in _re_lang.findall(contents) if l] + linguas.extend(ls) + return linguas + + ############################################################################# ############################################################################# from SCons.Builder import BuilderBase + + ############################################################################# class _POFileBuilder(BuilderBase): - """ `PO` file builder. - - This is multi-target single-source builder. In typical situation the source - is single `POT` file, e.g. `messages.pot`, and there are multiple `PO` - targets to be updated from this `POT`. We must run - `SCons.Builder.BuilderBase._execute()` separatelly for each target to track - dependencies separatelly for each target file. + """ `PO` file builder. - **NOTE**: if we call `SCons.Builder.BuilderBase._execute(.., target, ...)` - with target being list of all targets, all targets would be rebuilt each time - one of the targets from this list is missing. This would happen, for example, - when new language `ll` enters `LINGUAS_FILE` (at this moment there is no - `ll.po` file yet). To avoid this, we override - `SCons.Builder.BuilerBase._execute()` and call it separatelly for each - target. Here we also append to the target list the languages read from - `LINGUAS_FILE`. - """ - # - #* The argument for overriding _execute(): We must use environment with - # builder overrides applied (see BuilderBase.__init__(). Here it comes for - # free. - #* The argument against using 'emitter': The emitter is called too late - # by BuilderBase._execute(). If user calls, for example: - # - # env.POUpdate(LINGUAS_FILE = 'LINGUAS') - # - # the builder throws error, because it is called with target=None, - # source=None and is trying to "generate" sources or target list first. - # If user calls - # - # env.POUpdate(['foo', 'baz'], LINGUAS_FILE = 'LINGUAS') - # - # the env.BuilderWrapper() calls our builder with target=None, - # source=['foo', 'baz']. The BuilderBase._execute() then splits execution - # and execute iterativelly (recursion) self._execute(None, source[i]). - # After that it calls emitter (which is quite too late). The emitter is - # also called in each iteration, what makes things yet worse. - def __init__(self, env, **kw): - if not 'suffix' in kw: - kw['suffix'] = '$POSUFFIX' - if not 'src_suffix' in kw: - kw['src_suffix'] = '$POTSUFFIX' - if not 'src_builder' in kw: - kw['src_builder'] = '_POTUpdateBuilder' - if not 'single_source' in kw: - kw['single_source'] = True - alias = None - if 'target_alias' in kw: - alias = kw['target_alias'] - del kw['target_alias'] - if not 'target_factory' in kw: - kw['target_factory'] = _POTargetFactory(env, alias=alias).File - BuilderBase.__init__(self, **kw) - - def _execute(self, env, target, source, *args, **kw): - """ Execute builder's actions. + This is multi-target single-source builder. In typical situation the source + is single `POT` file, e.g. `messages.pot`, and there are multiple `PO` + targets to be updated from this `POT`. We must run + `SCons.Builder.BuilderBase._execute()` separatelly for each target to track + dependencies separatelly for each target file. - Here we append to `target` the languages read from `$LINGUAS_FILE` and - apply `SCons.Builder.BuilderBase._execute()` separatelly to each target. - The arguments and return value are same as for - `SCons.Builder.BuilderBase._execute()`. + **NOTE**: if we call `SCons.Builder.BuilderBase._execute(.., target, ...)` + with target being list of all targets, all targets would be rebuilt each time + one of the targets from this list is missing. This would happen, for example, + when new language `ll` enters `LINGUAS_FILE` (at this moment there is no + `ll.po` file yet). To avoid this, we override + `SCons.Builder.BuilerBase._execute()` and call it separatelly for each + target. Here we also append to the target list the languages read from + `LINGUAS_FILE`. """ - import SCons.Util - import SCons.Node - linguas_files = None - if 'LINGUAS_FILE' in env and env['LINGUAS_FILE']: - linguas_files = env['LINGUAS_FILE'] - # This prevents endless recursion loop (we'll be invoked once for - # each target appended here, we must not extend the list again). - env['LINGUAS_FILE'] = None - linguas = _read_linguas_from_files(env,linguas_files) - if SCons.Util.is_List(target): - target.extend(linguas) - elif target is not None: - target = [target] + linguas - else: - target = linguas - if not target: - # Let the SCons.BuilderBase to handle this patologic situation - return BuilderBase._execute( self, env, target, source, *args, **kw) - # The rest is ours - if not SCons.Util.is_List(target): - target = [ target ] - result = [] - for tgt in target: - r = BuilderBase._execute( self, env, [tgt], source, *args, **kw) - result.extend(r) - if linguas_files is not None: - env['LINGUAS_FILE'] = linguas_files - return SCons.Node.NodeList(result) + + # + # * The argument for overriding _execute(): We must use environment with + # builder overrides applied (see BuilderBase.__init__(). Here it comes for + # free. + # * The argument against using 'emitter': The emitter is called too late + # by BuilderBase._execute(). If user calls, for example: + # + # env.POUpdate(LINGUAS_FILE = 'LINGUAS') + # + # the builder throws error, because it is called with target=None, + # source=None and is trying to "generate" sources or target list first. + # If user calls + # + # env.POUpdate(['foo', 'baz'], LINGUAS_FILE = 'LINGUAS') + # + # the env.BuilderWrapper() calls our builder with target=None, + # source=['foo', 'baz']. The BuilderBase._execute() then splits execution + # and execute iterativelly (recursion) self._execute(None, source[i]). + # After that it calls emitter (which is quite too late). The emitter is + # also called in each iteration, what makes things yet worse. + def __init__(self, env, **kw): + if not 'suffix' in kw: + kw['suffix'] = '$POSUFFIX' + if not 'src_suffix' in kw: + kw['src_suffix'] = '$POTSUFFIX' + if not 'src_builder' in kw: + kw['src_builder'] = '_POTUpdateBuilder' + if not 'single_source' in kw: + kw['single_source'] = True + alias = None + if 'target_alias' in kw: + alias = kw['target_alias'] + del kw['target_alias'] + if not 'target_factory' in kw: + kw['target_factory'] = _POTargetFactory(env, alias=alias).File + BuilderBase.__init__(self, **kw) + + def _execute(self, env, target, source, *args, **kw): + """ Execute builder's actions. + + Here we append to `target` the languages read from `$LINGUAS_FILE` and + apply `SCons.Builder.BuilderBase._execute()` separatelly to each target. + The arguments and return value are same as for + `SCons.Builder.BuilderBase._execute()`. + """ + import SCons.Util + import SCons.Node + linguas_files = None + if 'LINGUAS_FILE' in env and env['LINGUAS_FILE']: + linguas_files = env['LINGUAS_FILE'] + # This prevents endless recursion loop (we'll be invoked once for + # each target appended here, we must not extend the list again). + env['LINGUAS_FILE'] = None + linguas = _read_linguas_from_files(env, linguas_files) + if SCons.Util.is_List(target): + target.extend(linguas) + elif target is not None: + target = [target] + linguas + else: + target = linguas + if not target: + # Let the SCons.BuilderBase to handle this patologic situation + return BuilderBase._execute(self, env, target, source, *args, **kw) + # The rest is ours + if not SCons.Util.is_List(target): + target = [target] + result = [] + for tgt in target: + r = BuilderBase._execute(self, env, [tgt], source, *args, **kw) + result.extend(r) + if linguas_files is not None: + env['LINGUAS_FILE'] = linguas_files + return SCons.Node.NodeList(result) + + ############################################################################# import SCons.Environment + + ############################################################################# def _translate(env, target=None, source=SCons.Environment._null, *args, **kw): - """ Function for `Translate()` pseudo-builder """ - if target is None: target = [] - pot = env.POTUpdate(None, source, *args, **kw) - po = env.POUpdate(target, pot, *args, **kw) - return po + """ Function for `Translate()` pseudo-builder """ + if target is None: target = [] + pot = env.POTUpdate(None, source, *args, **kw) + po = env.POUpdate(target, pot, *args, **kw) + return po + + ############################################################################# ############################################################################# class RPaths(object): - """ Callable object, which returns pathnames relative to SCons current - working directory. - - It seems like `SCons.Node.FS.Base.get_path()` returns absolute paths - for nodes that are outside of current working directory (`env.fs.getcwd()`). - Here, we often have `SConscript`, `POT` and `PO` files within `po/` - directory and source files (e.g. `*.c`) outside of it. When generating `POT` - template file, references to source files are written to `POT` template, so - a translator may later quickly jump to appropriate source file and line from - its `PO` editor (e.g. `poedit`). Relative paths in `PO` file are usually - interpreted by `PO` editor as paths relative to the place, where `PO` file - lives. The absolute paths would make resultant `POT` file nonportable, as - the references would be correct only on the machine, where `POT` file was - recently re-created. For such reason, we need a function, which always - returns relative paths. This is the purpose of `RPaths` callable object. - - The `__call__` method returns paths relative to current working directory, but - we assume, that *xgettext(1)* is run from the directory, where target file is - going to be created. - - Note, that this may not work for files distributed over several hosts or - across different drives on windows. We assume here, that single local - filesystem holds both source files and target `POT` templates. - - Intended use of `RPaths` - in `xgettext.py`:: - - def generate(env): - from GettextCommon import RPaths - ... - sources = '$( ${_concat( "", SOURCES, "", __env__, XgettextRPaths, TARGET, SOURCES)} $)' - env.Append( - ... - XGETTEXTCOM = 'XGETTEXT ... ' + sources, - ... - XgettextRPaths = RPaths(env) - ) - """ - # NOTE: This callable object returns pathnames of dirs/files relative to - # current working directory. The pathname remains relative also for entries - # that are outside of current working directory (node, that - # SCons.Node.FS.File and siblings return absolute path in such case). For - # simplicity we compute path relative to current working directory, this - # seems be enough for our purposes (don't need TARGET variable and - # SCons.Defaults.Variable_Caller stuff). + """ Callable object, which returns pathnames relative to SCons current + working directory. - def __init__(self, env): - """ Initialize `RPaths` callable object. - - **Arguments**: - - - *env* - a `SCons.Environment.Environment` object, defines *current - working dir*. + It seems like `SCons.Node.FS.Base.get_path()` returns absolute paths + for nodes that are outside of current working directory (`env.fs.getcwd()`). + Here, we often have `SConscript`, `POT` and `PO` files within `po/` + directory and source files (e.g. `*.c`) outside of it. When generating `POT` + template file, references to source files are written to `POT` template, so + a translator may later quickly jump to appropriate source file and line from + its `PO` editor (e.g. `poedit`). Relative paths in `PO` file are usually + interpreted by `PO` editor as paths relative to the place, where `PO` file + lives. The absolute paths would make resultant `POT` file nonportable, as + the references would be correct only on the machine, where `POT` file was + recently re-created. For such reason, we need a function, which always + returns relative paths. This is the purpose of `RPaths` callable object. + + The `__call__` method returns paths relative to current working directory, but + we assume, that *xgettext(1)* is run from the directory, where target file is + going to be created. + + Note, that this may not work for files distributed over several hosts or + across different drives on windows. We assume here, that single local + filesystem holds both source files and target `POT` templates. + + Intended use of `RPaths` - in `xgettext.py`:: + + def generate(env): + from GettextCommon import RPaths + ... + sources = '$( ${_concat( "", SOURCES, "", __env__, XgettextRPaths, TARGET, SOURCES)} $)' + env.Append( + ... + XGETTEXTCOM = 'XGETTEXT ... ' + sources, + ... + XgettextRPaths = RPaths(env) + ) """ - self.env = env - # FIXME: I'm not sure, how it should be implemented (what the *args are in - # general, what is **kw). - def __call__(self, nodes, *args, **kw): - """ Return nodes' paths (strings) relative to current working directory. - - **Arguments**: + # NOTE: This callable object returns pathnames of dirs/files relative to + # current working directory. The pathname remains relative also for entries + # that are outside of current working directory (node, that + # SCons.Node.FS.File and siblings return absolute path in such case). For + # simplicity we compute path relative to current working directory, this + # seems be enough for our purposes (don't need TARGET variable and + # SCons.Defaults.Variable_Caller stuff). - - *nodes* ([`SCons.Node.FS.Base`]) - list of nodes. - - *args* - currently unused. - - *kw* - currently unused. + def __init__(self, env): + """ Initialize `RPaths` callable object. + + **Arguments**: + + - *env* - a `SCons.Environment.Environment` object, defines *current + working dir*. + """ + self.env = env + + # FIXME: I'm not sure, how it should be implemented (what the *args are in + # general, what is **kw). + def __call__(self, nodes, *args, **kw): + """ Return nodes' paths (strings) relative to current working directory. + + **Arguments**: + + - *nodes* ([`SCons.Node.FS.Base`]) - list of nodes. + - *args* - currently unused. + - *kw* - currently unused. + + **Returns**: + + - Tuple of strings, which represent paths relative to current working + directory (for given environment). + """ + import os + import SCons.Node.FS + rpaths = () + cwd = self.env.fs.getcwd().get_abspath() + for node in nodes: + rpath = None + if isinstance(node, SCons.Node.FS.Base): + rpath = os.path.relpath(node.get_abspath(), cwd) + # FIXME: Other types possible here? + if rpath is not None: + rpaths += (rpath,) + return rpaths - **Returns**: - - Tuple of strings, which represent paths relative to current working - directory (for given environment). - """ - import os - import SCons.Node.FS - rpaths = () - cwd = self.env.fs.getcwd().get_abspath() - for node in nodes: - rpath = None - if isinstance(node, SCons.Node.FS.Base): - rpath = os.path.relpath(node.get_abspath(), cwd) - # FIXME: Other types possible here? - if rpath is not None: - rpaths += (rpath,) - return rpaths ############################################################################# - + ############################################################################# def _init_po_files(target, source, env): - """ Action function for `POInit` builder. """ - nop = lambda target, source, env : 0 - if 'POAUTOINIT' in env: - autoinit = env['POAUTOINIT'] - else: - autoinit = False - # Well, if everything outside works well, this loop should do single - # iteration. Otherwise we are rebuilding all the targets even, if just - # one has changed (but is this our fault?). - for tgt in target: - if not tgt.exists(): - if autoinit: - action = SCons.Action.Action('$MSGINITCOM', '$MSGINITCOMSTR') - else: - msg = 'File ' + repr(str(tgt)) + ' does not exist. ' \ - + 'If you are a translator, you can create it through: \n' \ - + '$MSGINITCOM' - action = SCons.Action.Action(nop, msg) - status = action([tgt], source, env) - if status: return status - return 0 + """ Action function for `POInit` builder. """ + nop = lambda target, source, env: 0 + if 'POAUTOINIT' in env: + autoinit = env['POAUTOINIT'] + else: + autoinit = False + # Well, if everything outside works well, this loop should do single + # iteration. Otherwise we are rebuilding all the targets even, if just + # one has changed (but is this our fault?). + for tgt in target: + if not tgt.exists(): + if autoinit: + action = SCons.Action.Action('$MSGINITCOM', '$MSGINITCOMSTR') + else: + msg = 'File ' + repr(str(tgt)) + ' does not exist. ' \ + + 'If you are a translator, you can create it through: \n' \ + + '$MSGINITCOM' + action = SCons.Action.Action(nop, msg) + status = action([tgt], source, env) + if status: return status + return 0 + + ############################################################################# ############################################################################# def _detect_xgettext(env): - """ Detects *xgettext(1)* binary """ - if 'XGETTEXT' in env: - return env['XGETTEXT'] - xgettext = env.Detect('xgettext'); - if xgettext: - return xgettext - raise SCons.Errors.StopError(XgettextNotFound,"Could not detect xgettext") - return None + """ Detects *xgettext(1)* binary """ + if 'XGETTEXT' in env: + return env['XGETTEXT'] + xgettext = env.Detect('xgettext'); + if xgettext: + return xgettext + raise SCons.Errors.StopError(XgettextNotFound, "Could not detect xgettext") + return None + + ############################################################################# def _xgettext_exists(env): - return _detect_xgettext(env) + return _detect_xgettext(env) + + ############################################################################# ############################################################################# def _detect_msginit(env): - """ Detects *msginit(1)* program. """ - if 'MSGINIT' in env: - return env['MSGINIT'] - msginit = env.Detect('msginit'); - if msginit: - return msginit - raise SCons.Errors.StopError(MsginitNotFound, "Could not detect msginit") - return None + """ Detects *msginit(1)* program. """ + if 'MSGINIT' in env: + return env['MSGINIT'] + msginit = env.Detect('msginit'); + if msginit: + return msginit + raise SCons.Errors.StopError(MsginitNotFound, "Could not detect msginit") + return None + + ############################################################################# def _msginit_exists(env): - return _detect_msginit(env) + return _detect_msginit(env) + + ############################################################################# ############################################################################# def _detect_msgmerge(env): - """ Detects *msgmerge(1)* program. """ - if 'MSGMERGE' in env: - return env['MSGMERGE'] - msgmerge = env.Detect('msgmerge'); - if msgmerge: - return msgmerge - raise SCons.Errors.StopError(MsgmergeNotFound, "Could not detect msgmerge") - return None + """ Detects *msgmerge(1)* program. """ + if 'MSGMERGE' in env: + return env['MSGMERGE'] + msgmerge = env.Detect('msgmerge'); + if msgmerge: + return msgmerge + raise SCons.Errors.StopError(MsgmergeNotFound, "Could not detect msgmerge") + return None + + ############################################################################# def _msgmerge_exists(env): - return _detect_msgmerge(env) + return _detect_msgmerge(env) + + ############################################################################# ############################################################################# def _detect_msgfmt(env): - """ Detects *msgmfmt(1)* program. """ - if 'MSGFMT' in env: - return env['MSGFMT'] - msgfmt = env.Detect('msgfmt'); - if msgfmt: - return msgfmt - raise SCons.Errors.StopError(MsgfmtNotFound, "Could not detect msgfmt") - return None + """ Detects *msgmfmt(1)* program. """ + if 'MSGFMT' in env: + return env['MSGFMT'] + msgfmt = env.Detect('msgfmt'); + if msgfmt: + return msgfmt + raise SCons.Errors.StopError(MsgfmtNotFound, "Could not detect msgfmt") + return None + + ############################################################################# def _msgfmt_exists(env): - return _detect_msgfmt(env) + return _detect_msgfmt(env) + + ############################################################################# ############################################################################# def tool_list(platform, env): - """ List tools that shall be generated by top-level `gettext` tool """ - return [ 'xgettext', 'msginit', 'msgmerge', 'msgfmt' ] -############################################################################# + """ List tools that shall be generated by top-level `gettext` tool """ + return ['xgettext', 'msginit', 'msgmerge', 'msgfmt'] +############################################################################# diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b14eba18..b60cd5b4 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -1,3 +1,6 @@ +""" +Common helper functions for working with the Microsoft tool chain. +""" # # __COPYRIGHT__ # @@ -24,10 +27,6 @@ from __future__ import print_function __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -__doc__ = """ -Common helper functions for working with the Microsoft tool chain. -""" - import copy import os import subprocess @@ -36,17 +35,17 @@ import re import SCons.Util -logfile = os.environ.get('SCONS_MSCOMMON_DEBUG') -if logfile == '-': - def debug(x): - print(x) -elif logfile: +LOGFILE = os.environ.get('SCONS_MSCOMMON_DEBUG') +if LOGFILE == '-': + def debug(message): + print(message) +elif LOGFILE: try: import logging except ImportError: - debug = lambda x: open(logfile, 'a').write(x + '\n') + debug = lambda message: open(LOGFILE, 'a').write(message + '\n') else: - logging.basicConfig(filename=logfile, level=logging.DEBUG) + logging.basicConfig(filename=LOGFILE, level=logging.DEBUG) debug = logging.debug else: debug = lambda x: None @@ -76,7 +75,7 @@ def is_win64(): # I structured these tests to make it easy to add new ones or # add exceptions in the future, because this is a bit fragile. _is_win64 = False - if os.environ.get('PROCESSOR_ARCHITECTURE','x86') != 'x86': + if os.environ.get('PROCESSOR_ARCHITECTURE', 'x86') != 'x86': _is_win64 = True if os.environ.get('PROCESSOR_ARCHITEW6432'): _is_win64 = True @@ -124,11 +123,20 @@ def normalize_env(env, keys, force=False): # This shouldn't be necessary, since the default environment should include system32, # but keep this here to be safe, since it's needed to find reg.exe which the MSVC # bat scripts use. - sys32_dir = os.path.join(os.environ.get("SystemRoot", os.environ.get("windir",r"C:\Windows\system32")),"System32") + sys32_dir = os.path.join(os.environ.get("SystemRoot", + os.environ.get("windir", r"C:\Windows\system32")), + "System32") if sys32_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_dir + # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized" + # error starting with Visual Studio 2017, although the script still + # seems to work anyway. + sys32_wbem_dir = os.path.join(sys32_dir, 'Wbem') + if sys32_wbem_dir not in normenv['PATH']: + normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir + debug("PATH: %s"%normenv['PATH']) return normenv @@ -145,11 +153,13 @@ def get_output(vcbat, args = None, env = None): # execution to work. This list should really be either directly # controlled by vc.py, or else derived from the common_tools_var # settings in vs.py. - vars = [ + vs_vc_vars = [ 'COMSPEC', -# VS100 and VS110: Still set, but modern MSVC setup scripts will -# discard these if registry has values. However Intel compiler setup -# script still requires these as of 2013/2014. + # VS100 and VS110: Still set, but modern MSVC setup scripts will + # discard these if registry has values. However Intel compiler setup + # script still requires these as of 2013/2014. + 'VS140COMNTOOLS', + 'VS120COMNTOOLS', 'VS110COMNTOOLS', 'VS100COMNTOOLS', 'VS90COMNTOOLS', @@ -158,22 +168,22 @@ def get_output(vcbat, args = None, env = None): 'VS70COMNTOOLS', 'VS60COMNTOOLS', ] - env['ENV'] = normalize_env(env['ENV'], vars, force=False) + env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False) if args: debug("Calling '%s %s'" % (vcbat, args)) popen = SCons.Action._subproc(env, - '"%s" %s & set' % (vcbat, args), - stdin = 'devnull', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + '"%s" %s & set' % (vcbat, args), + stdin='devnull', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) else: debug("Calling '%s'" % vcbat) popen = SCons.Action._subproc(env, - '"%s" & set' % vcbat, - stdin = 'devnull', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + '"%s" & set' % vcbat, + stdin='devnull', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) # Use the .stdout and .stderr attributes directly because the # .communicate() method uses the threading module on Windows @@ -196,10 +206,14 @@ def get_output(vcbat, args = None, env = None): output = stdout.decode("mbcs") return output -def parse_output(output, keep = ("INCLUDE", "LIB", "LIBPATH", "PATH")): +def parse_output(output, keep=("INCLUDE", "LIB", "LIBPATH", "PATH")): + """ + Parse output from running visual c++/studios vcvarsall.bat and running set + To capture the values listed in keep + """ + # dkeep is a dict associating key: path_list, where key is one item from # keep, and pat_list the associated list of paths - dkeep = dict([(i, []) for i in keep]) # rdk will keep the regex to match the .bat file output line starts @@ -208,21 +222,21 @@ def parse_output(output, keep = ("INCLUDE", "LIB", "LIBPATH", "PATH")): rdk[i] = re.compile('%s=(.*)' % i, re.I) def add_env(rmatch, key, dkeep=dkeep): - plist = rmatch.group(1).split(os.pathsep) - for p in plist: + path_list = rmatch.group(1).split(os.pathsep) + for path in path_list: # Do not add empty paths (when a var ends with ;) - if p: + if path: # XXX: For some reason, VC98 .bat file adds "" around the PATH # values, and it screws up the environment later, so we strip # it. - p = p.strip('"') - dkeep[key].append(p) + path = path.strip('"') + dkeep[key].append(str(path)) for line in output.splitlines(): - for k,v in list(rdk.items()): - m = v.match(line) - if m: - add_env(m, k) + for k, value in rdk.items(): + match = value.match(line) + if match: + add_env(match, k) return dkeep diff --git a/src/engine/SCons/Tool/MSCommon/sdk.py b/src/engine/SCons/Tool/MSCommon/sdk.py index 18455994..43b0272d 100644 --- a/src/engine/SCons/Tool/MSCommon/sdk.py +++ b/src/engine/SCons/Tool/MSCommon/sdk.py @@ -164,6 +164,12 @@ SDK70VCSetupScripts = { 'x86' : r'bin\vcvars32.bat', 'x86_ia64' : r'bin\vcvarsx86_ia64.bat', 'ia64' : r'bin\vcvarsia64.bat'} +SDK100VCSetupScripts = {'x86' : r'bin\vcvars32.bat', + 'amd64' : r'bin\vcvars64.bat', + 'x86_amd64': r'bin\x86_amd64\vcvarsx86_amd64.bat', + 'x86_arm' : r'bin\x86_arm\vcvarsx86_arm.bat'} + + # The list of support SDKs which we know how to detect. # # The first SDK found in the list is the one used by default if there @@ -172,6 +178,16 @@ SDK70VCSetupScripts = { 'x86' : r'bin\vcvars32.bat', # # If you update this list, update the documentation in Tool/mssdk.xml. SupportedSDKList = [ + WindowsSDK('10.0', + sanity_check_file=r'bin\SetEnv.Cmd', + include_subdir='include', + lib_subdir={ + 'x86' : ['lib'], + 'x86_64' : [r'lib\x64'], + 'ia64' : [r'lib\ia64'], + }, + vc_setup_scripts = SDK70VCSetupScripts, + ), WindowsSDK('7.1', sanity_check_file=r'bin\SetEnv.Cmd', include_subdir='include', diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index 588fe98c..c76afacf 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -37,6 +37,7 @@ __doc__ = """Module for Visual C/C++ detection and configuration. import SCons.compat import SCons.Util +import subprocess import os import platform from string import digits as string_digits @@ -135,9 +136,11 @@ def get_host_target(env): # If you update this, update SupportedVSList in Tool/MSCommon/vs.py, and the # MSVC_VERSION documentation in Tool/msvc.xml. -_VCVER = ["14.0", "14.0Exp", "12.0", "12.0Exp", "11.0", "11.0Exp", "10.0", "10.0Exp", "9.0", "9.0Exp","8.0", "8.0Exp","7.1", "7.0", "6.0"] +_VCVER = ["14.1", "14.0", "14.0Exp", "12.0", "12.0Exp", "11.0", "11.0Exp", "10.0", "10.0Exp", "9.0", "9.0Exp","8.0", "8.0Exp","7.1", "7.0", "6.0"] _VCVER_TO_PRODUCT_DIR = { + '14.1' : [ + (SCons.Util.HKEY_LOCAL_MACHINE, r'')], # Visual Studio 2017 doesn't set this registry key anymore '14.0' : [ (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VisualStudio\14.0\Setup\VC\ProductDir')], '14.0Exp' : [ @@ -185,18 +188,17 @@ _VCVER_TO_PRODUCT_DIR = { } def msvc_version_to_maj_min(msvc_version): - msvc_version_numeric = ''.join([x for x in msvc_version if x in string_digits + '.']) t = msvc_version_numeric.split(".") if not len(t) == 2: - raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) + raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) try: - maj = int(t[0]) - min = int(t[1]) - return maj, min + maj = int(t[0]) + min = int(t[1]) + return maj, min except ValueError as e: - raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) + raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) def is_host_target_supported(host_target, msvc_version): """Return True if the given (host, target) tuple is supported given the @@ -223,6 +225,29 @@ def is_host_target_supported(host_target, msvc_version): return True + +def find_vc_pdir_vswhere(msvc_version): + """ + Find the vswhere.exe install. + Run it asking for specified version and get MSVS install location + :param msvc_version: + :return: MSVC install dir + """ + vswhere_path = os.path.join( + 'C:\\', + 'Program Files (x86)', + 'Microsoft Visual Studio', + 'Installer', + 'vswhere.exe' + ) + vswhere_cmd = [vswhere_path, '-version', msvc_version, '-property', 'installationPath'] + + sp = subprocess.Popen(vswhere_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + vsdir, err = sp.communicate() + vc_pdir = os.path.join(vsdir.rstrip(), 'VC') + return vc_pdir + + def find_vc_pdir(msvc_version): """Try to find the product directory for the given version. @@ -241,16 +266,19 @@ def find_vc_pdir(msvc_version): for hkroot, key in hkeys: try: comps = None - if common.is_win64(): - try: - # ordinally at win64, try Wow6432Node first. - comps = common.read_reg(root + 'Wow6432Node\\' + key, hkroot) - except SCons.Util.WinError as e: - # at Microsoft Visual Studio for Python 2.7, value is not in Wow6432Node - pass - if not comps: - # not Win64, or Microsoft Visual Studio for Python 2.7 - comps = common.read_reg(root + key, hkroot) + if not key: + comps = find_vc_pdir_vswhere(msvc_version) + else: + if common.is_win64(): + try: + # ordinally at win64, try Wow6432Node first. + comps = common.read_reg(root + 'Wow6432Node\\' + key, hkroot) + except SCons.Util.WinError as e: + # at Microsoft Visual Studio for Python 2.7, value is not in Wow6432Node + pass + if not comps: + # not Win64, or Microsoft Visual Studio for Python 2.7 + comps = common.read_reg(root + key, hkroot) except SCons.Util.WinError as e: debug('find_vc_dir(): no VC registry key {}'.format(repr(key))) else: @@ -282,8 +310,10 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): elif vernum < 7: pdir = os.path.join(pdir, "Bin") batfilename = os.path.join(pdir, "vcvars32.bat") - else: # >= 8 + elif 8 <= vernum <= 14: batfilename = os.path.join(pdir, "vcvarsall.bat") + else: # vernum >= 14.1 VS2017 and above + batfilename = os.path.join(pdir, "Auxiliary", "Build", "vcvarsall.bat") if not os.path.exists(batfilename): debug("Not found: %s" % batfilename) @@ -435,6 +465,14 @@ def msvc_find_valid_batch_script(env,version): (host_target, version) SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) arg = _HOST_TARGET_ARCH_TO_BAT_ARCH[host_target] + + # Get just version numbers + maj, min = msvc_version_to_maj_min(version) + # VS2015+ + if maj >= 14: + if env.get('MSVC_UWP_APP') == '1': + # Initialize environment variables with store/universal paths + arg += ' store' # Try to locate a batch file for this host/target platform combo try: @@ -515,7 +553,7 @@ def msvc_setup_env(env): SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) return None - for k, v in list(d.items()): + for k, v in d.items(): debug('vc.py:msvc_setup_env() env:%s -> %s'%(k,v)) env.PrependENVPath(k, v, delete_existing=True) diff --git a/src/engine/SCons/Tool/MSCommon/vs.py b/src/engine/SCons/Tool/MSCommon/vs.py index 4bda4060..f9382fbd 100644 --- a/src/engine/SCons/Tool/MSCommon/vs.py +++ b/src/engine/SCons/Tool/MSCommon/vs.py @@ -199,10 +199,21 @@ class VisualStudio(object): # Tool/MSCommon/vc.py, and the MSVC_VERSION documentation in Tool/msvc.xml. SupportedVSList = [ + # Visual Studio 2017 + VisualStudio('14.1', + vc_version='14.1', + sdk_version='10.0A', + hkeys=[], + common_tools_var='VS150COMNTOOLS', + executable_path=r'Common7\IDE\devenv.com', + batch_file_path=r'VC\Auxiliary\Build\vsvars32.bat', + supported_arch=['x86', 'amd64', "arm"], + ), + # Visual Studio 2015 VisualStudio('14.0', vc_version='14.0', - sdk_version='10.0A', + sdk_version='10.0', hkeys=[r'Microsoft\VisualStudio\14.0\Setup\VS\ProductDir'], common_tools_var='VS140COMNTOOLS', executable_path=r'Common7\IDE\devenv.com', @@ -545,7 +556,7 @@ def msvs_setup_env(env): env['ENV'] = save_ENV vars = parse_output(output, vars) - for k, v in list(vars.items()): + for k, v in vars.items(): env.PrependENVPath(k, v, delete_existing=1) def query_versions(): diff --git a/src/engine/SCons/Tool/Perforce.py b/src/engine/SCons/Tool/Perforce.py deleted file mode 100644 index c8cf931f..00000000 --- a/src/engine/SCons/Tool/Perforce.py +++ /dev/null @@ -1,99 +0,0 @@ -"""SCons.Tool.Perforce.py - -Tool-specific initialization for Perforce Source Code Management system. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os - -import SCons.Action -import SCons.Builder -import SCons.Node.FS -import SCons.Util - - -# Variables that we want to import from the base OS environment. -_import_env = [ 'P4PORT', 'P4CLIENT', 'P4USER', 'USER', 'USERNAME', 'P4PASSWD', - 'P4CHARSET', 'P4LANGUAGE', 'SystemRoot' ] - -PerforceAction = SCons.Action.Action('$P4COM', '$P4COMSTR') - -def generate(env): - """Add a Builder factory function and construction variables for - Perforce to an Environment.""" - - def PerforceFactory(env=env): - """ """ - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The Perforce() factory is deprecated and there is no replacement.""") - return SCons.Builder.Builder(action = PerforceAction, env = env) - - env.Perforce = PerforceFactory - - env['P4'] = 'p4' - env['P4FLAGS'] = SCons.Util.CLVar('') - env['P4COM'] = '$P4 $P4FLAGS sync $TARGET' - try: - environ = env['ENV'] - except KeyError: - environ = {} - env['ENV'] = environ - - # Perforce seems to use the PWD environment variable rather than - # calling getcwd() for itself, which is odd. If no PWD variable - # is present, p4 WILL call getcwd, but this seems to cause problems - # with good ol' Windows's tilde-mangling for long file names. - environ['PWD'] = env.Dir('#').get_abspath() - - for var in _import_env: - v = os.environ.get(var) - if v: - environ[var] = v - - if SCons.Util.can_read_reg: - # If we can read the registry, add the path to Perforce to our environment. - try: - k=SCons.Util.RegOpenKeyEx(SCons.Util.hkey_mod.HKEY_LOCAL_MACHINE, - 'Software\\Perforce\\environment') - val, tok = SCons.Util.RegQueryValueEx(k, 'P4INSTROOT') - SCons.Util.AddPathIfNotExists(environ, 'PATH', val) - except SCons.Util.RegError: - # Can't detect where Perforce is, hope the user has it set in the - # PATH. - pass - -def exists(env): - return env.Detect('p4') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/Perforce.xml b/src/engine/SCons/Tool/Perforce.xml deleted file mode 100644 index 918bbc95..00000000 --- a/src/engine/SCons/Tool/Perforce.xml +++ /dev/null @@ -1,129 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<tool name="Perforce"> -<summary> -<para> -Sets construction variables for interacting with the -Perforce source code management system. -</para> -</summary> -<sets> -<item>P4</item> -<item>P4FLAGS</item> -<item>P4COM</item> -</sets> -<uses> -<item>P4COMSTR</item> -</uses> -</tool> - -<cvar name="P4"> -<summary> -<para> -The Perforce executable. -</para> -</summary> -</cvar> - -<cvar name="P4COM"> -<summary> -<para> -The command line used to -fetch source files from Perforce. -</para> -</summary> -</cvar> - -<cvar name="P4COMSTR"> -<summary> -<para> -The string displayed when -fetching a source file from Perforce. -If this is not set, then &cv-link-P4COM; (the command line) is displayed. -</para> -</summary> -</cvar> - -<cvar name="P4FLAGS"> -<summary> -<para> -General options that are passed to Perforce. -</para> -</summary> -</cvar> - -<scons_function name="Perforce"> -<arguments signature="env"> -() -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -from the Perforce source code management system. -The returned Builder -is intended to be passed to the -&f-SourceCode; -function. -</para> - -<para> -This function is deprecated. For details, see the entry for the -&f-SourceCode; -function. -</para> - -<para> -Example: -</para> - -<example_commands> -env.SourceCode('.', env.Perforce()) -</example_commands> - -<para> -Perforce uses a number of external -environment variables for its operation. -Consequently, this function adds the -following variables from the user's external environment -to the construction environment's -ENV dictionary: -P4CHARSET, -P4CLIENT, -P4LANGUAGE, -P4PASSWD, -P4PORT, -P4USER, -SystemRoot, -USER, -and -USERNAME. -</para> -</summary> -</scons_function> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/RCS.py b/src/engine/SCons/Tool/RCS.py deleted file mode 100644 index e24b89a3..00000000 --- a/src/engine/SCons/Tool/RCS.py +++ /dev/null @@ -1,63 +0,0 @@ -"""SCons.Tool.RCS.py - -Tool-specific initialization for RCS. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import SCons.Action -import SCons.Builder -import SCons.Util - -def generate(env): - """Add a Builder factory function and construction variables for - RCS to an Environment.""" - - def RCSFactory(env=env): - """ """ - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The RCS() factory is deprecated and there is no replacement.""") - act = SCons.Action.Action('$RCS_COCOM', '$RCS_COCOMSTR') - return SCons.Builder.Builder(action = act, env = env) - - env.RCS = RCSFactory - - env['RCS'] = 'rcs' - env['RCS_CO'] = 'co' - env['RCS_COFLAGS'] = SCons.Util.CLVar('') - env['RCS_COCOM'] = '$RCS_CO $RCS_COFLAGS $TARGET' - -def exists(env): - return env.Detect('rcs') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/RCS.xml b/src/engine/SCons/Tool/RCS.xml deleted file mode 100644 index 760f5c0a..00000000 --- a/src/engine/SCons/Tool/RCS.xml +++ /dev/null @@ -1,142 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<tool name="RCS"> -<summary> -<para> -Sets construction variables for the interaction -with the Revision Control System. -</para> -</summary> -<sets> -<item>RCS</item> -<item>RCS_CO</item> -<item>RCS_COFLAGS</item> -<item>RCS_COCOM</item> -</sets> -<uses> -<item>RCS_COCOMSTR</item> -</uses> -</tool> - -<cvar name="RCS"> -<summary> -<para> -The RCS executable. -Note that this variable is not actually used -for the command to fetch source files from RCS; -see the -&cv-link-RCS_CO; -construction variable, below. -</para> -</summary> -</cvar> - -<cvar name="RCS_CO"> -<summary> -<para> -The RCS "checkout" executable, -used to fetch source files from RCS. -</para> -</summary> -</cvar> - -<cvar name="RCS_COCOM"> -<summary> -<para> -The command line used to -fetch (checkout) source files from RCS. -</para> -</summary> -</cvar> - -<cvar name="RCS_COCOMSTR"> -<summary> -<para> -The string displayed when fetching -a source file from RCS. -If this is not set, then &cv-link-RCS_COCOM; -(the command line) is displayed. -</para> -</summary> -</cvar> - -<cvar name="RCS_COFLAGS"> -<summary> -<para> -Options that are passed to the &cv-link-RCS_CO; command. -</para> -</summary> -</cvar> - -<scons_function name="RCS"> -<arguments signature="env"> -() -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -from RCS. -The returned Builder -is intended to be passed to the -&f-SourceCode; -function: -</para> - -<para> -This function is deprecated. For details, see the entry for the -&f-SourceCode; -function. -</para> - -<para> -Examples: -</para> - -<example_commands> -env.SourceCode('.', env.RCS()) -</example_commands> - -<para> -Note that -&scons; -will fetch source files -from RCS subdirectories automatically, -so configuring RCS -as demonstrated in the above example -should only be necessary if -you are fetching from -RCS,v -files in the same -directory as the source files, -or if you need to explicitly specify RCS -for a specific subdirectory. -</para> -</summary> -</scons_function> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/SCCS.py b/src/engine/SCons/Tool/SCCS.py deleted file mode 100644 index 92ded51d..00000000 --- a/src/engine/SCons/Tool/SCCS.py +++ /dev/null @@ -1,63 +0,0 @@ -"""SCons.Tool.SCCS.py - -Tool-specific initialization for SCCS. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import SCons.Action -import SCons.Builder -import SCons.Util - -def generate(env): - """Add a Builder factory function and construction variables for - SCCS to an Environment.""" - - def SCCSFactory(env=env): - """ """ - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The SCCS() factory is deprecated and there is no replacement.""") - act = SCons.Action.Action('$SCCSCOM', '$SCCSCOMSTR') - return SCons.Builder.Builder(action = act, env = env) - - env.SCCS = SCCSFactory - - env['SCCS'] = 'sccs' - env['SCCSFLAGS'] = SCons.Util.CLVar('') - env['SCCSGETFLAGS'] = SCons.Util.CLVar('') - env['SCCSCOM'] = '$SCCS $SCCSFLAGS get $SCCSGETFLAGS $TARGET' - -def exists(env): - return env.Detect('sccs') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/SCCS.xml b/src/engine/SCons/Tool/SCCS.xml deleted file mode 100644 index de22177b..00000000 --- a/src/engine/SCons/Tool/SCCS.xml +++ /dev/null @@ -1,133 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<tool name="SCCS"> -<summary> -<para> -Sets construction variables for interacting with the -Source Code Control System. -</para> -</summary> -<sets> -<item>SCCS</item> -<item>SCCSFLAGS</item> -<item>SCCSGETFLAGS</item> -<item>SCCSCOM</item> -</sets> -<uses> -<item>SCCSCOMSTR</item> -</uses> -</tool> - -<cvar name="SCCS"> -<summary> -<para> -The SCCS executable. -</para> -</summary> -</cvar> - -<cvar name="SCCSCOM"> -<summary> -<para> -The command line used to -fetch source files from SCCS. -</para> -</summary> -</cvar> - -<cvar name="SCCSCOMSTR"> -<summary> -<para> -The string displayed when fetching -a source file from a CVS repository. -If this is not set, then &cv-link-SCCSCOM; -(the command line) is displayed. -</para> -</summary> -</cvar> - -<cvar name="SCCSFLAGS"> -<summary> -<para> -General options that are passed to SCCS. -</para> -</summary> -</cvar> - -<cvar name="SCCSGETFLAGS"> -<summary> -<para> -Options that are passed specifically to the SCCS "get" subcommand. -This can be set, for example, to -<option>-e</option> -to check out editable files from SCCS. -</para> -</summary> -</cvar> - -<scons_function name="SCCS"> -<arguments signature="env"> -() -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -from SCCS. -The returned Builder -is intended to be passed to the -&f-link-SourceCode; -function. -</para> - -<para> -Example: -</para> - -<example_commands> -env.SourceCode('.', env.SCCS()) -</example_commands> - -<para> -Note that -&scons; -will fetch source files -from SCCS subdirectories automatically, -so configuring SCCS -as demonstrated in the above example -should only be necessary if -you are fetching from -<filename>s.SCCS</filename> -files in the same -directory as the source files, -or if you need to explicitly specify SCCS -for a specific subdirectory. -</para> -</summary> -</scons_function> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/Subversion.py b/src/engine/SCons/Tool/Subversion.py deleted file mode 100644 index f1b3708f..00000000 --- a/src/engine/SCons/Tool/Subversion.py +++ /dev/null @@ -1,70 +0,0 @@ -"""SCons.Tool.Subversion.py - -Tool-specific initialization for Subversion. - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os.path - -import SCons.Action -import SCons.Builder -import SCons.Util - -def generate(env): - """Add a Builder factory function and construction variables for - Subversion to an Environment.""" - - def SubversionFactory(repos, module='', env=env): - """ """ - # fail if repos is not an absolute path name? - import SCons.Warnings as W - W.warn(W.DeprecatedSourceCodeWarning, """The Subversion() factory is deprecated and there is no replacement.""") - if module != '': - module = os.path.join(module, '') - act = SCons.Action.Action('$SVNCOM', '$SVNCOMSTR') - return SCons.Builder.Builder(action = act, - env = env, - SVNREPOSITORY = repos, - SVNMODULE = module) - - env.Subversion = SubversionFactory - - env['SVN'] = 'svn' - env['SVNFLAGS'] = SCons.Util.CLVar('') - env['SVNCOM'] = '$SVN $SVNFLAGS cat $SVNREPOSITORY/$SVNMODULE$TARGET > $TARGET' - -def exists(env): - return env.Detect('svn') - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/Subversion.xml b/src/engine/SCons/Tool/Subversion.xml deleted file mode 100644 index 7c851b5f..00000000 --- a/src/engine/SCons/Tool/Subversion.xml +++ /dev/null @@ -1,135 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -__COPYRIGHT__ - -This file is processed by the bin/SConsDoc.py module. -See its __doc__ string for a discussion of the format. ---> - -<!DOCTYPE sconsdoc [ -<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> -%scons; -<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> -%builders-mod; -<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> -%functions-mod; -<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> -%tools-mod; -<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> -%variables-mod; -]> - -<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> - -<!-- -<tool name="Subversion"> -<summary> -<para> -Sets construction variables for interacting with Subversion. -</para> -</summary> -<sets> -<item>SVN</item> -<item>SVNFLAGS</item> -<item>SVNCOM</item> -</sets> -<uses> -<item>SVNCOMSTR</item> -</uses> -</tool> ---> - -<!-- -<cvar name="SVN"> -<summary> -<para> -The Subversion executable (usually named -<command>svn</command>). -</para> -</summary> -</cvar> ---> - -<!-- -<cvar name="SVNCOM"> -<summary> -<para> -The command line used to -fetch source files from a Subversion repository. -</para> -</summary> -</cvar> ---> - -<!-- -<cvar name="SVNFLAGS"> -<summary> -<para> -General options that are passed to Subversion. -</para> -</summary> -</cvar> ---> - -<!-- -<scons_function name="Subversion"> -<arguments signature="global"> -(repository, module) -</arguments> -<summary> -<para> -A factory function that -returns a Builder object -to be used to fetch source files -from the specified Subversion -<varname>repository</varname>. -The returned Builder -is intended to be passed to the -&f-link-SourceCode; -function. -</para> - -<para> -The optional specified -<varname>module</varname> -will be added to the beginning -of all repository path names; -this can be used, in essence, -to strip initial directory names -from the repository path names, -so that you only have to -replicate part of the repository -directory hierarchy in your -local build directory. -</para> - -<para> -This function is deprecated, see the entry for the -&f-SourceCode; -function. -</para> - -<para> -Example: -</para> - -<example_commands> -# Will fetch foo/bar/src.c -# from /usr/local/Subversion/foo/bar/src.c. -env.SourceCode('.', env.Subversion('file:///usr/local/Subversion')) - -# Will fetch bar/src.c -# from /usr/local/Subversion/foo/bar/src.c. -env.SourceCode('.', env.Subversion('file:///usr/local/Subversion', 'foo')) - -# Will fetch src.c -# from /usr/local/Subversion/foo/bar/src.c. -env.SourceCode('.', env.Subversion('file:///usr/local/Subversion', 'foo/bar')) -</example_commands> -</summary> -</scons_function> ---> - -</sconsdoc> diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 0e5f3644..1ab43e7e 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -97,9 +97,20 @@ for suffix in LaTeXSuffixes: SourceFileScanner.add_scanner(suffix, LaTeXScanner) SourceFileScanner.add_scanner(suffix, PDFLaTeXScanner) + +# Tool aliases are needed for those tools whos module names also +# occur in the python standard library. This causes module shadowing and +# can break using python library functions under python3 +TOOL_ALIASES = { + 'gettext':'gettext_tool', + 'clang++': 'clangxx', +} + class Tool(object): def __init__(self, name, toolpath=[], **kw): - self.name = name + + # Rename if there's a TOOL_ALIAS for this tool + self.name = TOOL_ALIASES.get(name,name) self.toolpath = toolpath + DefaultToolpath # remember these so we can merge them into the call self.init_kw = kw @@ -110,25 +121,35 @@ class Tool(object): if hasattr(module, 'options'): self.options = module.options + def _load_dotted_module_py2(self, short_name, full_name, searchpaths=None): + splitname = short_name.split('.') + index = 0 + srchpths = searchpaths + for item in splitname: + file, path, desc = imp.find_module(item, srchpths) + mod = imp.load_module(full_name, file, path, desc) + srchpths = [path] + return mod, file + def _tool_module(self): - # TODO: Interchange zipimport with normal initialization for better error reporting oldpythonpath = sys.path sys.path = self.toolpath + sys.path + # sys.stderr.write("Tool:%s\nPATH:%s\n"%(self.name,sys.path)) - - if sys.version_info[0] < 3: + if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] in (0,1,2,3,4)): # Py 2 code try: try: - file, path, desc = imp.find_module(self.name, self.toolpath) + file = None try: - return imp.load_module(self.name, file, path, desc) - + mod, file = self._load_dotted_module_py2(self.name, self.name, self.toolpath) + return mod finally: if file: file.close() except ImportError as e: - if str(e)!="No module named %s"%self.name: + splitname = self.name.split('.') + if str(e)!="No module named %s"%splitname[0]: raise SCons.Errors.EnvironmentError(e) try: import zipimport @@ -143,34 +164,81 @@ class Tool(object): pass finally: sys.path = oldpythonpath - else: + elif sys.version_info[1] > 4: + # From: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path/67692#67692 + # import importlib.util + # spec = importlib.util.spec_from_file_location("module.name", "/path/to/file.py") + # foo = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(foo) + # foo.MyClass() # Py 3 code - try: - # Try site_tools first - return importlib.import_module(self.name) - except ImportError as e: - # Then try modules in main distribution - try: - return importlib.import_module('SCons.Tool.'+self.name) - except ImportError as e: - if str(e) != "No module named %s" % self.name: - raise SCons.Errors.EnvironmentError(e) - try: - import zipimport - except ImportError: - pass - else: - for aPath in self.toolpath: - try: - importer = zipimport.zipimporter(aPath) - return importer.load_module(self.name) - except ImportError as e: - pass - finally: - sys.path = oldpythonpath + # import pdb; pdb.set_trace() + import importlib.util + + # sys.stderr.write("toolpath:%s\n" % self.toolpath) + # sys.stderr.write("SCONS.TOOL path:%s\n" % sys.modules['SCons.Tool'].__path__) + debug = False + spec = None + found_name = self.name + add_to_scons_tools_namespace = False + for path in self.toolpath: + sepname = self.name.replace('.', os.path.sep) + file_path = os.path.join(path, "%s.py"%sepname) + file_package = os.path.join(path, sepname) + + if debug: sys.stderr.write("Trying:%s %s\n"%(file_path, file_package)) + + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(self.name, file_path) + if debug: print("file_Path:%s FOUND"%file_path) + break + elif os.path.isdir(file_package): + file_package = os.path.join(file_package, '__init__.py') + spec = importlib.util.spec_from_file_location(self.name, file_package) + if debug: print("PACKAGE:%s Found"%file_package) + break + + else: + continue + + if spec is None: + if debug: sys.stderr.write("NO SPEC :%s\n"%self.name) + spec = importlib.util.find_spec("."+self.name, package='SCons.Tool') + if spec: + found_name = 'SCons.Tool.'+self.name + add_to_scons_tools_namespace = True + if debug: sys.stderr.write("Spec Found? .%s :%s\n"%(self.name, spec)) + + if spec is None: + error_string = "No module named %s"%self.name + raise SCons.Errors.EnvironmentError(error_string) + + module = importlib.util.module_from_spec(spec) + if module is None: + if debug: print("MODULE IS NONE:%s"%self.name) + error_string = "No module named %s"%self.name + raise SCons.Errors.EnvironmentError(error_string) + + # Don't reload a tool we already loaded. + sys_modules_value = sys.modules.get(found_name,False) + if sys_modules_value and sys_modules_value.__file__ == spec.origin: + return sys.modules[found_name] + else: + # Not sure what to do in the case that there already + # exists sys.modules[self.name] but the source file is + # different.. ? + module = spec.loader.load_module(spec.name) + + sys.modules[found_name] = module + if add_to_scons_tools_namespace: + # If we found it in SCons.Tool, then add it to the module + setattr(SCons.Tool, self.name, module) + + return module + sys.path = oldpythonpath full_name = 'SCons.Tool.' + self.name try: @@ -179,8 +247,7 @@ class Tool(object): try: smpath = sys.modules['SCons.Tool'].__path__ try: - file, path, desc = imp.find_module(self.name, smpath) - module = imp.load_module(full_name, file, path, desc) + module, file = self._load_dotted_module_py2(self.name, full_name, smpath) setattr(SCons.Tool, self.name, module) if file: file.close() @@ -291,7 +358,7 @@ def _call_linker_cb(env, callback, args, result = None): if Verbose: print('_call_linker_cb: args=%r' % args) print('_call_linker_cb: callback=%r' % callback) - + try: cbfun = env['LINKCALLBACKS'][callback] except (KeyError, TypeError): @@ -423,7 +490,7 @@ class _LibInfoGeneratorBase(object): def generate_versioned_lib_info(self, env, args, result = None, **kw): callback = self.get_versioned_lib_info_generator(**kw) - return _call_linker_cb(env, callback, args, result) + return _call_linker_cb(env, callback, args, result) class _LibPrefixGenerator(_LibInfoGeneratorBase): """Library prefix generator, used as target_prefix in SharedLibrary and @@ -534,7 +601,7 @@ ImpLibSymlinkGenerator = _LibSymlinkGenerator('ImpLib') class _LibNameGenerator(_LibInfoGeneratorBase): """Generates "unmangled" library name from a library file node. - + Generally, it's thought to revert modifications done by prefix/suffix generators (_LibPrefixGenerator/_LibSuffixGenerator) used by a library builder. For example, on gnulink the suffix generator used by SharedLibrary @@ -544,7 +611,7 @@ class _LibNameGenerator(_LibInfoGeneratorBase): "$SHLIBSUFFIX" in the node's basename. So that, if $SHLIBSUFFIX is ".so", $SHLIBVERSION is "0.1.2" and the node path is "/foo/bar/libfoo.so.0.1.2", the _LibNameGenerator shall return "libfoo.so". Other link tools may - implement it's own way of library name unmangling. + implement it's own way of library name unmangling. """ def __init__(self, libtype): super(_LibNameGenerator, self).__init__(libtype, 'Name') @@ -585,7 +652,7 @@ LdModNameGenerator = _LibNameGenerator('LdMod') ImpLibNameGenerator = _LibNameGenerator('ImpLib') class _LibSonameGenerator(_LibInfoGeneratorBase): - """Library soname generator. Returns library soname (e.g. libfoo.so.0) for + """Library soname generator. Returns library soname (e.g. libfoo.so.0) for a given node (e.g. /foo/bar/libfoo.so.0.1.2)""" def __init__(self, libtype): super(_LibSonameGenerator, self).__init__(libtype, 'Soname') @@ -648,7 +715,7 @@ def EmitLibSymlinks(env, symlinks, libnode, **kw): clean_targets = kw.get('clean_targets', []) if not SCons.Util.is_List(clean_targets): clean_targets = [ clean_targets ] - + for link, linktgt in symlinks: env.SideEffect(link, linktgt) if(Verbose): @@ -663,7 +730,7 @@ def CreateLibSymlinks(env, symlinks): form [ (link, linktarget), ... ], where link and linktarget are SCons nodes. """ - + Verbose = False for link, linktgt in symlinks: linktgt = link.get_dir().rel_path(linktgt) @@ -705,10 +772,11 @@ def LibSymlinksStrFun(target, source, env, *args): else: cmd += ": %s" % linkstr return cmd - + LibSymlinksAction = SCons.Action.Action(LibSymlinksActionFunction, LibSymlinksStrFun) + def createSharedLibBuilder(env): """This is a utility function that creates the SharedLibrary Builder in an Environment if it is not there already. @@ -1040,7 +1108,7 @@ def tool_list(platform, env): "prefer Microsoft tools on Windows" linkers = ['mslink', 'gnulink', 'ilink', 'linkloc', 'ilink32' ] c_compilers = ['msvc', 'mingw', 'gcc', 'intelc', 'icl', 'icc', 'cc', 'bcc32' ] - cxx_compilers = ['msvc', 'intelc', 'icc', 'g++', 'c++', 'bcc32' ] + cxx_compilers = ['msvc', 'intelc', 'icc', 'g++', 'cxx', 'bcc32' ] assemblers = ['masm', 'nasm', 'gas', '386asm' ] fortran_compilers = ['gfortran', 'g77', 'ifl', 'cvf', 'f95', 'f90', 'fortran'] ars = ['mslib', 'ar', 'tlib'] @@ -1049,7 +1117,7 @@ def tool_list(platform, env): "prefer IBM tools on OS/2" linkers = ['ilink', 'gnulink', ]#'mslink'] c_compilers = ['icc', 'gcc',]# 'msvc', 'cc'] - cxx_compilers = ['icc', 'g++',]# 'msvc', 'c++'] + cxx_compilers = ['icc', 'g++',]# 'msvc', 'cxx'] assemblers = ['nasm',]# 'masm', 'gas'] fortran_compilers = ['ifl', 'g77'] ars = ['ar',]# 'mslib'] @@ -1057,7 +1125,7 @@ def tool_list(platform, env): "prefer MIPSPro on IRIX" linkers = ['sgilink', 'gnulink'] c_compilers = ['sgicc', 'gcc', 'cc'] - cxx_compilers = ['sgic++', 'g++', 'c++'] + cxx_compilers = ['sgicxx', 'g++', 'cxx'] assemblers = ['as', 'gas'] fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran'] ars = ['sgiar'] @@ -1065,7 +1133,7 @@ def tool_list(platform, env): "prefer Forte tools on SunOS" linkers = ['sunlink', 'gnulink'] c_compilers = ['suncc', 'gcc', 'cc'] - cxx_compilers = ['sunc++', 'g++', 'c++'] + cxx_compilers = ['suncxx', 'g++', 'cxx'] assemblers = ['as', 'gas'] fortran_compilers = ['sunf95', 'sunf90', 'sunf77', 'f95', 'f90', 'f77', 'gfortran', 'g77', 'fortran'] @@ -1074,7 +1142,7 @@ def tool_list(platform, env): "prefer aCC tools on HP-UX" linkers = ['hplink', 'gnulink'] c_compilers = ['hpcc', 'gcc', 'cc'] - cxx_compilers = ['hpc++', 'g++', 'c++'] + cxx_compilers = ['hpcxx', 'g++', 'cxx'] assemblers = ['as', 'gas'] fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran'] ars = ['ar'] @@ -1082,7 +1150,7 @@ def tool_list(platform, env): "prefer AIX Visual Age tools on AIX" linkers = ['aixlink', 'gnulink'] c_compilers = ['aixcc', 'gcc', 'cc'] - cxx_compilers = ['aixc++', 'g++', 'c++'] + cxx_compilers = ['aixcxx', 'g++', 'cxx'] assemblers = ['as', 'gas'] fortran_compilers = ['f95', 'f90', 'aixf77', 'g77', 'fortran'] ars = ['ar'] @@ -1090,7 +1158,7 @@ def tool_list(platform, env): "prefer GNU tools on Mac OS X, except for some linkers and IBM tools" linkers = ['applelink', 'gnulink'] c_compilers = ['gcc', 'cc'] - cxx_compilers = ['g++', 'c++'] + cxx_compilers = ['g++', 'cxx'] assemblers = ['as'] fortran_compilers = ['gfortran', 'f95', 'f90', 'g77'] ars = ['ar'] @@ -1098,7 +1166,7 @@ def tool_list(platform, env): "prefer GNU tools on Cygwin, except for a platform-specific linker" linkers = ['cyglink', 'mslink', 'ilink'] c_compilers = ['gcc', 'msvc', 'intelc', 'icc', 'cc'] - cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'c++'] + cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'cxx'] assemblers = ['gas', 'nasm', 'masm'] fortran_compilers = ['gfortran', 'g77', 'ifort', 'ifl', 'f95', 'f90', 'f77'] ars = ['ar', 'mslib'] @@ -1106,7 +1174,7 @@ def tool_list(platform, env): "prefer GNU tools on all other platforms" linkers = ['gnulink', 'mslink', 'ilink'] c_compilers = ['gcc', 'msvc', 'intelc', 'icc', 'cc'] - cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'c++'] + cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'cxx'] assemblers = ['gas', 'nasm', 'masm'] fortran_compilers = ['gfortran', 'g77', 'ifort', 'ifl', 'f95', 'f90', 'f77'] ars = ['ar', 'mslib'] @@ -1137,7 +1205,7 @@ def tool_list(platform, env): fortran_compiler = FindTool(fortran_compilers, env) or fortran_compilers[0] ar = FindTool(ars, env) or ars[0] - d_compilers = ['dmd', 'gdc', 'ldc'] + d_compilers = ['dmd', 'ldc', 'gdc'] d_compiler = FindTool(d_compilers, env) or d_compilers[0] other_tools = FindAllTools(other_plat_tools + [ @@ -1156,9 +1224,6 @@ def tool_list(platform, env): 'tex', 'latex', 'pdflatex', 'pdftex', # Archivers 'tar', 'zip', - # SourceCode factories - 'BitKeeper', 'CVS', 'Perforce', - 'RCS', 'SCCS', # 'Subversion', ], env) tools = ([linker, c_compiler, cxx_compiler, @@ -1172,4 +1237,3 @@ def tool_list(platform, env): # indent-tabs-mode:nil # End: # vim: set expandtab tabstop=4 shiftwidth=4: - diff --git a/src/engine/SCons/Tool/aixcxx.py b/src/engine/SCons/Tool/aixcxx.py index f03f763e..58d9ecca 100644 --- a/src/engine/SCons/Tool/aixcxx.py +++ b/src/engine/SCons/Tool/aixcxx.py @@ -37,7 +37,9 @@ import os.path import SCons.Platform.aix -cplusplus = __import__('c++', globals(), locals(), []) +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('cxx', globals(), locals(), []) packages = ['vacpp.cmp.core', 'vacpp.cmp.batch', 'vacpp.cmp.C', 'ibmcxx.cmp'] diff --git a/src/engine/SCons/Tool/aixlink.py b/src/engine/SCons/Tool/aixlink.py index 3117c551..fed43428 100644 --- a/src/engine/SCons/Tool/aixlink.py +++ b/src/engine/SCons/Tool/aixlink.py @@ -40,7 +40,10 @@ import SCons.Util from . import aixcc from . import link -cplusplus = __import__('c++', globals(), locals(), []) +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('cxx', globals(), locals(), []) + def smart_linkflags(source, target, env, for_signature): if cplusplus.iscplusplus(source): @@ -65,7 +68,7 @@ def exists(env): # TODO: sync with link.smart_link() to choose a linker linkers = { 'CXX': ['aixc++'], 'CC': ['aixcc'] } alltools = [] - for langvar, linktools in list(linkers.items()): + for langvar, linktools in linkers.items(): if langvar in env: # use CC over CXX when user specified CC but not CXX return SCons.Tool.FindTool(linktools, env) alltools.extend(linktools) diff --git a/src/engine/SCons/Tool/applelink.py b/src/engine/SCons/Tool/applelink.py index ba955a4e..c5f43763 100644 --- a/src/engine/SCons/Tool/applelink.py +++ b/src/engine/SCons/Tool/applelink.py @@ -51,6 +51,14 @@ def generate(env): env['SHLINKFLAGS'] = SCons.Util.CLVar('$LINKFLAGS -dynamiclib') env['SHLINKCOM'] = env['SHLINKCOM'] + ' $_FRAMEWORKPATH $_FRAMEWORKS $FRAMEWORKSFLAGS' + + # TODO: Work needed to generate versioned shared libraries + # Leaving this commented out, and also going to disable versioned library checking for now + # see: http://docstore.mik.ua/orelly/unix3/mac/ch05_04.htm for proper naming + #link._setup_versioned_lib_variables(env, tool = 'applelink')#, use_soname = use_soname) + #env['LINKCALLBACKS'] = link._versioned_lib_callbacks() + + # override the default for loadable modules, which are different # on OS X than dynamic shared libs. echoing what XCode does for # pre/suffixes: diff --git a/src/engine/SCons/Tool/clang.py b/src/engine/SCons/Tool/clang.py new file mode 100644 index 00000000..177e6b24 --- /dev/null +++ b/src/engine/SCons/Tool/clang.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8; -*- + +"""SCons.Tool.clang + +Tool-specific initialization for clang. + +There normally shouldn't be any need to import this module directly. +It will usually be imported through the generic SCons.Tool.Tool() +selection method. + +""" + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +# __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +# Based on SCons/Tool/gcc.py by Paweł Tomulik 2014 as a separate tool. +# Brought into the SCons mainline by Russel Winder 2017. + +import os +import re +import subprocess +import sys + +import SCons.Util +import SCons.Tool.cc + +compilers = ['clang'] + +def generate(env): + """Add Builders and construction variables for clang to an Environment.""" + SCons.Tool.cc.generate(env) + + env['CC'] = env.Detect(compilers) or 'clang' + if env['PLATFORM'] in ['cygwin', 'win32']: + env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS') + else: + env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS -fPIC') + # determine compiler version + if env['CC']: + #pipe = SCons.Action._subproc(env, [env['CC'], '-dumpversion'], + pipe = SCons.Action._subproc(env, [env['CC'], '--version'], + stdin='devnull', + stderr='devnull', + stdout=subprocess.PIPE) + if pipe.wait() != 0: return + # clang -dumpversion is of no use + line = pipe.stdout.readline() + if sys.version_info[0] > 2: + line = line.decode() + match = re.search(r'clang +version +([0-9]+(?:\.[0-9]+)+)', line) + if match: + env['CCVERSION'] = match.group(1) + +def exists(env): + return env.Detect(compilers) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/clang.xml b/src/engine/SCons/Tool/clang.xml new file mode 100644 index 00000000..e2e50d42 --- /dev/null +++ b/src/engine/SCons/Tool/clang.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +__COPYRIGHT__ + +This file is processed by the bin/SConsDoc.py module. +See its __doc__ string for a discussion of the format. +--> + +<!DOCTYPE sconsdoc [ +<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> +%scons; +<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> +%builders-mod; +<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> +%functions-mod; +<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> +%tools-mod; +<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> +%variables-mod; +]> + +<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> + +<tool name="clang"> +<summary> +<para> +Set construction variables for the Clang C compiler. +</para> +</summary> +<sets> +<item>CC</item> +<item>SHCCFLAGS</item> +<item>CCVERSION</item> +</sets> +</tool> + +</sconsdoc> diff --git a/src/engine/SCons/Tool/clangxx.py b/src/engine/SCons/Tool/clangxx.py new file mode 100644 index 00000000..dd501af7 --- /dev/null +++ b/src/engine/SCons/Tool/clangxx.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8; -*- + +"""SCons.Tool.clang++ + +Tool-specific initialization for clang++. + +There normally shouldn't be any need to import this module directly. +It will usually be imported through the generic SCons.Tool.Tool() +selection method. + +""" + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +# __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +# Based on SCons/Tool/g++.py by Paweł Tomulik 2014 as a separate tool. +# Brought into the SCons mainline by Russel Winder 2017. + +import os.path +import re +import subprocess +import sys + +import SCons.Tool +import SCons.Util +import SCons.Tool.cxx + +compilers = ['clang++'] + +def generate(env): + """Add Builders and construction variables for clang++ to an Environment.""" + static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + + SCons.Tool.cxx.generate(env) + + env['CXX'] = env.Detect(compilers) or 'clang++' + + # platform specific settings + if env['PLATFORM'] == 'aix': + env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS -mminimal-toc') + env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1 + env['SHOBJSUFFIX'] = '$OBJSUFFIX' + elif env['PLATFORM'] == 'hpux': + env['SHOBJSUFFIX'] = '.pic.o' + elif env['PLATFORM'] == 'sunos': + env['SHOBJSUFFIX'] = '.pic.o' + # determine compiler version + if env['CXX']: + pipe = SCons.Action._subproc(env, [env['CXX'], '--version'], + stdin='devnull', + stderr='devnull', + stdout=subprocess.PIPE) + if pipe.wait() != 0: return + # clang -dumpversion is of no use + line = pipe.stdout.readline() + if sys.version_info[0] > 2: + line = line.decode() + match = re.search(r'clang +version +([0-9]+(?:\.[0-9]+)+)', line) + if match: + env['CXXVERSION'] = match.group(1) + +def exists(env): + return env.Detect(compilers) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Tool/clangxx.xml b/src/engine/SCons/Tool/clangxx.xml new file mode 100644 index 00000000..2c1c2ca5 --- /dev/null +++ b/src/engine/SCons/Tool/clangxx.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +__COPYRIGHT__ + +This file is processed by the bin/SConsDoc.py module. +See its __doc__ string for a discussion of the format. +--> + +<!DOCTYPE sconsdoc [ +<!ENTITY % scons SYSTEM '../../../../doc/scons.mod'> +%scons; +<!ENTITY % builders-mod SYSTEM '../../../../doc/generated/builders.mod'> +%builders-mod; +<!ENTITY % functions-mod SYSTEM '../../../../doc/generated/functions.mod'> +%functions-mod; +<!ENTITY % tools-mod SYSTEM '../../../../doc/generated/tools.mod'> +%tools-mod; +<!ENTITY % variables-mod SYSTEM '../../../../doc/generated/variables.mod'> +%variables-mod; +]> + +<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> + +<tool name="clangxx"> +<summary> +<para> +Set construction variables for the Clang C++ compiler. +</para> +</summary> +<sets> +<item>CXX</item> +<item>SHCXXFLAGS</item> +<item>STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME</item> +<item>SHOBJSUFFIX</item> +<item>CXXVERSION</item> +</sets> +</tool> + +</sconsdoc> diff --git a/src/engine/SCons/Tool/dmd.py b/src/engine/SCons/Tool/dmd.py index 37229360..7c142eb1 100644 --- a/src/engine/SCons/Tool/dmd.py +++ b/src/engine/SCons/Tool/dmd.py @@ -1,3 +1,5 @@ +from __future__ import print_function + """SCons.Tool.dmd Tool-specific initialization for the Digital Mars D compiler. @@ -9,13 +11,6 @@ Originally coded by Andy Friesen (andy@ikagames.com) Evolved by Russel Winder (russel@winder.org.uk) 2010-02-07 onwards -There are a number of problems with this script at this point in time. -The one that irritates the most is the Windows linker setup. The D -linker doesn't have a way to add lib paths on the commandline, as far -as I can see. You have to specify paths relative to the SConscript or -use absolute paths. To hack around it, add '#/blah'. This will link -blah.lib from the directory where SConstruct resides. - Compiler variables: DC - The name of the D compiler to use. Defaults to dmd or gdmd, whichever is found. @@ -69,7 +64,7 @@ import SCons.Defaults import SCons.Scanner.D import SCons.Tool -import SCons.Tool.DCommon +import SCons.Tool.DCommon as DCommon def generate(env): @@ -80,7 +75,7 @@ def generate(env): static_obj.add_emitter('.d', SCons.Defaults.StaticObjectEmitter) shared_obj.add_emitter('.d', SCons.Defaults.SharedObjectEmitter) - env['DC'] = env.Detect(['dmd', 'gdmd']) + env['DC'] = env.Detect(['dmd', 'ldmd2', 'gdmd']) or 'dmd' env['DCOM'] = '$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -c -of$TARGET $SOURCES' env['_DINCFLAGS'] = '${_concat(DINCPREFIX, DPATH, DINCSUFFIX, __env__, RDirs, TARGET, SOURCE)}' env['_DVERFLAGS'] = '${_concat(DVERPREFIX, DVERSIONS, DVERSUFFIX, __env__)}' @@ -96,7 +91,7 @@ def generate(env): env['DDEBUG'] = [] if env['DC']: - SCons.Tool.DCommon.addDPATHToEnv(env, env['DC']) + DCommon.addDPATHToEnv(env, env['DC']) env['DINCPREFIX'] = '-I' env['DINCSUFFIX'] = '' @@ -124,18 +119,17 @@ def generate(env): env['DLIBDIRSUFFIX'] = '' env['_DLIBDIRFLAGS'] = '${_concat(DLIBDIRPREFIX, LIBPATH, DLIBDIRSUFFIX, __env__, RDirs, TARGET, SOURCE)}' - env['DLIB'] = 'lib' if env['PLATFORM'] == 'win32' else 'ar cr' env['DLIBCOM'] = '$DLIB $_DLIBFLAGS {0}$TARGET $SOURCES $_DLIBFLAGS'.format('-c ' if env['PLATFORM'] == 'win32' else '') - #env['_DLIBFLAGS'] = '${_concat(DLIBFLAGPREFIX, DLIBFLAGS, DLIBFLAGSUFFIX, __env__)}' + # env['_DLIBFLAGS'] = '${_concat(DLIBFLAGPREFIX, DLIBFLAGS, DLIBFLAGSUFFIX, __env__)}' env['DLIBFLAGPREFIX'] = '-' env['DLIBFLAGSUFFIX'] = '' # __RPATH is set to $_RPATH in the platform specification if that # platform supports it. - env['DRPATHPREFIX'] = '-L-rpath=' + env['DRPATHPREFIX'] = '-L-rpath,' if env['PLATFORM'] == 'darwin' else '-L-rpath=' env['DRPATHSUFFIX'] = '' env['_DRPATH'] = '${_concat(DRPATHPREFIX, RPATH, DRPATHSUFFIX, __env__)}' @@ -150,12 +144,16 @@ def generate(env): env['DSHLIBVERSION'] = '$SHLIBVERSION' env['DSHLIBVERSIONFLAGS'] = [] - SCons.Tool.createStaticLibBuilder(env) + env['BUILDERS']['ProgramAllAtOnce'] = SCons.Builder.Builder( + action='$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -of$TARGET $DLINKFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS', + emitter=DCommon.allAtOnceEmitter, + ) def exists(env): return env.Detect(['dmd', 'gdmd']) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Tool/dmd.xml b/src/engine/SCons/Tool/dmd.xml index f8936c12..8fb22ced 100644 --- a/src/engine/SCons/Tool/dmd.xml +++ b/src/engine/SCons/Tool/dmd.xml @@ -32,10 +32,6 @@ Sets construction variables for D language compiler DMD. <sets> <item>DC</item> <item>DCOM</item> -<item>_DINCFLAGS</item> -<item>_DVERFLAGS</item> -<item>_DDEBUGFLAGS</item> -<item>_DFLAGS</item> <item>SHDC</item> <item>SHDCOM</item> <item>DPATH</item> @@ -59,21 +55,321 @@ Sets construction variables for D language compiler DMD. <item>SHDLINKCOM</item> <item>DLIBLINKPREFIX</item> <item>DLIBLINKSUFFIX</item> -<item>_DLIBFLAGS</item> <item>DLIBDIRPREFIX</item> <item>DLIBDIRSUFFIX</item> -<item>_DLIBDIRFLAGS</item> <item>DLIB</item> <item>DLIBCOM</item> -<item>_DLIBFLAGS</item> <item>DLIBFLAGPREFIX</item> <item>DLIBFLAGSUFFIX</item> +<item>DLINKFLAGPREFIX</item> +<item>DLINKFLAGSUFFIX</item> <item>RPATHPREFIX</item> <item>RPATHSUFFIX</item> -<item>_RPATH</item> </sets> <uses> </uses> </tool> +<cvar name="DC"> +<summary> +<para> +The D compiler to use. +</para> +</summary> +</cvar> + +<cvar name="DCOM"> +<summary> +<para> + The command line used to compile a D file to an object file. + Any options specified in the &cv-link-DFLAGS; construction variable + is included on this command line. +</para> +</summary> +</cvar> + +<cvar name="DDEBUG"> +<summary> +<para> + List of debug tags to enable when compiling. +</para> +</summary> +</cvar> + +<cvar name="DFLAGS"> +<summary> +<para> + General options that are passed to the D compiler. +</para> +</summary> +</cvar> + +<cvar name="DLIB"> +<summary> +<para> + Name of the lib tool to use for D codes. +</para> +</summary> +</cvar> + +<cvar name="DLIBCOM"> +<summary> +<para> + The command line to use when creating libraries. +</para> +</summary> +</cvar> + +<cvar name="DLINK"> +<summary> +<para> + Name of the linker to use for linking systems including D sources. +</para> +</summary> +</cvar> + +<cvar name="DLINKCOM"> +<summary> +<para> + The command line to use when linking systems including D sources. +</para> +</summary> +</cvar> + +<cvar name="DLINKFLAGS"> +<summary> +<para> +List of linker flags. +</para> +</summary> +</cvar> + +<cvar name="DPATH"> +<summary> +<para> + List of paths to search for import modules. +</para> +</summary> +</cvar> + +<cvar name="DVERSIONS"> +<summary> +<para> + List of version tags to enable when compiling. +</para> +</summary> +</cvar> + + + +<cvar name="SHDC"> +<summary> +<para> + The name of the compiler to use when compiling D source + destined to be in a shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDCOM"> +<summary> +<para> + The command line to use when compiling code to be part of shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDLINK"> +<summary> +<para> + The linker to use when creating shared objects for code bases + include D sources. +</para> +</summary> +</cvar> + +<cvar name="SHDLINKCOM"> +<summary> +<para> + The command line to use when generating shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDLINKFLAGS"> +<summary> +<para> + The list of flags to use when generating a shared object. +</para> +</summary> +</cvar> + +<!-- The following are re-added because their removal breaks doc validation --> +<cvar name="DVERSUFFIX"> + <summary> + <para> + DVERSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DVERPREFIX"> + <summary> + <para> + DVERPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DLINKFLAGSUFFIX"> + <summary> + <para> + DLINKFLAGSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DLINKFLAGPREFIX"> + <summary> + <para> + DLINKFLAGPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBLINKSUFFIX"> + <summary> + <para> + DLIBLINKSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBLINKPREFIX"> + <summary> + <para> + DLIBLINKPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBFLAGSUFFIX"> + <summary> + <para> + DLIBFLAGSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBFLAGPREFIX"> + <summary> + <para> + DLIBFLAGPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBDIRSUFFIX"> + <summary> + <para> + DLIBLINKSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DLIBDIRPREFIX"> + <summary> + <para> + DLIBLINKPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DINCSUFFIX"> + <summary> + <para> + DLIBFLAGSUFFIX. + </para> + </summary> +</cvar> + + +<cvar name="DINCPREFIX"> + <summary> + <para> + DINCPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DFLAGSUFFIX"> + <summary> + <para> + DFLAGSUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DFLAGPREFIX"> + <summary> + <para> + DFLAGPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DFILESUFFIX"> + <summary> + <para> + DFILESUFFIX. + </para> + </summary> +</cvar> + +<cvar name="DDEBUGPREFIX"> + <summary> + <para> + DDEBUGPREFIX. + </para> + </summary> +</cvar> + +<cvar name="DDEBUGSUFFIX"> + <summary> + <para> + DDEBUGSUFFIX. + </para> + </summary> +</cvar> + + +<!-- Validation fix to here --> + + +<builder name="ProgramAllAtOnce"> + <summary> + <para> + Builds an executable from D sources without first creating individual + objects for each file. + </para> + <para> + D sources can be compiled file-by-file as C and C++ source are, and + D is integrated into the &scons; Object and Program builders for + this model of build. D codes can though do whole source + meta-programming (some of the testing frameworks do this). For this + it is imperative that all sources are compiled and linked in a single call of + the D compiler. This builder serves that purpose. + </para> + <example_commands> + env.ProgramAllAtOnce('executable', ['mod_a.d, mod_b.d', 'mod_c.d']) + </example_commands> + <para> + This command will compile the modules mod_a, mod_b, and mod_c in a + single compilation process without first creating object files for + the modules. Some of the D compilers will create executable.o others + will not. + </para> + </summary> +</builder> + </sconsdoc> diff --git a/src/engine/SCons/Tool/docbook/__init__.py b/src/engine/SCons/Tool/docbook/__init__.py index 8a7b2e7a..d60789d0 100644 --- a/src/engine/SCons/Tool/docbook/__init__.py +++ b/src/engine/SCons/Tool/docbook/__init__.py @@ -43,6 +43,8 @@ import SCons.Script import SCons.Tool import SCons.Util + +__debug_tool_location = False # Get full path to this script scriptpath = os.path.dirname(os.path.realpath(__file__)) @@ -157,6 +159,11 @@ def __create_output_dir(base_dir): # # Supported command line tools and their call "signature" # +xsltproc_com_priority = ['xsltproc', 'saxon', 'saxon-xslt', 'xalan'] + +# TODO: Set minimum version of saxon-xslt to be 8.x (lower than this only supports xslt 1.0. +# see: http://saxon.sourceforge.net/saxon6.5.5/ +# see: http://saxon.sourceforge.net/ xsltproc_com = {'xsltproc' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE', 'saxon' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE $DOCBOOK_XSLTPROCPARAMS', 'saxon-xslt' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE $DOCBOOK_XSLTPROCPARAMS', @@ -166,19 +173,27 @@ fop_com = {'fop' : '$DOCBOOK_FOP $DOCBOOK_FOPFLAGS -fo $SOURCE -pdf $TARGET', 'xep' : '$DOCBOOK_FOP $DOCBOOK_FOPFLAGS -valid -fo $SOURCE -pdf $TARGET', 'jw' : '$DOCBOOK_FOP $DOCBOOK_FOPFLAGS -f docbook -b pdf $SOURCE -o $TARGET'} -def __detect_cl_tool(env, chainkey, cdict): +def __detect_cl_tool(env, chainkey, cdict, cpriority=None): """ Helper function, picks a command line tool from the list and initializes its environment variables. """ if env.get(chainkey,'') == '': clpath = '' - for cltool in cdict: + + if cpriority is None: + cpriority = cdict.keys() + for cltool in cpriority: + if __debug_tool_location: + print("DocBook: Looking for %s"%cltool) clpath = env.WhereIs(cltool) if clpath: + if __debug_tool_location: + print("DocBook: Found:%s"%cltool) env[chainkey] = clpath if not env[chainkey + 'COM']: env[chainkey + 'COM'] = cdict[cltool] + break def _detect(env): """ @@ -192,10 +207,10 @@ def _detect(env): if ((not has_libxml2 and not has_lxml) or (prefer_xsltproc)): # Try to find the XSLT processors - __detect_cl_tool(env, 'DOCBOOK_XSLTPROC', xsltproc_com) + __detect_cl_tool(env, 'DOCBOOK_XSLTPROC', xsltproc_com, xsltproc_com_priority) __detect_cl_tool(env, 'DOCBOOK_XMLLINT', xmllint_com) - __detect_cl_tool(env, 'DOCBOOK_FOP', fop_com) + __detect_cl_tool(env, 'DOCBOOK_FOP', fop_com, ['fop','xep','jw']) # # Scanners diff --git a/src/engine/SCons/Tool/dvipdf.py b/src/engine/SCons/Tool/dvipdf.py index 374b9c58..f1e95131 100644 --- a/src/engine/SCons/Tool/dvipdf.py +++ b/src/engine/SCons/Tool/dvipdf.py @@ -87,7 +87,7 @@ def PDFEmitter(target, source, env): """ def strip_suffixes(n): return not SCons.Util.splitext(str(n))[1] in ['.aux', '.log'] - source = list(filter(strip_suffixes, source)) + source = [src for src in source if strip_suffixes(src)] return (target, source) def generate(env): diff --git a/src/engine/SCons/Tool/gdc.py b/src/engine/SCons/Tool/gdc.py index 32199b3d..a8e037ca 100644 --- a/src/engine/SCons/Tool/gdc.py +++ b/src/engine/SCons/Tool/gdc.py @@ -1,3 +1,5 @@ +from __future__ import print_function + """SCons.Tool.gdc Tool-specific initialization for the GDC compiler. @@ -52,7 +54,7 @@ import SCons.Action import SCons.Defaults import SCons.Tool -import SCons.Tool.DCommon +import SCons.Tool.DCommon as DCommon def generate(env): @@ -63,7 +65,7 @@ def generate(env): static_obj.add_emitter('.d', SCons.Defaults.StaticObjectEmitter) shared_obj.add_emitter('.d', SCons.Defaults.SharedObjectEmitter) - env['DC'] = env.Detect('gdc') + env['DC'] = env.Detect('gdc') or 'gdc' env['DCOM'] = '$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -c -o $TARGET $SOURCES' env['_DINCFLAGS'] = '${_concat(DINCPREFIX, DPATH, DINCSUFFIX, __env__, RDirs, TARGET, SOURCE)}' env['_DVERFLAGS'] = '${_concat(DVERPREFIX, DVERSIONS, DVERSUFFIX, __env__)}' @@ -79,7 +81,7 @@ def generate(env): env['DDEBUG'] = [] if env['DC']: - SCons.Tool.DCommon.addDPATHToEnv(env, env['DC']) + DCommon.addDPATHToEnv(env, env['DC']) env['DINCPREFIX'] = '-I' env['DINCSUFFIX'] = '' @@ -96,7 +98,7 @@ def generate(env): env['DLINKCOM'] = '$DLINK -o $TARGET $DLINKFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS' env['DSHLINK'] = '$DC' - env['DSHLINKFLAGS'] = SCons.Util.CLVar('$DLINKFLAGS -shared') + env['DSHLINKFLAGS'] = SCons.Util.CLVar('$DLINKFLAGS -shared -shared-libphobos') env['SHDLINKCOM'] = '$DLINK -o $TARGET $DSHLINKFLAGS $__DSHLIBVERSIONFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS' env['DLIB'] = 'lib' if env['PLATFORM'] == 'win32' else 'ar cr' @@ -126,12 +128,16 @@ def generate(env): env['DSHLIBVERSION'] = '$SHLIBVERSION' env['DSHLIBVERSIONFLAGS'] = '$SHLIBVERSIONFLAGS' - SCons.Tool.createStaticLibBuilder(env) + env['BUILDERS']['ProgramAllAtOnce'] = SCons.Builder.Builder( + action='$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -o $TARGET $DLINKFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS', + emitter=DCommon.allAtOnceEmitter, + ) def exists(env): return env.Detect('gdc') + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Tool/gdc.xml b/src/engine/SCons/Tool/gdc.xml index 20544b00..5ef1f9a7 100644 --- a/src/engine/SCons/Tool/gdc.xml +++ b/src/engine/SCons/Tool/gdc.xml @@ -32,10 +32,6 @@ Sets construction variables for the D language compiler GDC. <sets> <item>DC</item> <item>DCOM</item> -<item>_DINCFLAGS</item> -<item>_DVERFLAGS</item> -<item>_DDEBUGFLAGS</item> -<item>_DFLAGS</item> <item>SHDC</item> <item>SHDCOM</item> <item>DPATH</item> @@ -57,16 +53,18 @@ Sets construction variables for the D language compiler GDC. <item>SHDLINK</item> <item>SHDLINKFLAGS</item> <item>SHDLINKCOM</item> +<item>DLIBLINKPREFIX</item> +<item>DLIBLINKSUFFIX</item> +<item>DLIBDIRPREFIX</item> +<item>DLIBDIRSUFFIX</item> <item>DLIB</item> <item>DLIBCOM</item> -<item>_DLIBFLAGS</item> <item>DLIBFLAGPREFIX</item> <item>DLIBFLAGSUFFIX</item> <item>DLINKFLAGPREFIX</item> <item>DLINKFLAGSUFFIX</item> <item>RPATHPREFIX</item> <item>RPATHSUFFIX</item> -<item>_RPATH</item> </sets> <uses> </uses> @@ -75,7 +73,7 @@ Sets construction variables for the D language compiler GDC. <cvar name="DC"> <summary> <para> -DC. +The D compiler to use. </para> </summary> </cvar> @@ -83,7 +81,9 @@ DC. <cvar name="DCOM"> <summary> <para> -DCOM. + The command line used to compile a D file to an object file. + Any options specified in the &cv-link-DFLAGS; construction variable + is included on this command line. </para> </summary> </cvar> @@ -91,39 +91,7 @@ DCOM. <cvar name="DDEBUG"> <summary> <para> -DDEBUG. -</para> -</summary> -</cvar> - -<cvar name="DDEBUGPREFIX"> -<summary> -<para> -DDEBUGPREFIX. -</para> -</summary> -</cvar> - -<cvar name="DDEBUGSUFFIX"> -<summary> -<para> -DDEBUGSUFFIX. -</para> -</summary> -</cvar> - -<cvar name="DFILESUFFIX"> -<summary> -<para> -DFILESUFFIX. -</para> -</summary> -</cvar> - -<cvar name="DFLAGPREFIX"> -<summary> -<para> -DFLAGPREFIX. + List of debug tags to enable when compiling. </para> </summary> </cvar> @@ -131,31 +99,7 @@ DFLAGPREFIX. <cvar name="DFLAGS"> <summary> <para> -DFLAGS. -</para> -</summary> -</cvar> - -<cvar name="DFLAGSUFFIX"> -<summary> -<para> -DFLAGSUFFIX. -</para> -</summary> -</cvar> - -<cvar name="DINCPREFIX"> -<summary> -<para> -DINCPREFIX. -</para> -</summary> -</cvar> - -<cvar name="DINCSUFFIX"> -<summary> -<para> -DINCSUFFIX. + General options that are passed to the D compiler. </para> </summary> </cvar> @@ -163,7 +107,7 @@ DINCSUFFIX. <cvar name="DLIB"> <summary> <para> -DLIB. + Name of the lib tool to use for D codes. </para> </summary> </cvar> @@ -171,55 +115,7 @@ DLIB. <cvar name="DLIBCOM"> <summary> <para> -DLIBCOM. -</para> -</summary> -</cvar> - -<cvar name="DLIBDIRPREFIX"> -<summary> -<para> -DLIBDIRPREFIX. -</para> -</summary> -</cvar> - -<cvar name="DLIBDIRSUFFIX"> -<summary> -<para> -DLIBDIRSUFFIX. -</para> -</summary> -</cvar> - -<cvar name="DLIBFLAGPREFIX"> -<summary> -<para> -DLIBFLAGPREFIX. -</para> -</summary> -</cvar> - -<cvar name="DLIBFLAGSUFFIX"> -<summary> -<para> -DLIBFLAGSUFFIX. -</para> -</summary> -</cvar> - -<cvar name="DLIBLINKPREFIX"> -<summary> -<para> -DLIBLINKPREFIX. -</para> -</summary> -</cvar> - -<cvar name="DLIBLINKSUFFIX"> -<summary> -<para> -DLIBLINKSUFFIX. + The command line to use when creating libraries. </para> </summary> </cvar> @@ -227,7 +123,7 @@ DLIBLINKSUFFIX. <cvar name="DLINK"> <summary> <para> -DLINK. + Name of the linker to use for linking systems including D sources. </para> </summary> </cvar> @@ -235,15 +131,7 @@ DLINK. <cvar name="DLINKCOM"> <summary> <para> -DLINKCOM. -</para> -</summary> -</cvar> - -<cvar name="DLINKFLAGPREFIX"> -<summary> -<para> -DLINKFLAGPREFIX. + The command line to use when linking systems including D sources. </para> </summary> </cvar> @@ -251,15 +139,7 @@ DLINKFLAGPREFIX. <cvar name="DLINKFLAGS"> <summary> <para> -DLINKFLAGS. -</para> -</summary> -</cvar> - -<cvar name="DLINKFLAGSUFFIX"> -<summary> -<para> -DLINKFLAGSUFFIX. +List of linker flags. </para> </summary> </cvar> @@ -267,15 +147,7 @@ DLINKFLAGSUFFIX. <cvar name="DPATH"> <summary> <para> -DPATH. -</para> -</summary> -</cvar> - -<cvar name="DVERPREFIX"> -<summary> -<para> -DVERPREFIX. + List of paths to search for import modules. </para> </summary> </cvar> @@ -283,15 +155,7 @@ DVERPREFIX. <cvar name="DVERSIONS"> <summary> <para> -DVERSIONS. -</para> -</summary> -</cvar> - -<cvar name="DVERSUFFIX"> -<summary> -<para> -DVERSUFFIX. + List of version tags to enable when compiling. </para> </summary> </cvar> @@ -299,7 +163,8 @@ DVERSUFFIX. <cvar name="SHDC"> <summary> <para> -SHDC. + The name of the compiler to use when compiling D source + destined to be in a shared objects. </para> </summary> </cvar> @@ -307,7 +172,7 @@ SHDC. <cvar name="SHDCOM"> <summary> <para> -SHDCOM. + The command line to use when compiling code to be part of shared objects. </para> </summary> </cvar> @@ -315,7 +180,8 @@ SHDCOM. <cvar name="SHDLINK"> <summary> <para> -SHDLINK. + The linker to use when creating shared objects for code bases + include D sources. </para> </summary> </cvar> @@ -323,7 +189,7 @@ SHDLINK. <cvar name="SHDLINKCOM"> <summary> <para> -SHDLINKCOM. + The command line to use when generating shared objects. </para> </summary> </cvar> @@ -331,57 +197,35 @@ SHDLINKCOM. <cvar name="SHDLINKFLAGS"> <summary> <para> -SHDLINKFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DDEBUGFLAGS"> -<summary> -<para> -_DDEBUGFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DFLAGS"> -<summary> -<para> -_DFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DINCFLAGS"> -<summary> -<para> -_DINCFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DLIBDIRFLAGS"> -<summary> -<para> -_DLIBDIRFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DLIBFLAGS"> -<summary> -<para> -_DLIBFLAGS. -</para> -</summary> -</cvar> - -<cvar name="_DVERFLAGS"> -<summary> -<para> -_DVERFLAGS. -</para> -</summary> -</cvar> + The list of flags to use when generating a shared object. +</para> +</summary> +</cvar> + +<builder name="ProgramAllAtOnce"> + <summary> + <para> + Builds an executable from D sources without first creating individual + objects for each file. + </para> + <para> + D sources can be compiled file-by-file as C and C++ source are, and + D is integrated into the &scons; Object and Program builders for + this model of build. D codes can though do whole source + meta-programming (some of the testing frameworks do this). For this + it is imperative that all sources are compiled and linked in a single call of + the D compiler. This builder serves that purpose. + </para> + <example_commands> + env.ProgramAllAtOnce('executable', ['mod_a.d, mod_b.d', 'mod_c.d']) + </example_commands> + <para> + This command will compile the modules mod_a, mod_b, and mod_c in a + single compilation process without first creating object files for + the modules. Some of the D compilers will create executable.o others + will not. + </para> + </summary> +</builder> </sconsdoc> diff --git a/src/engine/SCons/Tool/gettext.py b/src/engine/SCons/Tool/gettext_tool.py index 6031e49f..6031e49f 100644 --- a/src/engine/SCons/Tool/gettext.py +++ b/src/engine/SCons/Tool/gettext_tool.py diff --git a/src/engine/SCons/Tool/gnulink.py b/src/engine/SCons/Tool/gnulink.py index cf1ce859..b1d50886 100644 --- a/src/engine/SCons/Tool/gnulink.py +++ b/src/engine/SCons/Tool/gnulink.py @@ -67,7 +67,7 @@ def exists(env): # TODO: sync with link.smart_link() to choose a linker linkers = { 'CXX': ['g++'], 'CC': ['gcc'] } alltools = [] - for langvar, linktools in list(linkers.items()): + for langvar, linktools in linkers.items(): if langvar in env: # use CC over CXX when user specified CC but not CXX return SCons.Tool.FindTool(linktools, env) alltools.extend(linktools) diff --git a/src/engine/SCons/Tool/gxx.py b/src/engine/SCons/Tool/gxx.py index c5eb5797..5bca179f 100644 --- a/src/engine/SCons/Tool/gxx.py +++ b/src/engine/SCons/Tool/gxx.py @@ -41,7 +41,7 @@ import SCons.Tool import SCons.Util from . import gcc -cplusplus = __import__(__package__+'.c++', globals(), locals(), ['*']) +import cxx compilers = ['g++'] @@ -52,7 +52,7 @@ def generate(env): if 'CXX' not in env: env['CXX'] = env.Detect(compilers) or compilers[0] - cplusplus.generate(env) + cxx.generate(env) # platform specific settings if env['PLATFORM'] == 'aix': diff --git a/src/engine/SCons/Tool/hpcxx.py b/src/engine/SCons/Tool/hpcxx.py index 2b8ed3fc..1ba91981 100644 --- a/src/engine/SCons/Tool/hpcxx.py +++ b/src/engine/SCons/Tool/hpcxx.py @@ -37,7 +37,10 @@ import os.path import SCons.Util -cplusplus = __import__('c++', globals(), locals(), []) +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('cxx', globals(), locals(), []) + acc = None diff --git a/src/engine/SCons/Tool/install.py b/src/engine/SCons/Tool/install.py index ee157531..c0a193bc 100644 --- a/src/engine/SCons/Tool/install.py +++ b/src/engine/SCons/Tool/install.py @@ -305,6 +305,7 @@ def InstallBuilderWrapper(env, target=None, source=None, dir=None, **kw): tgt.extend(BaseInstallBuilder(env, target, src, **kw)) return tgt + def InstallAsBuilderWrapper(env, target=None, source=None, **kw): result = [] for src, tgt in map(lambda x, y: (x, y), source, target): @@ -313,6 +314,7 @@ def InstallAsBuilderWrapper(env, target=None, source=None, **kw): BaseVersionedInstallBuilder = None + def InstallVersionedBuilderWrapper(env, target=None, source=None, dir=None, **kw): if target and dir: import SCons.Errors @@ -344,6 +346,7 @@ def InstallVersionedBuilderWrapper(env, target=None, source=None, dir=None, **kw added = None + def generate(env): from SCons.Script import AddOption, GetOption diff --git a/src/engine/SCons/Tool/ipkg.py b/src/engine/SCons/Tool/ipkg.py index 452cbb9d..df77eea9 100644 --- a/src/engine/SCons/Tool/ipkg.py +++ b/src/engine/SCons/Tool/ipkg.py @@ -44,20 +44,26 @@ def generate(env): try: bld = env['BUILDERS']['Ipkg'] except KeyError: - bld = SCons.Builder.Builder( action = '$IPKGCOM', - suffix = '$IPKGSUFFIX', - source_scanner = None, - target_scanner = None) + bld = SCons.Builder.Builder(action='$IPKGCOM', + suffix='$IPKGSUFFIX', + source_scanner=None, + target_scanner=None) env['BUILDERS']['Ipkg'] = bld - env['IPKG'] = 'ipkg-build' - env['IPKGCOM'] = '$IPKG $IPKGFLAGS ${SOURCE}' - env['IPKGUSER'] = os.popen('id -un').read().strip() - env['IPKGGROUP'] = os.popen('id -gn').read().strip() - env['IPKGFLAGS'] = SCons.Util.CLVar('-o $IPKGUSER -g $IPKGGROUP') + + env['IPKG'] = 'ipkg-build' + env['IPKGCOM'] = '$IPKG $IPKGFLAGS ${SOURCE}' + + if env.WhereIs('id'): + env['IPKGUSER'] = os.popen('id -un').read().strip() + env['IPKGGROUP'] = os.popen('id -gn').read().strip() + env['IPKGFLAGS'] = SCons.Util.CLVar('-o $IPKGUSER -g $IPKGGROUP') env['IPKGSUFFIX'] = '.ipk' def exists(env): + """ + Can we find the tool + """ return env.Detect('ipkg-build') # Local Variables: diff --git a/src/engine/SCons/Tool/ldc.py b/src/engine/SCons/Tool/ldc.py index ade95db4..b10bb75a 100644 --- a/src/engine/SCons/Tool/ldc.py +++ b/src/engine/SCons/Tool/ldc.py @@ -1,7 +1,9 @@ +from __future__ import print_function + """SCons.Tool.ldc Tool-specific initialization for the LDC compiler. -(http://www.dsource.org/projects/ldc) +(https://github.com/ldc-developers/ldc) Developed by Russel Winder (russel@winder.org.uk) 2012-05-09 onwards @@ -57,7 +59,7 @@ import SCons.Defaults import SCons.Scanner.D import SCons.Tool -import SCons.Tool.DCommon +import SCons.Tool.DCommon as DCommon def generate(env): @@ -68,7 +70,7 @@ def generate(env): static_obj.add_emitter('.d', SCons.Defaults.StaticObjectEmitter) shared_obj.add_emitter('.d', SCons.Defaults.SharedObjectEmitter) - env['DC'] = env.Detect('ldc2') + env['DC'] = env.Detect('ldc2') or 'ldc2' env['DCOM'] = '$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -c -of=$TARGET $SOURCES' env['_DINCFLAGS'] = '${_concat(DINCPREFIX, DPATH, DINCSUFFIX, __env__, RDirs, TARGET, SOURCE)}' env['_DVERFLAGS'] = '${_concat(DVERPREFIX, DVERSIONS, DVERSUFFIX, __env__)}' @@ -84,7 +86,7 @@ def generate(env): env['DDEBUG'] = [] if env['DC']: - SCons.Tool.DCommon.addDPATHToEnv(env, env['DC']) + DCommon.addDPATHToEnv(env, env['DC']) env['DINCPREFIX'] = '-I=' env['DINCSUFFIX'] = '' @@ -102,32 +104,29 @@ def generate(env): env['DSHLINK'] = '$DC' env['DSHLINKFLAGS'] = SCons.Util.CLVar('$DLINKFLAGS -shared -defaultlib=phobos2-ldc') - # Hack for Fedora the packages of which use the wrong name :-( - if os.path.exists('/usr/lib64/libphobos-ldc.so') or os.path.exists('/usr/lib32/libphobos-ldc.so') or os.path.exists('/usr/lib/libphobos-ldc.so') : - env['DSHLINKFLAGS'] = SCons.Util.CLVar('$DLINKFLAGS -shared -defaultlib=phobos-ldc') - env['SHDLINKCOM'] = '$DLINK -of=$TARGET $DSHLINKFLAGS $__DSHLIBVERSIONFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS' + + env['SHDLINKCOM'] = '$DLINK -of=$TARGET $DSHLINKFLAGS $__DSHLIBVERSIONFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS -L-ldruntime-ldc' env['DLIBLINKPREFIX'] = '' if env['PLATFORM'] == 'win32' else '-L-l' env['DLIBLINKSUFFIX'] = '.lib' if env['PLATFORM'] == 'win32' else '' - #env['_DLIBFLAGS'] = '${_concat(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, __env__, RDirs, TARGET, SOURCE)}' + # env['_DLIBFLAGS'] = '${_concat(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, __env__, RDirs, TARGET, SOURCE)}' env['_DLIBFLAGS'] = '${_stripixes(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__)}' env['DLIBDIRPREFIX'] = '-L-L' env['DLIBDIRSUFFIX'] = '' env['_DLIBDIRFLAGS'] = '${_concat(DLIBDIRPREFIX, LIBPATH, DLIBDIRSUFFIX, __env__, RDirs, TARGET, SOURCE)}' - env['DLIB'] = 'lib' if env['PLATFORM'] == 'win32' else 'ar cr' env['DLIBCOM'] = '$DLIB $_DLIBFLAGS {0}$TARGET $SOURCES $_DLIBFLAGS'.format('-c ' if env['PLATFORM'] == 'win32' else '') - #env['_DLIBFLAGS'] = '${_concat(DLIBFLAGPREFIX, DLIBFLAGS, DLIBFLAGSUFFIX, __env__)}' + # env['_DLIBFLAGS'] = '${_concat(DLIBFLAGPREFIX, DLIBFLAGS, DLIBFLAGSUFFIX, __env__)}' env['DLIBFLAGPREFIX'] = '-' env['DLIBFLAGSUFFIX'] = '' # __RPATH is set to $_RPATH in the platform specification if that # platform supports it. - env['DRPATHPREFIX'] = '-L-rpath=' + env['DRPATHPREFIX'] = '-L-Wl,-rpath,' if env['PLATFORM'] == 'darwin' else '-L-rpath=' env['DRPATHSUFFIX'] = '' env['_DRPATH'] = '${_concat(DRPATHPREFIX, RPATH, DRPATHSUFFIX, __env__)}' @@ -142,12 +141,16 @@ def generate(env): env['DSHLIBVERSION'] = '$SHLIBVERSION' env['DSHLIBVERSIONFLAGS'] = [] - SCons.Tool.createStaticLibBuilder(env) + env['BUILDERS']['ProgramAllAtOnce'] = SCons.Builder.Builder( + action='$DC $_DINCFLAGS $_DVERFLAGS $_DDEBUGFLAGS $_DFLAGS -of=$TARGET $DLINKFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS', + emitter=DCommon.allAtOnceEmitter, + ) def exists(env): return env.Detect('ldc2') + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Tool/ldc.xml b/src/engine/SCons/Tool/ldc.xml index f07144bd..9593f410 100644 --- a/src/engine/SCons/Tool/ldc.xml +++ b/src/engine/SCons/Tool/ldc.xml @@ -32,10 +32,6 @@ Sets construction variables for the D language compiler LDC2. <sets> <item>DC</item> <item>DCOM</item> -<item>_DINCFLAGS</item> -<item>_DVERFLAGS</item> -<item>_DDEBUGFLAGS</item> -<item>_DFLAGS</item> <item>SHDC</item> <item>SHDCOM</item> <item>DPATH</item> @@ -59,23 +55,177 @@ Sets construction variables for the D language compiler LDC2. <item>SHDLINKCOM</item> <item>DLIBLINKPREFIX</item> <item>DLIBLINKSUFFIX</item> -<item>_DLIBFLAGS</item> <item>DLIBDIRPREFIX</item> <item>DLIBDIRSUFFIX</item> -<item>_DLIBDIRFLAGS</item> <item>DLIB</item> <item>DLIBCOM</item> -<item>_DLIBFLAGS</item> <item>DLIBFLAGPREFIX</item> <item>DLIBFLAGSUFFIX</item> <item>DLINKFLAGPREFIX</item> <item>DLINKFLAGSUFFIX</item> <item>RPATHPREFIX</item> <item>RPATHSUFFIX</item> -<item>_RPATH</item> </sets> <uses> </uses> </tool> +<cvar name="DC"> +<summary> +<para> +The D compiler to use. +</para> +</summary> +</cvar> + +<cvar name="DCOM"> +<summary> +<para> + The command line used to compile a D file to an object file. + Any options specified in the &cv-link-DFLAGS; construction variable + is included on this command line. +</para> +</summary> +</cvar> + +<cvar name="DDEBUG"> +<summary> +<para> + List of debug tags to enable when compiling. +</para> +</summary> +</cvar> + +<cvar name="DFLAGS"> +<summary> +<para> + General options that are passed to the D compiler. +</para> +</summary> +</cvar> + +<cvar name="DLIB"> +<summary> +<para> + Name of the lib tool to use for D codes. +</para> +</summary> +</cvar> + +<cvar name="DLIBCOM"> +<summary> +<para> + The command line to use when creating libraries. +</para> +</summary> +</cvar> + +<cvar name="DLINK"> +<summary> +<para> + Name of the linker to use for linking systems including D sources. +</para> +</summary> +</cvar> + +<cvar name="DLINKCOM"> +<summary> +<para> + The command line to use when linking systems including D sources. +</para> +</summary> +</cvar> + +<cvar name="DLINKFLAGS"> +<summary> +<para> +List of linker flags. +</para> +</summary> +</cvar> + +<cvar name="DPATH"> +<summary> +<para> + List of paths to search for import modules. +</para> +</summary> +</cvar> + +<cvar name="DVERSIONS"> +<summary> +<para> + List of version tags to enable when compiling. +</para> +</summary> +</cvar> + +<cvar name="SHDC"> +<summary> +<para> + The name of the compiler to use when compiling D source + destined to be in a shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDCOM"> +<summary> +<para> + The command line to use when compiling code to be part of shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDLINK"> +<summary> +<para> + The linker to use when creating shared objects for code bases + include D sources. +</para> +</summary> +</cvar> + +<cvar name="SHDLINKCOM"> +<summary> +<para> + The command line to use when generating shared objects. +</para> +</summary> +</cvar> + +<cvar name="SHDLINKFLAGS"> +<summary> +<para> + The list of flags to use when generating a shared object. +</para> +</summary> +</cvar> + +<builder name="ProgramAllAtOnce"> + <summary> + <para> + Builds an executable from D sources without first creating individual + objects for each file. + </para> + <para> + D sources can be compiled file-by-file as C and C++ source are, and + D is integrated into the &scons; Object and Program builders for + this model of build. D codes can though do whole source + meta-programming (some of the testing frameworks do this). For this + it is imperative that all sources are compiled and linked in a single call of + the D compiler. This builder serves that purpose. + </para> + <example_commands> + env.ProgramAllAtOnce('executable', ['mod_a.d, mod_b.d', 'mod_c.d']) + </example_commands> + <para> + This command will compile the modules mod_a, mod_b, and mod_c in a + single compilation process without first creating object files for + the modules. Some of the D compilers will create executable.o others + will not. + </para> + </summary> +</builder> + </sconsdoc> diff --git a/src/engine/SCons/Tool/link.py b/src/engine/SCons/Tool/link.py index ae2c4b81..07e92507 100644 --- a/src/engine/SCons/Tool/link.py +++ b/src/engine/SCons/Tool/link.py @@ -46,7 +46,10 @@ import SCons.Warnings from SCons.Tool.FortranCommon import isfortran from SCons.Tool.DCommon import isD -cplusplus = __import__(__package__+'.c++', globals(), locals(), ['*']) + +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +# cplusplus = __import__(__package__+'.cxx', globals(), locals(), ['*']) issued_mixed_link_warning = False @@ -240,8 +243,10 @@ def _versioned_lib_callbacks(): 'VersionedLdModSoname' : _versioned_ldmod_soname, }.copy() -# Setup all variables required by the versioning machinery def _setup_versioned_lib_variables(env, **kw): + """ + Setup all variables required by the versioning machinery + """ tool = None try: tool = kw['tool'] diff --git a/src/engine/SCons/Tool/linkloc.py b/src/engine/SCons/Tool/linkloc.py index bd643f7a..c73852b7 100644 --- a/src/engine/SCons/Tool/linkloc.py +++ b/src/engine/SCons/Tool/linkloc.py @@ -52,8 +52,8 @@ def repl_linker_command(m): # Replaces any linker command file directives (e.g. "@foo.lnk") with # the actual contents of the file. try: - f=open(m.group(2), "r") - return m.group(1) + f.read() + with open(m.group(2), "r") as f: + return m.group(1) + f.read() except IOError: # the linker should return an error if it can't # find the linker command file so we will remain quiet. diff --git a/src/engine/SCons/Tool/midl.py b/src/engine/SCons/Tool/midl.py index 7a59e334..ed9ea94a 100644 --- a/src/engine/SCons/Tool/midl.py +++ b/src/engine/SCons/Tool/midl.py @@ -43,22 +43,22 @@ from .MSCommon import msvc_exists def midl_emitter(target, source, env): """Produces a list of outputs from the MIDL compiler""" - base, ext = SCons.Util.splitext(str(target[0])) + base, _ = SCons.Util.splitext(str(target[0])) tlb = target[0] incl = base + '.h' interface = base + '_i.c' - t = [tlb, incl, interface] + targets = [tlb, incl, interface] midlcom = env['MIDLCOM'] if midlcom.find('/proxy') != -1: proxy = base + '_p.c' - t.append(proxy) + targets.append(proxy) if midlcom.find('/dlldata') != -1: dlldata = base + '_data.c' - t.append(dlldata) - - return (t,source) + targets.append(dlldata) + + return (targets, source) idl_scanner = SCons.Scanner.IDL.IDLScan() diff --git a/src/engine/SCons/Tool/msvc.xml b/src/engine/SCons/Tool/msvc.xml index 2b4619e4..1823a895 100644 --- a/src/engine/SCons/Tool/msvc.xml +++ b/src/engine/SCons/Tool/msvc.xml @@ -352,6 +352,8 @@ constructor; setting it later has no effect. <para> Valid values for Windows are +<literal>14.0</literal>, +<literal>14.0Exp</literal>, <literal>12.0</literal>, <literal>12.0Exp</literal>, <literal>11.0</literal>, @@ -440,4 +442,27 @@ For example, if you want to compile 64-bit binaries, you would set </summary> </cvar> +<cvar name="MSVC_UWP_APP"> +<summary> +<para> +Build libraries for a Universal Windows Platform (UWP) Application. +</para> + +<para> +If &cv-MSVC_UWP_APP; is set, the Visual Studio environment will be set up to point +to the Windows Store compatible libraries and Visual Studio runtimes. In doing so, +any libraries that are built will be able to be used in a UWP App and published +to the Windows Store. +This flag will only have an effect with Visual Studio 2015+. +This variable must be passed as an argument to the Environment() +constructor; setting it later has no effect. +</para> + +<para> +Valid values are '1' or '0' +</para> + +</summary> +</cvar> + </sconsdoc> diff --git a/src/engine/SCons/Tool/msvs.py b/src/engine/SCons/Tool/msvs.py index 939668e9..e1175e0a 100644 --- a/src/engine/SCons/Tool/msvs.py +++ b/src/engine/SCons/Tool/msvs.py @@ -87,7 +87,7 @@ def _generateGUID(slnfile, name): # Normalize the slnfile path to a Windows path (\ separators) so # the generated file has a consistent GUID even if we generate # it on a non-Windows platform. - m.update(ntpath.normpath(str(slnfile)) + str(name)) + m.update(bytearray(ntpath.normpath(str(slnfile)) + str(name),'utf-8')) solution = m.hexdigest().upper() # convert most of the signature to GUID form (discard the rest) solution = "{" + solution[:8] + "-" + solution[8:12] + "-" + solution[12:16] + "-" + solution[16:20] + "-" + solution[20:32] + "}" @@ -213,7 +213,7 @@ class _UserGenerator(object): if self.createfile: dbg_settings = dict(list(zip(variants, dbg_settings))) - for var, src in list(dbg_settings.items()): + for var, src in dbg_settings.items(): # Update only expected keys trg = {} for key in [k for k in list(self.usrdebg.keys()) if k in src]: @@ -303,7 +303,7 @@ class _GenerateV7User(_UserGenerator): debug = self.configs[kind].debug if debug: debug_settings = '\n'.join(['\t\t\t\t%s="%s"' % (key, xmlify(value)) - for key, value in list(debug.items()) + for key, value in debug.items() if value is not None]) self.usrfile.write(self.usrconf % locals()) self.usrfile.write('\t</Configurations>\n</VisualStudioUserFile>') @@ -365,7 +365,7 @@ class _GenerateV10User(_UserGenerator): debug = self.configs[kind].debug if debug: debug_settings = '\n'.join(['\t\t<%s>%s</%s>' % (key, xmlify(value), key) - for key, value in list(debug.items()) + for key, value in debug.items() if value is not None]) self.usrfile.write(self.usrconf % locals()) self.usrfile.write('</Project>') @@ -645,10 +645,10 @@ class _GenerateV6DSP(_DSPGenerator): if self.nokeep == 0: # now we pickle some data and add it to the file -- MSDEV will ignore it. pdata = pickle.dumps(self.configs,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write(pdata + '\n') pdata = pickle.dumps(self.sources,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write(pdata + '\n') def PrintSourceFiles(self): @@ -917,14 +917,14 @@ class _GenerateV7DSP(_DSPGenerator, _GenerateV7User): if self.nokeep == 0: # now we pickle some data and add it to the file -- MSDEV will ignore it. pdata = pickle.dumps(self.configs,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write('<!-- SCons Data:\n' + pdata + '\n') pdata = pickle.dumps(self.sources,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write(pdata + '-->\n') def printSources(self, hierarchy, commonprefix): - sorteditems = sorted(list(hierarchy.items()), key=lambda a: a[0].lower()) + sorteditems = sorted(hierarchy.items(), key=lambda a: a[0].lower()) # First folders, then files for key, value in sorteditems: @@ -1236,14 +1236,14 @@ class _GenerateV10DSP(_DSPGenerator, _GenerateV10User): if self.nokeep == 0: # now we pickle some data and add it to the file -- MSDEV will ignore it. pdata = pickle.dumps(self.configs,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write('<!-- SCons Data:\n' + pdata + '\n') pdata = pickle.dumps(self.sources,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) + pdata = base64.encodestring(pdata).decode() self.file.write(pdata + '-->\n') def printFilters(self, hierarchy, name): - sorteditems = sorted(list(hierarchy.items()), key = lambda a: a[0].lower()) + sorteditems = sorted(hierarchy.items(), key = lambda a: a[0].lower()) for key, value in sorteditems: if SCons.Util.is_Dict(value): @@ -1260,7 +1260,7 @@ class _GenerateV10DSP(_DSPGenerator, _GenerateV10User): 'Resource Files': 'None', 'Other Files': 'None'} - sorteditems = sorted(list(hierarchy.items()), key = lambda a: a[0].lower()) + sorteditems = sorted(hierarchy.items(), key = lambda a: a[0].lower()) # First folders, then files for key, value in sorteditems: @@ -1610,8 +1610,9 @@ class _GenerateV7DSW(_DSWGenerator): self.file.write('EndGlobal\n') if self.nokeep == 0: pdata = pickle.dumps(self.configs,PICKLE_PROTOCOL) - pdata = base64.encodestring(pdata) - self.file.write(pdata + '\n') + pdata = base64.encodestring(pdata).decode() + self.file.write(pdata) + self.file.write('\n') def Build(self): try: diff --git a/src/engine/SCons/Tool/msvs.xml b/src/engine/SCons/Tool/msvs.xml index e85b27cd..c4701e1e 100644 --- a/src/engine/SCons/Tool/msvs.xml +++ b/src/engine/SCons/Tool/msvs.xml @@ -127,22 +127,24 @@ compilation error messages displayed in the Visual Studio console output window. This can be remedied by adding the Visual C/C++ <literal>/FC</literal> compiler option to the &cv-link-CCFLAGS; variable so that the compiler will print the full path name of any files that cause compilation errors. </para> -<para> Example usage: </para> <example_commands>barsrcs = ['bar.cpp'], -barincs = ['bar.h'], +<para> Example usage: </para> + <example_commands> +barsrcs = ['bar.cpp'] +barincs = ['bar.h'] barlocalincs = ['StdAfx.h'] barresources = ['bar.rc','resource.h'] barmisc = ['bar_readme.txt'] dll = env.SharedLibrary(target = 'bar.dll', source = barsrcs) - +buildtarget = [s for s in dll if str(s).endswith('dll')] env.MSVSProject(target = 'Bar' + env['MSVSPROJECTSUFFIX'], srcs = barsrcs, incs = barincs, localincs = barlocalincs, resources = barresources, misc = barmisc, - buildtarget = dll, + buildtarget = buildtarget, variant = 'Release') </example_commands> <para>Starting with version 2.4 of diff --git a/src/engine/SCons/Tool/packaging/__init__.py b/src/engine/SCons/Tool/packaging/__init__.py index 17279383..dc4b80da 100644 --- a/src/engine/SCons/Tool/packaging/__init__.py +++ b/src/engine/SCons/Tool/packaging/__init__.py @@ -72,7 +72,7 @@ def Tag(env, target, source, *more_tags, **kw_tags): target=env.Flatten(target) for t in target: - for (k,v) in list(kw_tags.items()): + for (k,v) in kw_tags.items(): # all file tags have to start with PACKAGING_, so we can later # differentiate between "normal" object attributes and the # packaging attributes. As the user should not be bothered with @@ -233,7 +233,7 @@ def copy_attr(f1, f2): """ copyit = lambda x: not hasattr(f2, x) and x[:10] == 'PACKAGING_' if f1._tags: - pattrs = list(filter(copyit, f1._tags)) + pattrs = [tag for tag in f1._tags if copyit(tag)] for attr in pattrs: f2.Tag(attr, f1.GetTag(attr)) @@ -288,7 +288,7 @@ def stripinstallbuilder(target, source, env): (file.builder.name=="InstallBuilder" or\ file.builder.name=="InstallAsBuilder")) - if len(list(filter(has_no_install_location, source))): + if len([src for src in source if has_no_install_location(src)]): warn(Warning, "there are files to package which have no\ InstallBuilder attached, this might lead to irreproducible packages") diff --git a/src/engine/SCons/Tool/packaging/msi.py b/src/engine/SCons/Tool/packaging/msi.py index 73d3567e..c25f856b 100644 --- a/src/engine/SCons/Tool/packaging/msi.py +++ b/src/engine/SCons/Tool/packaging/msi.py @@ -172,7 +172,7 @@ def generate_guids(root): # find all XMl nodes matching the key, retrieve their attribute, hash their # subtree, convert hash to string and add as a attribute to the xml node. - for (key,value) in list(needs_id.items()): + for (key,value) in needs_id.items(): node_list = root.getElementsByTagName(key) attribute = value for node in node_list: @@ -335,7 +335,7 @@ def build_wxsfile_file_section(root, files, NAME, VERSION, VENDOR, filename_set, } # fill in the default tags given above. - for k,v in [ (k, v) for (k,v) in list(h.items()) if not hasattr(file, k) ]: + for k,v in [ (k, v) for (k,v) in h.items() if not hasattr(file, k) ]: setattr( file, k, v ) File = factory.createElement( 'File' ) @@ -382,7 +382,7 @@ def build_wxsfile_features_section(root, files, NAME, VERSION, SUMMARY, id_set): Feature.attributes['Description'] = escape( SUMMARY ) Feature.attributes['Display'] = 'expand' - for (feature, files) in list(create_feature_dict(files).items()): + for (feature, files) in create_feature_dict(files).items(): SubFeature = factory.createElement('Feature') SubFeature.attributes['Level'] = '1' diff --git a/src/engine/SCons/Tool/packaging/rpm.py b/src/engine/SCons/Tool/packaging/rpm.py index 0931c77b..a3481a43 100644 --- a/src/engine/SCons/Tool/packaging/rpm.py +++ b/src/engine/SCons/Tool/packaging/rpm.py @@ -332,7 +332,7 @@ class SimpleTagCompiler(object): international = [t for t in replacements if is_international(t[0])] for key, replacement in international: try: - x = [t for t in list(values.items()) if strip_country_code(t[0]) == key] + x = [t for t in values.items() if strip_country_code(t[0]) == key] int_values_for_key = [(get_country_code(t[0]),t[1]) for t in x] for v in int_values_for_key: str = str + replacement % v diff --git a/src/engine/SCons/Tool/qt.py b/src/engine/SCons/Tool/qt.py index f01fff6a..5f99054f 100644 --- a/src/engine/SCons/Tool/qt.py +++ b/src/engine/SCons/Tool/qt.py @@ -59,7 +59,11 @@ SCons.Warnings.enableWarningClass(ToolQtWarning) header_extensions = [".h", ".hxx", ".hpp", ".hh"] if SCons.Util.case_sensitive_suffixes('.h', '.H'): header_extensions.append('.H') -cplusplus = __import__('c++', globals(), locals(), []) + +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('cxx', globals(), locals(), []) + cxx_suffixes = cplusplus.CXXSuffixes def checkMocIncluded(target, source, env): diff --git a/src/engine/SCons/Tool/rpm.py b/src/engine/SCons/Tool/rpm.py index b7d65a8c..5198f84a 100644 --- a/src/engine/SCons/Tool/rpm.py +++ b/src/engine/SCons/Tool/rpm.py @@ -71,7 +71,7 @@ def build_rpm(target, source, env): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output = handle.stdout.read() + output = SCons.Util.to_str(handle.stdout.read()) status = handle.wait() if status: diff --git a/src/engine/SCons/Tool/rpmutils.py b/src/engine/SCons/Tool/rpmutils.py index d4db4178..e2d69978 100644 --- a/src/engine/SCons/Tool/rpmutils.py +++ b/src/engine/SCons/Tool/rpmutils.py @@ -42,6 +42,8 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import platform import subprocess +import SCons.Util + # Start of rpmrc dictionaries (Marker, don't change or remove!) os_canon = { 'AIX' : ['AIX','5'], @@ -444,6 +446,7 @@ def defaultMachine(use_rpm_default=True): try: # This should be the most reliable way to get the default arch rmachine = subprocess.check_output(['rpm', '--eval=%_target_cpu'], shell=False).rstrip() + rmachine = SCons.Util.to_str(rmachine) except Exception as e: # Something went wrong, try again by looking up platform.machine() return defaultMachine(False) @@ -520,7 +523,7 @@ def updateRpmDicts(rpmrc, pyfile): if l.startswith('# Start of rpmrc dictionaries'): pm = 1 # Write data sections to single dictionaries - for key, entries in list(data.items()): + for key, entries in data.items(): out.write("%s = {\n" % key) for arch in sorted(entries.keys()): out.write(" '%s' : ['%s'],\n" % (arch, "','".join(entries[arch]))) diff --git a/src/engine/SCons/Tool/sgicxx.py b/src/engine/SCons/Tool/sgicxx.py index 35547ac9..48b04ea9 100644 --- a/src/engine/SCons/Tool/sgicxx.py +++ b/src/engine/SCons/Tool/sgicxx.py @@ -35,7 +35,10 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.Util -cplusplus = __import__('c++', globals(), locals(), []) +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('cxx', globals(), locals(), []) + def generate(env): """Add Builders and construction variables for SGI MIPS C++ to an Environment.""" diff --git a/src/engine/SCons/Tool/suncxx.py b/src/engine/SCons/Tool/suncxx.py index 49f90eaf..29226eaa 100644 --- a/src/engine/SCons/Tool/suncxx.py +++ b/src/engine/SCons/Tool/suncxx.py @@ -39,7 +39,9 @@ import os import re import subprocess -cplusplus = __import__('c++', globals(), locals(), []) +import SCons.Tool.cxx +cplusplus = SCons.Tool.cxx +#cplusplus = __import__('c++', globals(), locals(), []) package_info = {} diff --git a/src/engine/SCons/Tool/swig.py b/src/engine/SCons/Tool/swig.py index 9935de87..da4472f7 100644 --- a/src/engine/SCons/Tool/swig.py +++ b/src/engine/SCons/Tool/swig.py @@ -142,8 +142,8 @@ def _get_swig_version(env, swig): if pipe.wait() != 0: return # MAYBE: out = SCons.Util.to_str (pipe.stdout.read()) - out = pipe.stdout.read() - match = re.search(b'SWIG Version\s+(\S+).*', out, re.MULTILINE) + out = SCons.Util.to_str(pipe.stdout.read()) + match = re.search('SWIG Version\s+(\S+).*', out, re.MULTILINE) if match: if verbose: print("Version is:%s"%match.group(1)) return match.group(1) diff --git a/src/engine/SCons/Tool/tex.py b/src/engine/SCons/Tool/tex.py index 85bd41fb..8e09d56d 100644 --- a/src/engine/SCons/Tool/tex.py +++ b/src/engine/SCons/Tool/tex.py @@ -297,7 +297,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None logfilename = targetbase + '.log' logContent = '' if os.path.isfile(logfilename): - logContent = open(logfilename, "rb").read() + logContent = open(logfilename, "r").read() # Read the fls file to find all .aux files @@ -305,7 +305,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None flsContent = '' auxfiles = [] if os.path.isfile(flsfilename): - flsContent = open(flsfilename, "rb").read() + flsContent = open(flsfilename, "r").read() auxfiles = openout_aux_re.findall(flsContent) # remove duplicates dups = {} @@ -315,7 +315,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None bcffiles = [] if os.path.isfile(flsfilename): - flsContent = open(flsfilename, "rb").read() + flsContent = open(flsfilename, "r").read() bcffiles = openout_bcf_re.findall(flsContent) # remove duplicates dups = {} @@ -338,7 +338,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None already_bibtexed.append(auxfilename) target_aux = os.path.join(targetdir, auxfilename) if os.path.isfile(target_aux): - content = open(target_aux, "rb").read() + content = open(target_aux, "r").read() if content.find("bibdata") != -1: if Verbose: print("Need to run bibtex on ",auxfilename) @@ -361,7 +361,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None already_bibtexed.append(bcffilename) target_bcf = os.path.join(targetdir, bcffilename) if os.path.isfile(target_bcf): - content = open(target_bcf, "rb").read() + content = open(target_bcf, "r").read() if content.find("bibdata") != -1: if Verbose: print("Need to run biber on ",bcffilename) @@ -813,7 +813,7 @@ def tex_emitter_core(target, source, env, graphics_extensions): # read fls file to get all other files that latex creates and will read on the next pass # remove files from list that we explicitly dealt with above if os.path.isfile(flsfilename): - content = open(flsfilename, "rb").read() + content = open(flsfilename, "r").read() out_files = openout_re.findall(content) myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf'] for filename in out_files[:]: diff --git a/src/engine/SCons/Tool/textfile.py b/src/engine/SCons/Tool/textfile.py index e79badf2..9e2327af 100644 --- a/src/engine/SCons/Tool/textfile.py +++ b/src/engine/SCons/Tool/textfile.py @@ -53,7 +53,15 @@ import re from SCons.Node import Node from SCons.Node.Python import Value -from SCons.Util import is_String, is_Sequence, is_Dict +from SCons.Util import is_String, is_Sequence, is_Dict, to_bytes, PY3 + + +if PY3: + TEXTFILE_FILE_WRITE_MODE = 'w' +else: + TEXTFILE_FILE_WRITE_MODE = 'wb' + +LINESEP = '\n' def _do_subst(node, subs): """ @@ -64,62 +72,82 @@ def _do_subst(node, subs): 1.2345 and so forth. """ contents = node.get_text_contents() - if not subs: return contents - for (k,v) in subs: - contents = re.sub(k, v, contents) + if subs: + for (k, val) in subs: + contents = re.sub(k, val, contents) + + if 'b' in TEXTFILE_FILE_WRITE_MODE: + try: + contents = bytearray(contents, 'utf-8') + except UnicodeDecodeError: + # contents is already utf-8 encoded python 2 str i.e. a byte array + contents = bytearray(contents) + return contents + def _action(target, source, env): + # prepare the line separator linesep = env['LINESEPARATOR'] if linesep is None: - linesep = os.linesep + linesep = LINESEP # os.linesep elif is_String(linesep): pass elif isinstance(linesep, Value): linesep = linesep.get_text_contents() else: - raise SCons.Errors.UserError( - 'unexpected type/class for LINESEPARATOR: %s' - % repr(linesep), None) + raise SCons.Errors.UserError('unexpected type/class for LINESEPARATOR: %s' + % repr(linesep), None) + + if 'b' in TEXTFILE_FILE_WRITE_MODE: + linesep = to_bytes(linesep) # create a dictionary to use for the substitutions if 'SUBST_DICT' not in env: subs = None # no substitutions else: - d = env['SUBST_DICT'] - if is_Dict(d): - d = list(d.items()) - elif is_Sequence(d): + subst_dict = env['SUBST_DICT'] + if is_Dict(subst_dict): + subst_dict = list(subst_dict.items()) + elif is_Sequence(subst_dict): pass else: raise SCons.Errors.UserError('SUBST_DICT must be dict or sequence') subs = [] - for (k,v) in d: - if callable(v): - v = v() - if is_String(v): - v = env.subst(v) + for (k, value) in subst_dict: + if callable(value): + value = value() + if is_String(value): + value = env.subst(value) else: - v = str(v) - subs.append((k,v)) + value = str(value) + subs.append((k, value)) # write the file try: - fd = open(target[0].get_path(), "wb") - except (OSError,IOError) as e: + if SCons.Util.PY3: + target_file = open(target[0].get_path(), TEXTFILE_FILE_WRITE_MODE, newline='') + else: + target_file = open(target[0].get_path(), TEXTFILE_FILE_WRITE_MODE) + except (OSError, IOError): raise SCons.Errors.UserError("Can't write target file %s" % target[0]) + # separate lines by 'linesep' only if linesep is not empty lsep = None - for s in source: - if lsep: fd.write(lsep) - fd.write(_do_subst(s, subs)) + for line in source: + if lsep: + target_file.write(lsep) + + target_file.write(_do_subst(line, subs)) lsep = linesep - fd.close() + target_file.close() + def _strfunc(target, source, env): return "Creating '%s'" % target[0] + def _convert_list_R(newlist, sources): for elem in sources: if is_Sequence(elem): @@ -128,6 +156,8 @@ def _convert_list_R(newlist, sources): newlist.append(elem) else: newlist.append(Value(elem)) + + def _convert_list(target, source, env): if len(target) != 1: raise SCons.Errors.UserError("Only one target file allowed") @@ -135,29 +165,31 @@ def _convert_list(target, source, env): _convert_list_R(newlist, source) return target, newlist + _common_varlist = ['SUBST_DICT', 'LINESEPARATOR'] _text_varlist = _common_varlist + ['TEXTFILEPREFIX', 'TEXTFILESUFFIX'] _text_builder = SCons.Builder.Builder( - action = SCons.Action.Action(_action, _strfunc, varlist = _text_varlist), - source_factory = Value, - emitter = _convert_list, - prefix = '$TEXTFILEPREFIX', - suffix = '$TEXTFILESUFFIX', - ) + action=SCons.Action.Action(_action, _strfunc, varlist=_text_varlist), + source_factory=Value, + emitter=_convert_list, + prefix='$TEXTFILEPREFIX', + suffix='$TEXTFILESUFFIX', +) _subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'TEXTFILESUFFIX'] _subst_builder = SCons.Builder.Builder( - action = SCons.Action.Action(_action, _strfunc, varlist = _subst_varlist), - source_factory = SCons.Node.FS.File, - emitter = _convert_list, - prefix = '$SUBSTFILEPREFIX', - suffix = '$SUBSTFILESUFFIX', - src_suffix = ['.in'], - ) + action=SCons.Action.Action(_action, _strfunc, varlist=_subst_varlist), + source_factory=SCons.Node.FS.File, + emitter=_convert_list, + prefix='$SUBSTFILEPREFIX', + suffix='$SUBSTFILESUFFIX', + src_suffix=['.in'], +) + def generate(env): - env['LINESEPARATOR'] = os.linesep + env['LINESEPARATOR'] = LINESEP # os.linesep env['BUILDERS']['Textfile'] = _text_builder env['TEXTFILEPREFIX'] = '' env['TEXTFILESUFFIX'] = '.txt' @@ -165,6 +197,7 @@ def generate(env): env['SUBSTFILEPREFIX'] = '' env['SUBSTFILESUFFIX'] = '' + def exists(env): return 1 diff --git a/src/engine/SCons/Tool/xgettext.py b/src/engine/SCons/Tool/xgettext.py index e9b49b7b..2c0ce405 100644 --- a/src/engine/SCons/Tool/xgettext.py +++ b/src/engine/SCons/Tool/xgettext.py @@ -26,311 +26,333 @@ Tool specific initialization of `xgettext` tool. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + ############################################################################# class _CmdRunner(object): - """ Callabe object, which runs shell command storing its stdout and stderr to - variables. It also provides `strfunction()` method, which shall be used by - scons Action objects to print command string. """ - - def __init__(self, command, commandstr = None): - self.out = None - self.err = None - self.status = None - self.command = command - self.commandstr = commandstr - - def __call__(self, target, source, env): - import SCons.Action - import subprocess - import os - import sys - kw = { - 'stdin' : 'devnull', - 'stdout' : subprocess.PIPE, - 'stderr' : subprocess.PIPE, - 'universal_newlines' : True, - 'shell' : True - } - command = env.subst(self.command, target = target, source = source) - proc = SCons.Action._subproc(env, command, **kw) - self.out, self.err = proc.communicate() - self.status = proc.wait() - if self.err: - sys.stderr.write(unicode(self.err)) - return self.status - - def strfunction(self, target, source, env): - import os - comstr = self.commandstr - if env.subst(comstr, target = target, source = source) == "": - comstr = self.command - s = env.subst(comstr, target = target, source = source) - return s + """ Callable object, which runs shell command storing its stdout and stderr to + variables. It also provides `strfunction()` method, which shall be used by + scons Action objects to print command string. """ + + def __init__(self, command, commandstr=None): + self.out = None + self.err = None + self.status = None + self.command = command + self.commandstr = commandstr + + def __call__(self, target, source, env): + import SCons.Action + import subprocess + import os + import sys + kw = { + 'stdin': 'devnull', + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, + 'universal_newlines': True, + 'shell': True + } + command = env.subst(self.command, target=target, source=source) + proc = SCons.Action._subproc(env, command, **kw) + self.out, self.err = proc.communicate() + self.status = proc.wait() + if self.err: + sys.stderr.write(unicode(self.err)) + return self.status + + def strfunction(self, target, source, env): + import os + comstr = self.commandstr + if env.subst(comstr, target=target, source=source) == "": + comstr = self.command + s = env.subst(comstr, target=target, source=source) + return s + + ############################################################################# ############################################################################# def _update_pot_file(target, source, env): - """ Action function for `POTUpdate` builder """ - import re - import os - import SCons.Action - nop = lambda target, source, env : 0 - - # Save scons cwd and os cwd (NOTE: they may be different. After the job, we - # revert each one to its original state). - save_cwd = env.fs.getcwd() - save_os_cwd = os.getcwd() - chdir = target[0].dir - chdir_str = repr(chdir.get_abspath()) - # Print chdir message (employ SCons.Action.Action for that. It knows better - # than me how to to this correctly). - env.Execute(SCons.Action.Action(nop, "Entering " + chdir_str)) - # Go to target's directory and do our job - env.fs.chdir(chdir, 1) # Go into target's directory - try: - cmd = _CmdRunner('$XGETTEXTCOM', '$XGETTEXTCOMSTR') - action = SCons.Action.Action(cmd, strfunction=cmd.strfunction) - status = action([ target[0] ], source, env) - except: - # Something went wrong. + """ Action function for `POTUpdate` builder """ + import re + import os + import SCons.Action + nop = lambda target, source, env: 0 + + # Save scons cwd and os cwd (NOTE: they may be different. After the job, we + # revert each one to its original state). + save_cwd = env.fs.getcwd() + save_os_cwd = os.getcwd() + chdir = target[0].dir + chdir_str = repr(chdir.get_abspath()) + # Print chdir message (employ SCons.Action.Action for that. It knows better + # than me how to to this correctly). + env.Execute(SCons.Action.Action(nop, "Entering " + chdir_str)) + # Go to target's directory and do our job + env.fs.chdir(chdir, 1) # Go into target's directory + try: + cmd = _CmdRunner('$XGETTEXTCOM', '$XGETTEXTCOMSTR') + action = SCons.Action.Action(cmd, strfunction=cmd.strfunction) + status = action([target[0]], source, env) + except: + # Something went wrong. + env.Execute(SCons.Action.Action(nop, "Leaving " + chdir_str)) + # Revert working dirs to previous state and re-throw exception. + env.fs.chdir(save_cwd, 0) + os.chdir(save_os_cwd) + raise + # Print chdir message. env.Execute(SCons.Action.Action(nop, "Leaving " + chdir_str)) - # Revert working dirs to previous state and re-throw exception. + # Revert working dirs to previous state. env.fs.chdir(save_cwd, 0) os.chdir(save_os_cwd) - raise - # Print chdir message. - env.Execute(SCons.Action.Action(nop, "Leaving " + chdir_str)) - # Revert working dirs to previous state. - env.fs.chdir(save_cwd, 0) - os.chdir(save_os_cwd) - # If the command was not successfull, return error code. - if status: return status - - new_content = cmd.out - - if not new_content: - # When xgettext finds no internationalized messages, no *.pot is created - # (because we don't want to bother translators with empty POT files). - needs_update = False - explain = "no internationalized messages encountered" - else: - if target[0].exists(): - # If the file already exists, it's left unaltered unless its messages - # are outdated (w.r.t. to these recovered by xgettext from sources). - old_content = target[0].get_text_contents() - re_cdate = re.compile(r'^"POT-Creation-Date: .*"$[\r\n]?', re.M) - old_content_nocdate = re.sub(re_cdate,"",old_content) - new_content_nocdate = re.sub(re_cdate,"",new_content) - if(old_content_nocdate == new_content_nocdate): - # Messages are up-to-date + # If the command was not successfull, return error code. + if status: return status + + new_content = cmd.out + + if not new_content: + # When xgettext finds no internationalized messages, no *.pot is created + # (because we don't want to bother translators with empty POT files). needs_update = False - explain = "messages in file found to be up-to-date" - else: - # Messages are outdated - needs_update = True - explain = "messages in file were outdated" + explain = "no internationalized messages encountered" else: - # No POT file found, create new one - needs_update = True - explain = "new file" - if needs_update: - # Print message employing SCons.Action.Action for that. - msg = "Writing " + repr(str(target[0])) + " (" + explain + ")" - env.Execute(SCons.Action.Action(nop, msg)) - f = open(str(target[0]),"w") - f.write(new_content) - f.close() - return 0 - else: - # Print message employing SCons.Action.Action for that. - msg = "Not writing " + repr(str(target[0])) + " (" + explain + ")" - env.Execute(SCons.Action.Action(nop, msg)) - return 0 + if target[0].exists(): + # If the file already exists, it's left unaltered unless its messages + # are outdated (w.r.t. to these recovered by xgettext from sources). + old_content = target[0].get_text_contents() + re_cdate = re.compile(r'^"POT-Creation-Date: .*"$[\r\n]?', re.M) + old_content_nocdate = re.sub(re_cdate, "", old_content) + new_content_nocdate = re.sub(re_cdate, "", new_content) + if (old_content_nocdate == new_content_nocdate): + # Messages are up-to-date + needs_update = False + explain = "messages in file found to be up-to-date" + else: + # Messages are outdated + needs_update = True + explain = "messages in file were outdated" + else: + # No POT file found, create new one + needs_update = True + explain = "new file" + if needs_update: + # Print message employing SCons.Action.Action for that. + msg = "Writing " + repr(str(target[0])) + " (" + explain + ")" + env.Execute(SCons.Action.Action(nop, msg)) + f = open(str(target[0]), "w") + f.write(new_content) + f.close() + return 0 + else: + # Print message employing SCons.Action.Action for that. + msg = "Not writing " + repr(str(target[0])) + " (" + explain + ")" + env.Execute(SCons.Action.Action(nop, msg)) + return 0 + + ############################################################################# ############################################################################# from SCons.Builder import BuilderBase + + ############################################################################# class _POTBuilder(BuilderBase): - def _execute(self, env, target, source, *args): - if not target: - if 'POTDOMAIN' in env and env['POTDOMAIN']: - domain = env['POTDOMAIN'] - else: - domain = 'messages' - target = [ domain ] - return BuilderBase._execute(self, env, target, source, *args) + def _execute(self, env, target, source, *args): + if not target: + if 'POTDOMAIN' in env and env['POTDOMAIN']: + domain = env['POTDOMAIN'] + else: + domain = 'messages' + target = [domain] + return BuilderBase._execute(self, env, target, source, *args) + + ############################################################################# ############################################################################# -def _scan_xgettext_from_files(target, source, env, files = None, path = None): - """ Parses `POTFILES.in`-like file and returns list of extracted file names. - """ - import re - import SCons.Util - import SCons.Node.FS - - if files is None: +def _scan_xgettext_from_files(target, source, env, files=None, path=None): + """ Parses `POTFILES.in`-like file and returns list of extracted file names. + """ + import re + import SCons.Util + import SCons.Node.FS + + if files is None: + return 0 + if not SCons.Util.is_List(files): + files = [files] + + if path is None: + if 'XGETTEXTPATH' in env: + path = env['XGETTEXTPATH'] + else: + path = [] + if not SCons.Util.is_List(path): + path = [path] + + path = SCons.Util.flatten(path) + + dirs = () + for p in path: + if not isinstance(p, SCons.Node.FS.Base): + if SCons.Util.is_String(p): + p = env.subst(p, source=source, target=target) + p = env.arg2nodes(p, env.fs.Dir) + dirs += tuple(p) + # cwd is the default search path (when no path is defined by user) + if not dirs: + dirs = (env.fs.getcwd(),) + + # Parse 'POTFILE.in' files. + re_comment = re.compile(r'^#[^\n\r]*$\r?\n?', re.M) + re_emptyln = re.compile(r'^[ \t\r]*$\r?\n?', re.M) + re_trailws = re.compile(r'[ \t\r]+$') + for f in files: + # Find files in search path $XGETTEXTPATH + if isinstance(f, SCons.Node.FS.Base) and f.rexists(): + contents = f.get_text_contents() + contents = re_comment.sub("", contents) + contents = re_emptyln.sub("", contents) + contents = re_trailws.sub("", contents) + depnames = contents.splitlines() + for depname in depnames: + depfile = SCons.Node.FS.find_file(depname, dirs) + if not depfile: + depfile = env.arg2nodes(depname, dirs[0].File) + env.Depends(target, depfile) return 0 - if not SCons.Util.is_List(files): - files = [ files ] - if path is None: - if 'XGETTEXTPATH' in env: - path = env['XGETTEXTPATH'] - else: - path = [] - if not SCons.Util.is_List(path): - path = [ path ] - - path = SCons.Util.flatten(path) - - dirs = () - for p in path: - if not isinstance(p, SCons.Node.FS.Base): - if SCons.Util.is_String(p): - p = env.subst(p, source = source, target = target) - p = env.arg2nodes(p, env.fs.Dir) - dirs += tuple(p) - # cwd is the default search path (when no path is defined by user) - if not dirs: - dirs = (env.fs.getcwd(),) - - # Parse 'POTFILE.in' files. - re_comment = re.compile(r'^#[^\n\r]*$\r?\n?', re.M) - re_emptyln = re.compile(r'^[ \t\r]*$\r?\n?', re.M) - re_trailws = re.compile(r'[ \t\r]+$') - for f in files: - # Find files in search path $XGETTEXTPATH - if isinstance(f, SCons.Node.FS.Base) and f.rexists(): - contents = f.get_text_contents() - contents = re_comment.sub("", contents) - contents = re_emptyln.sub("", contents) - contents = re_trailws.sub("", contents) - depnames = contents.splitlines() - for depname in depnames: - depfile = SCons.Node.FS.find_file(depname, dirs) - if not depfile: - depfile = env.arg2nodes(depname, dirs[0].File) - env.Depends(target, depfile) - return 0 + ############################################################################# ############################################################################# def _pot_update_emitter(target, source, env): - """ Emitter function for `POTUpdate` builder """ - from SCons.Tool.GettextCommon import _POTargetFactory - import SCons.Util - import SCons.Node.FS - - if 'XGETTEXTFROM' in env: - xfrom = env['XGETTEXTFROM'] - else: + """ Emitter function for `POTUpdate` builder """ + from SCons.Tool.GettextCommon import _POTargetFactory + import SCons.Util + import SCons.Node.FS + + if 'XGETTEXTFROM' in env: + xfrom = env['XGETTEXTFROM'] + else: + return target, source + if not SCons.Util.is_List(xfrom): + xfrom = [xfrom] + + xfrom = SCons.Util.flatten(xfrom) + + # Prepare list of 'POTFILE.in' files. + files = [] + for xf in xfrom: + if not isinstance(xf, SCons.Node.FS.Base): + if SCons.Util.is_String(xf): + # Interpolate variables in strings + xf = env.subst(xf, source=source, target=target) + xf = env.arg2nodes(xf) + files.extend(xf) + if files: + env.Depends(target, files) + _scan_xgettext_from_files(target, source, env, files) return target, source - if not SCons.Util.is_List(xfrom): - xfrom = [ xfrom ] - - xfrom = SCons.Util.flatten(xfrom) - - # Prepare list of 'POTFILE.in' files. - files = [] - for xf in xfrom: - if not isinstance(xf, SCons.Node.FS.Base): - if SCons.Util.is_String(xf): - # Interpolate variables in strings - xf = env.subst(xf, source = source, target = target) - xf = env.arg2nodes(xf) - files.extend(xf) - if files: - env.Depends(target, files) - _scan_xgettext_from_files(target, source, env, files) - return target, source + + ############################################################################# ############################################################################# from SCons.Environment import _null + + ############################################################################# def _POTUpdateBuilderWrapper(env, target=None, source=_null, **kw): - return env._POTUpdateBuilder(target, source, **kw) + return env._POTUpdateBuilder(target, source, **kw) + + ############################################################################# ############################################################################# def _POTUpdateBuilder(env, **kw): - """ Creates `POTUpdate` builder object """ - import SCons.Action - from SCons.Tool.GettextCommon import _POTargetFactory - kw['action'] = SCons.Action.Action(_update_pot_file, None) - kw['suffix'] = '$POTSUFFIX' - kw['target_factory'] = _POTargetFactory(env, alias='$POTUPDATE_ALIAS').File - kw['emitter'] = _pot_update_emitter - return _POTBuilder(**kw) + """ Creates `POTUpdate` builder object """ + import SCons.Action + from SCons.Tool.GettextCommon import _POTargetFactory + kw['action'] = SCons.Action.Action(_update_pot_file, None) + kw['suffix'] = '$POTSUFFIX' + kw['target_factory'] = _POTargetFactory(env, alias='$POTUPDATE_ALIAS').File + kw['emitter'] = _pot_update_emitter + return _POTBuilder(**kw) + + ############################################################################# ############################################################################# -def generate(env,**kw): - """ Generate `xgettext` tool """ - import SCons.Util - from SCons.Tool.GettextCommon import RPaths, _detect_xgettext - - try: - env['XGETTEXT'] = _detect_xgettext(env) - except: - env['XGETTEXT'] = 'xgettext' - # NOTE: sources="$SOURCES" would work as well. However, we use following - # construction to convert absolute paths provided by scons onto paths - # relative to current working dir. Note, that scons expands $SOURCE(S) to - # absolute paths for sources $SOURCE(s) outside of current subtree (e.g. in - # "../"). With source=$SOURCE these absolute paths would be written to the - # resultant *.pot file (and its derived *.po files) as references to lines in - # source code (e.g. referring lines in *.c files). Such references would be - # correct (e.g. in poedit) only on machine on which *.pot was generated and - # would be of no use on other hosts (having a copy of source code located - # in different place in filesystem). - sources = '$( ${_concat( "", SOURCES, "", __env__, XgettextRPaths, TARGET' \ - + ', SOURCES)} $)' - - # NOTE: the output from $XGETTEXTCOM command must go to stdout, not to a file. - # This is required by the POTUpdate builder's action. - xgettextcom = '$XGETTEXT $XGETTEXTFLAGS $_XGETTEXTPATHFLAGS' \ - + ' $_XGETTEXTFROMFLAGS -o - ' + sources - - xgettextpathflags = '$( ${_concat( XGETTEXTPATHPREFIX, XGETTEXTPATH' \ - + ', XGETTEXTPATHSUFFIX, __env__, RDirs, TARGET, SOURCES)} $)' - xgettextfromflags = '$( ${_concat( XGETTEXTFROMPREFIX, XGETTEXTFROM' \ - + ', XGETTEXTFROMSUFFIX, __env__, target=TARGET, source=SOURCES)} $)' - - env.SetDefault( - _XGETTEXTDOMAIN = '${TARGET.filebase}', - XGETTEXTFLAGS = [ ], - XGETTEXTCOM = xgettextcom, - XGETTEXTCOMSTR = '', - XGETTEXTPATH = [ ], - XGETTEXTPATHPREFIX = '-D', - XGETTEXTPATHSUFFIX = '', - XGETTEXTFROM = None, - XGETTEXTFROMPREFIX = '-f', - XGETTEXTFROMSUFFIX = '', - _XGETTEXTPATHFLAGS = xgettextpathflags, - _XGETTEXTFROMFLAGS = xgettextfromflags, - POTSUFFIX = ['.pot'], - POTUPDATE_ALIAS = 'pot-update', - XgettextRPaths = RPaths(env) - ) - env.Append( BUILDERS = { - '_POTUpdateBuilder' : _POTUpdateBuilder(env) - } ) - env.AddMethod(_POTUpdateBuilderWrapper, 'POTUpdate') - env.AlwaysBuild(env.Alias('$POTUPDATE_ALIAS')) +def generate(env, **kw): + """ Generate `xgettext` tool """ + import SCons.Util + from SCons.Tool.GettextCommon import RPaths, _detect_xgettext + + try: + env['XGETTEXT'] = _detect_xgettext(env) + except: + env['XGETTEXT'] = 'xgettext' + # NOTE: sources="$SOURCES" would work as well. However, we use following + # construction to convert absolute paths provided by scons onto paths + # relative to current working dir. Note, that scons expands $SOURCE(S) to + # absolute paths for sources $SOURCE(s) outside of current subtree (e.g. in + # "../"). With source=$SOURCE these absolute paths would be written to the + # resultant *.pot file (and its derived *.po files) as references to lines in + # source code (e.g. referring lines in *.c files). Such references would be + # correct (e.g. in poedit) only on machine on which *.pot was generated and + # would be of no use on other hosts (having a copy of source code located + # in different place in filesystem). + sources = '$( ${_concat( "", SOURCES, "", __env__, XgettextRPaths, TARGET' \ + + ', SOURCES)} $)' + + # NOTE: the output from $XGETTEXTCOM command must go to stdout, not to a file. + # This is required by the POTUpdate builder's action. + xgettextcom = '$XGETTEXT $XGETTEXTFLAGS $_XGETTEXTPATHFLAGS' \ + + ' $_XGETTEXTFROMFLAGS -o - ' + sources + + xgettextpathflags = '$( ${_concat( XGETTEXTPATHPREFIX, XGETTEXTPATH' \ + + ', XGETTEXTPATHSUFFIX, __env__, RDirs, TARGET, SOURCES)} $)' + xgettextfromflags = '$( ${_concat( XGETTEXTFROMPREFIX, XGETTEXTFROM' \ + + ', XGETTEXTFROMSUFFIX, __env__, target=TARGET, source=SOURCES)} $)' + + env.SetDefault( + _XGETTEXTDOMAIN='${TARGET.filebase}', + XGETTEXTFLAGS=[], + XGETTEXTCOM=xgettextcom, + XGETTEXTCOMSTR='', + XGETTEXTPATH=[], + XGETTEXTPATHPREFIX='-D', + XGETTEXTPATHSUFFIX='', + XGETTEXTFROM=None, + XGETTEXTFROMPREFIX='-f', + XGETTEXTFROMSUFFIX='', + _XGETTEXTPATHFLAGS=xgettextpathflags, + _XGETTEXTFROMFLAGS=xgettextfromflags, + POTSUFFIX=['.pot'], + POTUPDATE_ALIAS='pot-update', + XgettextRPaths=RPaths(env) + ) + env.Append(BUILDERS={ + '_POTUpdateBuilder': _POTUpdateBuilder(env) + }) + env.AddMethod(_POTUpdateBuilderWrapper, 'POTUpdate') + env.AlwaysBuild(env.Alias('$POTUPDATE_ALIAS')) + + ############################################################################# ############################################################################# def exists(env): - """ Check, whether the tool exists """ - from SCons.Tool.GettextCommon import _xgettext_exists - try: - return _xgettext_exists(env) - except: - return False + """ Check, whether the tool exists """ + from SCons.Tool.GettextCommon import _xgettext_exists + try: + return _xgettext_exists(env) + except: + return False + ############################################################################# # Local Variables: diff --git a/src/engine/SCons/Tool/zip.py b/src/engine/SCons/Tool/zip.py index 2613bfca..23d540f3 100644 --- a/src/engine/SCons/Tool/zip.py +++ b/src/engine/SCons/Tool/zip.py @@ -40,31 +40,23 @@ import SCons.Defaults import SCons.Node.FS import SCons.Util -try: - import zipfile - internal_zip = 1 -except ImportError: - internal_zip = 0 - -if internal_zip: - zipcompression = zipfile.ZIP_DEFLATED - def zip(target, source, env): - compression = env.get('ZIPCOMPRESSION', 0) - zf = zipfile.ZipFile(str(target[0]), 'w', compression) - for s in source: - if s.isdir(): - for dirpath, dirnames, filenames in os.walk(str(s)): - for fname in filenames: - path = os.path.join(dirpath, fname) - if os.path.isfile(path): - zf.write(path, os.path.relpath(path, str(env.get('ZIPROOT', '')))) - else: - zf.write(str(s), os.path.relpath(str(s), str(env.get('ZIPROOT', '')))) - zf.close() -else: - zipcompression = 0 - zip = "$ZIP $ZIPFLAGS ${TARGET.abspath} $SOURCES" - +import zipfile + +zipcompression = zipfile.ZIP_DEFLATED +def zip(target, source, env): + compression = env.get('ZIPCOMPRESSION', 0) + zf = zipfile.ZipFile(str(target[0]), 'w', compression) + for s in source: + if s.isdir(): + for dirpath, dirnames, filenames in os.walk(str(s)): + for fname in filenames: + path = os.path.join(dirpath, fname) + if os.path.isfile(path): + + zf.write(path, os.path.relpath(path, str(env.get('ZIPROOT', '')))) + else: + zf.write(str(s), os.path.relpath(str(s), str(env.get('ZIPROOT', '')))) + zf.close() zipAction = SCons.Action.Action(zip, varlist=['ZIPCOMPRESSION']) @@ -91,7 +83,7 @@ def generate(env): env['ZIPROOT'] = SCons.Util.CLVar('') def exists(env): - return internal_zip or env.Detect('zip') + return True # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index 7653acd3..846d06c9 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -31,6 +31,10 @@ import sys import copy import re import types +import codecs +import pprint + +PY3 = sys.version_info[0] == 3 try: from UserDict import UserDict @@ -42,6 +46,8 @@ try: except ImportError as e: from collections import UserList +from collections import Iterable + try: from UserString import UserString except ImportError as e: @@ -131,6 +137,22 @@ class NodeList(UserList): >>> someList.strip() [ 'foo', 'bar' ] """ + +# def __init__(self, initlist=None): +# self.data = [] +# # print("TYPE:%s"%type(initlist)) +# if initlist is not None: +# # XXX should this accept an arbitrary sequence? +# if type(initlist) == type(self.data): +# self.data[:] = initlist +# elif isinstance(initlist, (UserList, NodeList)): +# self.data[:] = initlist.data[:] +# elif isinstance(initlist, Iterable): +# self.data = list(initlist) +# else: +# self.data = [ initlist,] + + def __nonzero__(self): return len(self.data) != 0 @@ -151,6 +173,25 @@ class NodeList(UserList): result = [getattr(x, name) for x in self.data] return self.__class__(result) + def __getitem__(self, index): + """ + This comes for free on py2, + but py3 slices of NodeList are returning a list + breaking slicing nodelist and refering to + properties and methods on contained object + """ +# return self.__class__(self.data[index]) + + if isinstance(index, slice): + # Expand the slice object using range() + # limited by number of items in self.data + indices = index.indices(len(self.data)) + return self.__class__([self[x] for x in + range(*indices)]) + else: + # Return one item of the tart + return self.data[index] + _get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') @@ -225,15 +266,15 @@ def render_tree(root, child_func, prune=0, margin=[0], visited=None): visited[rname] = 1 for i in range(len(children)): - margin.append(i<len(children)-1) - retval = retval + render_tree(children[i], child_func, prune, margin, visited -) + margin.append(i < len(children)-1) + retval = retval + render_tree(children[i], child_func, prune, margin, visited) margin.pop() return retval IDX = lambda N: N and 1 or 0 + def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): """ Print a tree of nodes. This is like render_tree, except it prints @@ -252,6 +293,7 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): rname = str(root) + # Initialize 'visited' dict, if required if visited is None: visited = {} @@ -270,7 +312,7 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): ' N = no clean\n' + ' H = no cache\n' + '\n') - sys.stdout.write(unicode(legend)) + sys.stdout.write(legend) tags = ['['] tags.append(' E'[IDX(root.exists())]) @@ -295,10 +337,10 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): children = child_func(root) if prune and rname in visited and children: - sys.stdout.write(''.join(tags + margins + ['+-[', rname, ']']) + u'\n') + sys.stdout.write(''.join(tags + margins + ['+-[', rname, ']']) + '\n') return - sys.stdout.write(''.join(tags + margins + ['+-', rname]) + u'\n') + sys.stdout.write(''.join(tags + margins + ['+-', rname]) + '\n') visited[rname] = 1 @@ -455,7 +497,13 @@ def to_String_for_signature(obj, to_String_for_subst=to_String_for_subst, try: f = obj.for_signature except AttributeError: - return to_String_for_subst(obj) + if isinstance(obj, dict): + # pprint will output dictionary in key sorted order + # with py3.5 the order was randomized. In general depending on dictionary order + # which was undefined until py3.6 (where it's by insertion order) was not wise. + return pprint.pformat(obj, width=1000000) + else: + return to_String_for_subst(obj) else: return f() @@ -477,7 +525,7 @@ _semi_deepcopy_dispatch = d = {} def semi_deepcopy_dict(x, exclude = [] ): copy = {} - for key, val in list(x.items()): + for key, val in x.items(): # The regular Python copy.deepcopy() also deepcopies the key, # as follows: # @@ -1037,7 +1085,7 @@ class OrderedDict(UserDict): if key not in self._keys: self._keys.append(key) def update(self, dict): - for (key, val) in list(dict.items()): + for (key, val) in dict.items(): self.__setitem__(key, val) def values(self): @@ -1059,7 +1107,7 @@ class Selector(OrderedDict): # Try to perform Environment substitution on the keys of # the dictionary before giving up. s_dict = {} - for (k,v) in list(self.items()): + for (k,v) in self.items(): if k is not None: s_k = env.subst(k) if s_k in s_dict: @@ -1416,9 +1464,9 @@ def AddMethod(obj, function, name=None): self.z = x + y AddMethod(f, A, "add") a.add(2, 4) - print a.z + print(a.z) AddMethod(lambda self, i: self.l[i], a, "listIndex") - print a.listIndex(5) + print(a.listIndex(5)) """ if name is None: name = function.__name__ @@ -1434,11 +1482,8 @@ def AddMethod(obj, function, name=None): else: method = MethodType(function, obj, obj.__class__) else: - # "obj" is a class, so it gets an unbound method. - if sys.version_info[:2] > (3, 2): - method = MethodType(function, None) - else: - method = MethodType(function, None, obj) + # Handle classes + method = function setattr(obj, name, method) @@ -1454,13 +1499,15 @@ def RenameFunction(function, name): md5 = False + + def MD5signature(s): return str(s) + def MD5filesignature(fname, chunksize=65536): - f = open(fname, "rb") - result = f.read() - f.close() + with open(fname, "rb") as f: + result = f.read() return result try: @@ -1470,9 +1517,15 @@ except ImportError: else: if hasattr(hashlib, 'md5'): md5 = True + def MD5signature(s): m = hashlib.md5() - m.update(to_bytes(str(s))) + + try: + m.update(to_bytes(s)) + except TypeError as e: + m.update(to_bytes(str(s))) + return m.hexdigest() def MD5filesignature(fname, chunksize=65536): @@ -1482,7 +1535,7 @@ else: blck = f.read(chunksize) if not blck: break - m.update(to_bytes (str(blck))) + m.update(to_bytes(blck)) f.close() return m.hexdigest() @@ -1559,7 +1612,7 @@ class NullSeq(Null): del __revision__ def to_bytes (s): - if isinstance (s, bytes) or bytes is str: + if isinstance (s, (bytes, bytearray)) or bytes is str: return s return bytes (s, 'utf-8') diff --git a/src/engine/SCons/UtilTests.py b/src/engine/SCons/UtilTests.py index 9ebb9246..0d1b7bb6 100644 --- a/src/engine/SCons/UtilTests.py +++ b/src/engine/SCons/UtilTests.py @@ -186,12 +186,17 @@ class UtilTestCase(unittest.TestCase): try: node, expect, withtags = self.tree_case_1() - sys.stdout = io.StringIO() + if sys.version_info.major < 3: + IOStream = io.BytesIO + else: + IOStream = io.StringIO + + sys.stdout = IOStream() print_tree(node, get_children) actual = sys.stdout.getvalue() assert expect == actual, (expect, actual) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, showtags=1) actual = sys.stdout.getvalue() assert withtags == actual, (withtags, actual) @@ -200,12 +205,12 @@ class UtilTestCase(unittest.TestCase): # the same as the default (see above) node, expect, withtags = self.tree_case_2(prune=0) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, 0) actual = sys.stdout.getvalue() assert expect == actual, (expect, actual) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, 0, showtags=1) actual = sys.stdout.getvalue() assert withtags == actual, (withtags, actual) @@ -213,7 +218,7 @@ class UtilTestCase(unittest.TestCase): # Test output with prune=1 node, expect, withtags = self.tree_case_2(prune=1) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, 1) actual = sys.stdout.getvalue() assert expect == actual, (expect, actual) @@ -222,12 +227,12 @@ class UtilTestCase(unittest.TestCase): # again. This wasn't possible in version 2.4.1 and earlier # due to a bug in print_tree (visited was set to {} as default # parameter) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, 1) actual = sys.stdout.getvalue() assert expect == actual, (expect, actual) - sys.stdout = io.StringIO() + sys.stdout = IOStream() print_tree(node, get_children, 1, showtags=1) actual = sys.stdout.getvalue() assert withtags == actual, (withtags, actual) @@ -237,7 +242,11 @@ class UtilTestCase(unittest.TestCase): def test_is_Dict(self): assert is_Dict({}) assert is_Dict(UserDict()) - assert is_Dict(os.environ) + + # os.environ is not a dictionary in python 3 + if sys.version_info < (3,0): + assert is_Dict(os.environ) + try: class mydict(dict): pass @@ -723,7 +732,7 @@ class UtilTestCase(unittest.TestCase): def test_LogicalLines(self): """Test the LogicalLines class""" - content = r""" + content = u""" foo \ bar \ baz @@ -732,7 +741,7 @@ bling \ bling \ bling bling """ - fobj = io.StringIO(unicode(content)) + fobj = io.StringIO(content) lines = LogicalLines(fobj).readlines() assert lines == [ '\n', diff --git a/src/engine/SCons/Variables/EnumVariableTests.py b/src/engine/SCons/Variables/EnumVariableTests.py index edc29738..931dfe2b 100644 --- a/src/engine/SCons/Variables/EnumVariableTests.py +++ b/src/engine/SCons/Variables/EnumVariableTests.py @@ -124,7 +124,7 @@ class EnumVariableTestCase(unittest.TestCase): 'C' : ['C', 'three', 'three'], } - for k, l in list(table.items()): + for k, l in table.items(): x = o0.converter(k) assert x == l[0], "o0 got %s, expected %s" % (x, l[0]) x = o1.converter(k) @@ -188,7 +188,7 @@ class EnumVariableTestCase(unittest.TestCase): 'no_v' : [invalid, invalid, invalid], } - for v, l in list(table.items()): + for v, l in table.items(): l[0](o0, v) l[1](o1, v) l[2](o2, v) diff --git a/src/engine/SCons/Variables/VariablesTests.py b/src/engine/SCons/Variables/VariablesTests.py index 7f2a978d..7ee66caf 100644 --- a/src/engine/SCons/Variables/VariablesTests.py +++ b/src/engine/SCons/Variables/VariablesTests.py @@ -49,6 +49,14 @@ class Environment(object): return key in self.dict +def cmp(a, b): + """ + Define cmp because it's no longer available in python3 + Works under python 2 as well + """ + return (a > b) - (a < b) + + def check(key, value, env): assert int(value) == 6 * 9, "key %s = %s" % (key, repr(value)) @@ -57,7 +65,7 @@ def check(key, value, env): def checkSave(file, expected): gdict = {} ldict = {} - exec(open(file, 'rU').read(), gdict, ldict) + exec(open(file, 'r').read(), gdict, ldict) assert expected == ldict, "%s\n...not equal to...\n%s" % (expected, ldict) class VariablesTestCase(unittest.TestCase): diff --git a/src/engine/SCons/Variables/__init__.py b/src/engine/SCons/Variables/__init__.py index ce3541cd..1fe44351 100644 --- a/src/engine/SCons/Variables/__init__.py +++ b/src/engine/SCons/Variables/__init__.py @@ -175,7 +175,9 @@ class Variables(object): sys.path.insert(0, dir) try: values['__name__'] = filename - exec(open(filename, 'rU').read(), {}, values) + with open(filename, 'r') as f: + contents = f.read() + exec(contents, {}, values) finally: if dir: del sys.path[0] @@ -185,7 +187,7 @@ class Variables(object): if args is None: args = self.args - for arg, value in list(args.items()): + for arg, value in args.items(): added = False for option in self.options: if arg in list(option.aliases) + [ option.key ]: diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index 477b67c7..59a1a94d 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -146,6 +146,15 @@ except AttributeError: del _UserString +import shutil +try: + shutil.SameFileError +except AttributeError: + class SameFileError(Exception): + pass + + shutil.SameFileError = SameFileError + def with_metaclass(meta, *bases): """ Function from jinja2/_compat.py. License: BSD. diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 18f154a0..2e20017d 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -72,7 +72,7 @@ cpp_lines_dict = { # the corresponding compiled regular expression that fetches the arguments # we care about. Table = {} -for op_list, expr in list(cpp_lines_dict.items()): +for op_list, expr in cpp_lines_dict.items(): e = re.compile(expr) for op in op_list: Table[op] = e @@ -312,7 +312,7 @@ class PreProcessor(object): t = self.tuples.pop(0) # Uncomment to see the list of tuples being processed (e.g., # to validate the CPP lines are being translated correctly). - #print t + #print(t) self.dispatch_table[t[0]](t) return self.finalize_result(fname) @@ -379,7 +379,8 @@ class PreProcessor(object): return None def read_file(self, file): - return open(file).read() + with open(file) as f: + return f.read() # Start and stop processing include lines. @@ -510,7 +511,7 @@ class PreProcessor(object): t = self.resolve_include(t) include_file = self.find_include_file(t) if include_file: - #print "include_file =", include_file + #print("include_file =", include_file) self.result.append(include_file) contents = self.read_file(include_file) new_tuples = [('scons_current_file', include_file)] + \ @@ -547,7 +548,7 @@ class PreProcessor(object): """ s = t[1] while not s[0] in '<"': - #print "s =", s + #print("s =", s) try: s = self.cpp_namespace[s] except KeyError: diff --git a/src/engine/SCons/dblite.py b/src/engine/SCons/dblite.py index 588a7ab1..4ef2222d 100644 --- a/src/engine/SCons/dblite.py +++ b/src/engine/SCons/dblite.py @@ -12,245 +12,263 @@ from SCons.compat import PICKLE_PROTOCOL keep_all_files = 00000 ignore_corrupt_dbfiles = 0 + def corruption_warning(filename): print("Warning: Discarding corrupt database:", filename) -try: unicode + +try: + unicode except NameError: def is_string(s): return isinstance(s, str) else: def is_string(s): return type(s) in (str, unicode) + + def is_bytes(s): - return isinstance (s, bytes) + return isinstance(s, bytes) + + try: unicode('a') except NameError: - def unicode(s): return s + def unicode(s): + return s dblite_suffix = '.dblite' -if bytes is not str: - dblite_suffix += '.p3' + +# TODO: Does commenting this out break switching from py2/3? +# if bytes is not str: +# dblite_suffix += '.p3' tmp_suffix = '.tmp' + + class dblite(object): + """ + Squirrel away references to the functions in various modules + that we'll use when our __del__() method calls our sync() method + during shutdown. We might get destroyed when Python is in the midst + of tearing down the different modules we import in an essentially + arbitrary order, and some of the various modules's global attributes + may already be wiped out from under us. - # Squirrel away references to the functions in various modules - # that we'll use when our __del__() method calls our sync() method - # during shutdown. We might get destroyed when Python is in the midst - # of tearing down the different modules we import in an essentially - # arbitrary order, and some of the various modules's global attributes - # may already be wiped out from under us. - # - # See the discussion at: - # http://mail.python.org/pipermail/python-bugs-list/2003-March/016877.html - - _open = open - _pickle_dump = staticmethod(pickle.dump) - _pickle_protocol = PICKLE_PROTOCOL - _os_chmod = os.chmod - try: - _os_chown = os.chown - except AttributeError: - _os_chown = None - _os_rename = os.rename - _os_unlink = os.unlink - _shutil_copyfile = shutil.copyfile - _time_time = time.time - - def __init__(self, file_base_name, flag, mode): - assert flag in (None, "r", "w", "c", "n") - if (flag is None): flag = "r" - base, ext = os.path.splitext(file_base_name) - if ext == dblite_suffix: - # There's already a suffix on the file name, don't add one. - self._file_name = file_base_name - self._tmp_name = base + tmp_suffix - else: - self._file_name = file_base_name + dblite_suffix - self._tmp_name = file_base_name + tmp_suffix - self._flag = flag - self._mode = mode - self._dict = {} - self._needs_sync = 00000 - if self._os_chown is not None and (os.geteuid()==0 or os.getuid()==0): - # running as root; chown back to current owner/group when done - try: - statinfo = os.stat(self._file_name) - self._chown_to = statinfo.st_uid - self._chgrp_to = statinfo.st_gid - except OSError as e: - # db file doesn't exist yet. - # Check os.environ for SUDO_UID, use if set - self._chown_to = int(os.environ.get('SUDO_UID', -1)) - self._chgrp_to = int(os.environ.get('SUDO_GID', -1)) - else: - self._chown_to = -1 # don't chown - self._chgrp_to = -1 # don't chgrp - if (self._flag == "n"): - self._open(self._file_name, "wb", self._mode) - else: - try: - f = self._open(self._file_name, "rb") - except IOError as e: - if (self._flag != "c"): - raise e - self._open(self._file_name, "wb", self._mode) - else: - p = f.read() - if (len(p) > 0): - try: - self._dict = pickle.loads(p) - except (pickle.UnpicklingError, EOFError, KeyError): - # Note how we catch KeyErrors too here, which might happen - # when we don't have cPickle available (default pickle - # throws it). - if (ignore_corrupt_dbfiles == 0): raise - if (ignore_corrupt_dbfiles == 1): - corruption_warning(self._file_name) - - def close(self): - if (self._needs_sync): - self.sync() - - def __del__(self): - self.close() - - def sync(self): - self._check_writable() - f = self._open(self._tmp_name, "wb", self._mode) - self._pickle_dump(self._dict, f, self._pickle_protocol) - f.close() - # Windows doesn't allow renaming if the file exists, so unlink - # it first, chmod'ing it to make sure we can do so. On UNIX, we - # may not be able to chmod the file if it's owned by someone else - # (e.g. from a previous run as root). We should still be able to - # unlink() the file if the directory's writable, though, so ignore - # any OSError exception thrown by the chmod() call. - try: self._os_chmod(self._file_name, 0o777) - except OSError: pass - self._os_unlink(self._file_name) - self._os_rename(self._tmp_name, self._file_name) - if self._os_chown is not None and self._chown_to > 0: # don't chown to root or -1 - try: - self._os_chown(self._file_name, self._chown_to, self._chgrp_to) - except OSError: - pass - self._needs_sync = 00000 - if (keep_all_files): - self._shutil_copyfile( - self._file_name, - self._file_name + "_" + str(int(self._time_time()))) + See the discussion at: + http://mail.python.org/pipermail/python-bugs-list/2003-March/016877.html + """ + + _open = open + _pickle_dump = staticmethod(pickle.dump) + _pickle_protocol = PICKLE_PROTOCOL + _os_chmod = os.chmod + try: + _os_chown = os.chown + except AttributeError: + _os_chown = None + _os_rename = os.rename + _os_unlink = os.unlink + _shutil_copyfile = shutil.copyfile + _time_time = time.time + + def __init__(self, file_base_name, flag, mode): + assert flag in (None, "r", "w", "c", "n") + if (flag is None): flag = "r" + base, ext = os.path.splitext(file_base_name) + if ext == dblite_suffix: + # There's already a suffix on the file name, don't add one. + self._file_name = file_base_name + self._tmp_name = base + tmp_suffix + else: + self._file_name = file_base_name + dblite_suffix + self._tmp_name = file_base_name + tmp_suffix + self._flag = flag + self._mode = mode + self._dict = {} + self._needs_sync = 00000 + if self._os_chown is not None and (os.geteuid() == 0 or os.getuid() == 0): + # running as root; chown back to current owner/group when done + try: + statinfo = os.stat(self._file_name) + self._chown_to = statinfo.st_uid + self._chgrp_to = statinfo.st_gid + except OSError as e: + # db file doesn't exist yet. + # Check os.environ for SUDO_UID, use if set + self._chown_to = int(os.environ.get('SUDO_UID', -1)) + self._chgrp_to = int(os.environ.get('SUDO_GID', -1)) + else: + self._chown_to = -1 # don't chown + self._chgrp_to = -1 # don't chgrp + if (self._flag == "n"): + self._open(self._file_name, "wb", self._mode) + else: + try: + f = self._open(self._file_name, "rb") + except IOError as e: + if (self._flag != "c"): + raise e + self._open(self._file_name, "wb", self._mode) + else: + p = f.read() + if (len(p) > 0): + try: + self._dict = pickle.loads(p) + except (pickle.UnpicklingError, EOFError, KeyError): + # Note how we catch KeyErrors too here, which might happen + # when we don't have cPickle available (default pickle + # throws it). + if (ignore_corrupt_dbfiles == 0): raise + if (ignore_corrupt_dbfiles == 1): + corruption_warning(self._file_name) + + def close(self): + if (self._needs_sync): + self.sync() - def _check_writable(self): - if (self._flag == "r"): - raise IOError("Read-only database: %s" % self._file_name) + def __del__(self): + self.close() - def __getitem__(self, key): - return self._dict[key] + def sync(self): + self._check_writable() + f = self._open(self._tmp_name, "wb", self._mode) + self._pickle_dump(self._dict, f, self._pickle_protocol) + f.close() + # Windows doesn't allow renaming if the file exists, so unlink + # it first, chmod'ing it to make sure we can do so. On UNIX, we + # may not be able to chmod the file if it's owned by someone else + # (e.g. from a previous run as root). We should still be able to + # unlink() the file if the directory's writable, though, so ignore + # any OSError exception thrown by the chmod() call. + try: + self._os_chmod(self._file_name, 0o777) + except OSError: + pass + self._os_unlink(self._file_name) + self._os_rename(self._tmp_name, self._file_name) + if self._os_chown is not None and self._chown_to > 0: # don't chown to root or -1 + try: + self._os_chown(self._file_name, self._chown_to, self._chgrp_to) + except OSError: + pass + self._needs_sync = 00000 + if (keep_all_files): + self._shutil_copyfile( + self._file_name, + self._file_name + "_" + str(int(self._time_time()))) - def __setitem__(self, key, value): - self._check_writable() - if (not is_string(key)): - raise TypeError("key `%s' must be a string but is %s" % (key, type(key))) - if (not is_bytes(value)): - raise TypeError("value `%s' must be a bytes but is %s" % (value, type(value))) - self._dict[key] = value - self._needs_sync = 0o001 + def _check_writable(self): + if (self._flag == "r"): + raise IOError("Read-only database: %s" % self._file_name) - def keys(self): - return list(self._dict.keys()) + def __getitem__(self, key): + return self._dict[key] - def has_key(self, key): - return key in self._dict + def __setitem__(self, key, value): + self._check_writable() + if (not is_string(key)): + raise TypeError("key `%s' must be a string but is %s" % (key, type(key))) + if (not is_bytes(value)): + raise TypeError("value `%s' must be a bytes but is %s" % (value, type(value))) + self._dict[key] = value + self._needs_sync = 0o001 - def __contains__(self, key): - return key in self._dict + def keys(self): + return list(self._dict.keys()) - def iterkeys(self): - # Wrapping name in () prevents fixer from "fixing" this - return (self._dict.iterkeys)() + def has_key(self, key): + return key in self._dict - __iter__ = iterkeys + def __contains__(self, key): + return key in self._dict + + def iterkeys(self): + # Wrapping name in () prevents fixer from "fixing" this + return (self._dict.iterkeys)() + + __iter__ = iterkeys + + def __len__(self): + return len(self._dict) - def __len__(self): - return len(self._dict) def open(file, flag=None, mode=0o666): - return dblite(file, flag, mode) + return dblite(file, flag, mode) + def _exercise(): - db = open("tmp", "n") - assert len(db) == 0 - db["foo"] = "bar" - assert db["foo"] == "bar" - db[unicode("ufoo")] = unicode("ubar") - assert db[unicode("ufoo")] == unicode("ubar") - db.sync() - db = open("tmp", "c") - assert len(db) == 2, len(db) - assert db["foo"] == "bar" - db["bar"] = "foo" - assert db["bar"] == "foo" - db[unicode("ubar")] = unicode("ufoo") - assert db[unicode("ubar")] == unicode("ufoo") - db.sync() - db = open("tmp", "r") - assert len(db) == 4, len(db) - assert db["foo"] == "bar" - assert db["bar"] == "foo" - assert db[unicode("ufoo")] == unicode("ubar") - assert db[unicode("ubar")] == unicode("ufoo") - try: + db = open("tmp", "n") + assert len(db) == 0 + db["foo"] = "bar" + assert db["foo"] == "bar" + db[unicode("ufoo")] = unicode("ubar") + assert db[unicode("ufoo")] == unicode("ubar") + db.sync() + db = open("tmp", "c") + assert len(db) == 2, len(db) + assert db["foo"] == "bar" + db["bar"] = "foo" + assert db["bar"] == "foo" + db[unicode("ubar")] = unicode("ufoo") + assert db[unicode("ubar")] == unicode("ufoo") db.sync() - except IOError as e: - assert str(e) == "Read-only database: tmp.dblite" - else: - raise RuntimeError("IOError expected.") - db = open("tmp", "w") - assert len(db) == 4 - db["ping"] = "pong" - db.sync() - try: - db[(1,2)] = "tuple" - except TypeError as e: - assert str(e) == "key `(1, 2)' must be a string but is <type 'tuple'>", str(e) - else: - raise RuntimeError("TypeError exception expected") - try: - db["list"] = [1,2] - except TypeError as e: - assert str(e) == "value `[1, 2]' must be a string but is <type 'list'>", str(e) - else: - raise RuntimeError("TypeError exception expected") - db = open("tmp", "r") - assert len(db) == 5 - db = open("tmp", "n") - assert len(db) == 0 - dblite._open("tmp.dblite", "w") - db = open("tmp", "r") - dblite._open("tmp.dblite", "w").write("x") - try: db = open("tmp", "r") - except pickle.UnpicklingError: - pass - else: - raise RuntimeError("pickle exception expected.") - global ignore_corrupt_dbfiles - ignore_corrupt_dbfiles = 2 - db = open("tmp", "r") - assert len(db) == 0 - os.unlink("tmp.dblite") - try: + assert len(db) == 4, len(db) + assert db["foo"] == "bar" + assert db["bar"] == "foo" + assert db[unicode("ufoo")] == unicode("ubar") + assert db[unicode("ubar")] == unicode("ufoo") + try: + db.sync() + except IOError as e: + assert str(e) == "Read-only database: tmp.dblite" + else: + raise RuntimeError("IOError expected.") db = open("tmp", "w") - except IOError as e: - assert str(e) == "[Errno 2] No such file or directory: 'tmp.dblite'", str(e) - else: - raise RuntimeError("IOError expected.") + assert len(db) == 4 + db["ping"] = "pong" + db.sync() + try: + db[(1, 2)] = "tuple" + except TypeError as e: + assert str(e) == "key `(1, 2)' must be a string but is <type 'tuple'>", str(e) + else: + raise RuntimeError("TypeError exception expected") + try: + db["list"] = [1, 2] + except TypeError as e: + assert str(e) == "value `[1, 2]' must be a string but is <type 'list'>", str(e) + else: + raise RuntimeError("TypeError exception expected") + db = open("tmp", "r") + assert len(db) == 5 + db = open("tmp", "n") + assert len(db) == 0 + dblite._open("tmp.dblite", "w") + db = open("tmp", "r") + dblite._open("tmp.dblite", "w").write("x") + try: + db = open("tmp", "r") + except pickle.UnpicklingError: + pass + else: + raise RuntimeError("pickle exception expected.") + global ignore_corrupt_dbfiles + ignore_corrupt_dbfiles = 2 + db = open("tmp", "r") + assert len(db) == 0 + os.unlink("tmp.dblite") + try: + db = open("tmp", "w") + except IOError as e: + assert str(e) == "[Errno 2] No such file or directory: 'tmp.dblite'", str(e) + else: + raise RuntimeError("IOError expected.") + if (__name__ == "__main__"): - _exercise() + _exercise() # Local Variables: # tab-width:4 |