SERVER-45390 Bring Subst improvements downstream

1 files changed, 341 insertions, 306 deletions

diff --git a/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/Subst.py b/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/Subst.py
index c3920c9e3a3..04ee78f3444 100644
--- a/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/Subst.py
+++ b/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/Subst.py
@@ -57,6 +57,281 @@ def raise_exception(exception, target, s):
         raise SCons.Errors.UserError(msg)
 
 
+
+class Literal(object):
+    """A wrapper for a string.  If you use this object wrapped
+    around a string, then it will be interpreted as literal.
+    When passed to the command interpreter, all special
+    characters will be escaped."""
+    def __init__(self, lstr):
+        self.lstr = lstr
+
+    def __str__(self):
+        return self.lstr
+
+    def escape(self, escape_func):
+        return escape_func(self.lstr)
+
+    def for_signature(self):
+        return self.lstr
+
+    def is_literal(self):
+        return 1
+
+    def __eq__(self, other):
+        if not isinstance(other, Literal):
+            return False
+        return self.lstr == other.lstr
+
+    def __neq__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash(self.lstr)
+
+class SpecialAttrWrapper(object):
+    """This is a wrapper for what we call a 'Node special attribute.'
+    This is any of the attributes of a Node that we can reference from
+    Environment variable substitution, such as $TARGET.abspath or
+    $SOURCES[1].filebase.  We implement the same methods as Literal
+    so we can handle special characters, plus a for_signature method,
+    such that we can return some canonical string during signature
+    calculation to avoid unnecessary rebuilds."""
+
+    def __init__(self, lstr, for_signature=None):
+        """The for_signature parameter, if supplied, will be the
+        canonical string we return from for_signature().  Else
+        we will simply return lstr."""
+        self.lstr = lstr
+        if for_signature:
+            self.forsig = for_signature
+        else:
+            self.forsig = lstr
+
+    def __str__(self):
+        return self.lstr
+
+    def escape(self, escape_func):
+        return escape_func(self.lstr)
+
+    def for_signature(self):
+        return self.forsig
+
+    def is_literal(self):
+        return 1
+
+def quote_spaces(arg):
+    """Generic function for putting double quotes around any string that
+    has white space in it."""
+    if ' ' in arg or '\t' in arg:
+        return '"%s"' % arg
+    else:
+        return str(arg)
+
+class CmdStringHolder(collections.UserString):
+    """This is a special class used to hold strings generated by
+    scons_subst() and scons_subst_list().  It defines a special method
+    escape().  When passed a function with an escape algorithm for a
+    particular platform, it will return the contained string with the
+    proper escape sequences inserted.
+    """
+    def __init__(self, cmd, literal=None):
+        collections.UserString.__init__(self, cmd)
+        self.literal = literal
+
+    def is_literal(self):
+        return self.literal
+
+    def escape(self, escape_func, quote_func=quote_spaces):
+        """Escape the string with the supplied function.  The
+        function is expected to take an arbitrary string, then
+        return it with all special characters escaped and ready
+        for passing to the command interpreter.
+
+        After calling this function, the next call to str() will
+        return the escaped string.
+        """
+
+        if self.is_literal():
+            return escape_func(self.data)
+        elif ' ' in self.data or '\t' in self.data:
+            return quote_func(self.data)
+        else:
+            return self.data
+
+def escape_list(mylist, escape_func):
+    """Escape a list of arguments by running the specified escape_func
+    on every object in the list that has an escape() method."""
+    def escape(obj, escape_func=escape_func):
+        try:
+            e = obj.escape
+        except AttributeError:
+            return obj
+        else:
+            return e(escape_func)
+    return list(map(escape, mylist))
+
+class NLWrapper(object):
+    """A wrapper class that delays turning a list of sources or targets
+    into a NodeList until it's needed.  The specified function supplied
+    when the object is initialized is responsible for turning raw nodes
+    into proxies that implement the special attributes like .abspath,
+    .source, etc.  This way, we avoid creating those proxies just
+    "in case" someone is going to use $TARGET or the like, and only
+    go through the trouble if we really have to.
+
+    In practice, this might be a wash performance-wise, but it's a little
+    cleaner conceptually...
+    """
+
+    def __init__(self, list, func):
+        self.list = list
+        self.func = func
+    def _return_nodelist(self):
+        return self.nodelist
+    def _gen_nodelist(self):
+        mylist = self.list
+        if mylist is None:
+            mylist = []
+        elif not is_Sequence(mylist):
+            mylist = [mylist]
+        # The map(self.func) call is what actually turns
+        # a list into appropriate proxies.
+        self.nodelist = SCons.Util.NodeList(list(map(self.func, mylist)))
+        self._create_nodelist = self._return_nodelist
+        return self.nodelist
+    _create_nodelist = _gen_nodelist
+
+
+class Targets_or_Sources(collections.UserList):
+    """A class that implements $TARGETS or $SOURCES expansions by in turn
+    wrapping a NLWrapper.  This class handles the different methods used
+    to access the list, calling the NLWrapper to create proxies on demand.
+
+    Note that we subclass collections.UserList purely so that the
+    is_Sequence() function will identify an object of this class as
+    a list during variable expansion.  We're not really using any
+    collections.UserList methods in practice.
+    """
+    def __init__(self, nl):
+        self.nl = nl
+    def __getattr__(self, attr):
+        nl = self.nl._create_nodelist()
+        return getattr(nl, attr)
+    def __getitem__(self, i):
+        nl = self.nl._create_nodelist()
+        return nl[i]
+    def __getslice__(self, i, j):
+        nl = self.nl._create_nodelist()
+        i = max(i, 0); j = max(j, 0)
+        return nl[i:j]
+    def __str__(self):
+        nl = self.nl._create_nodelist()
+        return str(nl)
+    def __repr__(self):
+        nl = self.nl._create_nodelist()
+        return repr(nl)
+
+class Target_or_Source(object):
+    """A class that implements $TARGET or $SOURCE expansions by in turn
+    wrapping a NLWrapper.  This class handles the different methods used
+    to access an individual proxy Node, calling the NLWrapper to create
+    a proxy on demand.
+    """
+    def __init__(self, nl):
+        self.nl = nl
+    def __getattr__(self, attr):
+        nl = self.nl._create_nodelist()
+        try:
+            nl0 = nl[0]
+        except IndexError:
+            # If there is nothing in the list, then we have no attributes to
+            # pass through, so raise AttributeError for everything.
+            raise AttributeError("NodeList has no attribute: %s" % attr)
+        return getattr(nl0, attr)
+    def __str__(self):
+        nl = self.nl._create_nodelist()
+        if nl:
+            return str(nl[0])
+        return ''
+    def __repr__(self):
+        nl = self.nl._create_nodelist()
+        if nl:
+            return repr(nl[0])
+        return ''
+
+class NullNodeList(SCons.Util.NullSeq):
+  def __call__(self, *args, **kwargs): return ''
+  def __str__(self): return ''
+
+NullNodesList = NullNodeList()
+
+def subst_dict(target, source):
+    """Create a dictionary for substitution of special
+    construction variables.
+
+    This translates the following special arguments:
+
+    target - the target (object or array of objects),
+             used to generate the TARGET and TARGETS
+             construction variables
+
+    source - the source (object or array of objects),
+             used to generate the SOURCES and SOURCE
+             construction variables
+    """
+    dict = {}
+
+    if target:
+        def get_tgt_subst_proxy(thing):
+            try:
+                subst_proxy = thing.get_subst_proxy()
+            except AttributeError:
+                subst_proxy = thing # probably a string, just return it
+            return subst_proxy
+        tnl = NLWrapper(target, get_tgt_subst_proxy)
+        dict['TARGETS'] = Targets_or_Sources(tnl)
+        dict['TARGET'] = Target_or_Source(tnl)
+
+        # This is a total cheat, but hopefully this dictionary goes
+        # away soon anyway.  We just let these expand to $TARGETS
+        # because that's "good enough" for the use of ToolSurrogates
+        # (see test/ToolSurrogate.py) to generate documentation.
+        dict['CHANGED_TARGETS'] = '$TARGETS'
+        dict['UNCHANGED_TARGETS'] = '$TARGETS'
+    else:
+        dict['TARGETS'] = NullNodesList
+        dict['TARGET'] = NullNodesList
+
+    if source:
+        def get_src_subst_proxy(node):
+            try:
+                rfile = node.rfile
+            except AttributeError:
+                pass
+            else:
+                node = rfile()
+            try:
+                return node.get_subst_proxy()
+            except AttributeError:
+                return node     # probably a String, just return it
+        snl = NLWrapper(source, get_src_subst_proxy)
+        dict['SOURCES'] = Targets_or_Sources(snl)
+        dict['SOURCE'] = Target_or_Source(snl)
+
+        # This is a total cheat, but hopefully this dictionary goes
+        # away soon anyway.  We just let these expand to $TARGETS
+        # because that's "good enough" for the use of ToolSurrogates
+        # (see test/ToolSurrogate.py) to generate documentation.
+        dict['CHANGED_SOURCES'] = '$SOURCES'
+        dict['UNCHANGED_SOURCES'] = '$SOURCES'
+    else:
+        dict['SOURCES'] = NullNodesList
+        dict['SOURCE'] = NullNodesList
+
+    return dict
+
+
 class StringSubber(object):
     """A class to construct the results of a scons_subst() call.
 
@@ -100,23 +375,29 @@ class StringSubber(object):
                 if key[0] == '{' or '.' in key:
                     if key[0] == '{':
                         key = key[1:-1]
-                    try:
-                        s = eval(key, self.gvars, lvars)
-                    except KeyboardInterrupt:
-                        raise
-                    except Exception as e:
-                        if e.__class__ in AllowableExceptions:
-                            return ''
-                        raise_exception(e, lvars['TARGETS'], s)
+
+                # Store for error messages if we fail to expand the
+                # value
+                old_s = s
+                s = None
+                if key in lvars:
+                     s = lvars[key]
+                elif key in self.gvars:
+                     s = self.gvars[key]
                 else:
-                    if key in lvars:
-                        s = lvars[key]
-                    elif key in self.gvars:
-                        s = self.gvars[key]
-                    elif NameError not in AllowableExceptions:
-                        raise_exception(NameError(key), lvars['TARGETS'], s)
-                    else:
-                        return ''
+                     try:
+                          s = eval(key, self.gvars, lvars)
+                     except KeyboardInterrupt:
+                          raise
+                     except Exception as e:
+                          if e.__class__ in AllowableExceptions:
+                               return ''
+                          raise_exception(e, lvars['TARGETS'], old_s)
+
+                if s is None and NameError not in AllowableExceptions:
+                     raise_exception(NameError(key), lvars['TARGETS'], old_s)
+                elif s is None:
+                     return ''
 
                 # Before re-expanding the result, handle
                 # recursive expansion by copying the local
@@ -218,6 +499,21 @@ class ListSubber(collections.UserList):
         self.in_strip = None
         self.next_line()
 
+    def expanded(self, s):
+        """Determines if the string s requires further expansion.
+
+        Due to the implementation of ListSubber expand will call
+        itself 2 additional times for an already expanded string. This
+        method is used to determine if a string is already fully
+        expanded and if so exit the loop early to prevent these
+        recursive calls.
+        """
+        if not is_String(s) or isinstance(s, CmdStringHolder):
+            return False
+
+        s = str(s)  # in case it's a UserString
+        return _separate_args.findall(s) is None
+
     def expand(self, s, lvars, within_list):
         """Expand a single "token" as necessary, appending the
         expansion to the current result.
@@ -249,23 +545,35 @@ class ListSubber(collections.UserList):
                 if key[0] == '{' or key.find('.') >= 0:
                     if key[0] == '{':
                         key = key[1:-1]
-                    try:
-                        s = eval(key, self.gvars, lvars)
-                    except KeyboardInterrupt:
-                        raise
-                    except Exception as e:
-                        if e.__class__ in AllowableExceptions:
-                            return
-                        raise_exception(e, lvars['TARGETS'], s)
+
+                # Store for error messages if we fail to expand the
+                # value
+                old_s = s
+                s = None
+                if key in lvars:
+                     s = lvars[key]
+                elif key in self.gvars:
+                     s = self.gvars[key]
                 else:
-                    if key in lvars:
-                        s = lvars[key]
-                    elif key in self.gvars:
-                        s = self.gvars[key]
-                    elif NameError not in AllowableExceptions:
-                        raise_exception(NameError(), lvars['TARGETS'], s)
-                    else:
-                        return
+                     try:
+                         s = eval(key, self.gvars, lvars)
+                     except KeyboardInterrupt:
+                         raise
+                     except Exception as e:
+                         if e.__class__ in AllowableExceptions:
+                             return
+                         raise_exception(e, lvars['TARGETS'], old_s)
+
+                if s is None and NameError not in AllowableExceptions:
+                     raise_exception(NameError(), lvars['TARGETS'], old_s)
+                elif s is None:
+                     return
+
+                # If the string is already full expanded there's no
+                # need to continue recursion.
+                if self.expanded(s):
+                    self.append(s)
+                    return
 
                 # Before re-expanding the result, handle
                 # recursive expansion by copying the local
@@ -416,279 +724,6 @@ class ListSubber(collections.UserList):
         self.in_strip = None
 
 
-class Literal(object):
-    """A wrapper for a string.  If you use this object wrapped
-    around a string, then it will be interpreted as literal.
-    When passed to the command interpreter, all special
-    characters will be escaped."""
-    def __init__(self, lstr):
-        self.lstr = lstr
-
-    def __str__(self):
-        return self.lstr
-
-    def escape(self, escape_func):
-        return escape_func(self.lstr)
-
-    def for_signature(self):
-        return self.lstr
-
-    def is_literal(self):
-        return 1
-
-    def __eq__(self, other):
-        if not isinstance(other, Literal):
-            return False
-        return self.lstr == other.lstr
-
-    def __neq__(self, other):
-        return not self.__eq__(other)
-
-    def __hash__(self):
-        return hash(self.lstr)
-
-class SpecialAttrWrapper(object):
-    """This is a wrapper for what we call a 'Node special attribute.'
-    This is any of the attributes of a Node that we can reference from
-    Environment variable substitution, such as $TARGET.abspath or
-    $SOURCES[1].filebase.  We implement the same methods as Literal
-    so we can handle special characters, plus a for_signature method,
-    such that we can return some canonical string during signature
-    calculation to avoid unnecessary rebuilds."""
-
-    def __init__(self, lstr, for_signature=None):
-        """The for_signature parameter, if supplied, will be the
-        canonical string we return from for_signature().  Else
-        we will simply return lstr."""
-        self.lstr = lstr
-        if for_signature:
-            self.forsig = for_signature
-        else:
-            self.forsig = lstr
-
-    def __str__(self):
-        return self.lstr
-
-    def escape(self, escape_func):
-        return escape_func(self.lstr)
-
-    def for_signature(self):
-        return self.forsig
-
-    def is_literal(self):
-        return 1
-
-def quote_spaces(arg):
-    """Generic function for putting double quotes around any string that
-    has white space in it."""
-    if ' ' in arg or '\t' in arg:
-        return '"%s"' % arg
-    else:
-        return str(arg)
-
-class CmdStringHolder(collections.UserString):
-    """This is a special class used to hold strings generated by
-    scons_subst() and scons_subst_list().  It defines a special method
-    escape().  When passed a function with an escape algorithm for a
-    particular platform, it will return the contained string with the
-    proper escape sequences inserted.
-    """
-    def __init__(self, cmd, literal=None):
-        collections.UserString.__init__(self, cmd)
-        self.literal = literal
-
-    def is_literal(self):
-        return self.literal
-
-    def escape(self, escape_func, quote_func=quote_spaces):
-        """Escape the string with the supplied function.  The
-        function is expected to take an arbitrary string, then
-        return it with all special characters escaped and ready
-        for passing to the command interpreter.
-
-        After calling this function, the next call to str() will
-        return the escaped string.
-        """
-
-        if self.is_literal():
-            return escape_func(self.data)
-        elif ' ' in self.data or '\t' in self.data:
-            return quote_func(self.data)
-        else:
-            return self.data
-
-def escape_list(mylist, escape_func):
-    """Escape a list of arguments by running the specified escape_func
-    on every object in the list that has an escape() method."""
-    def escape(obj, escape_func=escape_func):
-        try:
-            e = obj.escape
-        except AttributeError:
-            return obj
-        else:
-            return e(escape_func)
-    return list(map(escape, mylist))
-
-class NLWrapper(object):
-    """A wrapper class that delays turning a list of sources or targets
-    into a NodeList until it's needed.  The specified function supplied
-    when the object is initialized is responsible for turning raw nodes
-    into proxies that implement the special attributes like .abspath,
-    .source, etc.  This way, we avoid creating those proxies just
-    "in case" someone is going to use $TARGET or the like, and only
-    go through the trouble if we really have to.
-
-    In practice, this might be a wash performance-wise, but it's a little
-    cleaner conceptually...
-    """
-
-    def __init__(self, list, func):
-        self.list = list
-        self.func = func
-    def _return_nodelist(self):
-        return self.nodelist
-    def _gen_nodelist(self):
-        mylist = self.list
-        if mylist is None:
-            mylist = []
-        elif not is_Sequence(mylist):
-            mylist = [mylist]
-        # The map(self.func) call is what actually turns
-        # a list into appropriate proxies.
-        self.nodelist = SCons.Util.NodeList(list(map(self.func, mylist)))
-        self._create_nodelist = self._return_nodelist
-        return self.nodelist
-    _create_nodelist = _gen_nodelist
-
-
-class Targets_or_Sources(collections.UserList):
-    """A class that implements $TARGETS or $SOURCES expansions by in turn
-    wrapping a NLWrapper.  This class handles the different methods used
-    to access the list, calling the NLWrapper to create proxies on demand.
-
-    Note that we subclass collections.UserList purely so that the
-    is_Sequence() function will identify an object of this class as
-    a list during variable expansion.  We're not really using any
-    collections.UserList methods in practice.
-    """
-    def __init__(self, nl):
-        self.nl = nl
-    def __getattr__(self, attr):
-        nl = self.nl._create_nodelist()
-        return getattr(nl, attr)
-    def __getitem__(self, i):
-        nl = self.nl._create_nodelist()
-        return nl[i]
-    def __getslice__(self, i, j):
-        nl = self.nl._create_nodelist()
-        i = max(i, 0); j = max(j, 0)
-        return nl[i:j]
-    def __str__(self):
-        nl = self.nl._create_nodelist()
-        return str(nl)
-    def __repr__(self):
-        nl = self.nl._create_nodelist()
-        return repr(nl)
-
-class Target_or_Source(object):
-    """A class that implements $TARGET or $SOURCE expansions by in turn
-    wrapping a NLWrapper.  This class handles the different methods used
-    to access an individual proxy Node, calling the NLWrapper to create
-    a proxy on demand.
-    """
-    def __init__(self, nl):
-        self.nl = nl
-    def __getattr__(self, attr):
-        nl = self.nl._create_nodelist()
-        try:
-            nl0 = nl[0]
-        except IndexError:
-            # If there is nothing in the list, then we have no attributes to
-            # pass through, so raise AttributeError for everything.
-            raise AttributeError("NodeList has no attribute: %s" % attr)
-        return getattr(nl0, attr)
-    def __str__(self):
-        nl = self.nl._create_nodelist()
-        if nl:
-            return str(nl[0])
-        return ''
-    def __repr__(self):
-        nl = self.nl._create_nodelist()
-        if nl:
-            return repr(nl[0])
-        return ''
-
-class NullNodeList(SCons.Util.NullSeq):
-  def __call__(self, *args, **kwargs): return ''
-  def __str__(self): return ''
-
-NullNodesList = NullNodeList()
-
-def subst_dict(target, source):
-    """Create a dictionary for substitution of special
-    construction variables.
-
-    This translates the following special arguments:
-
-    target - the target (object or array of objects),
-             used to generate the TARGET and TARGETS
-             construction variables
-
-    source - the source (object or array of objects),
-             used to generate the SOURCES and SOURCE
-             construction variables
-    """
-    dict = {}
-
-    if target:
-        def get_tgt_subst_proxy(thing):
-            try:
-                subst_proxy = thing.get_subst_proxy()
-            except AttributeError:
-                subst_proxy = thing # probably a string, just return it
-            return subst_proxy
-        tnl = NLWrapper(target, get_tgt_subst_proxy)
-        dict['TARGETS'] = Targets_or_Sources(tnl)
-        dict['TARGET'] = Target_or_Source(tnl)
-
-        # This is a total cheat, but hopefully this dictionary goes
-        # away soon anyway.  We just let these expand to $TARGETS
-        # because that's "good enough" for the use of ToolSurrogates
-        # (see test/ToolSurrogate.py) to generate documentation.
-        dict['CHANGED_TARGETS'] = '$TARGETS'
-        dict['UNCHANGED_TARGETS'] = '$TARGETS'
-    else:
-        dict['TARGETS'] = NullNodesList
-        dict['TARGET'] = NullNodesList
-
-    if source:
-        def get_src_subst_proxy(node):
-            try:
-                rfile = node.rfile
-            except AttributeError:
-                pass
-            else:
-                node = rfile()
-            try:
-                return node.get_subst_proxy()
-            except AttributeError:
-                return node     # probably a String, just return it
-        snl = NLWrapper(source, get_src_subst_proxy)
-        dict['SOURCES'] = Targets_or_Sources(snl)
-        dict['SOURCE'] = Target_or_Source(snl)
-
-        # This is a total cheat, but hopefully this dictionary goes
-        # away soon anyway.  We just let these expand to $TARGETS
-        # because that's "good enough" for the use of ToolSurrogates
-        # (see test/ToolSurrogate.py) to generate documentation.
-        dict['CHANGED_SOURCES'] = '$SOURCES'
-        dict['UNCHANGED_SOURCES'] = '$SOURCES'
-    else:
-        dict['SOURCES'] = NullNodesList
-        dict['SOURCE'] = NullNodesList
-
-    return dict
-
 # Constants for the "mode" parameter to scons_subst_list() and
 # scons_subst().  SUBST_RAW gives the raw command line.  SUBST_CMD
 # gives a command line suitable for passing to a shell.  SUBST_SIG
@@ -752,7 +787,7 @@ _list_remove = [ _rm_list, None, _remove_list ]
 #
 _dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
 _dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
-_separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str)
+_separate_args = re.compile(r'(%s|\s+|[^\s$]+|\$)' % _dollar_exps_str)
 
 # This regular expression is used to replace strings of multiple white
 # space characters in the string result from the scons_subst() function.