diff options
30 files changed, 455 insertions, 2136 deletions
diff --git a/QMTest/TestCommon.py b/QMTest/TestCommon.py index 4e90e168..c4a5373b 100644 --- a/QMTest/TestCommon.py +++ b/QMTest/TestCommon.py @@ -36,6 +36,8 @@ provided by the TestCommon class: test.must_contain('file', 'required text\n') + test.must_contain_all(output, input, ['title', find]) + test.must_contain_all_lines(output, lines, ['title', find]) test.must_contain_any_line(output, lines, ['title', find]) @@ -305,6 +307,36 @@ class TestCommon(TestCmd): print file_contents self.fail_test(not contains) + def must_contain_all(self, output, input, title=None, find=None): + """Ensures that the specified output string (first argument) + contains all of the specified input as a block (second argument). + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(line, output), to use when searching + for lines in the output. + """ + if find is None: + def find(o, i): + try: + return o.index(i) + except ValueError: + return None + + if is_List(output): + output = os.newline.join(output) + + if find(output, input) is None: + if title is None: + title = 'output' + print 'Missing expected input from %s:' % title + print input + print self.banner(title + ' ') + print output + self.fail_test() + def must_contain_all_lines(self, output, lines, title=None, find=None): """Ensures that the specified output string (first argument) contains all of the specified lines (second argument). diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index ef702611..a9fd6a0f 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -34,7 +34,7 @@ from TestCmd import PIPE # here provides some independent verification that what we packaged # conforms to what we expect. -default_version = '2.4.1' +default_version = '2.4.2.alpha.yyyymmdd' python_version_unsupported = (2, 3, 0) python_version_deprecated = (2, 7, 0) @@ -480,14 +480,14 @@ following packages will be built:: build/dist/scons-2.4.1-1.noarch.rpm build/dist/scons-2.4.1-1.src.rpm build/dist/scons-2.4.1.linux-i686.tar.gz - build/dist/scons-2.4.1.tar.gz - build/dist/scons-2.4.1.win32.exe - build/dist/scons-2.4.1.zip - build/dist/scons-doc-2.4.1.tar.gz - build/dist/scons-local-2.4.1.tar.gz - build/dist/scons-local-2.4.1.zip - build/dist/scons-src-2.4.1.tar.gz - build/dist/scons-src-2.4.1.zip + build/dist/scons-2.4.2.alpha.yyyymmdd.tar.gz + build/dist/scons-2.4.2.alpha.yyyymmdd.win32.exe + build/dist/scons-2.4.2.alpha.yyyymmdd.zip + build/dist/scons-doc-2.4.2.alpha.yyyymmdd.tar.gz + build/dist/scons-local-2.4.2.alpha.yyyymmdd.tar.gz + build/dist/scons-local-2.4.2.alpha.yyyymmdd.zip + build/dist/scons-src-2.4.2.alpha.yyyymmdd.tar.gz + build/dist/scons-src-2.4.2.alpha.yyyymmdd.zip build/dist/scons_2.4.1-1_all.deb The SConstruct file is supposed to be smart enough to avoid trying to build @@ -6,7 +6,7 @@ copyright_years = '2001 - 2015' # This gets inserted into the man pages to reflect the month of release. -month_year = 'November 2015' +month_year = 'MONTH YEAR' # # __COPYRIGHT__ @@ -43,7 +43,7 @@ import tempfile import bootstrap project = 'scons' -default_version = '2.4.1' +default_version = '2.4.2.alpha.yyyymmdd' copyright = "Copyright (c) %s The SCons Foundation" % copyright_years platform = distutils.util.get_platform() @@ -88,7 +88,6 @@ fakeroot = whereis('fakeroot') gzip = whereis('gzip') rpmbuild = whereis('rpmbuild') hg = os.path.exists('.hg') and whereis('hg') -svn = os.path.exists('.svn') and whereis('svn') unzip = whereis('unzip') zip = whereis('zip') @@ -117,16 +116,11 @@ if not version: version = default_version hg_status_lines = [] -svn_status_lines = [] if hg: cmd = "%s status --all 2> /dev/null" % hg hg_status_lines = os.popen(cmd, "r").readlines() -if svn: - cmd = "%s status --verbose 2> /dev/null" % svn - svn_status_lines = os.popen(cmd, "r").readlines() - revision = ARGUMENTS.get('REVISION', '') def generate_build_id(revision): return revision @@ -145,17 +139,6 @@ if not revision and hg: result = result + '[MODIFIED]' return result -if not revision and svn: - svn_info = os.popen("%s info 2> /dev/null" % svn, "r").read() - m = re.search('Revision: (\d+)', svn_info) - if m: - revision = m.group(1) - def generate_build_id(revision): - result = 'r' + revision - if [l for l in svn_status_lines if l[0] in 'ACDMR']: - result = result + '[MODIFIED]' - return result - checkpoint = ARGUMENTS.get('CHECKPOINT', '') if checkpoint: if checkpoint == 'd': @@ -236,7 +219,7 @@ command_line_variables = [ ("REVISION=", "The revision number of the source being built. " + "The default is the Subversion revision returned " + - "'svn info', with an appended string of " + + "'hg heads', with an appended string of " + "'[MODIFIED]' if there are any changes in the " + "working copy."), @@ -1234,12 +1217,8 @@ sfiles = None if hg_status_lines: slines = [l for l in hg_status_lines if l[0] in 'ACM'] sfiles = [l.split()[-1] for l in slines] -elif svn_status_lines: - slines = [l for l in svn_status_lines if l[0] in ' MA'] - sentries = [l.split()[-1] for l in slines] - sfiles = list(filter(os.path.isfile, sentries)) else: - print "Not building in a Mercurial or Subversion tree; skipping building src package." + print "Not building in a Mercurial tree; skipping building src package." if sfiles: remove_patterns = [ diff --git a/bootstrap.py b/bootstrap.py index f3bc1050..08df11db 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -27,6 +27,8 @@ import os.path import sys import glob import subprocess +import filecmp +import shutil __doc__ = """bootstrap.py @@ -127,7 +129,7 @@ def main(): def must_copy(dst, src): if not os.path.exists(dst): return 1 - return open(dst, 'rb').read() != open(src, 'rb').read() + return not filecmp.cmp(dst,src) # Note: We don't use the getopt module to process the command-line # arguments because we'd have to teach it about all of the SCons options. @@ -195,7 +197,8 @@ def main(): os.makedirs(dir) try: os.unlink(dst) except: pass - open(dst, 'wb').write( open(src, 'rb').read() ) + + shutil.copyfile(src,dst) if update_only: sys.exit(0) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 23fd49b8..9e9ec42f 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -4,20 +4,19 @@ Change Log - RELEASE VERSION/DATE TO BE FILLED IN LATER - From John Doe: - - Whatever John Doe did. - - From Anatoly Techtonik: - - Document SCons.Debug.caller_trace() behavior. - + From William Blevins: + - Added support for cross-language dependency scanning; + SCons now respects scanner keys for implicit dependencies. + - Resolved missing cross-language dependencies for + SWIG bindings (fixes #2264). + RELEASE 2.4.1 - Mon, 07 Nov 2015 10:37:21 -0700 From Arfrever Frehtes Taifersar Arahesis: - - Fix for Bug # 2791 - setup.py fails unnecessarily under Jython. + - Fix for Bug # 2791 - Setup.py fails unnecessarily under Jython. From Dirk Baechle: - Fixed license of SVG titlepage files in the context of Debian @@ -76,6 +75,7 @@ RELEASE 2.4.1 - Mon, 07 Nov 2015 10:37:21 -0700 - Fix to swig tool - respect env['SWIG'] provided by user. + RELEASE 2.4.0 - Mon, 21 Sep 2015 08:56:00 -0700 From Dirk Baechle: @@ -88,8 +88,6 @@ RELEASE 2.4.0 - Mon, 21 Sep 2015 08:56:00 -0700 From Andrew Featherstone - Fixed typo in SWIGPATH description - - RELEASE 2.3.6 - Mon, 31 Jul 2015 14:35:03 -0700 From Rob Smith: @@ -383,8 +381,6 @@ RELEASE 2.3.0 - Mon, 02 Mar 2013 13:22:29 -0400 Add builder InstallVersionedLib to create the required symlinks installing a versioned shared library. - - RELEASE 2.2.0 - Mon, 05 Aug 2012 15:37:48 +0000 From dubcanada on Bitbucket: diff --git a/src/RELEASE.txt b/src/RELEASE.txt index ed179849..3bff366e 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -1,4 +1,4 @@ - A new SCons checkpoint release, 2.4.2.alpha.yyyymmdd, is now available + A new SCons checkpoint release, 2.4.3.alpha.yyyymmdd, is now available on the SCons download page: http://www.scons.org/download.php diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in index f62d16e7..f7d5aa77 100644 --- a/src/engine/MANIFEST.in +++ b/src/engine/MANIFEST.in @@ -39,6 +39,7 @@ SCons/Scanner/IDL.py SCons/Scanner/LaTeX.py SCons/Scanner/Prog.py SCons/Scanner/RC.py +SCons/Scanner/SWIG.py SCons/SConf.py SCons/SConsign.py SCons/Script/__init__.py diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index f32b326d..3b7d06fe 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -32,6 +32,7 @@ import stat import sys import SCons.Action +import SCons.Warnings cache_enabled = True cache_debug = False diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index 744da5f6..c1bd9027 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -493,7 +493,7 @@ def __libversionflags(env, version_var, flags_var): ConstructionEnvironment = { 'BUILDERS' : {}, - 'SCANNERS' : [], + 'SCANNERS' : [ SCons.Tool.SourceFileScanner ], 'CONFIGUREDIR' : '#/.sconf_temp', 'CONFIGURELOG' : '#/config.log', 'CPPSUFFIXES' : SCons.Tool.CSuffixes, diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 98ed758b..59e57e3c 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -486,29 +486,15 @@ class Executor(object): each individual target, which is a hell of a lot more efficient. """ env = self.get_build_env() + path = self.get_build_scanner_path + kw = self.get_kw() # TODO(batch): scan by batches) deps = [] - if scanner: - for node in node_list: - node.disambiguate() - s = scanner.select(node) - if not s: - continue - path = self.get_build_scanner_path(s) - deps.extend(node.get_implicit_deps(env, s, path)) - else: - kw = self.get_kw() - for node in node_list: - node.disambiguate() - scanner = node.get_env_scanner(env, kw) - if not scanner: - continue - scanner = scanner.select(node) - if not scanner: - continue - path = self.get_build_scanner_path(scanner) - deps.extend(node.get_implicit_deps(env, scanner, path)) + + for node in node_list: + node.disambiguate() + deps.extend(node.get_implicit_deps(env, scanner, path, kw)) deps.extend(self.get_implicit_deps()) diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py index 9df0b2d1..f390319d 100644 --- a/src/engine/SCons/ExecutorTests.py +++ b/src/engine/SCons/ExecutorTests.py @@ -83,7 +83,9 @@ class MyNode(object): executor(self) def get_env_scanner(self, env, kw): return MyScanner('dep-') - def get_implicit_deps(self, env, scanner, path): + def get_implicit_deps(self, env, scanner, path, kw={}): + if not scanner: + scanner = self.get_env_scanner(env, kw) return [scanner.prefix + str(self)] def add_to_implicit(self, deps): self.implicit.extend(deps) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index a6471b49..1478419e 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -121,6 +121,8 @@ class Environment(object): self._dict.update(kw) def __getitem__(self, key): return self._dict[key] + def get(self, key, default = None): + return self._dict.get(key, default) def Dictionary(self, *args): return {} def Override(self, overrides): @@ -132,7 +134,12 @@ class Environment(object): def get_factory(self, factory): return factory or MyNode def get_scanner(self, scanner_key): - return self._dict['SCANNERS'][0] + try: + return self._dict['SCANNERS'][0] + except: + pass + + return [] class Builder(object): def __init__(self, env=None, is_explicit=1): @@ -185,7 +192,7 @@ class Scanner(object): def __call__(self, node): self.called = 1 return node.GetTag('found_includes') - def path(self, env, dir, target=None, source=None): + def path(self, env, dir=None, target=None, source=None, kw={}): return () def select(self, node): return self @@ -928,7 +935,7 @@ class NodeTestCase(unittest.TestCase): node.Tag('found_includes', [d1, d2]) # Simple return of the found includes - deps = node.get_implicit_deps(env, s, target) + deps = node.get_implicit_deps(env, s, s.path) assert deps == [d1, d2], deps # By default, our fake scanner recurses @@ -938,24 +945,24 @@ class NodeTestCase(unittest.TestCase): d1.Tag('found_includes', [e, f]) d2.Tag('found_includes', [e, f]) f.Tag('found_includes', [g]) - deps = node.get_implicit_deps(env, s, target) + deps = node.get_implicit_deps(env, s, s.path) assert deps == [d1, d2, e, f, g], list(map(str, deps)) # Recursive scanning eliminates duplicates e.Tag('found_includes', [f]) - deps = node.get_implicit_deps(env, s, target) + deps = node.get_implicit_deps(env, s, s.path) assert deps == [d1, d2, e, f, g], list(map(str, deps)) # Scanner method can select specific nodes to recurse def no_fff(nodes): return [n for n in nodes if str(n)[0] != 'f'] s.recurse_nodes = no_fff - deps = node.get_implicit_deps(env, s, target) + deps = node.get_implicit_deps(env, s, s.path) assert deps == [d1, d2, e, f], list(map(str, deps)) # Scanner method can short-circuit recursing entirely s.recurse_nodes = lambda nodes: [] - deps = node.get_implicit_deps(env, s, target) + deps = node.get_implicit_deps(env, s, s.path) assert deps == [d1, d2], list(map(str, deps)) def test_get_env_scanner(self): diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index f2d37c26..5aa96000 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -916,35 +916,56 @@ class Node(object): """ return [] - def get_implicit_deps(self, env, scanner, path): + def get_implicit_deps(self, env, initial_scanner, path_func, kw = {}): """Return a list of implicit dependencies for this node. This method exists to handle recursive invocation of the scanner on the implicit dependencies returned by the scanner, if the scanner's recursive flag says that we should. """ - if not scanner: - return [] - - # Give the scanner a chance to select a more specific scanner - # for this Node. - #scanner = scanner.select(self) - nodes = [self] seen = {} seen[self] = 1 - deps = [] - while nodes: - n = nodes.pop(0) - d = [x for x in n.get_found_includes(env, scanner, path) if x not in seen] - if d: - deps.extend(d) - for n in d: - seen[n] = 1 - nodes.extend(scanner.recurse_nodes(d)) + dependencies = [] - return deps + root_node_scanner = self._get_scanner(env, initial_scanner, None, kw) + while nodes: + node = nodes.pop(0) + + scanner = node._get_scanner(env, initial_scanner, root_node_scanner, kw) + + if not scanner: + continue + + path = path_func(scanner) + + included_deps = [x for x in node.get_found_includes(env, scanner, path) if x not in seen] + if included_deps: + dependencies.extend(included_deps) + for dep in included_deps: + seen[dep] = 1 + nodes.extend(scanner.recurse_nodes(included_deps)) + + return dependencies + + def _get_scanner(self, env, initial_scanner, root_node_scanner, kw): + if not initial_scanner: + # handle implicit scanner case + scanner = self.get_env_scanner(env, kw) + if scanner: + scanner = scanner.select(self) + else: + # handle explicit scanner case + scanner = initial_scanner.select(self) + + if not scanner: + # no scanner could be found for the given node's scanner key; + # thus, make an attempt at using a default. + scanner = root_node_scanner + + return scanner + def get_env_scanner(self, env, kw={}): return env.get_scanner(self.scanner_key()) diff --git a/src/engine/SCons/compat/_scons_io.py b/src/engine/SCons/Scanner/SWIG.py index 538afb77..2650e4bf 100644 --- a/src/engine/SCons/compat/_scons_io.py +++ b/src/engine/SCons/Scanner/SWIG.py @@ -1,3 +1,9 @@ +"""SCons.Scanner.SWIG + +This module implements the dependency scanner for SWIG code. + +""" + # # __COPYRIGHT__ # @@ -21,22 +27,16 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__doc__ = """ -io compatibility module for older (pre-2.6) Python versions - -This does not not NOT (repeat, *NOT*) provide complete io -functionality. It only wraps the portions of io functionality used -by SCons, in an interface that looks enough like io for our purposes. -""" - __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -# Use the "imp" module to protect the imports below from fixers. -import imp +import SCons.Scanner + +SWIGSuffixes = [ '.i' ] -_cStringIO = imp.load_module('cStringIO', *imp.find_module('cStringIO')) -StringIO = _cStringIO.StringIO -del _cStringIO +def SWIGScanner(): + expr = '^[ \t]*%[ \t]*(?:include|import|extern)[ \t]*(<|"?)([^>\s"]+)(?:>|"?)' + scanner = SCons.Scanner.ClassicCPP("SWIGScanner", ".i", "SWIGPATH", expr) + return scanner # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 73746871..05947983 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -51,6 +51,7 @@ import SCons.Scanner.C import SCons.Scanner.D import SCons.Scanner.LaTeX import SCons.Scanner.Prog +import SCons.Scanner.SWIG DefaultToolpath=[] @@ -60,6 +61,7 @@ LaTeXScanner = SCons.Scanner.LaTeX.LaTeXScanner() PDFLaTeXScanner = SCons.Scanner.LaTeX.PDFLaTeXScanner() ProgramScanner = SCons.Scanner.Prog.ProgramScanner() SourceFileScanner = SCons.Scanner.Base({}, name='SourceFileScanner') +SWIGScanner = SCons.Scanner.SWIG.SWIGScanner() CSuffixes = [".c", ".C", ".cxx", ".cpp", ".c++", ".cc", ".h", ".H", ".hxx", ".hpp", ".hh", @@ -73,12 +75,17 @@ IDLSuffixes = [".idl", ".IDL"] LaTeXSuffixes = [".tex", ".ltx", ".latex"] +SWIGSuffixes = ['.i'] + for suffix in CSuffixes: SourceFileScanner.add_scanner(suffix, CScanner) for suffix in DSuffixes: SourceFileScanner.add_scanner(suffix, DScanner) +for suffix in SWIGSuffixes: + SourceFileScanner.add_scanner(suffix, SWIGScanner) + # FIXME: what should be done here? Two scanners scan the same extensions, # but look for different files, e.g., "picture.eps" vs. "picture.pdf". # The builders for DVI and PDF explicitly reference their scanners diff --git a/src/engine/SCons/Tool/install.py b/src/engine/SCons/Tool/install.py index 9d5db9f8..4226d567 100644 --- a/src/engine/SCons/Tool/install.py +++ b/src/engine/SCons/Tool/install.py @@ -374,6 +374,7 @@ def generate(env): source_factory = env.fs.Entry, multi = 1, emitter = [ add_targets_to_INSTALLED_FILES, ], + source_scanner = SCons.Scanner.Base( {}, name = 'Install', recursive = False ), name = 'InstallBuilder') global BaseVersionedInstallBuilder diff --git a/src/engine/SCons/Tool/swig.py b/src/engine/SCons/Tool/swig.py index 44686ef0..309585de 100644 --- a/src/engine/SCons/Tool/swig.py +++ b/src/engine/SCons/Tool/swig.py @@ -39,7 +39,6 @@ import subprocess import SCons.Action import SCons.Defaults -import SCons.Scanner import SCons.Tool import SCons.Util import SCons.Node @@ -177,11 +176,6 @@ def generate(env): env['_SWIGINCFLAGS'] = '$( ${_concat(SWIGINCPREFIX, SWIGPATH, SWIGINCSUFFIX, __env__, RDirs, TARGET, SOURCE)} $)' env['SWIGCOM'] = '$SWIG -o $TARGET ${_SWIGOUTDIR} ${_SWIGINCFLAGS} $SWIGFLAGS $SOURCES' - expr = '^[ \t]*%[ \t]*(?:include|import|extern)[ \t]*(<|"?)([^>\s"]+)(?:>|"?)' - scanner = SCons.Scanner.ClassicCPP("SWIGScan", ".i", "SWIGPATH", expr) - - env.Append(SCANNERS = scanner) - def exists(env): swig = env.get('SWIG') or env.Detect(['swig']) return swig diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index c870fbc4..3d5c835e 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -103,65 +103,6 @@ except ImportError: # can fall back to using timestamp. pass -try: - set -except NameError: - # Pre-2.4 Python has no native set type - import_as('_scons_sets', 'sets') - import builtins, sets - builtins.set = sets.Set - - -try: - import collections -except ImportError: - # Pre-2.4 Python has no collections module. - import_as('_scons_collections', 'collections') -else: - try: - collections.UserDict - except AttributeError: - exec('from UserDict import UserDict as _UserDict') - collections.UserDict = _UserDict - del _UserDict - try: - collections.UserList - except AttributeError: - exec('from UserList import UserList as _UserList') - collections.UserList = _UserList - del _UserList - try: - collections.UserString - except AttributeError: - exec('from UserString import UserString as _UserString') - collections.UserString = _UserString - del _UserString - - -try: - import io -except ImportError: - # Pre-2.6 Python has no io module. - import_as('_scons_io', 'io') - - -try: - os.devnull -except AttributeError: - # Pre-2.4 Python has no os.devnull attribute - _names = sys.builtin_module_names - if 'posix' in _names: - os.devnull = '/dev/null' - elif 'nt' in _names: - os.devnull = 'nul' - os.path.devnull = os.devnull -try: - os.path.lexists -except AttributeError: - # Pre-2.4 Python has no os.path.lexists function - def lexists(path): - return os.path.exists(path) or os.path.islink(path) - os.path.lexists = lexists # When we're using the '-3' option during regression tests, importing @@ -185,32 +126,44 @@ rename_module('queue', 'Queue') rename_module('winreg', '_winreg') -try: - import subprocess -except ImportError: - # Pre-2.4 Python has no subprocess module. - import_as('_scons_subprocess', 'subprocess') - +# Python 3 moved builtin intern() to sys package +# To make porting easier, make intern always live +# in sys package (for python 2.7.x) try: sys.intern except AttributeError: - # Pre-2.6 Python has no sys.intern() function. + # We must be using python 2.7.x so monkey patch + # intern into the sys package import builtins - try: - sys.intern = builtins.intern - except AttributeError: - # Pre-2.x Python has no builtin intern() function. - def intern(x): - return x - sys.intern = intern - del intern + sys.intern = builtins.intern + + + +# Preparing for 3.x. UserDict, UserList, UserString are in +# collections for 3.x, but standalone in 2.7.x +import collections +try: + collections.UserDict +except AttributeError: + exec('from UserDict import UserDict as _UserDict') + collections.UserDict = _UserDict + del _UserDict + +try: + collections.UserList +except AttributeError: + exec('from UserList import UserList as _UserList') + collections.UserList = _UserList + del _UserList + try: - sys.maxsize + collections.UserString except AttributeError: - # Pre-2.6 Python has no sys.maxsize attribute - # Wrapping sys in () is silly, but protects it from 2to3 renames fixer - sys.maxsize = (sys).maxint + exec('from UserString import UserString as _UserString') + collections.UserString = _UserString + del _UserString + if os.environ.get('SCONS_HORRIBLE_REGRESSION_TEST_HACK') is not None: # We can't apply the 'callable' fixer until the floor is 2.6, but the diff --git a/src/engine/SCons/compat/_scons_collections.py b/src/engine/SCons/compat/_scons_collections.py deleted file mode 100644 index 1591b2e7..00000000 --- a/src/engine/SCons/compat/_scons_collections.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__doc__ = """ -collections compatibility module for older (pre-2.4) Python versions - -This does not not NOT (repeat, *NOT*) provide complete collections -functionality. It only wraps the portions of collections functionality -used by SCons, in an interface that looks enough like collections for -our purposes. -""" - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -# Use exec to hide old names from fixers. -exec("""if True: - from UserDict import UserDict - from UserList import UserList - from UserString import UserString""") - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/compat/_scons_sets.py b/src/engine/SCons/compat/_scons_sets.py deleted file mode 100644 index 0fde9941..00000000 --- a/src/engine/SCons/compat/_scons_sets.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Classes to represent arbitrary sets (including sets of sets). - -This module implements sets using dictionaries whose values are -ignored. The usual operations (union, intersection, deletion, etc.) -are provided as both methods and operators. - -Important: sets are not sequences! While they support 'x in s', -'len(s)', and 'for x in s', none of those operations are unique for -sequences; for example, mappings support all three as well. The -characteristic operation for sequences is subscripting with small -integers: s[i], for i in range(len(s)). Sets don't support -subscripting at all. Also, sequences allow multiple occurrences and -their elements have a definite order; sets on the other hand don't -record multiple occurrences and don't remember the order of element -insertion (which is why they don't support s[i]). - -The following classes are provided: - -BaseSet -- All the operations common to both mutable and immutable - sets. This is an abstract class, not meant to be directly - instantiated. - -Set -- Mutable sets, subclass of BaseSet; not hashable. - -ImmutableSet -- Immutable sets, subclass of BaseSet; hashable. - An iterable argument is mandatory to create an ImmutableSet. - -_TemporarilyImmutableSet -- A wrapper around a Set, hashable, - giving the same hash value as the immutable set equivalent - would have. Do not use this class directly. - -Only hashable objects can be added to a Set. In particular, you cannot -really add a Set as an element to another Set; if you try, what is -actually added is an ImmutableSet built from it (it compares equal to -the one you tried adding). - -When you ask if `x in y' where x is a Set and y is a Set or -ImmutableSet, x is wrapped into a _TemporarilyImmutableSet z, and -what's tested is actually `z in y'. - -""" - -# Code history: -# -# - Greg V. Wilson wrote the first version, using a different approach -# to the mutable/immutable problem, and inheriting from dict. -# -# - Alex Martelli modified Greg's version to implement the current -# Set/ImmutableSet approach, and make the data an attribute. -# -# - Guido van Rossum rewrote much of the code, made some API changes, -# and cleaned up the docstrings. -# -# - Raymond Hettinger added a number of speedups and other -# improvements. - -# protect this import from the fixers... -exec('from itertools import ifilterfalse as filterfalse') - -__all__ = ['BaseSet', 'Set', 'ImmutableSet'] - -class BaseSet(object): - """Common base class for mutable and immutable sets.""" - - __slots__ = ['_data'] - - # Constructor - - def __init__(self): - """This is an abstract class.""" - # Don't call this from a concrete subclass! - if self.__class__ is BaseSet: - raise TypeError("BaseSet is an abstract class. " - "Use Set or ImmutableSet.") - - # Standard protocols: __len__, __repr__, __str__, __iter__ - - def __len__(self): - """Return the number of elements of a set.""" - return len(self._data) - - def __repr__(self): - """Return string representation of a set. - - This looks like 'Set([<list of elements>])'. - """ - return self._repr() - - # __str__ is the same as __repr__ - __str__ = __repr__ - - def _repr(self, sort_them=False): - elements = list(self._data.keys()) - if sort_them: - elements.sort() - return '%s(%r)' % (self.__class__.__name__, elements) - - def __iter__(self): - """Return an iterator over the elements or a set. - - This is the keys iterator for the underlying dict. - """ - # Wrapping name in () prevents fixer from "fixing" this - return (self._data.iterkeys)() - - # Three-way comparison is not supported. However, because __eq__ is - # tried before __cmp__, if Set x == Set y, x.__eq__(y) returns True and - # then cmp(x, y) returns 0 (Python doesn't actually call __cmp__ in this - # case). - - def __cmp__(self, other): - raise TypeError("can't compare sets using cmp()") - - # Equality comparisons using the underlying dicts. Mixed-type comparisons - # are allowed here, where Set == z for non-Set z always returns False, - # and Set != z always True. This allows expressions like "x in y" to - # give the expected result when y is a sequence of mixed types, not - # raising a pointless TypeError just because y contains a Set, or x is - # a Set and y contain's a non-set ("in" invokes only __eq__). - # Subtle: it would be nicer if __eq__ and __ne__ could return - # NotImplemented instead of True or False. Then the other comparand - # would get a chance to determine the result, and if the other comparand - # also returned NotImplemented then it would fall back to object address - # comparison (which would always return False for __eq__ and always - # True for __ne__). However, that doesn't work, because this type - # *also* implements __cmp__: if, e.g., __eq__ returns NotImplemented, - # Python tries __cmp__ next, and the __cmp__ here then raises TypeError. - - def __eq__(self, other): - if isinstance(other, BaseSet): - return self._data == other._data - else: - return False - - def __ne__(self, other): - if isinstance(other, BaseSet): - return self._data != other._data - else: - return True - - # Copying operations - - def copy(self): - """Return a shallow copy of a set.""" - result = self.__class__() - result._data.update(self._data) - return result - - __copy__ = copy # For the copy module - - def __deepcopy__(self, memo): - """Return a deep copy of a set; used by copy module.""" - # This pre-creates the result and inserts it in the memo - # early, in case the deep copy recurses into another reference - # to this same set. A set can't be an element of itself, but - # it can certainly contain an object that has a reference to - # itself. - from copy import deepcopy - result = self.__class__() - memo[id(self)] = result - data = result._data - value = True - for elt in self: - data[deepcopy(elt, memo)] = value - return result - - # Standard set operations: union, intersection, both differences. - # Each has an operator version (e.g. __or__, invoked with |) and a - # method version (e.g. union). - # Subtle: Each pair requires distinct code so that the outcome is - # correct when the type of other isn't suitable. For example, if - # we did "union = __or__" instead, then Set().union(3) would return - # NotImplemented instead of raising TypeError (albeit that *why* it - # raises TypeError as-is is also a bit subtle). - - def __or__(self, other): - """Return the union of two sets as a new set. - - (I.e. all elements that are in either set.) - """ - if not isinstance(other, BaseSet): - return NotImplemented - return self.union(other) - - def union(self, other): - """Return the union of two sets as a new set. - - (I.e. all elements that are in either set.) - """ - result = self.__class__(self) - result._update(other) - return result - - def __and__(self, other): - """Return the intersection of two sets as a new set. - - (I.e. all elements that are in both sets.) - """ - if not isinstance(other, BaseSet): - return NotImplemented - return self.intersection(other) - - def intersection(self, other): - """Return the intersection of two sets as a new set. - - (I.e. all elements that are in both sets.) - """ - if not isinstance(other, BaseSet): - other = Set(other) - if len(self) <= len(other): - little, big = self, other - else: - little, big = other, self - common = iter(filter(big._data.has_key, little)) - return self.__class__(common) - - def __xor__(self, other): - """Return the symmetric difference of two sets as a new set. - - (I.e. all elements that are in exactly one of the sets.) - """ - if not isinstance(other, BaseSet): - return NotImplemented - return self.symmetric_difference(other) - - def symmetric_difference(self, other): - """Return the symmetric difference of two sets as a new set. - - (I.e. all elements that are in exactly one of the sets.) - """ - result = self.__class__() - data = result._data - value = True - selfdata = self._data - try: - otherdata = other._data - except AttributeError: - otherdata = Set(other)._data - for elt in filterfalse(otherdata.has_key, selfdata): - data[elt] = value - for elt in filterfalse(selfdata.has_key, otherdata): - data[elt] = value - return result - - def __sub__(self, other): - """Return the difference of two sets as a new Set. - - (I.e. all elements that are in this set and not in the other.) - """ - if not isinstance(other, BaseSet): - return NotImplemented - return self.difference(other) - - def difference(self, other): - """Return the difference of two sets as a new Set. - - (I.e. all elements that are in this set and not in the other.) - """ - result = self.__class__() - data = result._data - try: - otherdata = other._data - except AttributeError: - otherdata = Set(other)._data - value = True - for elt in filterfalse(otherdata.has_key, self): - data[elt] = value - return result - - # Membership test - - def __contains__(self, element): - """Report whether an element is a member of a set. - - (Called in response to the expression `element in self'.) - """ - try: - return element in self._data - except TypeError: - transform = getattr(element, "__as_temporarily_immutable__", None) - if transform is None: - raise # re-raise the TypeError exception we caught - return transform() in self._data - - # Subset and superset test - - def issubset(self, other): - """Report whether another set contains this set.""" - self._binary_sanity_check(other) - if len(self) > len(other): # Fast check for obvious cases - return False - for elt in filterfalse(other._data.has_key, self): - return False - return True - - def issuperset(self, other): - """Report whether this set contains another set.""" - self._binary_sanity_check(other) - if len(self) < len(other): # Fast check for obvious cases - return False - for elt in filterfalse(self._data.has_key, other): - return False - return True - - # Inequality comparisons using the is-subset relation. - __le__ = issubset - __ge__ = issuperset - - def __lt__(self, other): - self._binary_sanity_check(other) - return len(self) < len(other) and self.issubset(other) - - def __gt__(self, other): - self._binary_sanity_check(other) - return len(self) > len(other) and self.issuperset(other) - - # Assorted helpers - - def _binary_sanity_check(self, other): - # Check that the other argument to a binary operation is also - # a set, raising a TypeError otherwise. - if not isinstance(other, BaseSet): - raise TypeError("Binary operation only permitted between sets") - - def _compute_hash(self): - # Calculate hash code for a set by xor'ing the hash codes of - # the elements. This ensures that the hash code does not depend - # on the order in which elements are added to the set. This is - # not called __hash__ because a BaseSet should not be hashable; - # only an ImmutableSet is hashable. - result = 0 - for elt in self: - result ^= hash(elt) - return result - - def _update(self, iterable): - # The main loop for update() and the subclass __init__() methods. - data = self._data - - # Use the fast update() method when a dictionary is available. - if isinstance(iterable, BaseSet): - data.update(iterable._data) - return - - value = True - - if type(iterable) in (list, tuple, xrange): - # Optimized: we know that __iter__() and next() can't - # raise TypeError, so we can move 'try:' out of the loop. - it = iter(iterable) - while True: - try: - for element in it: - data[element] = value - return - except TypeError: - transform = getattr(element, "__as_immutable__", None) - if transform is None: - raise # re-raise the TypeError exception we caught - data[transform()] = value - else: - # Safe: only catch TypeError where intended - for element in iterable: - try: - data[element] = value - except TypeError: - transform = getattr(element, "__as_immutable__", None) - if transform is None: - raise # re-raise the TypeError exception we caught - data[transform()] = value - - -class ImmutableSet(BaseSet): - """Immutable set class.""" - - __slots__ = ['_hashcode'] - - # BaseSet + hashing - - def __init__(self, iterable=None): - """Construct an immutable set from an optional iterable.""" - self._hashcode = None - self._data = {} - if iterable is not None: - self._update(iterable) - - def __hash__(self): - if self._hashcode is None: - self._hashcode = self._compute_hash() - return self._hashcode - - def __getstate__(self): - return self._data, self._hashcode - - def __setstate__(self, state): - self._data, self._hashcode = state - -class Set(BaseSet): - """ Mutable set class.""" - - __slots__ = [] - - # BaseSet + operations requiring mutability; no hashing - - def __init__(self, iterable=None): - """Construct a set from an optional iterable.""" - self._data = {} - if iterable is not None: - self._update(iterable) - - def __getstate__(self): - # getstate's results are ignored if it is not - return self._data, - - def __setstate__(self, data): - self._data, = data - - def __hash__(self): - """A Set cannot be hashed.""" - # We inherit object.__hash__, so we must deny this explicitly - raise TypeError("Can't hash a Set, only an ImmutableSet.") - - # In-place union, intersection, differences. - # Subtle: The xyz_update() functions deliberately return None, - # as do all mutating operations on built-in container types. - # The __xyz__ spellings have to return self, though. - - def __ior__(self, other): - """Update a set with the union of itself and another.""" - self._binary_sanity_check(other) - self._data.update(other._data) - return self - - def union_update(self, other): - """Update a set with the union of itself and another.""" - self._update(other) - - def __iand__(self, other): - """Update a set with the intersection of itself and another.""" - self._binary_sanity_check(other) - self._data = (self & other)._data - return self - - def intersection_update(self, other): - """Update a set with the intersection of itself and another.""" - if isinstance(other, BaseSet): - self &= other - else: - self._data = (self.intersection(other))._data - - def __ixor__(self, other): - """Update a set with the symmetric difference of itself and another.""" - self._binary_sanity_check(other) - self.symmetric_difference_update(other) - return self - - def symmetric_difference_update(self, other): - """Update a set with the symmetric difference of itself and another.""" - data = self._data - value = True - if not isinstance(other, BaseSet): - other = Set(other) - if self is other: - self.clear() - for elt in other: - if elt in data: - del data[elt] - else: - data[elt] = value - - def __isub__(self, other): - """Remove all elements of another set from this set.""" - self._binary_sanity_check(other) - self.difference_update(other) - return self - - def difference_update(self, other): - """Remove all elements of another set from this set.""" - data = self._data - if not isinstance(other, BaseSet): - other = Set(other) - if self is other: - self.clear() - for elt in filter(data.has_key, other): - del data[elt] - - # Python dict-like mass mutations: update, clear - - def update(self, iterable): - """Add all values from an iterable (such as a list or file).""" - self._update(iterable) - - def clear(self): - """Remove all elements from this set.""" - self._data.clear() - - # Single-element mutations: add, remove, discard - - def add(self, element): - """Add an element to a set. - - This has no effect if the element is already present. - """ - try: - self._data[element] = True - except TypeError: - transform = getattr(element, "__as_immutable__", None) - if transform is None: - raise # re-raise the TypeError exception we caught - self._data[transform()] = True - - def remove(self, element): - """Remove an element from a set; it must be a member. - - If the element is not a member, raise a KeyError. - """ - try: - del self._data[element] - except TypeError: - transform = getattr(element, "__as_temporarily_immutable__", None) - if transform is None: - raise # re-raise the TypeError exception we caught - del self._data[transform()] - - def discard(self, element): - """Remove an element from a set if it is a member. - - If the element is not a member, do nothing. - """ - try: - self.remove(element) - except KeyError: - pass - - def pop(self): - """Remove and return an arbitrary set element.""" - return self._data.popitem()[0] - - def __as_immutable__(self): - # Return a copy of self as an immutable set - return ImmutableSet(self) - - def __as_temporarily_immutable__(self): - # Return self wrapped in a temporarily immutable set - return _TemporarilyImmutableSet(self) - - -class _TemporarilyImmutableSet(BaseSet): - # Wrap a mutable set as if it was temporarily immutable. - # This only supplies hashing and equality comparisons. - - def __init__(self, set): - self._set = set - self._data = set._data # Needed by ImmutableSet.__eq__() - - def __hash__(self): - return self._set._compute_hash() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/compat/_scons_subprocess.py b/src/engine/SCons/compat/_scons_subprocess.py deleted file mode 100644 index eebe53d3..00000000 --- a/src/engine/SCons/compat/_scons_subprocess.py +++ /dev/null @@ -1,1281 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* -os.popen* -popen2.* -commands.* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On UNIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On UNIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines two shortcut functions: - -call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - -check_call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete. If the - exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - -check_call() will raise CalledProcessError, if the called process -returns a non-zero return code. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional stdin argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -pid, sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode - else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen(cmd, mode='r', bufsize) -==> -pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen(cmd, mode='w', bufsize) -==> -pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - - -Replacing popen2.* ------------------- -Note: If the cmd argument to popen2 functions is a string, the command -is executed through /bin/sh. If it is a list, the command is directly -executed. - -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. - - -""" - -import sys -mswindows = (sys.platform == "win32") - -import os -import types -import traceback - -# Exception classes used by this module. -class CalledProcessError(Exception): - """This exception is raised when a process run by check_call() returns - a non-zero exit status. The exit status will be stored in the - returncode attribute.""" - def __init__(self, returncode, cmd): - self.returncode = returncode - self.cmd = cmd - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) - - -if mswindows: - try: - import threading - except ImportError: - # SCons: the threading module is only used by the communicate() - # method, which we don't actually use, so don't worry if we - # can't import it. - pass - import msvcrt - try: - # Try to get _subprocess - from _subprocess import * - class STARTUPINFO(object): - dwFlags = 0 - hStdInput = None - hStdOutput = None - hStdError = None - wShowWindow = 0 - class pywintypes(object): - error = IOError - except ImportError: - # If not there, then drop back to requiring pywin32 - # TODO: Should this be wrapped in try as well? To notify user to install - # pywin32 ? With URL to it? - import pywintypes - from win32api import GetStdHandle, STD_INPUT_HANDLE, \ - STD_OUTPUT_HANDLE, STD_ERROR_HANDLE - from win32api import GetCurrentProcess, DuplicateHandle, \ - GetModuleFileName, GetVersion - from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE - from win32pipe import CreatePipe - from win32process import CreateProcess, STARTUPINFO, \ - GetExitCodeProcess, STARTF_USESTDHANDLES, \ - STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE - from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0 - - -else: - import select - import errno - import fcntl - import pickle - - try: - fcntl.F_GETFD - except AttributeError: - fcntl.F_GETFD = 1 - - try: - fcntl.F_SETFD - except AttributeError: - fcntl.F_SETFD = 2 - -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "CalledProcessError"] - -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except KeyboardInterrupt: - raise # SCons: don't swallow keyboard interrupts -except: - MAXFD = 256 - -try: - isinstance(1, int) -except TypeError: - def is_int(obj): - return isinstance(obj, type(1)) - def is_int_or_long(obj): - return type(obj) in (type(1), type(1L)) -else: - def is_int(obj): - return isinstance(obj, int) - def is_int_or_long(obj): - return isinstance(obj, (int, long)) - -try: - types.StringTypes -except AttributeError: - try: - types.StringTypes = (str, unicode) - except NameError: - types.StringTypes = (str,) -def is_string(obj): - return isinstance(obj, types.StringTypes) - -_active = [] - -def _cleanup(): - for inst in _active[:]: - if inst.poll(_deadstate=sys.maxsize) >= 0: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass - -PIPE = -1 -STDOUT = -2 - - -def call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return apply(Popen, popenargs, kwargs).wait() - - -def check_call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete. If - the exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - """ - retcode = call(*popenargs, **kwargs) - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - if retcode: - raise CalledProcessError(retcode, cmd) - return retcode - - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backspaces. - result.append('\\' * len(bs_buf)*2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backspaces, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - -class Popen(object): - def __init__(self, args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - """Create new Popen instance.""" - _cleanup() - - self._child_created = False - if not is_int_or_long(bufsize): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - if close_fds: - raise ValueError("close_fds is not supported on Windows " - "platforms") - else: - # POSIX - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None - self.stdout = None - self.stderr = None - self.pid = None - self.returncode = None - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - if p2cwrite: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) - - - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data - - - def __del__(self): - if not self._child_created: - # We didn't get to successfully create a child process. - return - # In case the child hasn't been waited on, check if it's done. - self.poll(_deadstate=sys.maxsize) - if self.returncode is None and _active is not None: - # Child is still running, keep us alive until we can wait on it. - _active.append(self) - - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: - stdout = None - stderr = None - if self.stdin: - if input: - self.stdin.write(input) - self.stdin.close() - elif self.stdout: - stdout = self.stdout.read() - elif self.stderr: - stderr = self.stderr.read() - self.wait() - return (stdout, stderr) - - return self._communicate(input) - - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tupel with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) - - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - p2cread = GetStdHandle(STD_INPUT_HANDLE) - elif stdin == PIPE: - p2cread, p2cwrite = CreatePipe(None, 0) - # Detach and turn into fd - p2cwrite = p2cwrite.Detach() - p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0) - elif is_int(stdin): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) - elif stdout == PIPE: - c2pread, c2pwrite = CreatePipe(None, 0) - # Detach and turn into fd - c2pread = c2pread.Detach() - c2pread = msvcrt.open_osfhandle(c2pread, 0) - elif is_int(stdout): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = GetStdHandle(STD_ERROR_HANDLE) - elif stderr == PIPE: - errread, errwrite = CreatePipe(None, 0) - # Detach and turn into fd - errread = errread.Detach() - errread = msvcrt.open_osfhandle(errread, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif is_int(stderr): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _make_inheritable(self, handle): - """Return a duplicate of handle, which is inheritable""" - return DuplicateHandle(GetCurrentProcess(), handle, - GetCurrentProcess(), 0, 1, - DUPLICATE_SAME_ACCESS) - - - def _find_w9xpopen(self): - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (MS Windows version)""" - - if not isinstance(args, types.StringTypes): - args = list2cmdline(args) - - # Process startup details - if startupinfo is None: - startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags = startupinfo.dwFlags | STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - startupinfo.dwFlags = startupinfo.dwFlags | STARTF_USESHOWWINDOW - startupinfo.wShowWindow = SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = comspec + " /c " + args - if (GetVersion() >= 0x80000000L or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont - # kill children. - creationflags = creationflags | CREATE_NEW_CONSOLE - - # Start the process - try: - hp, ht, pid, tid = CreateProcess(executable, args, - # no special security - None, None, - # must inherit handles to pass std - # handles - 1, - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error, e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or simliar), but - # how can this be done from Python? - raise WindowsError(*e.args) - - # Retain the process handle, but close the thread handle - self._child_created = True - self._handle = hp - self.pid = pid - ht.Close() - - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread is not None: - p2cread.Close() - if c2pwrite is not None: - c2pwrite.Close() - if errwrite is not None: - errwrite.Close() - - - def poll(self, _deadstate=None): - """Check if child process has terminated. Returns returncode - attribute.""" - if self.returncode is None: - if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: - self.returncode = GetExitCodeProcess(self._handle) - return self.returncode - - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - obj = WaitForSingleObject(self._handle, INFINITE) - self.returncode = GetExitCodeProcess(self._handle) - return self.returncode - - - def _readerthread(self, fh, buffer): - buffer.append(fh.read()) - - - def _communicate(self, input): - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) - stderr_thread.start() - - if self.stdin: - if input is not None: - self.stdin.write(input) - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = stdout[0] - if stderr is not None: - stderr = stderr[0] - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - else: - # - # POSIX methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tupel with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = os.pipe() - elif is_int(stdin): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = os.pipe() - elif is_int(stdout): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = os.pipe() - elif stderr == STDOUT: - errwrite = c2pwrite - elif is_int(stderr): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _set_cloexec_flag(self, fd): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - - - def _close_fds(self, but): - for i in range(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except KeyboardInterrupt: - raise # SCons: don't swallow keyboard interrupts - except: - pass - - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (POSIX version)""" - - if is_string(args): - args = [args] - - if shell: - args = ["/bin/sh", "-c"] + args - - if executable is None: - executable = args[0] - - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = os.pipe() - self._set_cloexec_flag(errpipe_write) - - self.pid = os.fork() - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite: - os.close(p2cwrite) - if c2pread: - os.close(c2pread) - if errread: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - if p2cread: - os.dup2(p2cread, 0) - if c2pwrite: - os.dup2(c2pwrite, 1) - if errwrite: - os.dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the same - # fd more than once, or standard fds. - try: - set - except NameError: - # Fall-back for earlier Python versions, so epydoc - # can use this module directly to execute things. - if p2cread: - os.close(p2cread) - if c2pwrite and c2pwrite not in (p2cread,): - os.close(c2pwrite) - if errwrite and errwrite not in (p2cread, c2pwrite): - os.close(errwrite) - else: - for fd in set((p2cread, c2pwrite, errwrite))-set((0,1,2)): - if fd: os.close(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - apply(preexec_fn) - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except KeyboardInterrupt: - raise # SCons: don't swallow keyboard interrupts - - except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so it - # really doesn't matter what we return. - os._exit(255) - - # Parent - os.close(errpipe_write) - if p2cread and p2cwrite: - os.close(p2cread) - if c2pwrite and c2pread: - os.close(c2pwrite) - if errwrite and errread: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising exception - data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB - os.close(errpipe_read) - if data != "": - os.waitpid(self.pid, 0) - child_exception = pickle.loads(data) - raise child_exception - - - def _handle_exitstatus(self, sts): - if os.WIFSIGNALED(sts): - self.returncode = -os.WTERMSIG(sts) - elif os.WIFEXITED(sts): - self.returncode = os.WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - - def poll(self, _deadstate=None): - """Check if child process has terminated. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = os.waitpid(self.pid, os.WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except os.error: - if _deadstate is not None: - self.returncode = _deadstate - return self.returncode - - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - pid, sts = os.waitpid(self.pid, 0) - self._handle_exitstatus(sts) - return self.returncode - - - def _communicate(self, input): - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if input: - write_set.append(self.stdin) - else: - self.stdin.close() - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: - rlist, wlist, xlist = select.select(read_set, write_set, []) - - if self.stdin in wlist: - # When select has indicated that the file is writable, - # we can write up to PIPE_BUF bytes without risk - # blocking. POSIX defines PIPE_BUF >= 512 - m = memoryview(input)[input_offset:input_offset+512] - bytes_written = os.write(self.stdin.fileno(), m) - input_offset = input_offset + bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if data == "": - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if data == "": - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - if stderr is not None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print "Looking for 'hda'..." - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 4: Catch execution error - # - print - print "Trying a weird file..." - try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: - if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback - else: - print "Error", e.errno - else: - sys.stderr.write( "Gosh. No error.\n" ) - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print "Looking for 'PROMPT' in set output..." - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 2: Simple execution of program - # - print "Executing calc..." - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Fortran/FORTRANSUFFIXES.py b/test/Fortran/FORTRANSUFFIXES.py index 583b71b8..9673e6f9 100644 --- a/test/Fortran/FORTRANSUFFIXES.py +++ b/test/Fortran/FORTRANSUFFIXES.py @@ -56,51 +56,51 @@ env = Environment(FORTRANPATH = ['.'], env.Append(FORTRANSUFFIXES = ['.x']) env.Object(target = 'test1', source = 'test1.f') env.InstallAs('test1_f', 'test1.f') -env.InstallAs('test1_h', 'test1.h') env.InstallAs('test1_x', 'test1.x') +env.InstallAs('test2_f', 'test2.f') """ % locals()) test.write('test1.f', """\ test1.f 1 - INCLUDE 'test1.h' + INCLUDE 'test2.f' INCLUDE 'test1.x' """) -test.write('test1.h', """\ - test1.h 1 - INCLUDE 'foo.h' +test.write('test2.f', """\ + test2.f 1 + INCLUDE 'foo.f' """) test.write('test1.x', """\ test1.x 1 - INCLUDE 'foo.h' + INCLUDE 'foo.f' """) -test.write('foo.h', """\ - foo.h 1 +test.write('foo.f', """\ + foo.f 1 """) expect = test.wrap_stdout("""\ %(_python_)s myfc.py test1.o test1.f Install file: "test1.f" as "test1_f" -Install file: "test1.h" as "test1_h" Install file: "test1.x" as "test1_x" +Install file: "test2.f" as "test2_f" """ % locals()) test.run(arguments='.', stdout=expect) test.must_match('test1.o', """\ test1.f 1 - test1.h 1 - foo.h 1 + test2.f 1 + foo.f 1 test1.x 1 - foo.h 1 + foo.f 1 """) test.up_to_date(arguments='.') -test.write('foo.h', """\ - foo.h 2 +test.write('foo.f', """\ + foo.f 2 """) expect = test.wrap_stdout("""\ @@ -111,17 +111,17 @@ test.run(arguments='.', stdout=expect) test.must_match('test1.o', """\ test1.f 1 - test1.h 1 - foo.h 2 + test2.f 1 + foo.f 2 test1.x 1 - foo.h 2 + foo.f 2 """) test.up_to_date(arguments='.') test.write('test1.x', """\ test1.x 2 - INCLUDE 'foo.h' + INCLUDE 'foo.f' """) expect = test.wrap_stdout("""\ @@ -133,32 +133,32 @@ test.run(arguments='.', stdout=expect) test.must_match('test1.o', """\ test1.f 1 - test1.h 1 - foo.h 2 + test2.f 1 + foo.f 2 test1.x 2 - foo.h 2 + foo.f 2 """) test.up_to_date(arguments='.') -test.write('test1.h', """\ - test1.h 2 - INCLUDE 'foo.h' +test.write('test2.f', """\ + test2.f 2 + INCLUDE 'foo.f' """) expect = test.wrap_stdout("""\ %(_python_)s myfc.py test1.o test1.f -Install file: "test1.h" as "test1_h" +Install file: "test2.f" as "test2_f" """ % locals()) test.run(arguments='.', stdout=expect) test.must_match('test1.o', """\ test1.f 1 - test1.h 2 - foo.h 2 + test2.f 2 + foo.f 2 test1.x 2 - foo.h 2 + foo.f 2 """) test.up_to_date(arguments='.') diff --git a/test/IDL/IDLSUFFIXES.py b/test/IDL/IDLSUFFIXES.py index f71ceba5..0a9a50c2 100644 --- a/test/IDL/IDLSUFFIXES.py +++ b/test/IDL/IDLSUFFIXES.py @@ -60,11 +60,6 @@ test.up_to_date(arguments='.') test.write('foo.h', "foo.h 2\n") -test.run(arguments='.', stdout=test.wrap_stdout("""\ -Install file: "foo.idl" as "foo_idl" -Install file: "foo.x" as "foo_x" -""")) - test.up_to_date(arguments='.') test.pass_test() diff --git a/test/QT/manual.py b/test/QT/manual.py index ff38f32b..d911fb33 100644 --- a/test/QT/manual.py +++ b/test/QT/manual.py @@ -46,13 +46,15 @@ sources = ['aaa.cpp', 'bbb.cpp', 'ddd.cpp', 'eee.cpp', 'main.cpp'] # normal invocation sources.append(env.Moc('include/aaa.h')) -env.Moc('bbb.cpp') +moc = env.Moc('bbb.cpp') +env.Ignore( moc, moc ) sources.extend(env.Uic('ui/ccc.ui')[1:]) # manual target specification sources.append(env.Moc('moc-ddd.cpp', 'include/ddd.h', QT_MOCHPREFIX='')) # Watch out ! -env.Moc('moc_eee.cpp', 'eee.cpp') +moc = env.Moc('moc_eee.cpp', 'eee.cpp') +env.Ignore( moc, moc ) sources.extend(env.Uic(['include/uic_fff.hpp', 'fff.cpp', 'fff.moc.cpp'], 'ui/fff.ui')[1:]) diff --git a/test/SWIG/recursive-includes-cpp.py b/test/SWIG/recursive-includes-cpp.py new file mode 100644 index 00000000..364bd738 --- /dev/null +++ b/test/SWIG/recursive-includes-cpp.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify that SWIG include directives produce the correct dependencies +in cases of recursive inclusion. +""" + +import os +import TestSCons +from SCons.Defaults import DefaultEnvironment + +DefaultEnvironment( tools = [ 'swig' ] ) + +test = TestSCons.TestSCons() + +# Check for prerequisites of this test. +for pre_req in ['swig', 'python']: + if not test.where_is(pre_req): + test.skip_test('Can not find installed "' + pre_req + '", skipping test.%s' % os.linesep) + +test.write("recursive.h", """\ +/* An empty header file. */ +""") + +test.write("main.h", """\ +#include "recursive.h" +""") + +test.write("main.c", """\ +#include "main.h" +""") + +test.write("mod.i", """\ +%module mod + +%include "main.h" + +#include "main.h" +""") + +test.write('SConstruct', """\ +import distutils.sysconfig + +DefaultEnvironment( tools = [ 'swig' ] ) + +env = Environment( + SWIGFLAGS = [ + '-python' + ], + CPPPATH = [ + distutils.sysconfig.get_python_inc() + ], + SHLIBPREFIX = "" +) + +env.SharedLibrary( + 'mod.so', + [ + "mod.i", + "main.c", + ] +) +""") + +expectMain = """\ ++-main.os + +-main.c + +-main.h + +-recursive.h""" + +expectMod = """\ ++-mod_wrap.os + +-mod_wrap.c + | +-mod.i + | +-main.h + | +-recursive.h""" + +# Validate that the recursive dependencies are found with SWIG scanning first. +test.run( arguments = '--tree=all mod_wrap.os main.os' ) + +test.must_contain_all( test.stdout(), expectMain ) +test.must_contain_all( test.stdout(), expectMod ) + +# Validate that the recursive dependencies are found consistently. +test.run( arguments = '--tree=all main.os mod_wrap.os' ) + +test.must_contain_all( test.stdout(), expectMain ) +test.must_contain_all( test.stdout(), expectMod ) + +test.run() +test.up_to_date() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Scanner/CrossLanguageNoExtension.py b/test/Scanner/CrossLanguageNoExtension.py new file mode 100644 index 00000000..5bf205f9 --- /dev/null +++ b/test/Scanner/CrossLanguageNoExtension.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import TestSCons + +_python_ = TestSCons._python_ + +test = TestSCons.TestSCons() + +# Test behavior of Scanners when evaluating implicit dependencies +# for nodes that do not have mappings from their scanner_key +# to a scanner instance + +test.write('SConstruct', """ +import re + +include_re = re.compile(r'^include\s+(\S+)$', re.M) + +def scan(node, env, scanpaths, arg): + contents = node.get_text_contents() + includes = include_re.findall(contents) + return includes + +def kfile_scan(node, env, scanpaths, arg): + print 'kscan: ' + str(node) + return scan(node, env, scanpaths, arg) + +def k2file_scan(node, env, scanpaths, arg): + print 'k2scan: ' + str(node) + return scan(node, env, scanpaths, arg) + +kscan = Scanner(name = 'kfile', + function = kfile_scan, + argument = None, + skeys = ['.k'], + recursive = True) + +k2scan = Scanner(name = 'k2', + function = k2file_scan, + argument = None, + skeys = ['.k2']) + +k2scan2 = Scanner(name = 'k2', + function = k2file_scan, + argument = None, + skeys = ['']) + +env1 = Environment() +env1.Append(SCANNERS = [ kscan, k2scan ] ) +env1.Command( 'k', 'foo.k', Copy( '$TARGET', '$SOURCE' ) ) + +env2 = env1.Clone() +env2.Append(SCANNERS = [ k2scan2 ] ) +env2.Command( 'k2', 'foo.k', Copy( '$TARGET', '$SOURCE' ) ) +""") + +test.write('foo.k', +"""foo.k 1 line 1 +include xxx.k +include yyy +foo.k 1 line 4 +""") + +test.write('xxx.k', "xxx.k 1\n") +test.write('yyy', "yyy 1\n") +test.write('yyy.k2', "yyy.k2 1\n") + +expected_stdout = test.wrap_stdout("""\ +kscan: foo.k +kscan: xxx.k +kscan: yyy +Copy("k", "foo.k") +kscan: foo.k +kscan: xxx.k +k2scan: yyy +Copy("k2", "foo.k") +""") + +test.run(arguments='k k2', stdout=expected_stdout) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Scanner/generated.py b/test/Scanner/generated.py index 845111ce..b41c7c8a 100644 --- a/test/Scanner/generated.py +++ b/test/Scanner/generated.py @@ -338,6 +338,7 @@ class CScannerCounter(object): import SCons.Tool MyCScanner = CScannerCounter(SCons.Script.CScanner) SCons.Tool.SourceFileScanner.add_scanner('.c', MyCScanner) +SCons.Tool.SourceFileScanner.add_scanner('.h', MyCScanner) env = Environment(CPPPATH = ".") l = env.StaticLibrary("g", Split("libg_1.c libg_2.c libg_3.c")) diff --git a/test/explain/basic.py b/test/explain/basic.py index 5e31cfd1..1072ac4c 100644 --- a/test/explain/basic.py +++ b/test/explain/basic.py @@ -169,10 +169,18 @@ test.write(['src', 'file6.in'], "file6.in 1\n") test.write(['src', 'subdir', 'file7.in'], "subdir/file7.in 1\n") -args = '--debug=explain .' +args = '--debug=explain ..' expect = test.wrap_stdout("""\ +scons: building `%(inc_aaa)s' because it doesn't exist +Install file: "aaa" as "%(inc_aaa)s" +scons: building `%(inc_bbb_k)s' because it doesn't exist +Install file: "bbb.k" as "%(inc_bbb_k)s" +scons: building `%(inc_ddd)s' because it doesn't exist +Install file: "ddd" as "%(inc_ddd)s" +scons: building `%(inc_eee)s' because it doesn't exist +Install file: "eee.in" as "%(inc_eee)s" scons: building `file1' because it doesn't exist %(_python_)s %(cat_py)s file1 file1.in scons: building `file2' because it doesn't exist @@ -181,14 +189,6 @@ scons: building `file3' because it doesn't exist %(_python_)s %(cat_py)s file3 xxx yyy zzz scons: building `file4' because it doesn't exist %(_python_)s %(cat_py)s file4 - file4.in -scons: building `%(inc_aaa)s' because it doesn't exist -Install file: "aaa" as "%(inc_aaa)s" -scons: building `%(inc_ddd)s' because it doesn't exist -Install file: "ddd" as "%(inc_ddd)s" -scons: building `%(inc_eee)s' because it doesn't exist -Install file: "eee.in" as "%(inc_eee)s" -scons: building `%(inc_bbb_k)s' because it doesn't exist -Install file: "bbb.k" as "%(inc_bbb_k)s" scons: building `file5' because it doesn't exist %(_python_)s %(cat_py)s file5 file5.k scons: building `file6' because it doesn't exist @@ -236,6 +236,8 @@ test_value = '"second"' WriteInitialTest( locals() ) expect = test.wrap_stdout("""\ +scons: rebuilding `%(inc_bbb_k)s' because `bbb.k' changed +Install file: "bbb.k" as "%(inc_bbb_k)s" scons: rebuilding `file1' because `file1.in' changed %(_python_)s %(cat_py)s file1 file1.in scons: rebuilding `file2' because `yyy' changed @@ -244,11 +246,6 @@ scons: rebuilding `file3' because: `yyy' changed `zzz' changed %(_python_)s %(cat_py)s file3 xxx yyy zzz -scons: rebuilding `%(inc_bbb_k)s' because: - `%(inc_ddd)s' is no longer a dependency - `%(inc_eee)s' is no longer a dependency - `bbb.k' changed -Install file: "bbb.k" as "%(inc_bbb_k)s" scons: rebuilding `file5' because `%(inc_bbb_k)s' changed %(_python_)s %(cat_py)s file5 file5.k scons: rebuilding `file6' because AlwaysBuild() is specified diff --git a/test/explain/save-info.py b/test/explain/save-info.py index d2ffc7de..af4c3f58 100644 --- a/test/explain/save-info.py +++ b/test/explain/save-info.py @@ -141,7 +141,7 @@ file5.k 1 line 4 test.write(['src', 'subdir', 'file6.in'], "subdir/file6.in 1\n") # -test.run(chdir='src', arguments='.') +test.run(chdir='src', arguments='..') test.must_match(['src', 'file1'], "file1.in 1\n") test.must_match(['src', 'file2'], """\ @@ -176,10 +176,7 @@ scons: rebuilding `file3' because: `yyy' changed `zzz' changed %(_python_)s %(cat_py)s file3 xxx yyy zzz -scons: rebuilding `%(inc_bbb_k)s' because: - `%(inc_ddd)s' is no longer a dependency - `%(inc_eee)s' is no longer a dependency - `bbb.k' changed +scons: rebuilding `%(inc_bbb_k)s' because `bbb.k' changed Install file: "bbb.k" as "%(inc_bbb_k)s" scons: rebuilding `file5' because `%(inc_bbb_k)s' changed %(_python_)s %(cat_py)s file5 file5.k |