Update to waf 2.0.11

- Migrate pkg-config checks from atleast_version.
- Check ppoll with a fragment, since the function_name argument has been
  removed.

49 files changed, 5447 insertions, 4438 deletions

diff --git a/waflib/Build.py b/waflib/Build.py
index 032e15f3..1afcba64 100644
--- a/waflib/Build.py
+++ b/waflib/Build.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Classes related to the build phase (build, clean, install, step, etc)
@@ -14,14 +14,13 @@ try:
 	import cPickle
 except ImportError:
 	import pickle as cPickle
-from waflib import Runner, TaskGen, Utils, ConfigSet, Task, Logs, Options, Context, Errors
-import waflib.Node
+from waflib import Node, Runner, TaskGen, Utils, ConfigSet, Task, Logs, Options, Context, Errors
 
 CACHE_DIR = 'c4che'
-"""Location of the cache files"""
+"""Name of the cache directory"""
 
 CACHE_SUFFIX = '_cache.py'
-"""Suffix for the cache files"""
+"""ConfigSet cache files for variants are written under :py:attr:´waflib.Build.CACHE_DIR´ in the form ´variant_name´_cache.py"""
 
 INSTALL = 1337
 """Positive value '->' install, see :py:attr:`waflib.Build.BuildContext.is_install`"""
@@ -29,20 +28,19 @@ INSTALL = 1337
 UNINSTALL = -1337
 """Negative value '<-' uninstall, see :py:attr:`waflib.Build.BuildContext.is_install`"""
 
-SAVED_ATTRS = 'root node_deps raw_deps task_sigs'.split()
-"""Build class members to save between the runs (root, node_deps, raw_deps, task_sigs)"""
+SAVED_ATTRS = 'root node_sigs task_sigs imp_sigs raw_deps node_deps'.split()
+"""Build class members to save between the runs; these should be all dicts
+except for `root` which represents a :py:class:`waflib.Node.Node` instance
+"""
 
 CFG_FILES = 'cfg_files'
 """Files from the build directory to hash before starting the build (``config.h`` written during the configuration)"""
 
 POST_AT_ONCE = 0
-"""Post mode: all task generators are posted before the build really starts"""
+"""Post mode: all task generators are posted before any task executed"""
 
 POST_LAZY = 1
-"""Post mode: post the task generators group after group"""
-
-POST_BOTH = 2
-"""Post mode: post the task generators at once, then re-check them for each group"""
+"""Post mode: post the task generators group after group, the tasks in the next group are created when the tasks in the previous groups are done"""
 
 PROTOCOL = -1
 if sys.platform == 'cli':
@@ -61,48 +59,65 @@ class BuildContext(Context.Context):
 		"""Non-zero value when installing or uninstalling file"""
 
 		self.top_dir = kw.get('top_dir', Context.top_dir)
+		"""See :py:attr:`waflib.Context.top_dir`; prefer :py:attr:`waflib.Build.BuildContext.srcnode`"""
+
+		self.out_dir = kw.get('out_dir', Context.out_dir)
+		"""See :py:attr:`waflib.Context.out_dir`; prefer :py:attr:`waflib.Build.BuildContext.bldnode`"""
 
 		self.run_dir = kw.get('run_dir', Context.run_dir)
+		"""See :py:attr:`waflib.Context.run_dir`"""
 
-		self.post_mode = POST_AT_ONCE
-		"""post the task generators at once, group-by-group, or both"""
+		self.launch_dir = Context.launch_dir
+		"""See :py:attr:`waflib.Context.out_dir`; prefer :py:meth:`waflib.Build.BuildContext.launch_node`"""
 
-		# output directory - may be set until the nodes are considered
-		self.out_dir = kw.get('out_dir', Context.out_dir)
+		self.post_mode = POST_LAZY
+		"""Whether to post the task generators at once or group-by-group (default is group-by-group)"""
 
-		self.cache_dir = kw.get('cache_dir', None)
+		self.cache_dir = kw.get('cache_dir')
 		if not self.cache_dir:
 			self.cache_dir = os.path.join(self.out_dir, CACHE_DIR)
 
-		# map names to environments, the '' must be defined
 		self.all_envs = {}
+		"""Map names to :py:class:`waflib.ConfigSet.ConfigSet`, the empty string must map to the default environment"""
 
 		# ======================================= #
 		# cache variables
 
+		self.node_sigs = {}
+		"""Dict mapping build nodes to task identifier (uid), it indicates whether a task created a particular file (persists across builds)"""
+
 		self.task_sigs = {}
-		"""Signatures of the tasks (persists between build executions)"""
+		"""Dict mapping task identifiers (uid) to task signatures (persists across builds)"""
+
+		self.imp_sigs = {}
+		"""Dict mapping task identifiers (uid) to implicit task dependencies used for scanning targets (persists across builds)"""
 
 		self.node_deps = {}
-		"""Dict of node dependencies found by :py:meth:`waflib.Task.Task.scan` (persists between build executions)"""
+		"""Dict mapping task identifiers (uid) to node dependencies found by :py:meth:`waflib.Task.Task.scan` (persists across builds)"""
 
 		self.raw_deps = {}
-		"""Dict of custom data returned by :py:meth:`waflib.Task.Task.scan` (persists between build executions)"""
-
-		# list of folders that are already scanned
-		# so that we do not need to stat them one more time
-		self.cache_dir_contents = {}
+		"""Dict mapping task identifiers (uid) to custom data returned by :py:meth:`waflib.Task.Task.scan` (persists across builds)"""
 
 		self.task_gen_cache_names = {}
 
-		self.launch_dir = Context.launch_dir
-
 		self.jobs = Options.options.jobs
+		"""Amount of jobs to run in parallel"""
+
 		self.targets = Options.options.targets
+		"""List of targets to build (default: \*)"""
+
 		self.keep = Options.options.keep
+		"""Whether the build should continue past errors"""
+
 		self.progress_bar = Options.options.progress_bar
+		"""
+		Level of progress status:
 
-		############ stuff below has not been reviewed
+		0. normal output
+		1. progress bar
+		2. IDE output
+		3. No output at all
+		"""
 
 		# Manual dependencies.
 		self.deps_man = Utils.defaultdict(list)
@@ -118,16 +133,21 @@ class BuildContext(Context.Context):
 		"""
 		List containing lists of task generators
 		"""
+
 		self.group_names = {}
 		"""
 		Map group names to the group lists. See :py:meth:`waflib.Build.BuildContext.add_group`
 		"""
 
+		for v in SAVED_ATTRS:
+			if not hasattr(self, v):
+				setattr(self, v, {})
+
 	def get_variant_dir(self):
 		"""Getter for the variant_dir attribute"""
 		if not self.variant:
 			return self.out_dir
-		return os.path.join(self.out_dir, self.variant)
+		return os.path.join(self.out_dir, os.path.normpath(self.variant))
 	variant_dir = property(get_variant_dir, None)
 
 	def __call__(self, *k, **kw):
@@ -152,56 +172,22 @@ class BuildContext(Context.Context):
 		kw['bld'] = self
 		ret = TaskGen.task_gen(*k, **kw)
 		self.task_gen_cache_names = {} # reset the cache, each time
-		self.add_to_group(ret, group=kw.get('group', None))
+		self.add_to_group(ret, group=kw.get('group'))
 		return ret
 
-	def rule(self, *k, **kw):
+	def __copy__(self):
 		"""
-		Wrapper for creating a task generator using the decorator notation. The following code::
-
-			@bld.rule(
-				target = "foo"
-			)
-			def _(tsk):
-				print("bar")
-
-		is equivalent to::
+		Build contexts cannot be copied
 
-			def bar(tsk):
-				print("bar")
-
-			bld(
-				target = "foo",
-				rule = bar,
-			)
+		:raises: :py:class:`waflib.Errors.WafError`
 		"""
-		def f(rule):
-			ret = self(*k, **kw)
-			ret.rule = rule
-			return ret
-		return f
-
-	def __copy__(self):
-		"""Implemented to prevents copies of build contexts (raises an exception)"""
-		raise Errors.WafError('build contexts are not supposed to be copied')
-
-	def install_files(self, *k, **kw):
-		"""Actual implementation provided by :py:meth:`waflib.Build.InstallContext.install_files`"""
-		pass
-
-	def install_as(self, *k, **kw):
-		"""Actual implementation provided by :py:meth:`waflib.Build.InstallContext.install_as`"""
-		pass
-
-	def symlink_as(self, *k, **kw):
-		"""Actual implementation provided by :py:meth:`waflib.Build.InstallContext.symlink_as`"""
-		pass
+		raise Errors.WafError('build contexts cannot be copied')
 
 	def load_envs(self):
 		"""
 		The configuration command creates files of the form ``build/c4che/NAMEcache.py``. This method
 		creates a :py:class:`waflib.ConfigSet.ConfigSet` instance for each ``NAME`` by reading those
-		files. The config sets are then stored in the dict :py:attr:`waflib.Build.BuildContext.allenvs`.
+		files and stores them in :py:attr:`waflib.Build.BuildContext.allenvs`.
 		"""
 		node = self.root.find_node(self.cache_dir)
 		if not node:
@@ -217,21 +203,16 @@ class BuildContext(Context.Context):
 			self.all_envs[name] = env
 			for f in env[CFG_FILES]:
 				newnode = self.root.find_resource(f)
-				try:
-					h = Utils.h_file(newnode.abspath())
-				except (IOError, AttributeError):
-					Logs.error('cannot find %r' % f)
-					h = Utils.SIG_NIL
-				newnode.sig = h
+				if not newnode or not newnode.exists():
+					raise Errors.WafError('Missing configuration file %r, reconfigure the project!' % f)
 
 	def init_dirs(self):
 		"""
 		Initialize the project directory and the build directory by creating the nodes
 		:py:attr:`waflib.Build.BuildContext.srcnode` and :py:attr:`waflib.Build.BuildContext.bldnode`
-		corresponding to ``top_dir`` and ``variant_dir`` respectively. The ``bldnode`` directory will be
-		created if it does not exist.
+		corresponding to ``top_dir`` and ``variant_dir`` respectively. The ``bldnode`` directory is
+		created if necessary.
 		"""
-
 		if not (os.path.isabs(self.top_dir) and os.path.isabs(self.out_dir)):
 			raise Errors.WafError('The project was not configured: run "waf configure" first!')
 
@@ -241,12 +222,12 @@ class BuildContext(Context.Context):
 
 	def execute(self):
 		"""
-		Restore the data from previous builds and call :py:meth:`waflib.Build.BuildContext.execute_build`. Overrides from :py:func:`waflib.Context.Context.execute`
+		Restore data from previous builds and call :py:meth:`waflib.Build.BuildContext.execute_build`.
+		Overrides from :py:func:`waflib.Context.Context.execute`
 		"""
 		self.restore()
 		if not self.all_envs:
 			self.load_envs()
-
 		self.execute_build()
 
 	def execute_build(self):
@@ -259,7 +240,7 @@ class BuildContext(Context.Context):
 		* calling :py:meth:`waflib.Build.BuildContext.post_build` to call user build functions
 		"""
 
-		Logs.info("Waf: Entering directory `%s'" % self.variant_dir)
+		Logs.info("Waf: Entering directory `%s'", self.variant_dir)
 		self.recurse([self.run_dir])
 		self.pre_build()
 
@@ -270,65 +251,70 @@ class BuildContext(Context.Context):
 			self.compile()
 		finally:
 			if self.progress_bar == 1 and sys.stderr.isatty():
-				c = len(self.returned_tasks) or 1
+				c = self.producer.processed or 1
 				m = self.progress_line(c, c, Logs.colors.BLUE, Logs.colors.NORMAL)
 				Logs.info(m, extra={'stream': sys.stderr, 'c1': Logs.colors.cursor_off, 'c2' : Logs.colors.cursor_on})
-			Logs.info("Waf: Leaving directory `%s'" % self.variant_dir)
+			Logs.info("Waf: Leaving directory `%s'", self.variant_dir)
+		try:
+			self.producer.bld = None
+			del self.producer
+		except AttributeError:
+			pass
 		self.post_build()
 
 	def restore(self):
 		"""
-		Load the data from a previous run, sets the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`
+		Load data from a previous run, sets the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`
 		"""
 		try:
 			env = ConfigSet.ConfigSet(os.path.join(self.cache_dir, 'build.config.py'))
 		except EnvironmentError:
 			pass
 		else:
-			if env['version'] < Context.HEXVERSION:
-				raise Errors.WafError('Version mismatch! reconfigure the project')
-			for t in env['tools']:
+			if env.version < Context.HEXVERSION:
+				raise Errors.WafError('Project was configured with a different version of Waf, please reconfigure it')
+
+			for t in env.tools:
 				self.setup(**t)
 
 		dbfn = os.path.join(self.variant_dir, Context.DBFILE)
 		try:
 			data = Utils.readf(dbfn, 'rb')
-		except (IOError, EOFError):
+		except (EnvironmentError, EOFError):
 			# handle missing file/empty file
-			Logs.debug('build: Could not load the build cache %s (missing)' % dbfn)
+			Logs.debug('build: Could not load the build cache %s (missing)', dbfn)
 		else:
 			try:
-				waflib.Node.pickle_lock.acquire()
-				waflib.Node.Nod3 = self.node_class
+				Node.pickle_lock.acquire()
+				Node.Nod3 = self.node_class
 				try:
 					data = cPickle.loads(data)
 				except Exception as e:
-					Logs.debug('build: Could not pickle the build cache %s: %r' % (dbfn, e))
+					Logs.debug('build: Could not pickle the build cache %s: %r', dbfn, e)
 				else:
 					for x in SAVED_ATTRS:
-						setattr(self, x, data[x])
+						setattr(self, x, data.get(x, {}))
 			finally:
-				waflib.Node.pickle_lock.release()
+				Node.pickle_lock.release()
 
 		self.init_dirs()
 
 	def store(self):
 		"""
-		Store the data for next runs, sets the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`. Uses a temporary
+		Store data for next runs, set the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`. Uses a temporary
 		file to avoid problems on ctrl+c.
 		"""
-
 		data = {}
 		for x in SAVED_ATTRS:
 			data[x] = getattr(self, x)
 		db = os.path.join(self.variant_dir, Context.DBFILE)
 
 		try:
-			waflib.Node.pickle_lock.acquire()
-			waflib.Node.Nod3 = self.node_class
+			Node.pickle_lock.acquire()
+			Node.Nod3 = self.node_class
 			x = cPickle.dumps(data, PROTOCOL)
 		finally:
-			waflib.Node.pickle_lock.release()
+			Node.pickle_lock.release()
 
 		Utils.writef(db + '.tmp', x, m='wb')
 
@@ -346,29 +332,34 @@ class BuildContext(Context.Context):
 	def compile(self):
 		"""
 		Run the build by creating an instance of :py:class:`waflib.Runner.Parallel`
-		The cache file is not written if the build is up to date (no task executed).
+		The cache file is written when at least a task was executed.
+
+		:raises: :py:class:`waflib.Errors.BuildError` in case the build fails
 		"""
 		Logs.debug('build: compile()')
 
-		# use another object to perform the producer-consumer logic (reduce the complexity)
+		# delegate the producer-consumer logic to another object to reduce the complexity
 		self.producer = Runner.Parallel(self, self.jobs)
 		self.producer.biter = self.get_build_iterator()
-		self.returned_tasks = [] # not part of the API yet
 		try:
 			self.producer.start()
 		except KeyboardInterrupt:
-			self.store()
+			if self.is_dirty():
+				self.store()
 			raise
 		else:
-			if self.producer.dirty:
+			if self.is_dirty():
 				self.store()
 
 		if self.producer.error:
 			raise Errors.BuildError(self.producer.error)
 
+	def is_dirty(self):
+		return self.producer.dirty
+
 	def setup(self, tool, tooldir=None, funs=None):
 		"""
-		Import waf tools, used to import those accessed during the configuration::
+		Import waf tools defined during the configuration::
 
 			def configure(conf):
 				conf.load('glib2')
@@ -383,11 +374,13 @@ class BuildContext(Context.Context):
 		:param funs: unused variable
 		"""
 		if isinstance(tool, list):
-			for i in tool: self.setup(i, tooldir)
+			for i in tool:
+				self.setup(i, tooldir)
 			return
 
 		module = Context.load_tool(tool, tooldir)
-		if hasattr(module, "setup"): module.setup(self)
+		if hasattr(module, "setup"):
+			module.setup(self)
 
 	def get_env(self):
 		"""Getter for the env property"""
@@ -412,26 +405,28 @@ class BuildContext(Context.Context):
 
 		:param path: file path
 		:type path: string or :py:class:`waflib.Node.Node`
-		:param value: value to depend on
-		:type value: :py:class:`waflib.Node.Node`, string, or function returning a string
+		:param value: value to depend
+		:type value: :py:class:`waflib.Node.Node`, byte object, or function returning a byte object
 		"""
-		if path is None:
-			raise ValueError('Invalid input')
+		if not path:
+			raise ValueError('Invalid input path %r' % path)
 
-		if isinstance(path, waflib.Node.Node):
+		if isinstance(path, Node.Node):
 			node = path
 		elif os.path.isabs(path):
 			node = self.root.find_resource(path)
 		else:
 			node = self.path.find_resource(path)
+		if not node:
+			raise ValueError('Could not find the path %r' % path)
 
 		if isinstance(value, list):
-			self.deps_man[id(node)].extend(value)
+			self.deps_man[node].extend(value)
 		else:
-			self.deps_man[id(node)].append(value)
+			self.deps_man[node].append(value)
 
 	def launch_node(self):
-		"""Returns the launch directory as a :py:class:`waflib.Node.Node` object"""
+		"""Returns the launch directory as a :py:class:`waflib.Node.Node` object (cached)"""
 		try:
 			# private cache
 			return self.p_ln
@@ -441,11 +436,13 @@ class BuildContext(Context.Context):
 
 	def hash_env_vars(self, env, vars_lst):
 		"""
-		Hash configuration set variables::
+		Hashes configuration set variables::
 
 			def build(bld):
 				bld.hash_env_vars(bld.env, ['CXX', 'CC'])
 
+		This method uses an internal cache.
+
 		:param env: Configuration Set
 		:type env: :py:class:`waflib.ConfigSet.ConfigSet`
 		:param vars_lst: list of variables
@@ -469,21 +466,23 @@ class BuildContext(Context.Context):
 				pass
 
 		lst = [env[a] for a in vars_lst]
-		ret = Utils.h_list(lst)
+		cache[idx] = ret = Utils.h_list(lst)
 		Logs.debug('envhash: %s %r', Utils.to_hex(ret), lst)
-
-		cache[idx] = ret
-
 		return ret
 
 	def get_tgen_by_name(self, name):
 		"""
-		Retrieves a task generator from its name or its target name
-		the name must be unique::
+		Fetches a task generator by its name or its target attribute;
+		the name must be unique in a build::
 
 			def build(bld):
 				tg = bld(name='foo')
 				tg == bld.get_tgen_by_name('foo')
+
+		This method use a private internal cache.
+
+		:param name: Task generator name
+		:raises: :py:class:`waflib.Errors.WafError` in case there is no task genenerator by that name
 		"""
 		cache = self.task_gen_cache_names
 		if not cache:
@@ -500,9 +499,12 @@ class BuildContext(Context.Context):
 		except KeyError:
 			raise Errors.WafError('Could not find a task generator for the name %r' % name)
 
-	def progress_line(self, state, total, col1, col2):
+	def progress_line(self, idx, total, col1, col2):
 		"""
-		Compute the progress bar used by ``waf -p``
+		Computes a progress bar line displayed when running ``waf -p``
+
+		:returns: progress bar line
+		:rtype: string
 		"""
 		if not sys.stderr.isatty():
 			return ''
@@ -512,16 +514,16 @@ class BuildContext(Context.Context):
 		Utils.rot_idx += 1
 		ind = Utils.rot_chr[Utils.rot_idx % 4]
 
-		pc = (100.*state)/total
-		eta = str(self.timer)
-		fs = "[%%%dd/%%%dd][%%s%%2d%%%%%%s][%s][" % (n, n, ind)
-		left = fs % (state, total, col1, pc, col2)
-		right = '][%s%s%s]' % (col1, eta, col2)
+		pc = (100. * idx)/total
+		fs = "[%%%dd/%%d][%%s%%2d%%%%%%s][%s][" % (n, ind)
+		left = fs % (idx, total, col1, pc, col2)
+		right = '][%s%s%s]' % (col1, self.timer, col2)
 
 		cols = Logs.get_term_cols() - len(left) - len(right) + 2*len(col1) + 2*len(col2)
-		if cols < 7: cols = 7
+		if cols < 7:
+			cols = 7
 
-		ratio = ((cols*state)//total) - 1
+		ratio = ((cols * idx)//total) - 1
 
 		bar = ('='*ratio+'>').ljust(cols)
 		msg = Logs.indicator % (left, bar, right)
@@ -530,23 +532,23 @@ class BuildContext(Context.Context):
 
 	def declare_chain(self, *k, **kw):
 		"""
-		Wrapper for :py:func:`waflib.TaskGen.declare_chain` provided for convenience
+		Wraps :py:func:`waflib.TaskGen.declare_chain` for convenience
 		"""
 		return TaskGen.declare_chain(*k, **kw)
 
 	def pre_build(self):
-		"""Execute user-defined methods before the build starts, see :py:meth:`waflib.Build.BuildContext.add_pre_fun`"""
+		"""Executes user-defined methods before the build starts, see :py:meth:`waflib.Build.BuildContext.add_pre_fun`"""
 		for m in getattr(self, 'pre_funs', []):
 			m(self)
 
 	def post_build(self):
-		"""Executes the user-defined methods after the build is successful, see :py:meth:`waflib.Build.BuildContext.add_post_fun`"""
+		"""Executes user-defined methods after the build is successful, see :py:meth:`waflib.Build.BuildContext.add_post_fun`"""
 		for m in getattr(self, 'post_funs', []):
 			m(self)
 
 	def add_pre_fun(self, meth):
 		"""
-		Bind a method to execute after the scripts are read and before the build starts::
+		Binds a callback method to execute after the scripts are read and before the build starts::
 
 			def mycallback(bld):
 				print("Hello, world!")
@@ -561,7 +563,7 @@ class BuildContext(Context.Context):
 
 	def add_post_fun(self, meth):
 		"""
-		Bind a method to execute immediately after the build is successful::
+		Binds a callback method to execute immediately after the build is successful::
 
 			def call_ldconfig(bld):
 				bld.exec_command('/sbin/ldconfig')
@@ -577,7 +579,7 @@ class BuildContext(Context.Context):
 
 	def get_group(self, x):
 		"""
-		Get the group x, or return the current group if x is None
+		Returns the build group named `x`, or the current group if `x` is None
 
 		:param x: name or number or None
 		:type x: string, int or None
@@ -591,14 +593,20 @@ class BuildContext(Context.Context):
 		return self.groups[x]
 
 	def add_to_group(self, tgen, group=None):
-		"""add a task or a task generator for the build"""
-		# paranoid
-		assert(isinstance(tgen, TaskGen.task_gen) or isinstance(tgen, Task.TaskBase))
+		"""Adds a task or a task generator to the build; there is no attempt to remove it if it was already added."""
+		assert(isinstance(tgen, TaskGen.task_gen) or isinstance(tgen, Task.Task))
 		tgen.bld = self
 		self.get_group(group).append(tgen)
 
 	def get_group_name(self, g):
-		"""name for the group g (utility)"""
+		"""
+		Returns the name of the input build group
+
+		:param g: build group object or build group index
+		:type g: integer or list
+		:return: name
+		:rtype: string
+		"""
 		if not isinstance(g, list):
 			g = self.groups[g]
 		for x in self.group_names:
@@ -608,7 +616,7 @@ class BuildContext(Context.Context):
 
 	def get_group_idx(self, tg):
 		"""
-		Index of the group containing the task generator given as argument::
+		Returns the index of the group containing the task generator given as argument::
 
 			def build(bld):
 				tg = bld(name='nada')
@@ -616,27 +624,28 @@ class BuildContext(Context.Context):
 
 		:param tg: Task generator object
 		:type tg: :py:class:`waflib.TaskGen.task_gen`
+		:rtype: int
 		"""
 		se = id(tg)
-		for i in range(len(self.groups)):
-			for t in self.groups[i]:
+		for i, tmp in enumerate(self.groups):
+			for t in tmp:
 				if id(t) == se:
 					return i
 		return None
 
 	def add_group(self, name=None, move=True):
 		"""
-		Add a new group of tasks/task generators. By default the new group becomes the default group for new task generators.
+		Adds a new group of tasks/task generators. By default the new group becomes
+		the default group for new task generators (make sure to create build groups in order).
 
 		:param name: name for this group
 		:type name: string
-		:param move: set the group created as default group (True by default)
+		:param move: set this new group as default group (True by default)
 		:type move: bool
+		:raises: :py:class:`waflib.Errors.WafError` if a group by the name given already exists
 		"""
-		#if self.groups and not self.groups[0].tasks:
-		#	error('add_group: an empty group is already present')
 		if name and name in self.group_names:
-			Logs.error('add_group: name %s already present' % name)
+			raise Errors.WafError('add_group: name %s already present', name)
 		g = []
 		self.group_names[name] = g
 		self.groups.append(g)
@@ -645,7 +654,8 @@ class BuildContext(Context.Context):
 
 	def set_group(self, idx):
 		"""
-		Set the current group to be idx: now new task generators will be added to this group by default::
+		Sets the build group at position idx as current so that newly added
+		task generators are added to this one by default::
 
 			def build(bld):
 				bld(rule='touch ${TGT}', target='foo.txt')
@@ -659,8 +669,8 @@ class BuildContext(Context.Context):
 		"""
 		if isinstance(idx, str):
 			g = self.group_names[idx]
-			for i in range(len(self.groups)):
-				if id(g) == id(self.groups[i]):
+			for i, tmp in enumerate(self.groups):
+				if id(g) == id(tmp):
 					self.current_group = i
 					break
 		else:
@@ -668,8 +678,11 @@ class BuildContext(Context.Context):
 
 	def total(self):
 		"""
-		Approximate task count: this value may be inaccurate if task generators are posted lazily (see :py:attr:`waflib.Build.BuildContext.post_mode`).
+		Approximate task count: this value may be inaccurate if task generators
+		are posted lazily (see :py:attr:`waflib.Build.BuildContext.post_mode`).
 		The value :py:attr:`waflib.Runner.Parallel.total` is updated during the task execution.
+
+		:rtype: int
 		"""
 		total = 0
 		for group in self.groups:
@@ -682,9 +695,16 @@ class BuildContext(Context.Context):
 
 	def get_targets(self):
 		"""
-		Return the task generator corresponding to the 'targets' list, used by :py:meth:`waflib.Build.BuildContext.get_build_iterator`::
+		This method returns a pair containing the index of the last build group to post,
+		and the list of task generator objects corresponding to the target names.
+
+		This is used internally by :py:meth:`waflib.Build.BuildContext.get_build_iterator`
+		to perform partial builds::
 
 			$ waf --targets=myprogram,myshlib
+
+		:return: the minimum build group index, and list of task generators
+		:rtype: tuple
 		"""
 		to_post = []
 		min_grp = 0
@@ -700,7 +720,7 @@ class BuildContext(Context.Context):
 
 	def get_all_task_gen(self):
 		"""
-		Utility method, returns a list of all task generators - if you need something more complicated, implement your own
+		Returns a list of all task generators for troubleshooting purposes.
 		"""
 		lst = []
 		for g in self.groups:
@@ -709,25 +729,24 @@ class BuildContext(Context.Context):
 
 	def post_group(self):
 		"""
-		Post the task generators from the group indexed by self.cur, used by :py:meth:`waflib.Build.BuildContext.get_build_iterator`
+		Post task generators from the group indexed by self.current_group; used internally
+		by :py:meth:`waflib.Build.BuildContext.get_build_iterator`
 		"""
+		def tgpost(tg):
+			try:
+				f = tg.post
+			except AttributeError:
+				pass
+			else:
+				f()
+
 		if self.targets == '*':
-			for tg in self.groups[self.cur]:
-				try:
-					f = tg.post
-				except AttributeError:
-					pass
-				else:
-					f()
+			for tg in self.groups[self.current_group]:
+				tgpost(tg)
 		elif self.targets:
-			if self.cur < self._min_grp:
-				for tg in self.groups[self.cur]:
-					try:
-						f = tg.post
-					except AttributeError:
-						pass
-					else:
-						f()
+			if self.current_group < self._min_grp:
+				for tg in self.groups[self.current_group]:
+					tgpost(tg)
 			else:
 				for tg in self._exact_tg:
 					tg.post()
@@ -737,55 +756,71 @@ class BuildContext(Context.Context):
 				Logs.warn('Building from the build directory, forcing --targets=*')
 				ln = self.srcnode
 			elif not ln.is_child_of(self.srcnode):
-				Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)' % (ln.abspath(), self.srcnode.abspath()))
+				Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath())
 				ln = self.srcnode
-			for tg in self.groups[self.cur]:
+
+			def is_post(tg, ln):
 				try:
-					f = tg.post
+					p = tg.path
 				except AttributeError:
 					pass
 				else:
-					if tg.path.is_child_of(ln):
-						f()
+					if p.is_child_of(ln):
+						return True
+
+			def is_post_group():
+				for i, g in enumerate(self.groups):
+					if i > self.current_group:
+						for tg in g:
+							if is_post(tg, ln):
+								return True
+
+			if self.post_mode == POST_LAZY and ln != self.srcnode:
+				# partial folder builds require all targets from a previous build group
+				if is_post_group():
+					ln = self.srcnode
+
+			for tg in self.groups[self.current_group]:
+				if is_post(tg, ln):
+					tgpost(tg)
 
 	def get_tasks_group(self, idx):
 		"""
-		Return all the tasks for the group of num idx, used by :py:meth:`waflib.Build.BuildContext.get_build_iterator`
+		Returns all task instances for the build group at position idx,
+		used internally by :py:meth:`waflib.Build.BuildContext.get_build_iterator`
+
+		:rtype: list of :py:class:`waflib.Task.Task`
 		"""
 		tasks = []
 		for tg in self.groups[idx]:
 			try:
 				tasks.extend(tg.tasks)
-			except AttributeError: # not a task generator, can be the case for installation tasks
+			except AttributeError: # not a task generator
 				tasks.append(tg)
 		return tasks
 
 	def get_build_iterator(self):
 		"""
-		Creates a generator object that returns lists of tasks executable in parallel (yield)
+		Creates a Python generator object that returns lists of tasks that may be processed in parallel.
 
-		:return: tasks which can be executed immediatly
-		:rtype: list of :py:class:`waflib.Task.TaskBase`
+		:return: tasks which can be executed immediately
+		:rtype: generator returning lists of :py:class:`waflib.Task.Task`
 		"""
-		self.cur = 0
-
 		if self.targets and self.targets != '*':
 			(self._min_grp, self._exact_tg) = self.get_targets()
 
-		global lazy_post
 		if self.post_mode != POST_LAZY:
-			while self.cur < len(self.groups):
+			for self.current_group, _ in enumerate(self.groups):
 				self.post_group()
-				self.cur += 1
-			self.cur = 0
 
-		while self.cur < len(self.groups):
+		for self.current_group, _ in enumerate(self.groups):
 			# first post the task generators for the group
 			if self.post_mode != POST_AT_ONCE:
 				self.post_group()
 
 			# then extract the tasks
-			tasks = self.get_tasks_group(self.cur)
+			tasks = self.get_tasks_group(self.current_group)
+
 			# if the constraints are set properly (ext_in/ext_out, before/after)
 			# the call to set_file_constraints may be removed (can be a 15% penalty on no-op rebuilds)
 			# (but leave set_file_constraints for the installation step)
@@ -796,141 +831,320 @@ class BuildContext(Context.Context):
 			Task.set_precedence_constraints(tasks)
 
 			self.cur_tasks = tasks
-			self.cur += 1
-			if not tasks: # return something else the build will stop
-				continue
-			yield tasks
+			if tasks:
+				yield tasks
+
 		while 1:
+			# the build stops once there are no tasks to process
 			yield []
 
-class inst(Task.Task):
+	def install_files(self, dest, files, **kw):
+		"""
+		Creates a task generator to install files on the system::
+
+			def build(bld):
+				bld.install_files('${DATADIR}', self.path.find_resource('wscript'))
+
+		:param dest: path representing the destination directory
+		:type dest: :py:class:`waflib.Node.Node` or string (absolute path)
+		:param files: input files
+		:type files: list of strings or list of :py:class:`waflib.Node.Node`
+		:param env: configuration set to expand *dest*
+		:type env: :py:class:`waflib.ConfigSet.ConfigSet`
+		:param relative_trick: preserve the folder hierarchy when installing whole folders
+		:type relative_trick: bool
+		:param cwd: parent node for searching srcfile, when srcfile is not an instance of :py:class:`waflib.Node.Node`
+		:type cwd: :py:class:`waflib.Node.Node`
+		:param postpone: execute the task immediately to perform the installation (False by default)
+		:type postpone: bool
+		"""
+		assert(dest)
+		tg = self(features='install_task', install_to=dest, install_from=files, **kw)
+		tg.dest = tg.install_to
+		tg.type = 'install_files'
+		if not kw.get('postpone', True):
+			tg.post()
+		return tg
+
+	def install_as(self, dest, srcfile, **kw):
+		"""
+		Creates a task generator to install a file on the system with a different name::
+
+			def build(bld):
+				bld.install_as('${PREFIX}/bin', 'myapp', chmod=Utils.O755)
+
+		:param dest: destination file
+		:type dest: :py:class:`waflib.Node.Node` or string (absolute path)
+		:param srcfile: input file
+		:type srcfile: string or :py:class:`waflib.Node.Node`
+		:param cwd: parent node for searching srcfile, when srcfile is not an instance of :py:class:`waflib.Node.Node`
+		:type cwd: :py:class:`waflib.Node.Node`
+		:param env: configuration set for performing substitutions in dest
+		:type env: :py:class:`waflib.ConfigSet.ConfigSet`
+		:param postpone: execute the task immediately to perform the installation (False by default)
+		:type postpone: bool
+		"""
+		assert(dest)
+		tg = self(features='install_task', install_to=dest, install_from=srcfile, **kw)
+		tg.dest = tg.install_to
+		tg.type = 'install_as'
+		if not kw.get('postpone', True):
+			tg.post()
+		return tg
+
+	def symlink_as(self, dest, src, **kw):
+		"""
+		Creates a task generator to install a symlink::
+
+			def build(bld):
+				bld.symlink_as('${PREFIX}/lib/libfoo.so', 'libfoo.so.1.2.3')
+
+		:param dest: absolute path of the symlink
+		:type dest: :py:class:`waflib.Node.Node` or string (absolute path)
+		:param src: link contents, which is a relative or absolute path which may exist or not
+		:type src: string
+		:param env: configuration set for performing substitutions in dest
+		:type env: :py:class:`waflib.ConfigSet.ConfigSet`
+		:param add: add the task created to a build group - set ``False`` only if the installation task is created after the build has started
+		:type add: bool
+		:param postpone: execute the task immediately to perform the installation
+		:type postpone: bool
+		:param relative_trick: make the symlink relative (default: ``False``)
+		:type relative_trick: bool
+		"""
+		assert(dest)
+		tg = self(features='install_task', install_to=dest, install_from=src, **kw)
+		tg.dest = tg.install_to
+		tg.type = 'symlink_as'
+		tg.link = src
+		# TODO if add: self.add_to_group(tsk)
+		if not kw.get('postpone', True):
+			tg.post()
+		return tg
+
+@TaskGen.feature('install_task')
+@TaskGen.before_method('process_rule', 'process_source')
+def process_install_task(self):
+	"""Creates the installation task for the current task generator; uses :py:func:`waflib.Build.add_install_task` internally."""
+	self.add_install_task(**self.__dict__)
+
+@TaskGen.taskgen_method
+def add_install_task(self, **kw):
+	"""
+	Creates the installation task for the current task generator, and executes it immediately if necessary
+
+	:returns: An installation task
+	:rtype: :py:class:`waflib.Build.inst`
+	"""
+	if not self.bld.is_install:
+		return
+	if not kw['install_to']:
+		return
+
+	if kw['type'] == 'symlink_as' and Utils.is_win32:
+		if kw.get('win32_install'):
+			kw['type'] = 'install_as'
+		else:
+			# just exit
+			return
+
+	tsk = self.install_task = self.create_task('inst')
+	tsk.chmod = kw.get('chmod', Utils.O644)
+	tsk.link = kw.get('link', '') or kw.get('install_from', '')
+	tsk.relative_trick = kw.get('relative_trick', False)
+	tsk.type = kw['type']
+	tsk.install_to = tsk.dest = kw['install_to']
+	tsk.install_from = kw['install_from']
+	tsk.relative_base = kw.get('cwd') or kw.get('relative_base', self.path)
+	tsk.install_user = kw.get('install_user')
+	tsk.install_group = kw.get('install_group')
+	tsk.init_files()
+	if not kw.get('postpone', True):
+		tsk.run_now()
+	return tsk
+
+@TaskGen.taskgen_method
+def add_install_files(self, **kw):
+	"""
+	Creates an installation task for files
+
+	:returns: An installation task
+	:rtype: :py:class:`waflib.Build.inst`
+	"""
+	kw['type'] = 'install_files'
+	return self.add_install_task(**kw)
+
+@TaskGen.taskgen_method
+def add_install_as(self, **kw):
+	"""
+	Creates an installation task for a single file
+
+	:returns: An installation task
+	:rtype: :py:class:`waflib.Build.inst`
+	"""
+	kw['type'] = 'install_as'
+	return self.add_install_task(**kw)
+
+@TaskGen.taskgen_method
+def add_symlink_as(self, **kw):
 	"""
-	Special task used for installing files and symlinks, it behaves both like a task
-	and like a task generator
+	Creates an installation task for a symbolic link
+
+	:returns: An installation task
+	:rtype: :py:class:`waflib.Build.inst`
 	"""
-	color = 'CYAN'
+	kw['type'] = 'symlink_as'
+	return self.add_install_task(**kw)
+
+class inst(Task.Task):
+	"""Task that installs files or symlinks; it is typically executed by :py:class:`waflib.Build.InstallContext` and :py:class:`waflib.Build.UnInstallContext`"""
+	def __str__(self):
+		"""Returns an empty string to disable the standard task display"""
+		return ''
 
 	def uid(self):
-		lst = [self.dest, self.path] + self.source
-		return Utils.h_list(repr(lst))
+		"""Returns a unique identifier for the task"""
+		lst = self.inputs + self.outputs + [self.link, self.generator.path.abspath()]
+		return Utils.h_list(lst)
 
-	def post(self):
+	def init_files(self):
 		"""
-		Same interface as in :py:meth:`waflib.TaskGen.task_gen.post`
+		Initializes the task input and output nodes
 		"""
-		buf = []
-		for x in self.source:
-			if isinstance(x, waflib.Node.Node):
-				y = x
-			else:
-				y = self.path.find_resource(x)
-				if not y:
-					if os.path.isabs(x):
-						y = self.bld.root.make_node(x)
-					else:
-						y = self.path.make_node(x)
-			buf.append(y)
-		self.inputs = buf
+		if self.type == 'symlink_as':
+			inputs = []
+		else:
+			inputs = self.generator.to_nodes(self.install_from)
+			if self.type == 'install_as':
+				assert len(inputs) == 1
+		self.set_inputs(inputs)
+
+		dest = self.get_install_path()
+		outputs = []
+		if self.type == 'symlink_as':
+			if self.relative_trick:
+				self.link = os.path.relpath(self.link, os.path.dirname(dest))
+			outputs.append(self.generator.bld.root.make_node(dest))
+		elif self.type == 'install_as':
+			outputs.append(self.generator.bld.root.make_node(dest))
+		else:
+			for y in inputs:
+				if self.relative_trick:
+					destfile = os.path.join(dest, y.path_from(self.relative_base))
+				else:
+					destfile = os.path.join(dest, y.name)
+				outputs.append(self.generator.bld.root.make_node(destfile))
+		self.set_outputs(outputs)
 
 	def runnable_status(self):
 		"""
 		Installation tasks are always executed, so this method returns either :py:const:`waflib.Task.ASK_LATER` or :py:const:`waflib.Task.RUN_ME`.
 		"""
 		ret = super(inst, self).runnable_status()
-		if ret == Task.SKIP_ME:
+		if ret == Task.SKIP_ME and self.generator.bld.is_install:
 			return Task.RUN_ME
 		return ret
 
-	def __str__(self):
-		"""Return an empty string to disable the display"""
-		return ''
-
-	def run(self):
-		"""The attribute 'exec_task' holds the method to execute"""
-		return self.generator.exec_task()
+	def post_run(self):
+		"""
+		Disables any post-run operations
+		"""
+		pass
 
 	def get_install_path(self, destdir=True):
 		"""
-		Installation path obtained from ``self.dest`` and prefixed by the destdir.
-		The variables such as '${PREFIX}/bin' are substituted.
+		Returns the destination path where files will be installed, pre-pending `destdir`.
+
+		:rtype: string
 		"""
-		dest = Utils.subst_vars(self.dest, self.env)
-		dest = dest.replace('/', os.sep)
+		if isinstance(self.install_to, Node.Node):
+			dest = self.install_to.abspath()
+		else:
+			dest = Utils.subst_vars(self.install_to, self.env)
 		if destdir and Options.options.destdir:
 			dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep))
 		return dest
 
-	def exec_install_files(self):
+	def copy_fun(self, src, tgt):
 		"""
-		Predefined method for installing files
+		Copies a file from src to tgt, preserving permissions and trying to work
+		around path limitations on Windows platforms. On Unix-like platforms,
+		the owner/group of the target file may be set through install_user/install_group
+
+		:param src: absolute path
+		:type src: string
+		:param tgt: absolute path
+		:type tgt: string
 		"""
-		destpath = self.get_install_path()
-		if not destpath:
-			raise Errors.WafError('unknown installation path %r' % self.generator)
-		for x, y in zip(self.source, self.inputs):
-			if self.relative_trick:
-				destfile = os.path.join(destpath, y.path_from(self.path))
-			else:
-				destfile = os.path.join(destpath, y.name)
-			self.generator.bld.do_install(y.abspath(), destfile, chmod=self.chmod, tsk=self)
+		# override this if you want to strip executables
+		# kw['tsk'].source is the task that created the files in the build
+		if Utils.is_win32 and len(tgt) > 259 and not tgt.startswith('\\\\?\\'):
+			tgt = '\\\\?\\' + tgt
+		shutil.copy2(src, tgt)
+		self.fix_perms(tgt)
 
-	def exec_install_as(self):
+	def rm_empty_dirs(self, tgt):
 		"""
-		Predefined method for installing one file with a given name
+		Removes empty folders recursively when uninstalling.
+
+		:param tgt: absolute path
+		:type tgt: string
 		"""
-		destfile = self.get_install_path()
-		self.generator.bld.do_install(self.inputs[0].abspath(), destfile, chmod=self.chmod, tsk=self)
+		while tgt:
+			tgt = os.path.dirname(tgt)
+			try:
+				os.rmdir(tgt)
+			except OSError:
+				break
 
-	def exec_symlink_as(self):
+	def run(self):
 		"""
-		Predefined method for installing a symlink
+		Performs file or symlink installation
 		"""
-		destfile = self.get_install_path()
-		src = self.link
-		if self.relative_trick:
-			src = os.path.relpath(src, os.path.dirname(destfile))
-		self.generator.bld.do_link(src, destfile, tsk=self)
-
-class InstallContext(BuildContext):
-	'''installs the targets on the system'''
-	cmd = 'install'
-
-	def __init__(self, **kw):
-		super(InstallContext, self).__init__(**kw)
+		is_install = self.generator.bld.is_install
+		if not is_install: # unnecessary?
+			return
 
-		# list of targets to uninstall for removing the empty folders after uninstalling
-		self.uninstall = []
-		self.is_install = INSTALL
+		for x in self.outputs:
+			if is_install == INSTALL:
+				x.parent.mkdir()
+		if self.type == 'symlink_as':
+			fun = is_install == INSTALL and self.do_link or self.do_unlink
+			fun(self.link, self.outputs[0].abspath())
+		else:
+			fun = is_install == INSTALL and self.do_install or self.do_uninstall
+			launch_node = self.generator.bld.launch_node()
+			for x, y in zip(self.inputs, self.outputs):
+				fun(x.abspath(), y.abspath(), x.path_from(launch_node))
 
-	def copy_fun(self, src, tgt, **kw):
-		# override this if you want to strip executables
-		# kw['tsk'].source is the task that created the files in the build
-		if Utils.is_win32 and len(tgt) > 259 and not tgt.startswith('\\\\?\\'):
-			tgt = '\\\\?\\' + tgt
-		shutil.copy2(src, tgt)
-		os.chmod(tgt, kw.get('chmod', Utils.O644))
+	def run_now(self):
+		"""
+		Try executing the installation task right now
 
-	def do_install(self, src, tgt, **kw):
+		:raises: :py:class:`waflib.Errors.TaskNotReady`
 		"""
-		Copy a file from src to tgt with given file permissions. The actual copy is not performed
-		if the source and target file have the same size and the same timestamps. When the copy occurs,
-		the file is first removed and then copied (prevent stale inodes).
+		status = self.runnable_status()
+		if status not in (Task.RUN_ME, Task.SKIP_ME):
+			raise Errors.TaskNotReady('Could not process %r: status %r' % (self, status))
+		self.run()
+		self.hasrun = Task.SUCCESS
 
-		This method is overridden in :py:meth:`waflib.Build.UninstallContext.do_install` to remove the file.
+	def do_install(self, src, tgt, lbl, **kw):
+		"""
+		Copies a file from src to tgt with given file permissions. The actual copy is only performed
+		if the source and target file sizes or timestamps differ. When the copy occurs,
+		the file is always first removed and then copied so as to prevent stale inodes.
 
 		:param src: file name as absolute path
 		:type src: string
 		:param tgt: file destination, as absolute path
 		:type tgt: string
+		:param lbl: file source description
+		:type lbl: string
 		:param chmod: installation mode
 		:type chmod: int
+		:raises: :py:class:`waflib.Errors.WafError` if the file cannot be written
 		"""
-		d, _ = os.path.split(tgt)
-		if not d:
-			raise Errors.WafError('Invalid installation given %r->%r' % (src, tgt))
-		Utils.check_dir(d)
-
-		srclbl = src.replace(self.srcnode.abspath() + os.sep, '')
 		if not Options.options.force:
 			# check if the file is already there to avoid a copy
 			try:
@@ -941,12 +1155,12 @@ class InstallContext(BuildContext):
 			else:
 				# same size and identical timestamps -> make no copy
 				if st1.st_mtime + 2 >= st2.st_mtime and st1.st_size == st2.st_size:
-					if not self.progress_bar:
-						Logs.info('- install %s (from %s)' % (tgt, srclbl))
+					if not self.generator.bld.progress_bar:
+						Logs.info('- install %s (from %s)', tgt, lbl)
 					return False
 
-		if not self.progress_bar:
-			Logs.info('+ install %s (from %s)' % (tgt, srclbl))
+		if not self.generator.bld.progress_bar:
+			Logs.info('+ install %s (from %s)', tgt, lbl)
 
 		# Give best attempt at making destination overwritable,
 		# like the 'install' utility used by 'make install' does.
@@ -962,190 +1176,66 @@ class InstallContext(BuildContext):
 			pass
 
 		try:
-			self.copy_fun(src, tgt, **kw)
-		except IOError:
-			try:
-				os.stat(src)
-			except EnvironmentError:
-				Logs.error('File %r does not exist' % src)
-			raise Errors.WafError('Could not install the file %r' % tgt)
+			self.copy_fun(src, tgt)
+		except EnvironmentError as e:
+			if not os.path.exists(src):
+				Logs.error('File %r does not exist', src)
+			elif not os.path.isfile(src):
+				Logs.error('Input %r is not a file', src)
+			raise Errors.WafError('Could not install the file %r' % tgt, e)
 
-	def do_link(self, src, tgt, **kw):
+	def fix_perms(self, tgt):
 		"""
-		Create a symlink from tgt to src.
+		Change the ownership of the file/folder/link pointed by the given path
+		This looks up for `install_user` or `install_group` attributes
+		on the task or on the task generator::
+
+			def build(bld):
+				bld.install_as('${PREFIX}/wscript',
+					'wscript',
+					install_user='nobody', install_group='nogroup')
+				bld.symlink_as('${PREFIX}/wscript_link',
+					Utils.subst_vars('${PREFIX}/wscript', bld.env),
+					install_user='nobody', install_group='nogroup')
+		"""
+		if not Utils.is_win32:
+			user = getattr(self, 'install_user', None) or getattr(self.generator, 'install_user', None)
+			group = getattr(self, 'install_group', None) or getattr(self.generator, 'install_group', None)
+			if user or group:
+				Utils.lchown(tgt, user or -1, group or -1)
+		if not os.path.islink(tgt):
+			os.chmod(tgt, self.chmod)
 
-		This method is overridden in :py:meth:`waflib.Build.UninstallContext.do_link` to remove the symlink.
+	def do_link(self, src, tgt, **kw):
+		"""
+		Creates a symlink from tgt to src.
 
 		:param src: file name as absolute path
 		:type src: string
 		:param tgt: file destination, as absolute path
 		:type tgt: string
 		"""
-		d, _ = os.path.split(tgt)
-		Utils.check_dir(d)
-
-		link = False
-		if not os.path.islink(tgt):
-			link = True
-		elif os.readlink(tgt) != src:
-			link = True
-
-		if link:
-			try: os.remove(tgt)
-			except OSError: pass
-			if not self.progress_bar:
-				Logs.info('+ symlink %s (to %s)' % (tgt, src))
-			os.symlink(src, tgt)
-		else:
-			if not self.progress_bar:
-				Logs.info('- symlink %s (to %s)' % (tgt, src))
-
-	def run_task_now(self, tsk, postpone):
-		"""
-		This method is called by :py:meth:`waflib.Build.InstallContext.install_files`,
-		:py:meth:`waflib.Build.InstallContext.install_as` and :py:meth:`waflib.Build.InstallContext.symlink_as` immediately
-		after the installation task is created. Its role is to force the immediate execution if necessary, that is when
-		``postpone=False`` was given.
-		"""
-		tsk.post()
-		if not postpone:
-			if tsk.runnable_status() == Task.ASK_LATER:
-				raise self.WafError('cannot post the task %r' % tsk)
-			tsk.run()
-			tsk.hasrun = True
-
-	def install_files(self, dest, files, env=None, chmod=Utils.O644, relative_trick=False, cwd=None, add=True, postpone=True, task=None):
-		"""
-		Create a task to install files on the system::
-
-			def build(bld):
-				bld.install_files('${DATADIR}', self.path.find_resource('wscript'))
-
-		:param dest: absolute path of the destination directory
-		:type dest: string
-		:param files: input files
-		:type files: list of strings or list of nodes
-		:param env: configuration set for performing substitutions in dest
-		:type env: Configuration set
-		:param relative_trick: preserve the folder hierarchy when installing whole folders
-		:type relative_trick: bool
-		:param cwd: parent node for searching srcfile, when srcfile is not a :py:class:`waflib.Node.Node`
-		:type cwd: :py:class:`waflib.Node.Node`
-		:param add: add the task created to a build group - set ``False`` only if the installation task is created after the build has started
-		:type add: bool
-		:param postpone: execute the task immediately to perform the installation
-		:type postpone: bool
-		"""
-		assert(dest)
-		tsk = inst(env=env or self.env)
-		tsk.bld = self
-		tsk.path = cwd or self.path
-		tsk.chmod = chmod
-		tsk.task = task
-		if isinstance(files, waflib.Node.Node):
-			tsk.source =  [files]
+		if os.path.islink(tgt) and os.readlink(tgt) == src:
+			if not self.generator.bld.progress_bar:
+				Logs.info('- symlink %s (to %s)', tgt, src)
 		else:
-			tsk.source = Utils.to_list(files)
-		tsk.dest = dest
-		tsk.exec_task = tsk.exec_install_files
-		tsk.relative_trick = relative_trick
-		if add: self.add_to_group(tsk)
-		self.run_task_now(tsk, postpone)
-		return tsk
-
-	def install_as(self, dest, srcfile, env=None, chmod=Utils.O644, cwd=None, add=True, postpone=True, task=None):
-		"""
-		Create a task to install a file on the system with a different name::
-
-			def build(bld):
-				bld.install_as('${PREFIX}/bin', 'myapp', chmod=Utils.O755)
-
-		:param dest: absolute path of the destination file
-		:type dest: string
-		:param srcfile: input file
-		:type srcfile: string or node
-		:param cwd: parent node for searching srcfile, when srcfile is not a :py:class:`waflib.Node.Node`
-		:type cwd: :py:class:`waflib.Node.Node`
-		:param env: configuration set for performing substitutions in dest
-		:type env: Configuration set
-		:param add: add the task created to a build group - set ``False`` only if the installation task is created after the build has started
-		:type add: bool
-		:param postpone: execute the task immediately to perform the installation
-		:type postpone: bool
-		"""
-		assert(dest)
-		tsk = inst(env=env or self.env)
-		tsk.bld = self
-		tsk.path = cwd or self.path
-		tsk.chmod = chmod
-		tsk.source = [srcfile]
-		tsk.task = task
-		tsk.dest = dest
-		tsk.exec_task = tsk.exec_install_as
-		if add: self.add_to_group(tsk)
-		self.run_task_now(tsk, postpone)
-		return tsk
-
-	def symlink_as(self, dest, src, env=None, cwd=None, add=True, postpone=True, relative_trick=False, task=None):
-		"""
-		Create a task to install a symlink::
-
-			def build(bld):
-				bld.symlink_as('${PREFIX}/lib/libfoo.so', 'libfoo.so.1.2.3')
-
-		:param dest: absolute path of the symlink
-		:type dest: string
-		:param src: absolute or relative path of the link
-		:type src: string
-		:param env: configuration set for performing substitutions in dest
-		:type env: Configuration set
-		:param add: add the task created to a build group - set ``False`` only if the installation task is created after the build has started
-		:type add: bool
-		:param postpone: execute the task immediately to perform the installation
-		:type postpone: bool
-		:param relative_trick: make the symlink relative (default: ``False``)
-		:type relative_trick: bool
-		"""
-		if Utils.is_win32:
-			# symlinks *cannot* work on that platform
-			# TODO waf 1.9 - replace by install_as
-			return
-		assert(dest)
-		tsk = inst(env=env or self.env)
-		tsk.bld = self
-		tsk.dest = dest
-		tsk.path = cwd or self.path
-		tsk.source = []
-		tsk.task = task
-		tsk.link = src
-		tsk.relative_trick = relative_trick
-		tsk.exec_task = tsk.exec_symlink_as
-		if add: self.add_to_group(tsk)
-		self.run_task_now(tsk, postpone)
-		return tsk
-
-class UninstallContext(InstallContext):
-	'''removes the targets installed'''
-	cmd = 'uninstall'
-
-	def __init__(self, **kw):
-		super(UninstallContext, self).__init__(**kw)
-		self.is_install = UNINSTALL
-
-	def rm_empty_dirs(self, tgt):
-		while tgt:
-			tgt = os.path.dirname(tgt)
 			try:
-				os.rmdir(tgt)
+				os.remove(tgt)
 			except OSError:
-				break
+				pass
+			if not self.generator.bld.progress_bar:
+				Logs.info('+ symlink %s (to %s)', tgt, src)
+			os.symlink(src, tgt)
+			self.fix_perms(tgt)
 
-	def do_install(self, src, tgt, **kw):
-		"""See :py:meth:`waflib.Build.InstallContext.do_install`"""
-		if not self.progress_bar:
-			Logs.info('- remove %s' % tgt)
+	def do_uninstall(self, src, tgt, lbl, **kw):
+		"""
+		See :py:meth:`waflib.Build.inst.do_install`
+		"""
+		if not self.generator.bld.progress_bar:
+			Logs.info('- remove %s', tgt)
 
-		self.uninstall.append(tgt)
+		#self.uninstall.append(tgt)
 		try:
 			os.remove(tgt)
 		except OSError as e:
@@ -1154,42 +1244,43 @@ class UninstallContext(InstallContext):
 					self.uninstall_error = True
 					Logs.warn('build: some files could not be uninstalled (retry with -vv to list them)')
 				if Logs.verbose > 1:
-					Logs.warn('Could not remove %s (error code %r)' % (e.filename, e.errno))
-
+					Logs.warn('Could not remove %s (error code %r)', e.filename, e.errno)
 		self.rm_empty_dirs(tgt)
 
-	def do_link(self, src, tgt, **kw):
-		"""See :py:meth:`waflib.Build.InstallContext.do_link`"""
+	def do_unlink(self, src, tgt, **kw):
+		"""
+		See :py:meth:`waflib.Build.inst.do_link`
+		"""
 		try:
-			if not self.progress_bar:
-				Logs.info('- remove %s' % tgt)
+			if not self.generator.bld.progress_bar:
+				Logs.info('- remove %s', tgt)
 			os.remove(tgt)
 		except OSError:
 			pass
-
 		self.rm_empty_dirs(tgt)
 
-	def execute(self):
-		"""
-		See :py:func:`waflib.Context.Context.execute`
-		"""
-		try:
-			# do not execute any tasks
-			def runnable_status(self):
-				return Task.SKIP_ME
-			setattr(Task.Task, 'runnable_status_back', Task.Task.runnable_status)
-			setattr(Task.Task, 'runnable_status', runnable_status)
+class InstallContext(BuildContext):
+	'''installs the targets on the system'''
+	cmd = 'install'
 
-			super(UninstallContext, self).execute()
-		finally:
-			setattr(Task.Task, 'runnable_status', Task.Task.runnable_status_back)
+	def __init__(self, **kw):
+		super(InstallContext, self).__init__(**kw)
+		self.is_install = INSTALL
+
+class UninstallContext(InstallContext):
+	'''removes the targets installed'''
+	cmd = 'uninstall'
+
+	def __init__(self, **kw):
+		super(UninstallContext, self).__init__(**kw)
+		self.is_install = UNINSTALL
 
 class CleanContext(BuildContext):
 	'''cleans the project'''
 	cmd = 'clean'
 	def execute(self):
 		"""
-		See :py:func:`waflib.Context.Context.execute`
+		See :py:func:`waflib.Build.BuildContext.execute`.
 		"""
 		self.restore()
 		if not self.all_envs:
@@ -1202,30 +1293,49 @@ class CleanContext(BuildContext):
 			self.store()
 
 	def clean(self):
-		"""Remove files from the build directory if possible, and reset the caches"""
+		"""
+		Remove most files from the build directory, and reset all caches.
+
+		Custom lists of files to clean can be declared as `bld.clean_files`.
+		For example, exclude `build/program/myprogram` from getting removed::
+
+			def build(bld):
+				bld.clean_files = bld.bldnode.ant_glob('**',
+					excl='.lock* config.log c4che/* config.h program/myprogram',
+					quiet=True, generator=True)
+		"""
 		Logs.debug('build: clean called')
 
-		if self.bldnode != self.srcnode:
+		if hasattr(self, 'clean_files'):
+			for n in self.clean_files:
+				n.delete()
+		elif self.bldnode != self.srcnode:
 			# would lead to a disaster if top == out
-			lst=[]
-			for e in self.all_envs.values():
-				lst.extend(self.root.find_or_declare(f) for f in e[CFG_FILES])
+			lst = []
+			for env in self.all_envs.values():
+				lst.extend(self.root.find_or_declare(f) for f in env[CFG_FILES])
 			for n in self.bldnode.ant_glob('**/*', excl='.lock* *conf_check_*/** config.log c4che/*', quiet=True):
 				if n in lst:
 					continue
 				n.delete()
 		self.root.children = {}
 
-		for v in 'node_deps task_sigs raw_deps'.split():
+		for v in SAVED_ATTRS:
+			if v == 'root':
+				continue
 			setattr(self, v, {})
 
 class ListContext(BuildContext):
 	'''lists the targets to execute'''
-
 	cmd = 'list'
+
 	def execute(self):
 		"""
-		See :py:func:`waflib.Context.Context.execute`.
+		In addition to printing the name of each build target,
+		a description column will include text for each task
+		generator which has a "description" field set.
+
+		See :py:func:`waflib.Build.BuildContext.execute`.
 		"""
 		self.restore()
 		if not self.all_envs:
@@ -1249,12 +1359,25 @@ class ListContext(BuildContext):
 		try:
 			# force the cache initialization
 			self.get_tgen_by_name('')
-		except Exception:
+		except Errors.WafError:
 			pass
-		lst = list(self.task_gen_cache_names.keys())
-		lst.sort()
-		for k in lst:
-			Logs.pprint('GREEN', k)
+
+		targets = sorted(self.task_gen_cache_names)
+
+		# figure out how much to left-justify, for largest target name
+		line_just = max(len(t) for t in targets) if targets else 0
+
+		for target in targets:
+			tgen = self.task_gen_cache_names[target]
+
+			# Support displaying the description for the target
+			# if it was set on the tgen
+			descript = getattr(tgen, 'description', '')
+			if descript:
+				target = target.ljust(line_just)
+				descript = ': %s' % descript
+
+			Logs.pprint('GREEN', target, label=descript)
 
 class StepContext(BuildContext):
 	'''executes tasks in a step-by-step fashion, for debugging'''
@@ -1266,7 +1389,8 @@ class StepContext(BuildContext):
 
 	def compile(self):
 		"""
-		Compile the tasks matching the input/output files given (regular expression matching). Derived from :py:meth:`waflib.Build.BuildContext.compile`::
+		Overrides :py:meth:`waflib.Build.BuildContext.compile` to perform a partial build
+		on tasks matching the input/output pattern given (regular expression matching)::
 
 			$ waf step --files=foo.c,bar.c,in:truc.c,out:bar.o
 			$ waf step --files=in:foo.cpp.1.o # link task only
@@ -1277,7 +1401,7 @@ class StepContext(BuildContext):
 			BuildContext.compile(self)
 			return
 
-		targets = None
+		targets = []
 		if self.targets and self.targets != '*':
 			targets = self.targets.split(',')
 
@@ -1296,25 +1420,32 @@ class StepContext(BuildContext):
 			for pat in self.files.split(','):
 				matcher = self.get_matcher(pat)
 				for tg in g:
-					if isinstance(tg, Task.TaskBase):
+					if isinstance(tg, Task.Task):
 						lst = [tg]
 					else:
 						lst = tg.tasks
 					for tsk in lst:
 						do_exec = False
-						for node in getattr(tsk, 'inputs', []):
+						for node in tsk.inputs:
 							if matcher(node, output=False):
 								do_exec = True
 								break
-						for node in getattr(tsk, 'outputs', []):
+						for node in tsk.outputs:
 							if matcher(node, output=True):
 								do_exec = True
 								break
 						if do_exec:
 							ret = tsk.run()
-							Logs.info('%s -> exit %r' % (str(tsk), ret))
+							Logs.info('%s -> exit %r', tsk, ret)
 
 	def get_matcher(self, pat):
+		"""
+		Converts a step pattern into a function
+
+		:param: pat: pattern of the form in:truc.c,out:bar.o
+		:returns: Python function that uses Node objects as inputs and returns matches
+		:rtype: function
+		"""
 		# this returns a function
 		inn = True
 		out = True
@@ -1335,9 +1466,9 @@ class StepContext(BuildContext):
 			pattern = re.compile(pat)
 
 		def match(node, output):
-			if output == True and not out:
+			if output and not out:
 				return False
-			if output == False and not inn:
+			if not output and not inn:
 				return False
 
 			if anode:
@@ -1346,3 +1477,15 @@ class StepContext(BuildContext):
 				return pattern.match(node.abspath())
 		return match
 
+class EnvContext(BuildContext):
+	"""Subclass EnvContext to create commands that require configuration data in 'env'"""
+	fun = cmd = None
+	def execute(self):
+		"""
+		See :py:func:`waflib.Build.BuildContext.execute`.
+		"""
+		self.restore()
+		if not self.all_envs:
+			self.load_envs()
+		self.recurse([self.run_dir])
+
diff --git a/waflib/ConfigSet.py b/waflib/ConfigSet.py
index a337a09b..b300bb56 100644
--- a/waflib/ConfigSet.py
+++ b/waflib/ConfigSet.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 
 ConfigSet: a special dict
 
-The values put in :py:class:`ConfigSet` must be lists
+The values put in :py:class:`ConfigSet` must be serializable (dicts, lists, strings)
 """
 
 import copy, re, os
@@ -15,7 +15,7 @@ re_imp = re.compile('^(#)*?([^#=]*?)\ =\ (.*?)$', re.M)
 
 class ConfigSet(object):
 	"""
-	A dict that honor serialization and parent relationships. The serialization format
+	A copy-on-write dict with human-readable serialized format. The serialization format
 	is human-readable (python-like) and performed by using eval() and repr().
 	For high performance prefer pickle. Do not store functions as they are not serializable.
 
@@ -39,17 +39,20 @@ class ConfigSet(object):
 
 	def __contains__(self, key):
 		"""
-		Enable the *in* syntax::
+		Enables the *in* syntax::
 
 			if 'foo' in env:
 				print(env['foo'])
 		"""
-		if key in self.table: return True
-		try: return self.parent.__contains__(key)
-		except AttributeError: return False # parent may not exist
+		if key in self.table:
+			return True
+		try:
+			return self.parent.__contains__(key)
+		except AttributeError:
+			return False # parent may not exist
 
 	def keys(self):
-		"""Dict interface (unknown purpose)"""
+		"""Dict interface"""
 		keys = set()
 		cur = self
 		while cur:
@@ -59,6 +62,9 @@ class ConfigSet(object):
 		keys.sort()
 		return keys
 
+	def __iter__(self):
+		return iter(self.keys())
+
 	def __str__(self):
 		"""Text representation of the ConfigSet (for debugging purposes)"""
 		return "\n".join(["%r %r" % (x, self.__getitem__(x)) for x in self.keys()])
@@ -73,7 +79,7 @@ class ConfigSet(object):
 		"""
 		try:
 			while 1:
-				x = self.table.get(key, None)
+				x = self.table.get(key)
 				if not x is None:
 					return x
 				self = self.parent
@@ -82,13 +88,13 @@ class ConfigSet(object):
 
 	def __setitem__(self, key, value):
 		"""
-		Dictionary interface: get value from key
+		Dictionary interface: set value from key
 		"""
 		self.table[key] = value
 
 	def __delitem__(self, key):
 		"""
-		Dictionary interface: get value from key
+		Dictionary interface: mark the value as missing
 		"""
 		self[key] = []
 
@@ -101,7 +107,7 @@ class ConfigSet(object):
 				conf.env['value']
 		"""
 		if name in self.__slots__:
-			return object.__getattr__(self, name)
+			return object.__getattribute__(self, name)
 		else:
 			return self[name]
 
@@ -152,7 +158,7 @@ class ConfigSet(object):
 
 	def detach(self):
 		"""
-		Detach self from its parent (if existing)
+		Detaches this instance from its parent (if present)
 
 		Modifying the parent :py:class:`ConfigSet` will not change the current object
 		Modifying this :py:class:`ConfigSet` will not modify the parent one.
@@ -171,18 +177,19 @@ class ConfigSet(object):
 
 	def get_flat(self, key):
 		"""
-		Return a value as a string. If the input is a list, the value returned is space-separated.
+		Returns a value as a string. If the input is a list, the value returned is space-separated.
 
 		:param key: key to use
 		:type key: string
 		"""
 		s = self[key]
-		if isinstance(s, str): return s
+		if isinstance(s, str):
+			return s
 		return ' '.join(s)
 
 	def _get_list_value_for_modification(self, key):
 		"""
-		Return a list value for further modification.
+		Returns a list value for further modification.
 
 		The list may be modified inplace and there is no need to do this afterwards::
 
@@ -191,16 +198,20 @@ class ConfigSet(object):
 		try:
 			value = self.table[key]
 		except KeyError:
-			try: value = self.parent[key]
-			except AttributeError: value = []
-			if isinstance(value, list):
-				value = value[:]
+			try:
+				value = self.parent[key]
+			except AttributeError:
+				value = []
 			else:
-				value = [value]
+				if isinstance(value, list):
+					# force a copy
+					value = value[:]
+				else:
+					value = [value]
+			self.table[key] = value
 		else:
 			if not isinstance(value, list):
-				value = [value]
-		self.table[key] = value
+				self.table[key] = value = [value]
 		return value
 
 	def append_value(self, var, val):
@@ -232,7 +243,7 @@ class ConfigSet(object):
 
 	def append_unique(self, var, val):
 		"""
-		Append a value to the specified item only if it's not already present::
+		Appends a value to the specified item only if it's not already present::
 
 			def build(bld):
 				bld.env.append_unique('CFLAGS', ['-O2', '-g'])
@@ -249,7 +260,7 @@ class ConfigSet(object):
 
 	def get_merged_dict(self):
 		"""
-		Compute the merged dictionary from the fusion of self and all its parent
+		Computes the merged dictionary from the fusion of self and all its parent
 
 		:rtype: a ConfigSet object
 		"""
@@ -257,8 +268,10 @@ class ConfigSet(object):
 		env = self
 		while 1:
 			table_list.insert(0, env.table)
-			try: env = env.parent
-			except AttributeError: break
+			try:
+				env = env.parent
+			except AttributeError:
+				break
 		merged_table = {}
 		for table in table_list:
 			merged_table.update(table)
@@ -266,7 +279,7 @@ class ConfigSet(object):
 
 	def store(self, filename):
 		"""
-		Write the :py:class:`ConfigSet` data into a file. See :py:meth:`ConfigSet.load` for reading such files.
+		Serializes the :py:class:`ConfigSet` data to a file. See :py:meth:`ConfigSet.load` for reading such files.
 
 		:param filename: file to use
 		:type filename: string
@@ -293,7 +306,7 @@ class ConfigSet(object):
 
 	def load(self, filename):
 		"""
-		Retrieve the :py:class:`ConfigSet` data from a file. See :py:meth:`ConfigSet.store` for writing such files
+		Restores contents from a file (current values are not cleared). Files are written using :py:meth:`ConfigSet.store`.
 
 		:param filename: file to use
 		:type filename: string
@@ -303,21 +316,20 @@ class ConfigSet(object):
 		for m in re_imp.finditer(code):
 			g = m.group
 			tbl[g(2)] = eval(g(3))
-		Logs.debug('env: %s' % str(self.table))
+		Logs.debug('env: %s', self.table)
 
 	def update(self, d):
 		"""
-		Dictionary interface: replace values from another dict
+		Dictionary interface: replace values with the ones from another dict
 
 		:param d: object to use the value from
 		:type d: dict-like object
 		"""
-		for k, v in d.items():
-			self[k] = v
+		self.table.update(d)
 
 	def stash(self):
 		"""
-		Store the object state, to provide a kind of transaction support::
+		Stores the object state to provide transactionality semantics::
 
 			env = ConfigSet()
 			env.stash()
diff --git a/waflib/Configure.py b/waflib/Configure.py
index 247173c8..d0a4793a 100644
--- a/waflib/Configure.py
+++ b/waflib/Configure.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Configuration system
@@ -12,15 +12,9 @@ A :py:class:`waflib.Configure.ConfigurationContext` instance is created when ``w
 * hold configuration routines such as ``find_program``, etc
 """
 
-import os, shlex, sys, time, re, shutil
+import os, re, shlex, shutil, sys, time, traceback
 from waflib import ConfigSet, Utils, Options, Logs, Context, Build, Errors
 
-BREAK    = 'break'
-"""In case of a configuration error, break"""
-
-CONTINUE = 'continue'
-"""In case of a configuration error, continue"""
-
 WAF_CONFIG_LOG = 'config.log'
 """Name of the configuration log file"""
 
@@ -157,7 +151,7 @@ class ConfigurationContext(Context.Context):
 		self.msg('Setting out to', self.bldnode.abspath())
 
 		if id(self.srcnode) == id(self.bldnode):
-			Logs.warn('Setting top == out (remember to use "update_outputs")')
+			Logs.warn('Setting top == out')
 		elif id(self.path) != id(self.srcnode):
 			if self.srcnode.is_child_of(self.path):
 				Logs.warn('Are you certain that you do not want to set top="." ?')
@@ -173,8 +167,9 @@ class ConfigurationContext(Context.Context):
 		# consider the current path as the root directory (see prepare_impl).
 		# to remove: use 'waf distclean'
 		env = ConfigSet.ConfigSet()
-		env['argv'] = sys.argv
-		env['options'] = Options.options.__dict__
+		env.argv = sys.argv
+		env.options = Options.options.__dict__
+		env.config_cmd = self.cmd
 
 		env.run_dir = Context.run_dir
 		env.top_dir = Context.top_dir
@@ -182,15 +177,15 @@ class ConfigurationContext(Context.Context):
 
 		# conf.hash & conf.files hold wscript files paths and hash
 		# (used only by Configure.autoconfig)
-		env['hash'] = self.hash
-		env['files'] = self.files
-		env['environ'] = dict(self.environ)
+		env.hash = self.hash
+		env.files = self.files
+		env.environ = dict(self.environ)
 
-		if not self.env.NO_LOCK_IN_RUN and not getattr(Options.options, 'no_lock_in_run'):
+		if not (self.env.NO_LOCK_IN_RUN or env.environ.get('NO_LOCK_IN_RUN') or getattr(Options.options, 'no_lock_in_run')):
 			env.store(os.path.join(Context.run_dir, Options.lockfile))
-		if not self.env.NO_LOCK_IN_TOP and not getattr(Options.options, 'no_lock_in_top'):
+		if not (self.env.NO_LOCK_IN_TOP or env.environ.get('NO_LOCK_IN_TOP') or getattr(Options.options, 'no_lock_in_top')):
 			env.store(os.path.join(Context.top_dir, Options.lockfile))
-		if not self.env.NO_LOCK_IN_OUT and not getattr(Options.options, 'no_lock_in_out'):
+		if not (self.env.NO_LOCK_IN_OUT or env.environ.get('NO_LOCK_IN_OUT') or getattr(Options.options, 'no_lock_in_out')):
 			env.store(os.path.join(Context.out_dir, Options.lockfile))
 
 	def prepare_env(self, env):
@@ -202,17 +197,17 @@ class ConfigurationContext(Context.Context):
 		"""
 		if not env.PREFIX:
 			if Options.options.prefix or Utils.is_win32:
-				env.PREFIX = Utils.sane_path(Options.options.prefix)
+				env.PREFIX = Options.options.prefix
 			else:
-				env.PREFIX = ''
+				env.PREFIX = '/'
 		if not env.BINDIR:
 			if Options.options.bindir:
-				env.BINDIR = Utils.sane_path(Options.options.bindir)
+				env.BINDIR = Options.options.bindir
 			else:
 				env.BINDIR = Utils.subst_vars('${PREFIX}/bin', env)
 		if not env.LIBDIR:
 			if Options.options.libdir:
-				env.LIBDIR = Utils.sane_path(Options.options.libdir)
+				env.LIBDIR = Options.options.libdir
 			else:
 				env.LIBDIR = Utils.subst_vars('${PREFIX}/lib%s' % Utils.lib64(), env)
 
@@ -228,38 +223,42 @@ class ConfigurationContext(Context.Context):
 			tmpenv = self.all_envs[key]
 			tmpenv.store(os.path.join(self.cachedir.abspath(), key + Build.CACHE_SUFFIX))
 
-	def load(self, input, tooldir=None, funs=None, with_sys_path=True):
+	def load(self, tool_list, tooldir=None, funs=None, with_sys_path=True, cache=False):
 		"""
 		Load Waf tools, which will be imported whenever a build is started.
 
-		:param input: waf tools to import
-		:type input: list of string
+		:param tool_list: waf tools to import
+		:type tool_list: list of string
 		:param tooldir: paths for the imports
 		:type tooldir: list of string
 		:param funs: functions to execute from the waf tools
 		:type funs: list of string
+		:param cache: whether to prevent the tool from running twice
+		:type cache: bool
 		"""
 
-		tools = Utils.to_list(input)
-		if tooldir: tooldir = Utils.to_list(tooldir)
+		tools = Utils.to_list(tool_list)
+		if tooldir:
+			tooldir = Utils.to_list(tooldir)
 		for tool in tools:
 			# avoid loading the same tool more than once with the same functions
 			# used by composite projects
 
-			mag = (tool, id(self.env), tooldir, funs)
-			if mag in self.tool_cache:
-				self.to_log('(tool %s is already loaded, skipping)' % tool)
-				continue
-			self.tool_cache.append(mag)
+			if cache:
+				mag = (tool, id(self.env), tooldir, funs)
+				if mag in self.tool_cache:
+					self.to_log('(tool %s is already loaded, skipping)' % tool)
+					continue
+				self.tool_cache.append(mag)
 
 			module = None
 			try:
 				module = Context.load_tool(tool, tooldir, ctx=self, with_sys_path=with_sys_path)
 			except ImportError as e:
-				self.fatal('Could not load the Waf tool %r from %r\n%s' % (tool, sys.path, e))
+				self.fatal('Could not load the Waf tool %r from %r\n%s' % (tool, getattr(e, 'waf_sys_path', sys.path), e))
 			except Exception as e:
 				self.to_log('imp %r (%r & %r)' % (tool, tooldir, funs))
-				self.to_log(Utils.ex_stack())
+				self.to_log(traceback.format_exc())
 				raise
 
 			if funs is not None:
@@ -267,8 +266,10 @@ class ConfigurationContext(Context.Context):
 			else:
 				func = getattr(module, 'configure', None)
 				if func:
-					if type(func) is type(Utils.readf): func(self)
-					else: self.eval_rules(func)
+					if type(func) is type(Utils.readf):
+						func(self)
+					else:
+						self.eval_rules(func)
 
 			self.tools.append({'tool':tool, 'tooldir':tooldir, 'funs':funs})
 
@@ -285,8 +286,7 @@ class ConfigurationContext(Context.Context):
 
 	def eval_rules(self, rules):
 		"""
-		Execute the configuration tests. The method :py:meth:`waflib.Configure.ConfigurationContext.err_handler`
-		is used to process the eventual exceptions
+		Execute configuration tests provided as list of functions to run
 
 		:param rules: list of configuration method names
 		:type rules: list of string
@@ -294,28 +294,9 @@ class ConfigurationContext(Context.Context):
 		self.rules = Utils.to_list(rules)
 		for x in self.rules:
 			f = getattr(self, x)
-			if not f: self.fatal("No such method '%s'." % x)
-			try:
-				f()
-			except Exception as e:
-				ret = self.err_handler(x, e)
-				if ret == BREAK:
-					break
-				elif ret == CONTINUE:
-					continue
-				else:
-					raise
-
-	def err_handler(self, fun, error):
-		"""
-		Error handler for the configuration tests, the default is to let the exception raise
-
-		:param fun: configuration test
-		:type fun: method
-		:param error: exception
-		:type error: exception
-		"""
-		pass
+			if not f:
+				self.fatal('No such configuration function %r' % x)
+			f()
 
 def conf(f):
 	"""
@@ -330,11 +311,7 @@ def conf(f):
 	:type f: function
 	"""
 	def fun(*k, **kw):
-		mandatory = True
-		if 'mandatory' in kw:
-			mandatory = kw['mandatory']
-			del kw['mandatory']
-
+		mandatory = kw.pop('mandatory', True)
 		try:
 			return f(*k, **kw)
 		except Errors.ConfigurationError:
@@ -347,7 +324,7 @@ def conf(f):
 	return f
 
 @conf
-def add_os_flags(self, var, dest=None, dup=True):
+def add_os_flags(self, var, dest=None, dup=False):
 	"""
 	Import operating system environment values into ``conf.env`` dict::
 
@@ -365,7 +342,6 @@ def add_os_flags(self, var, dest=None, dup=True):
 		flags = shlex.split(self.environ[var])
 	except KeyError:
 		return
-	# TODO: in waf 1.9, make dup=False the default
 	if dup or ''.join(flags) not in ''.join(Utils.to_list(self.env[dest or var])):
 		self.env.append_value(dest or var, flags)
 
@@ -377,21 +353,26 @@ def cmd_to_list(self, cmd):
 	:param cmd: command
 	:type cmd: a string or a list of string
 	"""
-	if isinstance(cmd, str) and cmd.find(' '):
-		try:
-			os.stat(cmd)
-		except OSError:
+	if isinstance(cmd, str):
+		if os.path.isfile(cmd):
+			# do not take any risk
+			return [cmd]
+		if os.sep == '/':
 			return shlex.split(cmd)
 		else:
-			return [cmd]
+			try:
+				return shlex.split(cmd, posix=False)
+			except TypeError:
+				# Python 2.5 on windows?
+				return shlex.split(cmd)
 	return cmd
 
 @conf
-def check_waf_version(self, mini='1.7.99', maxi='1.9.0', **kw):
+def check_waf_version(self, mini='1.9.99', maxi='2.1.0', **kw):
 	"""
 	Raise a Configuration error if the Waf version does not strictly match the given bounds::
 
-		conf.check_waf_version(mini='1.8.0', maxi='1.9.0')
+		conf.check_waf_version(mini='1.9.99', maxi='2.1.0')
 
 	:type  mini: number, tuple or string
 	:param mini: Minimum required version
@@ -413,7 +394,7 @@ def find_file(self, filename, path_list=[]):
 
 	:param filename: name of the file to search for
 	:param path_list: list of directories to search
-	:return: the first occurrence filename or '' if filename could not be found
+	:return: the first matching filename; else a configuration exception is raised
 	"""
 	for n in Utils.to_list(filename):
 		for d in Utils.to_list(path_list):
@@ -433,14 +414,17 @@ def find_program(self, filename, **kw):
 
 	:param path_list: paths to use for searching
 	:type param_list: list of string
-	:param var: store the result to conf.env[var], by default use filename.upper()
+	:param var: store the result to conf.env[var] where var defaults to filename.upper() if not provided; the result is stored as a list of strings
 	:type var: string
-	:param ext: list of extensions for the binary (do not add an extension for portability)
-	:type ext: list of string
+	:param value: obtain the program from the value passed exclusively
+	:type value: list or string (list is preferred)
+	:param exts: list of extensions for the binary (do not add an extension for portability)
+	:type exts: list of string
 	:param msg: name to display in the log, by default filename is used
 	:type msg: string
 	:param interpreter: interpreter for the program
 	:type interpreter: ConfigSet variable key
+	:raises: :py:class:`waflib.Errors.ConfigurationError`
 	"""
 
 	exts = kw.get('exts', Utils.is_win32 and '.exe,.com,.bat,.cmd' or ',.sh,.pl,.py')
@@ -462,18 +446,15 @@ def find_program(self, filename, **kw):
 	else:
 		path_list = environ.get('PATH', '').split(os.pathsep)
 
-	if var in environ:
-		filename = environ[var]
-		if os.path.isfile(filename):
-			# typical CC=/usr/bin/gcc waf configure build
-			ret = [filename]
-		else:
-			# case  CC='ccache gcc' waf configure build
-			ret = self.cmd_to_list(filename)
+	if kw.get('value'):
+		# user-provided in command-line options and passed to find_program
+		ret = self.cmd_to_list(kw['value'])
+	elif environ.get(var):
+		# user-provided in the os environment
+		ret = self.cmd_to_list(environ[var])
 	elif self.env[var]:
-		# set by the user in the wscript file
-		ret = self.env[var]
-		ret = self.cmd_to_list(ret)
+		# a default option in the wscript file
+		ret = self.cmd_to_list(self.env[var])
 	else:
 		if not ret:
 			ret = self.find_binary(filename, exts.split(','), path_list)
@@ -483,7 +464,6 @@ def find_program(self, filename, **kw):
 			ret = Utils.get_registry_app_path(Utils.winreg.HKEY_LOCAL_MACHINE, filename)
 		ret = self.cmd_to_list(ret)
 
-
 	if ret:
 		if len(ret) == 1:
 			retmsg = ret[0]
@@ -492,14 +472,14 @@ def find_program(self, filename, **kw):
 	else:
 		retmsg = False
 
-	self.msg("Checking for program '%s'" % msg, retmsg, **kw)
-	if not kw.get('quiet', None):
+	self.msg('Checking for program %r' % msg, retmsg, **kw)
+	if not kw.get('quiet'):
 		self.to_log('find program=%r paths=%r var=%r -> %r' % (filename, path_list, var, ret))
 
 	if not ret:
 		self.fatal(kw.get('errmsg', '') or 'Could not find the program %r' % filename)
 
-	interpreter = kw.get('interpreter', None)
+	interpreter = kw.get('interpreter')
 	if interpreter is None:
 		if not Utils.check_exe(ret[0], env=environ):
 			self.fatal('Program %r is not executable' % ret)
@@ -554,7 +534,6 @@ def run_build(self, *k, **kw):
 		$ waf configure --confcache
 
 	"""
-
 	lst = [str(v) for (p, v) in kw.items() if p != 'env']
 	h = Utils.h_list(lst)
 	dir = self.bldnode.abspath() + os.sep + (not Utils.is_win32 and '.' or '') + 'conf_check_' + Utils.to_hex(h)
@@ -573,9 +552,7 @@ def run_build(self, *k, **kw):
 	if cachemode == 1:
 		try:
 			proj = ConfigSet.ConfigSet(os.path.join(dir, 'cache_run_build'))
-		except OSError:
-			pass
-		except IOError:
+		except EnvironmentError:
 			pass
 		else:
 			ret = proj['cache_run_build']
@@ -588,7 +565,8 @@ def run_build(self, *k, **kw):
 	if not os.path.exists(bdir):
 		os.makedirs(bdir)
 
-	self.test_bld = bld = Build.BuildContext(top_dir=dir, out_dir=bdir)
+	cls_name = kw.get('run_build_cls') or getattr(self, 'run_build_cls', 'build')
+	self.test_bld = bld = Context.create_context(cls_name, top_dir=dir, out_dir=bdir)
 	bld.init_dirs()
 	bld.progress_bar = 0
 	bld.targets = '*'
@@ -597,17 +575,15 @@ def run_build(self, *k, **kw):
 	bld.all_envs.update(self.all_envs) # not really necessary
 	bld.env = kw['env']
 
-	# OMG huge hack
 	bld.kw = kw
 	bld.conf = self
 	kw['build_fun'](bld)
-
 	ret = -1
 	try:
 		try:
 			bld.compile()
 		except Errors.WafError:
-			ret = 'Test does not build: %s' % Utils.ex_stack()
+			ret = 'Test does not build: %s' % traceback.format_exc()
 			self.fatal(ret)
 		else:
 			ret = getattr(bld, 'retval', 0)
@@ -619,7 +595,6 @@ def run_build(self, *k, **kw):
 			proj.store(os.path.join(dir, 'cache_run_build'))
 		else:
 			shutil.rmtree(dir)
-
 	return ret
 
 @conf
@@ -635,7 +610,7 @@ def test(self, *k, **kw):
 		kw['env'] = self.env.derive()
 
 	# validate_c for example
-	if kw.get('validate', None):
+	if kw.get('validate'):
 		kw['validate'](kw)
 
 	self.start_msg(kw['msg'], **kw)
@@ -651,7 +626,7 @@ def test(self, *k, **kw):
 	else:
 		kw['success'] = ret
 
-	if kw.get('post_check', None):
+	if kw.get('post_check'):
 		ret = kw['post_check'](kw)
 
 	if ret:
@@ -661,5 +636,3 @@ def test(self, *k, **kw):
 		self.end_msg(self.ret_msg(kw['okmsg'], kw), **kw)
 	return ret
 
-
-
diff --git a/waflib/Context.py b/waflib/Context.py
index d9bacece..bb47c921 100644
--- a/waflib/Context.py
+++ b/waflib/Context.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2010-2016 (ita)
+# Thomas Nagy, 2010-2018 (ita)
 
 """
-Classes and functions required for waf commands
+Classes and functions enabling the command system
 """
 
 import os, re, imp, sys
@@ -11,16 +11,16 @@ from waflib import Utils, Errors, Logs
 import waflib.Node
 
 # the following 3 constants are updated on each new release (do not touch)
-HEXVERSION=0x1081600
+HEXVERSION=0x2000b00
 """Constant updated on new releases"""
 
-WAFVERSION="1.8.22"
+WAFVERSION="2.0.11"
 """Constant updated on new releases"""
 
-WAFREVISION="17d4d4faa52c454eb3580e482df69b2a80e19fa7"
+WAFREVISION="a97f6fb0941091b4966b625f15ec32fa783a8bec"
 """Git revision when the waf version is updated"""
 
-ABI = 98
+ABI = 20
 """Version of the build data cache file format (used in :py:const:`waflib.Context.DBFILE`)"""
 
 DBFILE = '.wafpickle-%s-%d-%d' % (sys.platform, sys.hexversion, ABI)
@@ -41,7 +41,6 @@ OUT  = 'out'
 WSCRIPT_FILE = 'wscript'
 """Name of the waf script files"""
 
-
 launch_dir = ''
 """Directory from which waf has been called"""
 run_dir = ''
@@ -53,23 +52,12 @@ out_dir = ''
 waf_dir = ''
 """Directory containing the waf modules"""
 
-local_repo = ''
-"""Local repository containing additional Waf tools (plugins)"""
-remote_repo = 'https://raw.githubusercontent.com/waf-project/waf/master/'
-"""
-Remote directory containing downloadable waf tools. The missing tools can be downloaded by using::
-
-	$ waf configure --download
-"""
-
-remote_locs = ['waflib/extras', 'waflib/Tools']
-"""
-Remote directories for use with :py:const:`waflib.Context.remote_repo`
-"""
+default_encoding = Utils.console_encoding()
+"""Encoding to use when reading outputs from other processes"""
 
 g_module = None
 """
-Module representing the main wscript file (see :py:const:`waflib.Context.run_dir`)
+Module representing the top-level wscript file (see :py:const:`waflib.Context.run_dir`)
 """
 
 STDOUT = 1
@@ -82,20 +70,20 @@ List of :py:class:`waflib.Context.Context` subclasses that can be used as waf co
 are added automatically by a metaclass.
 """
 
-
 def create_context(cmd_name, *k, **kw):
 	"""
-	Create a new :py:class:`waflib.Context.Context` instance corresponding to the given command.
+	Returns a new :py:class:`waflib.Context.Context` instance corresponding to the given command.
 	Used in particular by :py:func:`waflib.Scripting.run_command`
 
-	:param cmd_name: command
+	:param cmd_name: command name
 	:type cmd_name: string
 	:param k: arguments to give to the context class initializer
 	:type k: list
 	:param k: keyword arguments to give to the context class initializer
 	:type k: dict
+	:return: Context object
+	:rtype: :py:class:`waflib.Context.Context`
 	"""
-	global classes
 	for x in classes:
 		if x.cmd == cmd_name:
 			return x(*k, **kw)
@@ -105,14 +93,15 @@ def create_context(cmd_name, *k, **kw):
 
 class store_context(type):
 	"""
-	Metaclass for storing the command classes into the list :py:const:`waflib.Context.classes`
-	Context classes must provide an attribute 'cmd' representing the command to execute
+	Metaclass that registers command classes into the list :py:const:`waflib.Context.classes`
+	Context classes must provide an attribute 'cmd' representing the command name, and a function
+	attribute 'fun' representing the function name that the command uses.
 	"""
-	def __init__(cls, name, bases, dict):
-		super(store_context, cls).__init__(name, bases, dict)
+	def __init__(cls, name, bases, dct):
+		super(store_context, cls).__init__(name, bases, dct)
 		name = cls.__name__
 
-		if name == 'ctx' or name == 'Context':
+		if name in ('ctx', 'Context'):
 			return
 
 		try:
@@ -123,11 +112,10 @@ class store_context(type):
 		if not getattr(cls, 'fun', None):
 			cls.fun = cls.cmd
 
-		global classes
 		classes.insert(0, cls)
 
 ctx = store_context('ctx', (object,), {})
-"""Base class for the :py:class:`waflib.Context.Context` classes"""
+"""Base class for all :py:class:`waflib.Context.Context` classes"""
 
 class Context(ctx):
 	"""
@@ -138,7 +126,7 @@ class Context(ctx):
 		def foo(ctx):
 			print(ctx.__class__.__name__) # waflib.Context.Context
 
-	Subclasses must define the attribute 'cmd':
+	Subclasses must define the class attributes 'cmd' and 'fun':
 
 	:param cmd: command to execute as in ``waf cmd``
 	:type cmd: string
@@ -156,19 +144,18 @@ class Context(ctx):
 
 	tools = {}
 	"""
-	A cache for modules (wscript files) read by :py:meth:`Context.Context.load`
+	A module cache for wscript files; see :py:meth:`Context.Context.load`
 	"""
 
 	def __init__(self, **kw):
 		try:
 			rd = kw['run_dir']
 		except KeyError:
-			global run_dir
 			rd = run_dir
 
 		# binds the context to the nodes in use to avoid a context singleton
-		self.node_class = type("Nod3", (waflib.Node.Node,), {})
-		self.node_class.__module__ = "waflib.Node"
+		self.node_class = type('Nod3', (waflib.Node.Node,), {})
+		self.node_class.__module__ = 'waflib.Node'
 		self.node_class.ctx = self
 
 		self.root = self.node_class('', None)
@@ -179,18 +166,9 @@ class Context(ctx):
 		self.exec_dict = {'ctx':self, 'conf':self, 'bld':self, 'opt':self}
 		self.logger = None
 
-	def __hash__(self):
-		"""
-		Return a hash value for storing context objects in dicts or sets. The value is not persistent.
-
-		:return: hash value
-		:rtype: int
-		"""
-		return id(self)
-
 	def finalize(self):
 		"""
-		Use to free resources such as open files potentially held by the logger
+		Called to free resources such as logger files
 		"""
 		try:
 			logger = self.logger
@@ -202,11 +180,11 @@ class Context(ctx):
 
 	def load(self, tool_list, *k, **kw):
 		"""
-		Load a Waf tool as a module, and try calling the function named :py:const:`waflib.Context.Context.fun` from it.
-		A ``tooldir`` value may be provided as a list of module paths.
+		Loads a Waf tool as a module, and try calling the function named :py:const:`waflib.Context.Context.fun`
+		from it.  A ``tooldir`` argument may be provided as a list of module paths.
 
+		:param tool_list: list of Waf tool names to load
 		:type tool_list: list of string or space-separated string
-		:param tool_list: list of Waf tools to use
 		"""
 		tools = Utils.to_list(tool_list)
 		path = Utils.to_list(kw.get('tooldir', ''))
@@ -220,15 +198,16 @@ class Context(ctx):
 
 	def execute(self):
 		"""
-		Execute the command. Redefine this method in subclasses.
+		Here, it calls the function name in the top-level wscript file. Most subclasses
+		redefine this method to provide additional functionality.
 		"""
-		global g_module
 		self.recurse([os.path.dirname(g_module.root_path)])
 
 	def pre_recurse(self, node):
 		"""
-		Method executed immediately before a folder is read by :py:meth:`waflib.Context.Context.recurse`. The node given is set
-		as an attribute ``self.cur_script``, and as the current path ``self.path``
+		Method executed immediately before a folder is read by :py:meth:`waflib.Context.Context.recurse`.
+		The current script is bound as a Node object on ``self.cur_script``, and the current path
+		is bound to ``self.path``
 
 		:param node: script
 		:type node: :py:class:`waflib.Node.Node`
@@ -240,7 +219,7 @@ class Context(ctx):
 
 	def post_recurse(self, node):
 		"""
-		Restore ``self.cur_script`` and ``self.path`` right after :py:meth:`waflib.Context.Context.recurse` terminates.
+		Restores ``self.cur_script`` and ``self.path`` right after :py:meth:`waflib.Context.Context.recurse` terminates.
 
 		:param node: script
 		:type node: :py:class:`waflib.Node.Node`
@@ -251,10 +230,13 @@ class Context(ctx):
 
 	def recurse(self, dirs, name=None, mandatory=True, once=True, encoding=None):
 		"""
-		Run user code from the supplied list of directories.
+		Runs user-provided functions from the supplied list of directories.
 		The directories can be either absolute, or relative to the directory
-		of the wscript file. The methods :py:meth:`waflib.Context.Context.pre_recurse` and :py:meth:`waflib.Context.Context.post_recurse`
-		are called immediately before and after a script has been executed.
+		of the wscript file
+
+		The methods :py:meth:`waflib.Context.Context.pre_recurse` and
+		:py:meth:`waflib.Context.Context.post_recurse` are called immediately before
+		and after a script has been executed.
 
 		:param dirs: List of directories to visit
 		:type dirs: list of string or space-separated string
@@ -300,7 +282,7 @@ class Context(ctx):
 						if not user_function:
 							if not mandatory:
 								continue
-							raise Errors.WafError('No function %s defined in %s' % (name or self.fun, node.abspath()))
+							raise Errors.WafError('No function %r defined in %s' % (name or self.fun, node.abspath()))
 						user_function(self)
 					finally:
 						self.post_recurse(node)
@@ -313,25 +295,39 @@ class Context(ctx):
 						raise Errors.WafError('Cannot read the folder %r' % d)
 					raise Errors.WafError('No wscript file in directory %s' % d)
 
+	def log_command(self, cmd, kw):
+		if Logs.verbose:
+			fmt = os.environ.get('WAF_CMD_FORMAT')
+			if fmt == 'string':
+				if not isinstance(cmd, str):
+					cmd = Utils.shell_escape(cmd)
+			Logs.debug('runner: %r', cmd)
+			Logs.debug('runner_env: kw=%s', kw)
+
 	def exec_command(self, cmd, **kw):
 		"""
-		Execute a command and return the exit status. If the context has the attribute 'log',
-		capture and log the process stderr/stdout for logging purposes::
+		Runs an external process and returns the exit status::
 
 			def run(tsk):
 				ret = tsk.generator.bld.exec_command('touch foo.txt')
 				return ret
 
-		This method captures the standard/error outputs (Issue 1101), but it does not return the values
-		unlike :py:meth:`waflib.Context.Context.cmd_and_log`
+		If the context has the attribute 'log', then captures and logs the process stderr/stdout.
+		Unlike :py:meth:`waflib.Context.Context.cmd_and_log`, this method does not return the
+		stdout/stderr values captured.
 
 		:param cmd: command argument for subprocess.Popen
+		:type cmd: string or list
 		:param kw: keyword arguments for subprocess.Popen. The parameters input/timeout will be passed to wait/communicate.
+		:type kw: dict
+		:returns: process exit status
+		:rtype: integer
+		:raises: :py:class:`waflib.Errors.WafError` if an invalid executable is specified for a non-shell process
+		:raises: :py:class:`waflib.Errors.WafError` in case of execution failure
 		"""
 		subprocess = Utils.subprocess
 		kw['shell'] = isinstance(cmd, str)
-		Logs.debug('runner: %r' % (cmd,))
-		Logs.debug('runner_env: kw=%s' % kw)
+		self.log_command(cmd, kw)
 
 		if self.logger:
 			self.logger.info(cmd)
@@ -342,40 +338,42 @@ class Context(ctx):
 			kw['stderr'] = subprocess.PIPE
 
 		if Logs.verbose and not kw['shell'] and not Utils.check_exe(cmd[0]):
-			raise Errors.WafError("Program %s not found!" % cmd[0])
+			raise Errors.WafError('Program %s not found!' % cmd[0])
 
-		wargs = {}
+		cargs = {}
 		if 'timeout' in kw:
-			if kw['timeout'] is not None:
-				wargs['timeout'] = kw['timeout']
+			if sys.hexversion >= 0x3030000:
+				cargs['timeout'] = kw['timeout']
+				if not 'start_new_session' in kw:
+					kw['start_new_session'] = True
 			del kw['timeout']
 		if 'input' in kw:
 			if kw['input']:
-				wargs['input'] = kw['input']
+				cargs['input'] = kw['input']
 				kw['stdin'] = subprocess.PIPE
 			del kw['input']
 
+		if 'cwd' in kw:
+			if not isinstance(kw['cwd'], str):
+				kw['cwd'] = kw['cwd'].abspath()
+
+		encoding = kw.pop('decode_as', default_encoding)
+
 		try:
-			if kw['stdout'] or kw['stderr']:
-				p = subprocess.Popen(cmd, **kw)
-				(out, err) = p.communicate(**wargs)
-				ret = p.returncode
-			else:
-				out, err = (None, None)
-				ret = subprocess.Popen(cmd, **kw).wait(**wargs)
+			ret, out, err = Utils.run_process(cmd, kw, cargs)
 		except Exception as e:
 			raise Errors.WafError('Execution failure: %s' % str(e), ex=e)
 
 		if out:
 			if not isinstance(out, str):
-				out = out.decode(sys.stdout.encoding or 'iso8859-1')
+				out = out.decode(encoding, errors='replace')
 			if self.logger:
-				self.logger.debug('out: %s' % out)
+				self.logger.debug('out: %s', out)
 			else:
 				Logs.info(out, extra={'stream':sys.stdout, 'c1': ''})
 		if err:
 			if not isinstance(err, str):
-				err = err.decode(sys.stdout.encoding or 'iso8859-1')
+				err = err.decode(encoding, errors='replace')
 			if self.logger:
 				self.logger.error('err: %s' % err)
 			else:
@@ -385,9 +383,9 @@ class Context(ctx):
 
 	def cmd_and_log(self, cmd, **kw):
 		"""
-		Execute a command and return stdout/stderr if the execution is successful.
+		Executes a process and returns stdout/stderr if the execution is successful.
 		An exception is thrown when the exit status is non-0. In that case, both stderr and stdout
-		will be bound to the WafError object::
+		will be bound to the WafError object (configuration tests)::
 
 			def configure(conf):
 				out = conf.cmd_and_log(['echo', 'hello'], output=waflib.Context.STDOUT, quiet=waflib.Context.BOTH)
@@ -395,65 +393,69 @@ class Context(ctx):
 				(out, err) = conf.cmd_and_log(cmd, input='\\n'.encode(), output=waflib.Context.STDOUT)
 				try:
 					conf.cmd_and_log(['which', 'someapp'], output=waflib.Context.BOTH)
-				except Exception as e:
+				except Errors.WafError as e:
 					print(e.stdout, e.stderr)
 
 		:param cmd: args for subprocess.Popen
+		:type cmd: list or string
 		:param kw: keyword arguments for subprocess.Popen. The parameters input/timeout will be passed to wait/communicate.
+		:type kw: dict
+		:returns: a tuple containing the contents of stdout and stderr
+		:rtype: string
+		:raises: :py:class:`waflib.Errors.WafError` if an invalid executable is specified for a non-shell process
+		:raises: :py:class:`waflib.Errors.WafError` in case of execution failure; stdout/stderr/returncode are bound to the exception object
 		"""
 		subprocess = Utils.subprocess
 		kw['shell'] = isinstance(cmd, str)
-		Logs.debug('runner: %r' % (cmd,))
-
-		if 'quiet' in kw:
-			quiet = kw['quiet']
-			del kw['quiet']
-		else:
-			quiet = None
+		self.log_command(cmd, kw)
 
-		if 'output' in kw:
-			to_ret = kw['output']
-			del kw['output']
-		else:
-			to_ret = STDOUT
+		quiet = kw.pop('quiet', None)
+		to_ret = kw.pop('output', STDOUT)
 
 		if Logs.verbose and not kw['shell'] and not Utils.check_exe(cmd[0]):
-			raise Errors.WafError("Program %s not found!" % cmd[0])
+			raise Errors.WafError('Program %r not found!' % cmd[0])
 
 		kw['stdout'] = kw['stderr'] = subprocess.PIPE
 		if quiet is None:
 			self.to_log(cmd)
 
-		wargs = {}
+		cargs = {}
 		if 'timeout' in kw:
-			if kw['timeout'] is not None:
-				wargs['timeout'] = kw['timeout']
+			if sys.hexversion >= 0x3030000:
+				cargs['timeout'] = kw['timeout']
+				if not 'start_new_session' in kw:
+					kw['start_new_session'] = True
 			del kw['timeout']
 		if 'input' in kw:
 			if kw['input']:
-				wargs['input'] = kw['input']
+				cargs['input'] = kw['input']
 				kw['stdin'] = subprocess.PIPE
 			del kw['input']
 
+		if 'cwd' in kw:
+			if not isinstance(kw['cwd'], str):
+				kw['cwd'] = kw['cwd'].abspath()
+
+		encoding = kw.pop('decode_as', default_encoding)
+
 		try:
-			p = subprocess.Popen(cmd, **kw)
-			(out, err) = p.communicate(**wargs)
+			ret, out, err = Utils.run_process(cmd, kw, cargs)
 		except Exception as e:
 			raise Errors.WafError('Execution failure: %s' % str(e), ex=e)
 
 		if not isinstance(out, str):
-			out = out.decode(sys.stdout.encoding or 'iso8859-1')
+			out = out.decode(encoding, errors='replace')
 		if not isinstance(err, str):
-			err = err.decode(sys.stdout.encoding or 'iso8859-1')
+			err = err.decode(encoding, errors='replace')
 
 		if out and quiet != STDOUT and quiet != BOTH:
 			self.to_log('out: %s' % out)
 		if err and quiet != STDERR and quiet != BOTH:
 			self.to_log('err: %s' % err)
 
-		if p.returncode:
-			e = Errors.WafError('Command %r returned %r' % (cmd, p.returncode))
-			e.returncode = p.returncode
+		if ret:
+			e = Errors.WafError('Command %r returned %r' % (cmd, ret))
+			e.returncode = ret
 			e.stderr = err
 			e.stdout = out
 			raise e
@@ -466,7 +468,8 @@ class Context(ctx):
 
 	def fatal(self, msg, ex=None):
 		"""
-		Raise a configuration error to interrupt the execution immediately::
+		Prints an error message in red and stops command execution; this is
+		usually used in the configuration section::
 
 			def configure(conf):
 				conf.fatal('a requirement is missing')
@@ -475,24 +478,31 @@ class Context(ctx):
 		:type msg: string
 		:param ex: optional exception object
 		:type ex: exception
+		:raises: :py:class:`waflib.Errors.ConfigurationError`
 		"""
 		if self.logger:
 			self.logger.info('from %s: %s' % (self.path.abspath(), msg))
 		try:
-			msg = '%s\n(complete log in %s)' % (msg, self.logger.handlers[0].baseFilename)
-		except Exception:
+			logfile = self.logger.handlers[0].baseFilename
+		except AttributeError:
 			pass
+		else:
+			if os.environ.get('WAF_PRINT_FAILURE_LOG'):
+				# see #1930
+				msg = 'Log from (%s):\n%s\n' % (logfile, Utils.readf(logfile))
+			else:
+				msg = '%s\n(complete log in %s)' % (msg, logfile)
 		raise self.errors.ConfigurationError(msg, ex=ex)
 
 	def to_log(self, msg):
 		"""
-		Log some information to the logger (if present), or to stderr. If the message is empty,
-		it is not printed::
+		Logs information to the logger (if present), or to stderr.
+		Empty messages are not printed::
 
 			def build(bld):
 				bld.to_log('starting the build')
 
-		When in doubt, override this method, or provide a logger on the context class.
+		Provide a logger on the context class or override this method if necessary.
 
 		:param msg: message
 		:type msg: string
@@ -508,7 +518,7 @@ class Context(ctx):
 
 	def msg(self, *k, **kw):
 		"""
-		Print a configuration message of the form ``msg: result``.
+		Prints a configuration message of the form ``msg: result``.
 		The second part of the message will be in colors. The output
 		can be disabled easly by setting ``in_msg`` to a positive value::
 
@@ -536,7 +546,7 @@ class Context(ctx):
 		except KeyError:
 			result = k[1]
 
-		color = kw.get('color', None)
+		color = kw.get('color')
 		if not isinstance(color, str):
 			color = result and 'GREEN' or 'YELLOW'
 
@@ -544,12 +554,12 @@ class Context(ctx):
 
 	def start_msg(self, *k, **kw):
 		"""
-		Print the beginning of a 'Checking for xxx' message. See :py:meth:`waflib.Context.Context.msg`
+		Prints the beginning of a 'Checking for xxx' message. See :py:meth:`waflib.Context.Context.msg`
 		"""
-		if kw.get('quiet', None):
+		if kw.get('quiet'):
 			return
 
-		msg = kw.get('msg', None) or k[0]
+		msg = kw.get('msg') or k[0]
 		try:
 			if self.in_msg:
 				self.in_msg += 1
@@ -567,19 +577,19 @@ class Context(ctx):
 		Logs.pprint('NORMAL', "%s :" % msg.ljust(self.line_just), sep='')
 
 	def end_msg(self, *k, **kw):
-		"""Print the end of a 'Checking for' message. See :py:meth:`waflib.Context.Context.msg`"""
-		if kw.get('quiet', None):
+		"""Prints the end of a 'Checking for' message. See :py:meth:`waflib.Context.Context.msg`"""
+		if kw.get('quiet'):
 			return
 		self.in_msg -= 1
 		if self.in_msg:
 			return
 
-		result = kw.get('result', None) or k[0]
+		result = kw.get('result') or k[0]
 
 		defcolor = 'GREEN'
-		if result == True:
+		if result is True:
 			msg = 'ok'
-		elif result == False:
+		elif not result:
 			msg = 'not found'
 			defcolor = 'YELLOW'
 		else:
@@ -597,7 +607,17 @@ class Context(ctx):
 		Logs.pprint(color, msg)
 
 	def load_special_tools(self, var, ban=[]):
-		global waf_dir
+		"""
+		Loads third-party extensions modules for certain programming languages
+		by trying to list certain files in the extras/ directory. This method
+		is typically called once for a programming language group, see for
+		example :py:mod:`waflib.Tools.compiler_c`
+
+		:param var: glob expression, for example 'cxx\_\*.py'
+		:type var: string
+		:param ban: list of exact file names to exclude
+		:type ban: list of string
+		"""
 		if os.path.isdir(waf_dir):
 			lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
 			for x in lst:
@@ -608,12 +628,12 @@ class Context(ctx):
 			waflibs = PyZipFile(waf_dir)
 			lst = waflibs.namelist()
 			for x in lst:
-				if not re.match("waflib/extras/%s" % var.replace("*", ".*"), var):
+				if not re.match('waflib/extras/%s' % var.replace('*', '.*'), var):
 					continue
 				f = os.path.basename(x)
 				doban = False
 				for b in ban:
-					r = b.replace("*", ".*")
+					r = b.replace('*', '.*')
 					if re.match(r, f):
 						doban = True
 				if not doban:
@@ -622,13 +642,13 @@ class Context(ctx):
 
 cache_modules = {}
 """
-Dictionary holding already loaded modules, keyed by their absolute path.
+Dictionary holding already loaded modules (wscript), indexed by their absolute path.
 The modules are added automatically by :py:func:`waflib.Context.load_module`
 """
 
 def load_module(path, encoding=None):
 	"""
-	Load a source file as a python module.
+	Loads a wscript file as a python module. This method caches results in :py:attr:`waflib.Context.cache_modules`
 
 	:param path: file path
 	:type path: string
@@ -648,17 +668,17 @@ def load_module(path, encoding=None):
 
 	module_dir = os.path.dirname(path)
 	sys.path.insert(0, module_dir)
-
-	try    : exec(compile(code, path, 'exec'), module.__dict__)
-	finally: sys.path.remove(module_dir)
+	try:
+		exec(compile(code, path, 'exec'), module.__dict__)
+	finally:
+		sys.path.remove(module_dir)
 
 	cache_modules[path] = module
-
 	return module
 
 def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True):
 	"""
-	Import a Waf tool (python module), and store it in the dict :py:const:`waflib.Context.Context.tools`
+	Importx a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools`
 
 	:type  tool: string
 	:param tool: Name of the tool
@@ -672,14 +692,18 @@ def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True):
 	else:
 		tool = tool.replace('++', 'xx')
 
-	origSysPath = sys.path
-	if not with_sys_path: sys.path = []
+	if not with_sys_path:
+		back_path = sys.path
+		sys.path = []
 	try:
 		if tooldir:
 			assert isinstance(tooldir, list)
 			sys.path = tooldir + sys.path
 			try:
 				__import__(tool)
+			except ImportError as e:
+				e.waf_sys_path = list(sys.path)
+				raise
 			finally:
 				for d in tooldir:
 					sys.path.remove(d)
@@ -687,7 +711,8 @@ def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True):
 			Context.tools[tool] = ret
 			return ret
 		else:
-			if not with_sys_path: sys.path.insert(0, waf_dir)
+			if not with_sys_path:
+				sys.path.insert(0, waf_dir)
 			try:
 				for x in ('waflib.Tools.%s', 'waflib.extras.%s', 'waflib.%s', '%s'):
 					try:
@@ -695,13 +720,18 @@ def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True):
 						break
 					except ImportError:
 						x = None
-				if x is None: # raise an exception
+				else: # raise an exception
 					__import__(tool)
+			except ImportError as e:
+				e.waf_sys_path = list(sys.path)
+				raise
 			finally:
-				if not with_sys_path: sys.path.remove(waf_dir)
+				if not with_sys_path:
+					sys.path.remove(waf_dir)
 			ret = sys.modules[x % tool]
 			Context.tools[tool] = ret
 			return ret
 	finally:
-		if not with_sys_path: sys.path += origSysPath
+		if not with_sys_path:
+			sys.path += back_path
 
diff --git a/waflib/Errors.py b/waflib/Errors.py
index 104f7d82..bf75c1b6 100644
--- a/waflib/Errors.py
+++ b/waflib/Errors.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2010 (ita)
+# Thomas Nagy, 2010-2018 (ita)
 
 """
 Exceptions used in the Waf code
@@ -17,6 +17,7 @@ class WafError(Exception):
 		:param ex: exception causing this error (optional)
 		:type ex: exception
 		"""
+		Exception.__init__(self)
 		self.msg = msg
 		assert not isinstance(msg, Exception)
 
@@ -35,9 +36,7 @@ class WafError(Exception):
 		return str(self.msg)
 
 class BuildError(WafError):
-	"""
-	Errors raised during the build and install phases
-	"""
+	"""Error raised during the build and install phases"""
 	def __init__(self, error_tasks=[]):
 		"""
 		:param error_tasks: tasks that could not complete normally
@@ -47,24 +46,23 @@ class BuildError(WafError):
 		WafError.__init__(self, self.format_error())
 
 	def format_error(self):
-		"""format the error messages from the tasks that failed"""
+		"""Formats the error messages from the tasks that failed"""
 		lst = ['Build failed']
 		for tsk in self.tasks:
 			txt = tsk.format_error()
-			if txt: lst.append(txt)
+			if txt:
+				lst.append(txt)
 		return '\n'.join(lst)
 
 class ConfigurationError(WafError):
-	"""
-	Configuration exception raised in particular by :py:meth:`waflib.Context.Context.fatal`
-	"""
+	"""Configuration exception raised in particular by :py:meth:`waflib.Context.Context.fatal`"""
 	pass
 
 class TaskRescan(WafError):
-	"""task-specific exception type, trigger a signature recomputation"""
+	"""Task-specific exception type signalling required signature recalculations"""
 	pass
 
 class TaskNotReady(WafError):
-	"""task-specific exception type, raised when the task signature cannot be computed"""
+	"""Task-specific exception type signalling that task signatures cannot be computed"""
 	pass
 
diff --git a/waflib/Logs.py b/waflib/Logs.py
index b2880535..2a475169 100644
--- a/waflib/Logs.py
+++ b/waflib/Logs.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 logging, colors, terminal width and pretty-print
@@ -23,8 +23,15 @@ import logging
 LOG_FORMAT = os.environ.get('WAF_LOG_FORMAT', '%(asctime)s %(c1)s%(zone)s%(c2)s %(message)s')
 HOUR_FORMAT = os.environ.get('WAF_HOUR_FORMAT', '%H:%M:%S')
 
-zones = ''
+zones = []
+"""
+See :py:class:`waflib.Logs.log_filter`
+"""
+
 verbose = 0
+"""
+Global verbosity level, see :py:func:`waflib.Logs.debug` and :py:func:`waflib.Logs.error`
+"""
 
 colors_lst = {
 'USE' : True,
@@ -49,6 +56,15 @@ except NameError:
 	unicode = None
 
 def enable_colors(use):
+	"""
+	If *1* is given, then the system will perform a few verifications
+	before enabling colors, such as checking whether the interpreter
+	is running in a terminal. A value of zero will disable colors,
+	and a value above *1* will force colors.
+
+	:param use: whether to enable colors or not
+	:type use: integer
+	"""
 	if use == 1:
 		if not (sys.stderr.isatty() or sys.stdout.isatty()):
 			use = 0
@@ -74,15 +90,23 @@ except AttributeError:
 		return 80
 
 get_term_cols.__doc__ = """
-	Get the console width in characters.
+	Returns the console width in characters.
 
 	:return: the number of characters per line
 	:rtype: int
 	"""
 
 def get_color(cl):
-	if not colors_lst['USE']: return ''
-	return colors_lst.get(cl, '')
+	"""
+	Returns the ansi sequence corresponding to the given color name.
+	An empty string is returned when coloring is globally disabled.
+
+	:param cl: color name in capital letters
+	:type cl: string
+	"""
+	if colors_lst['USE']:
+		return colors_lst.get(cl, '')
+	return ''
 
 class color_dict(object):
 	"""attribute-based color access, eg: colors.PINK"""
@@ -96,7 +120,7 @@ colors = color_dict()
 re_log = re.compile(r'(\w+): (.*)', re.M)
 class log_filter(logging.Filter):
 	"""
-	The waf logs are of the form 'name: message', and can be filtered by 'waf --zones=name'.
+	Waf logs are of the form 'name: message', and can be filtered by 'waf --zones=name'.
 	For example, the following::
 
 		from waflib import Logs
@@ -106,17 +130,14 @@ class log_filter(logging.Filter):
 
 		$ waf --zones=test
 	"""
-	def __init__(self, name=None):
-		pass
+	def __init__(self, name=''):
+		logging.Filter.__init__(self, name)
 
 	def filter(self, rec):
 		"""
-		filter a record, adding the colors automatically
+		Filters log records by zone and by logging level
 
-		* error: red
-		* warning: yellow
-
-		:param rec: message to record
+		:param rec: log entry
 		"""
 		rec.zone = rec.module
 		if rec.levelno >= logging.INFO:
@@ -136,6 +157,9 @@ class log_filter(logging.Filter):
 class log_handler(logging.StreamHandler):
 	"""Dispatches messages to stderr/stdout depending on the severity level"""
 	def emit(self, record):
+		"""
+		Delegates the functionality to :py:meth:`waflib.Log.log_handler.emit_override`
+		"""
 		# default implementation
 		try:
 			try:
@@ -153,6 +177,9 @@ class log_handler(logging.StreamHandler):
 			self.handleError(record)
 
 	def emit_override(self, record, **kw):
+		"""
+		Writes the log record to the desired stream (stderr/stdout)
+		"""
 		self.terminator = getattr(record, 'terminator', '\n')
 		stream = self.stream
 		if unicode:
@@ -179,7 +206,10 @@ class formatter(logging.Formatter):
 		logging.Formatter.__init__(self, LOG_FORMAT, HOUR_FORMAT)
 
 	def format(self, rec):
-		"""Messages in warning, error or info mode are displayed in color by default"""
+		"""
+		Formats records and adds colors as needed. The records do not get
+		a leading hour format if the logging level is above *INFO*.
+		"""
 		try:
 			msg = rec.msg.decode('utf-8')
 		except Exception:
@@ -204,7 +234,10 @@ class formatter(logging.Formatter):
 			# and other terminal commands
 			msg = re.sub(r'\r(?!\n)|\x1B\[(K|.*?(m|h|l))', '', msg)
 
-		if rec.levelno >= logging.INFO: # ??
+		if rec.levelno >= logging.INFO:
+			# the goal of this is to format without the leading "Logs, hour" prefix
+			if rec.args:
+				return msg % rec.args
 			return msg
 
 		rec.msg = msg
@@ -217,19 +250,17 @@ log = None
 
 def debug(*k, **kw):
 	"""
-	Wrap logging.debug, the output is filtered for performance reasons
+	Wraps logging.debug and discards messages if the verbosity level :py:attr:`waflib.Logs.verbose` ≤ 0
 	"""
 	if verbose:
 		k = list(k)
 		k[0] = k[0].replace('\n', ' ')
-		global log
 		log.debug(*k, **kw)
 
 def error(*k, **kw):
 	"""
-	Wrap logging.errors, display the origin of the message when '-vv' is set
+	Wrap logging.errors, adds the stack trace when the verbosity level :py:attr:`waflib.Logs.verbose` ≥ 2
 	"""
-	global log
 	log.error(*k, **kw)
 	if verbose > 2:
 		st = traceback.extract_stack()
@@ -237,28 +268,27 @@ def error(*k, **kw):
 			st = st[:-1]
 			buf = []
 			for filename, lineno, name, line in st:
-				buf.append('  File "%s", line %d, in %s' % (filename, lineno, name))
+				buf.append('  File %r, line %d, in %s' % (filename, lineno, name))
 				if line:
 					buf.append('	%s' % line.strip())
-			if buf: log.error("\n".join(buf))
+			if buf:
+				log.error('\n'.join(buf))
 
 def warn(*k, **kw):
 	"""
-	Wrap logging.warn
+	Wraps logging.warn
 	"""
-	global log
 	log.warn(*k, **kw)
 
 def info(*k, **kw):
 	"""
-	Wrap logging.info
+	Wraps logging.info
 	"""
-	global log
 	log.info(*k, **kw)
 
 def init_log():
 	"""
-	Initialize the loggers globally
+	Initializes the logger :py:attr:`waflib.Logs.log`
 	"""
 	global log
 	log = logging.getLogger('waflib')
@@ -272,7 +302,7 @@ def init_log():
 
 def make_logger(path, name):
 	"""
-	Create a simple logger, which is often used to redirect the context command output::
+	Creates a simple logger, which is often used to redirect the context command output::
 
 		from waflib import Logs
 		bld.logger = Logs.make_logger('test.log', 'build')
@@ -292,7 +322,11 @@ def make_logger(path, name):
 	:type name: string
 	"""
 	logger = logging.getLogger(name)
-	hdlr = logging.FileHandler(path, 'w')
+	if sys.hexversion > 0x3000000:
+		encoding = sys.stdout.encoding
+	else:
+		encoding = None
+	hdlr = logging.FileHandler(path, 'w', encoding=encoding)
 	formatter = logging.Formatter('%(message)s')
 	hdlr.setFormatter(formatter)
 	logger.addHandler(hdlr)
@@ -301,7 +335,7 @@ def make_logger(path, name):
 
 def make_mem_logger(name, to_log, size=8192):
 	"""
-	Create a memory logger to avoid writing concurrently to the main logger
+	Creates a memory logger to avoid writing concurrently to the main logger
 	"""
 	from logging.handlers import MemoryHandler
 	logger = logging.getLogger(name)
@@ -315,7 +349,7 @@ def make_mem_logger(name, to_log, size=8192):
 
 def free_logger(logger):
 	"""
-	Free the resources held by the loggers created through make_logger or make_mem_logger.
+	Frees the resources held by the loggers created through make_logger or make_mem_logger.
 	This is used for file cleanup and for handler removal (logger objects are re-used).
 	"""
 	try:
@@ -327,7 +361,7 @@ def free_logger(logger):
 
 def pprint(col, msg, label='', sep='\n'):
 	"""
-	Print messages in color immediately on stderr::
+	Prints messages in color immediately on stderr::
 
 		from waflib import Logs
 		Logs.pprint('RED', 'Something bad just happened')
@@ -341,5 +375,5 @@ def pprint(col, msg, label='', sep='\n'):
 	:param sep: a string to append at the end (line separator)
 	:type sep: string
 	"""
-	info("%s%s%s %s" % (colors(col), msg, colors.NORMAL, label), extra={'terminator':sep})
+	info('%s%s%s %s', colors(col), msg, colors.NORMAL, label, extra={'terminator':sep})
 
diff --git a/waflib/Node.py b/waflib/Node.py
index fb3cd273..4ac1ea8a 100644
--- a/waflib/Node.py
+++ b/waflib/Node.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
-Node: filesystem structure, contains lists of nodes
+Node: filesystem structure
 
 #. Each file/folder is represented by exactly one node.
 
@@ -11,13 +11,14 @@ Node: filesystem structure, contains lists of nodes
    Unused class members can increase the `.wafpickle` file size sensibly.
 
 #. Node objects should never be created directly, use
-   the methods :py:func:`Node.make_node` or :py:func:`Node.find_node`
+   the methods :py:func:`Node.make_node` or :py:func:`Node.find_node` for the low-level operations
 
-#. The methods :py:func:`Node.find_resource`, :py:func:`Node.find_dir` :py:func:`Node.find_or_declare` should be
+#. The methods :py:func:`Node.find_resource`, :py:func:`Node.find_dir` :py:func:`Node.find_or_declare` must be
    used when a build context is present
 
-#. Each instance of :py:class:`waflib.Context.Context` has a unique :py:class:`Node` subclass.
-   (:py:class:`waflib.Node.Nod3`, see the :py:class:`waflib.Context.Context` initializer). A reference to the context owning a node is held as self.ctx
+#. Each instance of :py:class:`waflib.Context.Context` has a unique :py:class:`Node` subclass required for serialization.
+   (:py:class:`waflib.Node.Nod3`, see the :py:class:`waflib.Context.Context` initializer). A reference to the context
+   owning a node is held as *self.ctx*
 """
 
 import os, re, sys, shutil
@@ -29,6 +30,7 @@ exclude_regs = '''
 **/.#*
 **/%*%
 **/._*
+**/*.swp
 **/CVS
 **/CVS/**
 **/.cvsignore
@@ -59,99 +61,148 @@ Ant patterns for files and folders to exclude while doing the
 recursive traversal in :py:meth:`waflib.Node.Node.ant_glob`
 """
 
-# TODO remove in waf 1.9
-split_path = Utils.split_path
-split_path_unix = Utils.split_path_unix
-split_path_cygwin = Utils.split_path_cygwin
-split_path_win32 = Utils.split_path_win32
+def ant_matcher(s, ignorecase):
+	reflags = re.I if ignorecase else 0
+	ret = []
+	for x in Utils.to_list(s):
+		x = x.replace('\\', '/').replace('//', '/')
+		if x.endswith('/'):
+			x += '**'
+		accu = []
+		for k in x.split('/'):
+			if k == '**':
+				accu.append(k)
+			else:
+				k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+')
+				k = '^%s$' % k
+				try:
+					exp = re.compile(k, flags=reflags)
+				except Exception as e:
+					raise Errors.WafError('Invalid pattern: %s' % k, e)
+				else:
+					accu.append(exp)
+		ret.append(accu)
+	return ret
+
+def ant_sub_filter(name, nn):
+	ret = []
+	for lst in nn:
+		if not lst:
+			pass
+		elif lst[0] == '**':
+			ret.append(lst)
+			if len(lst) > 1:
+				if lst[1].match(name):
+					ret.append(lst[2:])
+			else:
+				ret.append([])
+		elif lst[0].match(name):
+			ret.append(lst[1:])
+	return ret
+
+def ant_sub_matcher(name, pats):
+	nacc = ant_sub_filter(name, pats[0])
+	nrej = ant_sub_filter(name, pats[1])
+	if [] in nrej:
+		nacc = []
+	return [nacc, nrej]
 
 class Node(object):
 	"""
-	This class is organized in two parts
+	This class is organized in two parts:
 
 	* The basic methods meant for filesystem access (compute paths, create folders, etc)
 	* The methods bound to a :py:class:`waflib.Build.BuildContext` (require ``bld.srcnode`` and ``bld.bldnode``)
-
-	The Node objects are not thread safe in any way.
 	"""
+
 	dict_class = dict
-	__slots__ = ('name', 'sig', 'children', 'parent', 'cache_abspath', 'cache_isdir', 'cache_sig')
+	"""
+	Subclasses can provide a dict class to enable case insensitivity for example.
+	"""
+
+	__slots__ = ('name', 'parent', 'children', 'cache_abspath', 'cache_isdir')
 	def __init__(self, name, parent):
+		"""
+		.. note:: Use :py:func:`Node.make_node` or :py:func:`Node.find_node` instead of calling this constructor
+		"""
 		self.name = name
 		self.parent = parent
-
 		if parent:
 			if name in parent.children:
 				raise Errors.WafError('node %s exists in the parent files %r already' % (name, parent))
 			parent.children[name] = self
 
 	def __setstate__(self, data):
-		"Deserializes from data"
+		"Deserializes node information, used for persistence"
 		self.name = data[0]
 		self.parent = data[1]
 		if data[2] is not None:
 			# Issue 1480
 			self.children = self.dict_class(data[2])
-		if data[3] is not None:
-			self.sig = data[3]
 
 	def __getstate__(self):
-		"Serialize the node info"
-		return (self.name, self.parent, getattr(self, 'children', None), getattr(self, 'sig', None))
+		"Serializes node information, used for persistence"
+		return (self.name, self.parent, getattr(self, 'children', None))
 
 	def __str__(self):
-		"String representation (name), for debugging purposes"
-		return self.name
+		"""
+		String representation (abspath), for debugging purposes
 
-	def __repr__(self):
-		"String representation (abspath), for debugging purposes"
+		:rtype: string
+		"""
 		return self.abspath()
 
-	def __hash__(self):
-		"Node hash, used for storage in dicts. This hash is not persistent."
-		return id(self)
+	def __repr__(self):
+		"""
+		String representation (abspath), for debugging purposes
 
-	def __eq__(self, node):
-		"Node comparison, based on the IDs"
-		return id(self) == id(node)
+		:rtype: string
+		"""
+		return self.abspath()
 
 	def __copy__(self):
-		"Implemented to prevent nodes from being copied (raises an exception)"
+		"""
+		Provided to prevent nodes from being copied
+
+		:raises: :py:class:`waflib.Errors.WafError`
+		"""
 		raise Errors.WafError('nodes are not supposed to be copied')
 
-	def read(self, flags='r', encoding='ISO8859-1'):
+	def read(self, flags='r', encoding='latin-1'):
 		"""
-		Return the contents of the file represented by this node::
+		Reads and returns the contents of the file represented by this node, see :py:func:`waflib.Utils.readf`::
 
 			def build(bld):
 				bld.path.find_node('wscript').read()
 
-		:type  fname: string
-		:param fname: Path to file
-		:type  m: string
-		:param m: Open mode
-		:rtype: string
+		:param flags: Open mode
+		:type  flags: string
+		:param encoding: encoding value for Python3
+		:type encoding: string
+		:rtype: string or bytes
 		:return: File contents
 		"""
 		return Utils.readf(self.abspath(), flags, encoding)
 
-	def write(self, data, flags='w', encoding='ISO8859-1'):
+	def write(self, data, flags='w', encoding='latin-1'):
 		"""
-		Write some text to the physical file represented by this node::
+		Writes data to the file represented by this node, see :py:func:`waflib.Utils.writef`::
 
 			def build(bld):
 				bld.path.make_node('foo.txt').write('Hello, world!')
 
-		:type  data: string
 		:param data: data to write
-		:type  flags: string
+		:type  data: string
 		:param flags: Write mode
+		:type  flags: string
+		:param encoding: encoding value for Python3
+		:type encoding: string
 		"""
 		Utils.writef(self.abspath(), data, flags, encoding)
 
 	def read_json(self, convert=True, encoding='utf-8'):
 		"""
-		Read and parse the contents of this node as JSON::
+		Reads and parses the contents of this node as JSON (Python ≥ 2.6)::
 
 			def build(bld):
 				bld.path.find_node('abc.json').read_json()
@@ -190,7 +241,7 @@ class Node(object):
 
 	def write_json(self, data, pretty=True):
 		"""
-		Writes a python object as JSON to disk. Files are always written as UTF8 as per the JSON standard::
+		Writes a python object as JSON to disk (Python ≥ 2.6) as UTF-8 data (JSON standard)::
 
 			def build(bld):
 				bld.path.find_node('xyz.json').write_json(199)
@@ -212,40 +263,71 @@ class Node(object):
 		output = json.dumps(data, indent=indent, separators=separators, sort_keys=sort_keys) + newline
 		self.write(output, encoding='utf-8')
 
+	def exists(self):
+		"""
+		Returns whether the Node is present on the filesystem
+
+		:rtype: bool
+		"""
+		return os.path.exists(self.abspath())
+
+	def isdir(self):
+		"""
+		Returns whether the Node represents a folder
+
+		:rtype: bool
+		"""
+		return os.path.isdir(self.abspath())
+
 	def chmod(self, val):
 		"""
-		Change file/dir permissions::
+		Changes the file/dir permissions::
 
 			def build(bld):
 				bld.path.chmod(493) # 0755
 		"""
 		os.chmod(self.abspath(), val)
 
-	def delete(self):
-		"""Delete the file/folder, and remove this node from the tree. Do not use this object after calling this method."""
+	def delete(self, evict=True):
+		"""
+		Removes the file/folder from the filesystem (equivalent to `rm -rf`), and remove this object from the Node tree.
+		Do not use this object after calling this method.
+		"""
 		try:
 			try:
-				if hasattr(self, 'children'):
+				if os.path.isdir(self.abspath()):
 					shutil.rmtree(self.abspath())
 				else:
 					os.remove(self.abspath())
-			except OSError as e:
+			except OSError:
 				if os.path.exists(self.abspath()):
-					raise e
+					raise
 		finally:
-			self.evict()
+			if evict:
+				self.evict()
 
 	def evict(self):
-		"""Internal - called when a node is removed"""
+		"""
+		Removes this node from the Node tree
+		"""
 		del self.parent.children[self.name]
 
 	def suffix(self):
-		"""Return the file extension"""
+		"""
+		Returns the file rightmost extension, for example `a.b.c.d → .d`
+
+		:rtype: string
+		"""
 		k = max(0, self.name.rfind('.'))
 		return self.name[k:]
 
 	def height(self):
-		"""Depth in the folder hierarchy from the filesystem root or from all the file drives"""
+		"""
+		Returns the depth in the folder hierarchy from the filesystem root or from all the file drives
+
+		:returns: filesystem depth
+		:rtype: integer
+		"""
 		d = self
 		val = -1
 		while d:
@@ -254,17 +336,23 @@ class Node(object):
 		return val
 
 	def listdir(self):
-		"""List the folder contents"""
+		"""
+		Lists the folder contents
+
+		:returns: list of file/folder names ordered alphabetically
+		:rtype: list of string
+		"""
 		lst = Utils.listdir(self.abspath())
 		lst.sort()
 		return lst
 
 	def mkdir(self):
 		"""
-		Create a folder represented by this node, creating intermediate nodes as needed
-		An exception will be raised only when the folder cannot possibly exist there
+		Creates a folder represented by this node. Intermediate folders are created as needed.
+
+		:raises: :py:class:`waflib.Errors.WafError` when the folder is missing
 		"""
-		if getattr(self, 'cache_isdir', None):
+		if self.isdir():
 			return
 
 		try:
@@ -278,26 +366,31 @@ class Node(object):
 			except OSError:
 				pass
 
-			if not os.path.isdir(self.abspath()):
-				raise Errors.WafError('Could not create the directory %s' % self.abspath())
+			if not self.isdir():
+				raise Errors.WafError('Could not create the directory %r' % self)
 
 			try:
 				self.children
 			except AttributeError:
 				self.children = self.dict_class()
 
-		self.cache_isdir = True
-
 	def find_node(self, lst):
 		"""
-		Find a node on the file system (files or folders), create intermediate nodes as needed
+		Finds a node on the file system (files or folders), and creates the corresponding Node objects if it exists
 
-		:param lst: path
+		:param lst: relative path
 		:type lst: string or list of string
+		:returns: The corresponding Node object or None if no entry was found on the filesystem
+		:rtype: :py:class:´waflib.Node.Node´
 		"""
 
 		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
+
+		if lst and lst[0].startswith('\\\\') and not self.parent:
+			node = self.ctx.root.make_node(lst[0])
+			node.cache_isdir = True
+			return node.find_node(lst[1:])
 
 		cur = self
 		for x in lst:
@@ -318,38 +411,26 @@ class Node(object):
 
 			# optimistic: create the node first then look if it was correct to do so
 			cur = self.__class__(x, cur)
-			try:
-				os.stat(cur.abspath())
-			except OSError:
+			if not cur.exists():
 				cur.evict()
 				return None
 
-		ret = cur
-
-		try:
-			os.stat(ret.abspath())
-		except OSError:
-			ret.evict()
+		if not cur.exists():
+			cur.evict()
 			return None
 
-		try:
-			while not getattr(cur.parent, 'cache_isdir', None):
-				cur = cur.parent
-				cur.cache_isdir = True
-		except AttributeError:
-			pass
-
-		return ret
+		return cur
 
 	def make_node(self, lst):
 		"""
-		Find or create a node without looking on the filesystem
+		Returns or creates a Node object corresponding to the input path without considering the filesystem.
 
-		:param lst: path
+		:param lst: relative path
 		:type lst: string or list of string
+		:rtype: :py:class:´waflib.Node.Node´
 		"""
 		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
 
 		cur = self
 		for x in lst:
@@ -357,24 +438,27 @@ class Node(object):
 				cur = cur.parent or cur
 				continue
 
-			if getattr(cur, 'children', {}):
-				if x in cur.children:
-					cur = cur.children[x]
-					continue
-			else:
+			try:
+				cur = cur.children[x]
+			except AttributeError:
 				cur.children = self.dict_class()
+			except KeyError:
+				pass
+			else:
+				continue
 			cur = self.__class__(x, cur)
 		return cur
 
 	def search_node(self, lst):
 		"""
-		Search for a node without looking on the filesystem
+		Returns a Node previously defined in the data structure. The filesystem is not considered.
 
-		:param lst: path
+		:param lst: relative path
 		:type lst: string or list of string
+		:rtype: :py:class:´waflib.Node.Node´ or None if there is no entry in the Node datastructure
 		"""
 		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
 
 		cur = self
 		for x in lst:
@@ -398,8 +482,9 @@ class Node(object):
 
 		:param node: path to use as a reference
 		:type node: :py:class:`waflib.Node.Node`
+		:returns: a relative path or an absolute one if that is better
+		:rtype: string
 		"""
-
 		c1 = self
 		c2 = node
 
@@ -419,7 +504,7 @@ class Node(object):
 			c2 = c2.parent
 			c2h -= 1
 
-		while id(c1) != id(c2):
+		while not c1 is c2:
 			lst.append(c1.name)
 			up += 1
 
@@ -427,17 +512,17 @@ class Node(object):
 			c2 = c2.parent
 
 		if c1.parent:
-			for i in range(up):
-				lst.append('..')
+			lst.extend(['..'] * up)
+			lst.reverse()
+			return os.sep.join(lst) or '.'
 		else:
-			if lst and not Utils.is_win32:
-				lst.append('')
-		lst.reverse()
-		return os.sep.join(lst) or '.'
+			return self.abspath()
 
 	def abspath(self):
 		"""
-		Absolute path. A cache is kept in the context as ``cache_node_abspath``
+		Returns the absolute path. A cache is kept in the context as ``cache_node_abspath``
+
+		:rtype: string
 		"""
 		try:
 			return self.cache_abspath
@@ -471,7 +556,7 @@ class Node(object):
 
 	def is_child_of(self, node):
 		"""
-		Does this node belong to the subtree node?::
+		Returns whether the object belongs to a subtree of the input node::
 
 			def build(bld):
 				node = bld.path.find_node('wscript')
@@ -479,17 +564,18 @@ class Node(object):
 
 		:param node: path to use as a reference
 		:type node: :py:class:`waflib.Node.Node`
+		:rtype: bool
 		"""
 		p = self
 		diff = self.height() - node.height()
 		while diff > 0:
 			diff -= 1
 			p = p.parent
-		return id(p) == id(node)
+		return p is node
 
-	def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remove=True):
+	def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remove=True, quiet=False):
 		"""
-		Semi-private and recursive method used by ant_glob.
+		Recursive method used by :py:meth:`waflib.Node.ant_glob`.
 
 		:param accept: function used for accepting/rejecting a node, returns the patterns that can be still accepted in recursion
 		:type accept: function
@@ -503,6 +589,10 @@ class Node(object):
 		:type src: bool
 		:param remove: remove files/folders that do not exist (True by default)
 		:type remove: bool
+		:param quiet: disable build directory traversal warnings (verbose mode)
+		:type quiet: bool
+		:returns: A generator object to iterate from
+		:rtype: iterator
 		"""
 		dircont = self.listdir()
 		dircont.sort()
@@ -523,25 +613,23 @@ class Node(object):
 
 				node = self.make_node([name])
 
-				isdir = os.path.isdir(node.abspath())
+				isdir = node.isdir()
 				if accepted:
 					if isdir:
 						if dir:
 							yield node
-					else:
-						if src:
-							yield node
+					elif src:
+						yield node
 
-				if getattr(node, 'cache_isdir', None) or isdir:
+				if isdir:
 					node.cache_isdir = True
 					if maxdepth:
-						for k in node.ant_iter(accept=accept, maxdepth=maxdepth - 1, pats=npats, dir=dir, src=src, remove=remove):
+						for k in node.ant_iter(accept=accept, maxdepth=maxdepth - 1, pats=npats, dir=dir, src=src, remove=remove, quiet=quiet):
 							yield k
-		raise StopIteration
 
 	def ant_glob(self, *k, **kw):
 		"""
-		This method is used for finding files across folders. It behaves like ant patterns:
+		Finds files across folders and returns Node objects:
 
 		* ``**/*`` find all files recursively
 		* ``**/*.class`` find all files ending by .class
@@ -550,14 +638,51 @@ class Node(object):
 		For example::
 
 			def configure(cfg):
-				cfg.path.ant_glob('**/*.cpp') # find all .cpp files
-				cfg.root.ant_glob('etc/*.txt') # using the filesystem root can be slow
-				cfg.path.ant_glob('*.cpp', excl=['*.c'], src=True, dir=False)
+				# find all .cpp files
+				cfg.path.ant_glob('**/*.cpp')
+				# find particular files from the root filesystem (can be slow)
+				cfg.root.ant_glob('etc/*.txt')
+				# simple exclusion rule example
+				cfg.path.ant_glob('*.c*', excl=['*.c'], src=True, dir=False)
+
+		For more information about the patterns, consult http://ant.apache.org/manual/dirtasks.html
+		Please remember that the '..' sequence does not represent the parent directory::
+
+			def configure(cfg):
+				cfg.path.ant_glob('../*.h') # incorrect
+				cfg.path.parent.ant_glob('*.h') # correct
+
+		The Node structure is itself a filesystem cache, so certain precautions must
+		be taken while matching files in the build or installation phases.
+		Nodes objects that do have a corresponding file or folder are garbage-collected by default.
+		This garbage collection is usually required to prevent returning files that do not
+		exist anymore. Yet, this may also remove Node objects of files that are yet-to-be built.
+
+		This typically happens when trying to match files in the build directory,
+		but there are also cases when files are created in the source directory.
+		Run ``waf -v`` to display any warnings, and try consider passing ``remove=False``
+		when matching files in the build directory.
 
-		For more information see http://ant.apache.org/manual/dirtasks.html
+		Since ant_glob can traverse both source and build folders, it is a best practice
+		to call this method only from the most specific build node::
+
+			def build(bld):
+				# traverses the build directory, may need ``remove=False``:
+				bld.path.ant_glob('project/dir/**/*.h')
+				# better, no accidental build directory traversal:
+				bld.path.find_node('project/dir').ant_glob('**/*.h') # best
+
+		In addition, files and folders are listed immediately. When matching files in the
+		build folders, consider passing ``generator=True`` so that the generator object
+		returned can defer computation to a later stage. For example::
+
+			def build(bld):
+				bld(rule='tar xvf ${SRC}', source='arch.tar')
+				bld.add_group()
+				gen = bld.bldnode.ant_glob("*.h", generator=True, remove=True)
+				# files will be listed only after the arch.tar is unpacked
+				bld(rule='ls ${SRC}', source=gen, name='XYZ')
 
-		The nodes that correspond to files and folders that do not exist will be removed. To prevent this
-		behaviour, pass 'remove=False'
 
 		:param incl: ant patterns or list of patterns to include
 		:type incl: string or list of strings
@@ -567,126 +692,90 @@ class Node(object):
 		:type dir: bool
 		:param src: return files (True by default)
 		:type src: bool
-		:param remove: remove files/folders that do not exist (True by default)
-		:type remove: bool
 		:param maxdepth: maximum depth of recursion
 		:type maxdepth: int
 		:param ignorecase: ignore case while matching (False by default)
 		:type ignorecase: bool
+		:param generator: Whether to evaluate the Nodes lazily
+		:type generator: bool
+		:param remove: remove files/folders that do not exist (True by default)
+		:type remove: bool
+		:param quiet: disable build directory traversal warnings (verbose mode)
+		:type quiet: bool
+		:returns: The corresponding Node objects as a list or as a generator object (generator=True)
+		:rtype: by default, list of :py:class:`waflib.Node.Node` instances
 		"""
-
 		src = kw.get('src', True)
-		dir = kw.get('dir', False)
-
+		dir = kw.get('dir')
 		excl = kw.get('excl', exclude_regs)
 		incl = k and k[0] or kw.get('incl', '**')
-		reflags = kw.get('ignorecase', 0) and re.I
-
-		def to_pat(s):
-			lst = Utils.to_list(s)
-			ret = []
-			for x in lst:
-				x = x.replace('\\', '/').replace('//', '/')
-				if x.endswith('/'):
-					x += '**'
-				lst2 = x.split('/')
-				accu = []
-				for k in lst2:
-					if k == '**':
-						accu.append(k)
-					else:
-						k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+')
-						k = '^%s$' % k
-						try:
-							#print "pattern", k
-							accu.append(re.compile(k, flags=reflags))
-						except Exception as e:
-							raise Errors.WafError("Invalid pattern: %s" % k, e)
-				ret.append(accu)
-			return ret
-
-		def filtre(name, nn):
-			ret = []
-			for lst in nn:
-				if not lst:
-					pass
-				elif lst[0] == '**':
-					ret.append(lst)
-					if len(lst) > 1:
-						if lst[1].match(name):
-							ret.append(lst[2:])
-					else:
-						ret.append([])
-				elif lst[0].match(name):
-					ret.append(lst[1:])
-			return ret
-
-		def accept(name, pats):
-			nacc = filtre(name, pats[0])
-			nrej = filtre(name, pats[1])
-			if [] in nrej:
-				nacc = []
-			return [nacc, nrej]
-
-		ret = [x for x in self.ant_iter(accept=accept, pats=[to_pat(incl), to_pat(excl)], maxdepth=kw.get('maxdepth', 25), dir=dir, src=src, remove=kw.get('remove', True))]
-		if kw.get('flat', False):
-			return ' '.join([x.path_from(self) for x in ret])
+		remove = kw.get('remove', True)
+		maxdepth = kw.get('maxdepth', 25)
+		ignorecase = kw.get('ignorecase', False)
+		quiet = kw.get('quiet', False)
+		pats = (ant_matcher(incl, ignorecase), ant_matcher(excl, ignorecase))
 
-		return ret
+		if kw.get('generator'):
+			return Utils.lazy_generator(self.ant_iter, (ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet))
+
+		it = self.ant_iter(ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet)
+		if kw.get('flat'):
+			# returns relative paths as a space-delimited string
+			# prefer Node objects whenever possible
+			return ' '.join(x.path_from(self) for x in it)
+		return list(it)
 
-	# --------------------------------------------------------------------------------
-	# the following methods require the source/build folders (bld.srcnode/bld.bldnode)
-	# using a subclass is a possibility, but is that really necessary?
-	# --------------------------------------------------------------------------------
+	# ----------------------------------------------------------------------------
+	# the methods below require the source/build folders (bld.srcnode/bld.bldnode)
 
 	def is_src(self):
 		"""
-		True if the node is below the source directory
-		note: !is_src does not imply is_bld()
+		Returns True if the node is below the source directory. Note that ``!is_src() ≠ is_bld()``
 
 		:rtype: bool
 		"""
 		cur = self
-		x = id(self.ctx.srcnode)
-		y = id(self.ctx.bldnode)
+		x = self.ctx.srcnode
+		y = self.ctx.bldnode
 		while cur.parent:
-			if id(cur) == y:
+			if cur is y:
 				return False
-			if id(cur) == x:
+			if cur is x:
 				return True
 			cur = cur.parent
 		return False
 
 	def is_bld(self):
 		"""
-		True if the node is below the build directory
-		note: !is_bld does not imply is_src
+		Returns True if the node is below the build directory. Note that ``!is_bld() ≠ is_src()``
 
 		:rtype: bool
 		"""
 		cur = self
-		y = id(self.ctx.bldnode)
+		y = self.ctx.bldnode
 		while cur.parent:
-			if id(cur) == y:
+			if cur is y:
 				return True
 			cur = cur.parent
 		return False
 
 	def get_src(self):
 		"""
-		Return the equivalent src node (or self if not possible)
+		Returns the corresponding Node object in the source directory (or self if already
+		under the source directory). Use this method only if the purpose is to create
+		a Node object (this is common with folders but not with files, see ticket 1937)
 
 		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		cur = self
-		x = id(self.ctx.srcnode)
-		y = id(self.ctx.bldnode)
+		x = self.ctx.srcnode
+		y = self.ctx.bldnode
 		lst = []
 		while cur.parent:
-			if id(cur) == y:
+			if cur is y:
 				lst.reverse()
-				return self.ctx.srcnode.make_node(lst)
-			if id(cur) == x:
+				return x.make_node(lst)
+			if cur is x:
 				return self
 			lst.append(cur.name)
 			cur = cur.parent
@@ -694,18 +783,20 @@ class Node(object):
 
 	def get_bld(self):
 		"""
-		Return the equivalent bld node (or self if not possible)
+		Return the corresponding Node object in the build directory (or self if already
+		under the build directory). Use this method only if the purpose is to create
+		a Node object (this is common with folders but not with files, see ticket 1937)
 
 		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		cur = self
-		x = id(self.ctx.srcnode)
-		y = id(self.ctx.bldnode)
+		x = self.ctx.srcnode
+		y = self.ctx.bldnode
 		lst = []
 		while cur.parent:
-			if id(cur) == y:
+			if cur is y:
 				return self
-			if id(cur) == x:
+			if cur is x:
 				lst.reverse()
 				return self.ctx.bldnode.make_node(lst)
 			lst.append(cur.name)
@@ -718,75 +809,66 @@ class Node(object):
 
 	def find_resource(self, lst):
 		"""
-		Try to find a declared build node or a source file
+		Use this method in the build phase to find source files corresponding to the relative path given.
 
-		:param lst: path
+		First it looks up the Node data structure to find any declared Node object in the build directory.
+		If None is found, it then considers the filesystem in the source directory.
+
+		:param lst: relative path
 		:type lst: string or list of string
+		:returns: the corresponding Node object or None
+		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
 
 		node = self.get_bld().search_node(lst)
 		if not node:
-			self = self.get_src()
-			node = self.find_node(lst)
-		if node:
-			if os.path.isdir(node.abspath()):
-				return None
+			node = self.get_src().find_node(lst)
+		if node and node.isdir():
+			return None
 		return node
 
 	def find_or_declare(self, lst):
 		"""
-		if 'self' is in build directory, try to return an existing node
-		if no node is found, go to the source directory
-		try to find an existing node in the source directory
-		if no node is found, create it in the build directory
+		Use this method in the build phase to declare output files which
+		are meant to be written in the build directory.
 
-		:param lst: path
+		This method creates the Node object and its parent folder
+		as needed.
+
+		:param lst: relative path
 		:type lst: string or list of string
 		"""
-		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
-
-		node = self.get_bld().search_node(lst)
-		if node:
-			if not os.path.isfile(node.abspath()):
-				node.sig = None
-				node.parent.mkdir()
-			return node
-		self = self.get_src()
-		node = self.find_node(lst)
-		if node:
-			if not os.path.isfile(node.abspath()):
-				node.sig = None
-				node.parent.mkdir()
-			return node
-		node = self.get_bld().make_node(lst)
+		if isinstance(lst, str) and os.path.isabs(lst):
+			node = self.ctx.root.make_node(lst)
+		else:
+			node = self.get_bld().make_node(lst)
 		node.parent.mkdir()
 		return node
 
 	def find_dir(self, lst):
 		"""
-		Search for a folder in the filesystem
+		Searches for a folder on the filesystem (see :py:meth:`waflib.Node.Node.find_node`)
 
-		:param lst: path
+		:param lst: relative path
 		:type lst: string or list of string
+		:returns: The corresponding Node object or None if there is no such folder
+		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		if isinstance(lst, str):
-			lst = [x for x in split_path(lst) if x and x != '.']
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
 
 		node = self.find_node(lst)
-		try:
-			if not os.path.isdir(node.abspath()):
-				return None
-		except (OSError, AttributeError):
-			# the node might be None, and raise an AttributeError
+		if node and not node.isdir():
 			return None
 		return node
 
 	# helpers for building things
 	def change_ext(self, ext, ext_in=None):
 		"""
+		Declares a build node with a distinct extension; this is uses :py:meth:`waflib.Node.Node.find_or_declare`
+
 		:return: A build node of the same path, but with a different extension
 		:rtype: :py:class:`waflib.Node.Node`
 		"""
@@ -803,39 +885,79 @@ class Node(object):
 		return self.parent.find_or_declare([name])
 
 	def bldpath(self):
-		"Path seen from the build directory default/src/foo.cpp"
+		"""
+		Returns the relative path seen from the build directory ``src/foo.cpp``
+
+		:rtype: string
+		"""
 		return self.path_from(self.ctx.bldnode)
 
 	def srcpath(self):
-		"Path seen from the source directory ../src/foo.cpp"
+		"""
+		Returns the relative path seen from the source directory ``../src/foo.cpp``
+
+		:rtype: string
+		"""
 		return self.path_from(self.ctx.srcnode)
 
 	def relpath(self):
-		"If a file in the build directory, bldpath, else srcpath"
+		"""
+		If a file in the build directory, returns :py:meth:`waflib.Node.Node.bldpath`,
+		else returns :py:meth:`waflib.Node.Node.srcpath`
+
+		:rtype: string
+		"""
 		cur = self
-		x = id(self.ctx.bldnode)
+		x = self.ctx.bldnode
 		while cur.parent:
-			if id(cur) == x:
+			if cur is x:
 				return self.bldpath()
 			cur = cur.parent
 		return self.srcpath()
 
 	def bld_dir(self):
-		"Build path without the file name"
+		"""
+		Equivalent to self.parent.bldpath()
+
+		:rtype: string
+		"""
 		return self.parent.bldpath()
 
+	def h_file(self):
+		"""
+		See :py:func:`waflib.Utils.h_file`
+
+		:return: a hash representing the file contents
+		:rtype: string or bytes
+		"""
+		return Utils.h_file(self.abspath())
+
 	def get_bld_sig(self):
 		"""
-		Node signature, assuming the file is in the build directory
+		Returns a signature (see :py:meth:`waflib.Node.Node.h_file`) for the purpose
+		of build dependency calculation. This method uses a per-context cache.
+
+		:return: a hash representing the object contents
+		:rtype: string or bytes
 		"""
+		# previous behaviour can be set by returning self.ctx.node_sigs[self] when a build node
 		try:
-			return self.cache_sig
+			cache = self.ctx.cache_sig
 		except AttributeError:
-			pass
-
-		if not self.is_bld() or self.ctx.bldnode is self.ctx.srcnode:
-			self.sig = Utils.h_file(self.abspath())
-		self.cache_sig = ret = self.sig
+			cache = self.ctx.cache_sig = {}
+		try:
+			ret = cache[self]
+		except KeyError:
+			p = self.abspath()
+			try:
+				ret = cache[self] = self.h_file()
+			except EnvironmentError:
+				if self.isdir():
+					# allow folders as build nodes, do not use the creation time
+					st = os.stat(p)
+					ret = cache[self] = Utils.h_list([p, st.st_ino, st.st_mode])
+					return ret
+				raise
 		return ret
 
 pickle_lock = Utils.threading.Lock()
diff --git a/waflib/Options.py b/waflib/Options.py
index fe2121e3..ad802d4b 100644
--- a/waflib/Options.py
+++ b/waflib/Options.py
@@ -1,66 +1,75 @@
 #!/usr/bin/env python
 # encoding: utf-8
 # Scott Newton, 2005 (scottn)
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 
 """
 Support for waf command-line options
 
-Provides default command-line options,
-as well as custom ones, used by the ``options`` wscript function.
-
+Provides default and command-line options, as well the command
+that reads the ``options`` wscript function.
 """
 
 import os, tempfile, optparse, sys, re
-from waflib import Logs, Utils, Context
+from waflib import Logs, Utils, Context, Errors
 
-cmds = 'distclean configure build install clean uninstall check dist distcheck'.split()
+options = optparse.Values()
 """
-Constant representing the default waf commands displayed in::
-
-	$ waf --help
-
-"""
-
-options = {}
-"""
-A dictionary representing the command-line options::
+A global dictionary representing user-provided command-line options::
 
 	$ waf --foo=bar
-
 """
 
 commands = []
 """
-List of commands to execute extracted from the command-line. This list is consumed during the execution, see :py:func:`waflib.Scripting.run_commands`.
+List of commands to execute extracted from the command-line. This list
+is consumed during the execution by :py:func:`waflib.Scripting.run_commands`.
 """
 
 envvars = []
 """
 List of environment variable declarations placed after the Waf executable name.
-These are detected by searching for "=" in the rest arguments.
+These are detected by searching for "=" in the remaining arguments.
+You probably do not want to use this.
 """
 
 lockfile = os.environ.get('WAFLOCK', '.lock-waf_%s_build' % sys.platform)
-platform = Utils.unversioned_sys_platform()
-
+"""
+Name of the lock file that marks a project as configured
+"""
 
 class opt_parser(optparse.OptionParser):
 	"""
 	Command-line options parser.
 	"""
-	def __init__(self, ctx):
-		optparse.OptionParser.__init__(self, conflict_handler="resolve", version='waf %s (%s)' % (Context.WAFVERSION, Context.WAFREVISION))
-
+	def __init__(self, ctx, allow_unknown=False):
+		optparse.OptionParser.__init__(self, conflict_handler='resolve', add_help_option=False,
+			version='waf %s (%s)' % (Context.WAFVERSION, Context.WAFREVISION))
 		self.formatter.width = Logs.get_term_cols()
 		self.ctx = ctx
+		self.allow_unknown = allow_unknown
+
+	def _process_args(self, largs, rargs, values):
+		"""
+		Custom _process_args to allow unknown options according to the allow_unknown status
+		"""
+		while rargs:
+			try:
+				optparse.OptionParser._process_args(self,largs,rargs,values)
+			except (optparse.BadOptionError, optparse.AmbiguousOptionError) as e:
+				if self.allow_unknown:
+					largs.append(e.opt_str)
+				else:
+					self.error(str(e))
 
 	def print_usage(self, file=None):
 		return self.print_help(file)
 
 	def get_usage(self):
 		"""
-		Return the message to print on ``waf --help``
+		Builds the message to print on ``waf --help``
+
+		:rtype: string
 		"""
 		cmds_str = {}
 		for cls in Context.classes:
@@ -96,10 +105,9 @@ Main commands (example: ./waf build -j4)
 
 class OptionsContext(Context.Context):
 	"""
-	Collect custom options from wscript files and parses the command line.
-	Set the global :py:const:`waflib.Options.commands` and :py:const:`waflib.Options.options` values.
+	Collects custom options from wscript files and parses the command line.
+	Sets the global :py:const:`waflib.Options.commands` and :py:const:`waflib.Options.options` values.
 	"""
-
 	cmd = 'options'
 	fun = 'options'
 
@@ -114,11 +122,18 @@ class OptionsContext(Context.Context):
 		jobs = self.jobs()
 		p = self.add_option
 		color = os.environ.get('NOCOLOR', '') and 'no' or 'auto'
+		if os.environ.get('CLICOLOR', '') == '0':
+			color = 'no'
+		elif os.environ.get('CLICOLOR_FORCE', '') == '1':
+			color = 'yes'
 		p('-c', '--color',    dest='colors',  default=color, action='store', help='whether to use colors (yes/no/auto) [default: auto]', choices=('yes', 'no', 'auto'))
-		p('-j', '--jobs',     dest='jobs',    default=jobs, type='int', help='amount of parallel jobs (%r)' % jobs)
+		p('-j', '--jobs',     dest='jobs',    default=jobs,  type='int', help='amount of parallel jobs (%r)' % jobs)
 		p('-k', '--keep',     dest='keep',    default=0,     action='count', help='continue despite errors (-kk to try harder)')
 		p('-v', '--verbose',  dest='verbose', default=0,     action='count', help='verbosity level -v -vv or -vvv [default: 0]')
 		p('--zones',          dest='zones',   default='',    action='store', help='debugging zones (task_gen, deps, tasks, etc)')
+		p('--profile',        dest='profile', default=0,     action='store_true', help=optparse.SUPPRESS_HELP)
+		p('--pdb',            dest='pdb',     default=0,     action='store_true', help=optparse.SUPPRESS_HELP)
+		p('-h', '--help',     dest='whelp',   default=0,     action='store_true', help="show this help message and exit")
 
 		gr = self.add_option_group('Configuration options')
 		self.option_groups['configure options'] = gr
@@ -132,7 +147,7 @@ class OptionsContext(Context.Context):
 
 		default_prefix = getattr(Context.g_module, 'default_prefix', os.environ.get('PREFIX'))
 		if not default_prefix:
-			if platform == 'win32':
+			if Utils.unversioned_sys_platform() == 'win32':
 				d = tempfile.gettempdir()
 				default_prefix = d[0].upper() + d[1:]
 				# win32 preserves the case, but gettempdir does not
@@ -161,8 +176,8 @@ class OptionsContext(Context.Context):
 
 	def jobs(self):
 		"""
-		Find the amount of cpu cores to set the default amount of tasks executed in parallel. At
-		runtime the options can be obtained from :py:const:`waflib.Options.options` ::
+		Finds the optimal amount of cpu cores to use for parallel jobs.
+		At runtime the options can be obtained from :py:const:`waflib.Options.options` ::
 
 			from waflib.Options import options
 			njobs = options.jobs
@@ -185,7 +200,7 @@ class OptionsContext(Context.Context):
 				if not count and os.name not in ('nt', 'java'):
 					try:
 						tmp = self.cmd_and_log(['sysctl', '-n', 'hw.ncpu'], quiet=0)
-					except Exception:
+					except Errors.WafError:
 						pass
 					else:
 						if re.match('^[0-9]+$', tmp):
@@ -198,21 +213,25 @@ class OptionsContext(Context.Context):
 
 	def add_option(self, *k, **kw):
 		"""
-		Wrapper for optparse.add_option::
+		Wraps ``optparse.add_option``::
 
 			def options(ctx):
-				ctx.add_option('-u', '--use', dest='use', default=False, action='store_true',
-					help='a boolean option')
+				ctx.add_option('-u', '--use', dest='use', default=False,
+					action='store_true', help='a boolean option')
+
+		:rtype: optparse option object
 		"""
 		return self.parser.add_option(*k, **kw)
 
 	def add_option_group(self, *k, **kw):
 		"""
-		Wrapper for optparse.add_option_group::
+		Wraps ``optparse.add_option_group``::
 
 			def options(ctx):
 				gr = ctx.add_option_group('some options')
 				gr.add_option('-u', '--use', dest='use', default=False, action='store_true')
+
+		:rtype: optparse option group object
 		"""
 		try:
 			gr = self.option_groups[k[0]]
@@ -223,13 +242,14 @@ class OptionsContext(Context.Context):
 
 	def get_option_group(self, opt_str):
 		"""
-		Wrapper for optparse.get_option_group::
+		Wraps ``optparse.get_option_group``::
 
 			def options(ctx):
 				gr = ctx.get_option_group('configure options')
 				gr.add_option('-o', '--out', action='store', default='',
 					help='build dir for the project', dest='out')
 
+		:rtype: optparse option group object
 		"""
 		try:
 			return self.option_groups[opt_str]
@@ -239,35 +259,84 @@ class OptionsContext(Context.Context):
 					return group
 			return None
 
-	def parse_args(self, _args=None):
-		"""
-		Parse arguments from a list (not bound to the command-line).
+	def sanitize_path(self, path, cwd=None):
+		if not cwd:
+			cwd = Context.launch_dir
+		p = os.path.expanduser(path)
+		p = os.path.join(cwd, p)
+		p = os.path.normpath(p)
+		p = os.path.abspath(p)
+		return p
 
-		:param _args: arguments
-		:type _args: list of strings
+	def parse_cmd_args(self, _args=None, cwd=None, allow_unknown=False):
+		"""
+		Just parse the arguments
 		"""
-		global options, commands, envvars
+		self.parser.allow_unknown = allow_unknown
 		(options, leftover_args) = self.parser.parse_args(args=_args)
-
+		envvars = []
+		commands = []
 		for arg in leftover_args:
 			if '=' in arg:
 				envvars.append(arg)
-			else:
+			elif arg != 'options':
 				commands.append(arg)
 
-		if options.destdir:
-			options.destdir = Utils.sane_path(options.destdir)
+		for name in 'top out destdir prefix bindir libdir'.split():
+			# those paths are usually expanded from Context.launch_dir
+			if getattr(options, name, None):
+				path = self.sanitize_path(getattr(options, name), cwd)
+				setattr(options, name, path)
+		return options, commands, envvars
+
+	def init_module_vars(self, arg_options, arg_commands, arg_envvars):
+		options.__dict__.clear()
+		del commands[:]
+		del envvars[:]
+
+		options.__dict__.update(arg_options.__dict__)
+		commands.extend(arg_commands)
+		envvars.extend(arg_envvars)
 
+		for var in envvars:
+			(name, value) = var.split('=', 1)
+			os.environ[name.strip()] = value
+
+	def init_logs(self, options, commands, envvars):
+		Logs.verbose = options.verbose
 		if options.verbose >= 1:
 			self.load('errcheck')
 
 		colors = {'yes' : 2, 'auto' : 1, 'no' : 0}[options.colors]
 		Logs.enable_colors(colors)
 
+		if options.zones:
+			Logs.zones = options.zones.split(',')
+			if not Logs.verbose:
+				Logs.verbose = 1
+		elif Logs.verbose > 0:
+			Logs.zones = ['runner']
+		if Logs.verbose > 2:
+			Logs.zones = ['*']
+
+	def parse_args(self, _args=None):
+		"""
+		Parses arguments from a list which is not necessarily the command-line.
+		Initializes the module variables options, commands and envvars
+		If help is requested, prints it and exit the application
+
+		:param _args: arguments
+		:type _args: list of strings
+		"""
+		options, commands, envvars = self.parse_cmd_args()
+		self.init_logs(options, commands, envvars)
+		self.init_module_vars(options, commands, envvars)
+
 	def execute(self):
 		"""
 		See :py:func:`waflib.Context.Context.execute`
 		"""
 		super(OptionsContext, self).execute()
 		self.parse_args()
+		Utils.alloc_process_pool(options.jobs)
 
diff --git a/waflib/Runner.py b/waflib/Runner.py
index b3087292..261084d2 100644
--- a/waflib/Runner.py
+++ b/waflib/Runner.py
@@ -1,98 +1,125 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Runner.py: Task scheduling and execution
-
 """
 
-import random, atexit
+import heapq, traceback
 try:
-	from queue import Queue
+	from queue import Queue, PriorityQueue
 except ImportError:
 	from Queue import Queue
+	try:
+		from Queue import PriorityQueue
+	except ImportError:
+		class PriorityQueue(Queue):
+			def _init(self, maxsize):
+				self.maxsize = maxsize
+				self.queue = []
+			def _put(self, item):
+				heapq.heappush(self.queue, item)
+			def _get(self):
+				return heapq.heappop(self.queue)
+
 from waflib import Utils, Task, Errors, Logs
 
-GAP = 10
+GAP = 5
 """
-Wait for free tasks if there are at least ``GAP * njobs`` in queue
+Wait for at least ``GAP * njobs`` before trying to enqueue more tasks to run
 """
 
-class TaskConsumer(Utils.threading.Thread):
-	"""
-	Task consumers belong to a pool of workers
+class PriorityTasks(object):
+	def __init__(self):
+		self.lst = []
+	def __len__(self):
+		return len(self.lst)
+	def __iter__(self):
+		return iter(self.lst)
+	def clear(self):
+		self.lst = []
+	def append(self, task):
+		heapq.heappush(self.lst, task)
+	def appendleft(self, task):
+		"Deprecated, do not use"
+		heapq.heappush(self.lst, task)
+	def pop(self):
+		return heapq.heappop(self.lst)
+	def extend(self, lst):
+		if self.lst:
+			for x in lst:
+				self.append(x)
+		else:
+			if isinstance(lst, list):
+				self.lst = lst
+				heapq.heapify(lst)
+			else:
+				self.lst = lst.lst
 
-	They wait for tasks in the queue and then use ``task.process(...)``
+class Consumer(Utils.threading.Thread):
 	"""
-	def __init__(self):
+	Daemon thread object that executes a task. It shares a semaphore with
+	the coordinator :py:class:`waflib.Runner.Spawner`. There is one
+	instance per task to consume.
+	"""
+	def __init__(self, spawner, task):
 		Utils.threading.Thread.__init__(self)
-		self.ready = Queue()
+		self.task = task
+		"""Task to execute"""
+		self.spawner = spawner
+		"""Coordinator object"""
+		self.setDaemon(1)
+		self.start()
+	def run(self):
 		"""
-		Obtain :py:class:`waflib.Task.TaskBase` instances from this queue.
+		Processes a single task
 		"""
+		try:
+			if not self.spawner.master.stop:
+				self.spawner.master.process_task(self.task)
+		finally:
+			self.spawner.sem.release()
+			self.spawner.master.out.put(self.task)
+			self.task = None
+			self.spawner = None
+
+class Spawner(Utils.threading.Thread):
+	"""
+	Daemon thread that consumes tasks from :py:class:`waflib.Runner.Parallel` producer and
+	spawns a consuming thread :py:class:`waflib.Runner.Consumer` for each
+	:py:class:`waflib.Task.Task` instance.
+	"""
+	def __init__(self, master):
+		Utils.threading.Thread.__init__(self)
+		self.master = master
+		""":py:class:`waflib.Runner.Parallel` producer instance"""
+		self.sem = Utils.threading.Semaphore(master.numjobs)
+		"""Bounded semaphore that prevents spawning more than *n* concurrent consumers"""
 		self.setDaemon(1)
 		self.start()
-
 	def run(self):
 		"""
-		Loop over the tasks to execute
+		Spawns new consumers to execute tasks by delegating to :py:meth:`waflib.Runner.Spawner.loop`
 		"""
 		try:
 			self.loop()
 		except Exception:
+			# Python 2 prints unnecessary messages when shutting down
+			# we also want to stop the thread properly
 			pass
-
 	def loop(self):
 		"""
-		Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call
-		:py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it.
+		Consumes task objects from the producer; ends when the producer has no more
+		task to provide.
 		"""
+		master = self.master
 		while 1:
-			tsk = self.ready.get()
-			if not isinstance(tsk, Task.TaskBase):
-				tsk(self)
-			else:
-				tsk.process()
-
-pool = Queue()
-"""
-Pool of task consumer objects
-"""
-
-def get_pool():
-	"""
-	Obtain a task consumer from :py:attr:`waflib.Runner.pool`.
-	Do not forget to put it back by using :py:func:`waflib.Runner.put_pool`
-	and reset properly (original waiting queue).
-
-	:rtype: :py:class:`waflib.Runner.TaskConsumer`
-	"""
-	try:
-		return pool.get(False)
-	except Exception:
-		return TaskConsumer()
-
-def put_pool(x):
-	"""
-	Return a task consumer to the thread pool :py:attr:`waflib.Runner.pool`
-
-	:param x: task consumer object
-	:type x: :py:class:`waflib.Runner.TaskConsumer`
-	"""
-	pool.put(x)
-
-def _free_resources():
-	global pool
-	lst = []
-	while pool.qsize():
-		lst.append(pool.get())
-	for x in lst:
-		x.ready.put(None)
-	for x in lst:
-		x.join()
-	pool = None
-atexit.register(_free_resources)
+			task = master.ready.get()
+			self.sem.acquire()
+			if not master.stop:
+				task.log_display(task.generator.bld)
+			Consumer(self, task)
 
 class Parallel(object):
 	"""
@@ -106,7 +133,7 @@ class Parallel(object):
 
 		self.numjobs = j
 		"""
-		Number of consumers in the pool
+		Amount of parallel consumers to use
 		"""
 
 		self.bld = bld
@@ -114,19 +141,25 @@ class Parallel(object):
 		Instance of :py:class:`waflib.Build.BuildContext`
 		"""
 
-		self.outstanding = []
-		"""List of :py:class:`waflib.Task.TaskBase` that may be ready to be executed"""
+		self.outstanding = PriorityTasks()
+		"""Heap of :py:class:`waflib.Task.Task` that may be ready to be executed"""
+
+		self.postponed = PriorityTasks()
+		"""Heap of :py:class:`waflib.Task.Task` which are not ready to run for non-DAG reasons"""
 
-		self.frozen = []
-		"""List of :py:class:`waflib.Task.TaskBase` that cannot be executed immediately"""
+		self.incomplete = set()
+		"""List of :py:class:`waflib.Task.Task` waiting for dependent tasks to complete (DAG)"""
+
+		self.ready = PriorityQueue(0)
+		"""List of :py:class:`waflib.Task.Task` ready to be executed by consumers"""
 
 		self.out = Queue(0)
-		"""List of :py:class:`waflib.Task.TaskBase` returned by the task consumers"""
+		"""List of :py:class:`waflib.Task.Task` returned by the task consumers"""
 
 		self.count = 0
 		"""Amount of tasks that may be processed by :py:class:`waflib.Runner.TaskConsumer`"""
 
-		self.processed = 1
+		self.processed = 0
 		"""Amount of tasks processed"""
 
 		self.stop = False
@@ -139,33 +172,44 @@ class Parallel(object):
 		"""Task iterator which must give groups of parallelizable tasks when calling ``next()``"""
 
 		self.dirty = False
-		"""Flag to indicate that tasks have been executed, and that the build cache must be saved (call :py:meth:`waflib.Build.BuildContext.store`)"""
+		"""
+		Flag that indicates that the build cache must be saved when a task was executed
+		(calls :py:meth:`waflib.Build.BuildContext.store`)"""
+
+		self.revdeps = Utils.defaultdict(set)
+		"""
+		The reverse dependency graph of dependencies obtained from Task.run_after
+		"""
+
+		self.spawner = Spawner(self)
+		"""
+		Coordinating daemon thread that spawns thread consumers
+		"""
 
 	def get_next_task(self):
 		"""
-		Obtain the next task to execute.
+		Obtains the next Task instance to run
 
-		:rtype: :py:class:`waflib.Task.TaskBase`
+		:rtype: :py:class:`waflib.Task.Task`
 		"""
 		if not self.outstanding:
 			return None
-		return self.outstanding.pop(0)
+		return self.outstanding.pop()
 
 	def postpone(self, tsk):
 		"""
-		A task cannot be executed at this point, put it in the list :py:attr:`waflib.Runner.Parallel.frozen`.
+		Adds the task to the list :py:attr:`waflib.Runner.Parallel.postponed`.
+		The order is scrambled so as to consume as many tasks in parallel as possible.
 
-		:param tsk: task
-		:type tsk: :py:class:`waflib.Task.TaskBase`
+		:param tsk: task instance
+		:type tsk: :py:class:`waflib.Task.Task`
 		"""
-		if random.randint(0, 1):
-			self.frozen.insert(0, tsk)
-		else:
-			self.frozen.append(tsk)
+		self.postponed.append(tsk)
 
 	def refill_task_list(self):
 		"""
-		Put the next group of tasks to execute in :py:attr:`waflib.Runner.Parallel.outstanding`.
+		Pulls a next group of tasks to execute in :py:attr:`waflib.Runner.Parallel.outstanding`.
+		Ensures that all tasks in the current build group are complete before processing the next one.
 		"""
 		while self.count > self.numjobs * GAP:
 			self.get_out()
@@ -173,132 +217,224 @@ class Parallel(object):
 		while not self.outstanding:
 			if self.count:
 				self.get_out()
-			elif self.frozen:
+				if self.outstanding:
+					break
+			elif self.postponed:
 				try:
 					cond = self.deadlock == self.processed
 				except AttributeError:
 					pass
 				else:
 					if cond:
-						msg = 'check the build order for the tasks'
-						for tsk in self.frozen:
-							if not tsk.run_after:
-								msg = 'check the methods runnable_status'
-								break
+						# The most common reason is conflicting build order declaration
+						# for example: "X run_after Y" and "Y run_after X"
+						# Another can be changing "run_after" dependencies while the build is running
+						# for example: updating "tsk.run_after" in the "runnable_status" method
 						lst = []
-						for tsk in self.frozen:
-							lst.append('%s\t-> %r' % (repr(tsk), [id(x) for x in tsk.run_after]))
-						raise Errors.WafError('Deadlock detected: %s%s' % (msg, ''.join(lst)))
+						for tsk in self.postponed:
+							deps = [id(x) for x in tsk.run_after if not x.hasrun]
+							lst.append('%s\t-> %r' % (repr(tsk), deps))
+							if not deps:
+								lst.append('\n  task %r dependencies are done, check its *runnable_status*?' % id(tsk))
+						raise Errors.WafError('Deadlock detected: check the task build order%s' % ''.join(lst))
 				self.deadlock = self.processed
 
-			if self.frozen:
-				self.outstanding += self.frozen
-				self.frozen = []
+			if self.postponed:
+				self.outstanding.extend(self.postponed)
+				self.postponed.clear()
 			elif not self.count:
-				self.outstanding.extend(next(self.biter))
-				self.total = self.bld.total()
-				break
+				if self.incomplete:
+					for x in self.incomplete:
+						for k in x.run_after:
+							if not k.hasrun:
+								break
+						else:
+							# dependency added after the build started without updating revdeps
+							self.incomplete.remove(x)
+							self.outstanding.append(x)
+							break
+					else:
+						raise Errors.WafError('Broken revdeps detected on %r' % self.incomplete)
+				else:
+					tasks = next(self.biter)
+					ready, waiting = self.prio_and_split(tasks)
+					self.outstanding.extend(ready)
+					self.incomplete.update(waiting)
+					self.total = self.bld.total()
+					break
 
 	def add_more_tasks(self, tsk):
 		"""
-		Tasks may be added dynamically during the build by binding them to the task :py:attr:`waflib.Task.TaskBase.more_tasks`
+		If a task provides :py:attr:`waflib.Task.Task.more_tasks`, then the tasks contained
+		in that list are added to the current build and will be processed before the next build group.
 
-		:param tsk: task
-		:type tsk: :py:attr:`waflib.Task.TaskBase`
+		The priorities for dependent tasks are not re-calculated globally
+
+		:param tsk: task instance
+		:type tsk: :py:attr:`waflib.Task.Task`
 		"""
 		if getattr(tsk, 'more_tasks', None):
-			self.outstanding += tsk.more_tasks
+			more = set(tsk.more_tasks)
+			groups_done = set()
+			def iteri(a, b):
+				for x in a:
+					yield x
+				for x in b:
+					yield x
+
+			# Update the dependency tree
+			# this assumes that task.run_after values were updated
+			for x in iteri(self.outstanding, self.incomplete):
+				for k in x.run_after:
+					if isinstance(k, Task.TaskGroup):
+						if k not in groups_done:
+							groups_done.add(k)
+							for j in k.prev & more:
+								self.revdeps[j].add(k)
+					elif k in more:
+						self.revdeps[k].add(x)
+
+			ready, waiting = self.prio_and_split(tsk.more_tasks)
+			self.outstanding.extend(ready)
+			self.incomplete.update(waiting)
 			self.total += len(tsk.more_tasks)
 
+	def mark_finished(self, tsk):
+		def try_unfreeze(x):
+			# DAG ancestors are likely to be in the incomplete set
+			# This assumes that the run_after contents have not changed
+			# after the build starts, else a deadlock may occur
+			if x in self.incomplete:
+				# TODO remove dependencies to free some memory?
+				# x.run_after.remove(tsk)
+				for k in x.run_after:
+					if not k.hasrun:
+						break
+				else:
+					self.incomplete.remove(x)
+					self.outstanding.append(x)
+
+		if tsk in self.revdeps:
+			for x in self.revdeps[tsk]:
+				if isinstance(x, Task.TaskGroup):
+					x.prev.remove(tsk)
+					if not x.prev:
+						for k in x.next:
+							# TODO necessary optimization?
+							k.run_after.remove(x)
+							try_unfreeze(k)
+						# TODO necessary optimization?
+						x.next = []
+				else:
+					try_unfreeze(x)
+			del self.revdeps[tsk]
+
+		if hasattr(tsk, 'semaphore'):
+			sem = tsk.semaphore
+			sem.release(tsk)
+			while sem.waiting and not sem.is_locked():
+				# take a frozen task, make it ready to run
+				x = sem.waiting.pop()
+				self._add_task(x)
+
 	def get_out(self):
 		"""
-		Obtain one task returned from the task consumers, and update the task count. Add more tasks if necessary through
-		:py:attr:`waflib.Runner.Parallel.add_more_tasks`.
+		Waits for a Task that task consumers add to :py:attr:`waflib.Runner.Parallel.out` after execution.
+		Adds more Tasks if necessary through :py:attr:`waflib.Runner.Parallel.add_more_tasks`.
 
-		:rtype: :py:attr:`waflib.Task.TaskBase`
+		:rtype: :py:attr:`waflib.Task.Task`
 		"""
 		tsk = self.out.get()
 		if not self.stop:
 			self.add_more_tasks(tsk)
+		self.mark_finished(tsk)
+
 		self.count -= 1
 		self.dirty = True
 		return tsk
 
 	def add_task(self, tsk):
 		"""
-		Pass a task to a consumer.
+		Enqueue a Task to :py:attr:`waflib.Runner.Parallel.ready` so that consumers can run them.
 
-		:param tsk: task
-		:type tsk: :py:attr:`waflib.Task.TaskBase`
+		:param tsk: task instance
+		:type tsk: :py:attr:`waflib.Task.Task`
 		"""
-		try:
-			self.pool
-		except AttributeError:
-			self.init_task_pool()
+		# TODO change in waf 2.1
 		self.ready.put(tsk)
 
-	def init_task_pool(self):
-		# lazy creation, and set a common pool for all task consumers
-		pool = self.pool = [get_pool() for i in range(self.numjobs)]
-		self.ready = Queue(0)
-		def setq(consumer):
-			consumer.ready = self.ready
-		for x in pool:
-			x.ready.put(setq)
-		return pool
-
-	def free_task_pool(self):
-		# return the consumers, setting a different queue for each of them
-		def setq(consumer):
-			consumer.ready = Queue(0)
-			self.out.put(self)
-		try:
-			pool = self.pool
-		except AttributeError:
-			pass
+	def _add_task(self, tsk):
+		if hasattr(tsk, 'semaphore'):
+			sem = tsk.semaphore
+			try:
+				sem.acquire(tsk)
+			except IndexError:
+				sem.waiting.add(tsk)
+				return
+
+		self.count += 1
+		self.processed += 1
+		if self.numjobs == 1:
+			tsk.log_display(tsk.generator.bld)
+			try:
+				self.process_task(tsk)
+			finally:
+				self.out.put(tsk)
 		else:
-			for x in pool:
-				self.ready.put(setq)
-			for x in pool:
-				self.get_out()
-			for x in pool:
-				put_pool(x)
-			self.pool = []
+			self.add_task(tsk)
+
+	def process_task(self, tsk):
+		"""
+		Processes a task and attempts to stop the build in case of errors
+		"""
+		tsk.process()
+		if tsk.hasrun != Task.SUCCESS:
+			self.error_handler(tsk)
 
 	def skip(self, tsk):
+		"""
+		Mark a task as skipped/up-to-date
+		"""
 		tsk.hasrun = Task.SKIPPED
+		self.mark_finished(tsk)
+
+	def cancel(self, tsk):
+		"""
+		Mark a task as failed because of unsatisfiable dependencies
+		"""
+		tsk.hasrun = Task.CANCELED
+		self.mark_finished(tsk)
 
 	def error_handler(self, tsk):
 		"""
-		Called when a task cannot be executed. The flag :py:attr:`waflib.Runner.Parallel.stop` is set, unless
-		the build is executed with::
+		Called when a task cannot be executed. The flag :py:attr:`waflib.Runner.Parallel.stop` is set,
+		unless the build is executed with::
 
 			$ waf build -k
 
-		:param tsk: task
-		:type tsk: :py:attr:`waflib.Task.TaskBase`
+		:param tsk: task instance
+		:type tsk: :py:attr:`waflib.Task.Task`
 		"""
-		if hasattr(tsk, 'scan') and hasattr(tsk, 'uid'):
-			# TODO waf 1.9 - this breaks encapsulation
-			key = (tsk.uid(), 'imp')
-			try:
-				del self.bld.task_sigs[key]
-			except KeyError:
-				pass
 		if not self.bld.keep:
 			self.stop = True
 		self.error.append(tsk)
 
 	def task_status(self, tsk):
+		"""
+		Obtains the task status to decide whether to run it immediately or not.
+
+		:return: the exit status, for example :py:attr:`waflib.Task.ASK_LATER`
+		:rtype: integer
+		"""
 		try:
 			return tsk.runnable_status()
 		except Exception:
 			self.processed += 1
-			tsk.err_msg = Utils.ex_stack()
+			tsk.err_msg = traceback.format_exc()
 			if not self.stop and self.bld.keep:
 				self.skip(tsk)
 				if self.bld.keep == 1:
-					# if -k stop at the first exception, if -kk try to go as far as possible
+					# if -k stop on the first exception, if -kk try to go as far as possible
 					if Logs.verbose > 1 or not self.error:
 						self.error.append(tsk)
 					self.stop = True
@@ -306,17 +442,20 @@ class Parallel(object):
 					if Logs.verbose > 1:
 						self.error.append(tsk)
 				return Task.EXCEPTION
-			tsk.hasrun = Task.EXCEPTION
 
+			tsk.hasrun = Task.EXCEPTION
 			self.error_handler(tsk)
+
 			return Task.EXCEPTION
 
 	def start(self):
 		"""
-		Give tasks to :py:class:`waflib.Runner.TaskConsumer` instances until the build finishes or the ``stop`` flag is set.
-		If only one job is used, then execute the tasks one by one, without consumers.
+		Obtains Task instances from the BuildContext instance and adds the ones that need to be executed to
+		:py:class:`waflib.Runner.Parallel.ready` so that the :py:class:`waflib.Runner.Spawner` consumer thread
+		has them executed. Obtains the executed Tasks back from :py:class:`waflib.Runner.Parallel.out`
+		and marks the build as failed by setting the ``stop`` flag.
+		If only one job is used, then executes the tasks one by one, without consumers.
 		"""
-
 		self.total = self.bld.total()
 
 		while not self.stop:
@@ -338,36 +477,135 @@ class Parallel(object):
 				self.processed += 1
 				continue
 
-			if self.stop: # stop immediately after a failure was detected
+			if self.stop: # stop immediately after a failure is detected
 				break
 
-
 			st = self.task_status(tsk)
 			if st == Task.RUN_ME:
-				tsk.position = (self.processed, self.total)
-				self.count += 1
-				tsk.master = self
-				self.processed += 1
-
-				if self.numjobs == 1:
-					tsk.process()
-				else:
-					self.add_task(tsk)
-			if st == Task.ASK_LATER:
+				self._add_task(tsk)
+			elif st == Task.ASK_LATER:
 				self.postpone(tsk)
 			elif st == Task.SKIP_ME:
 				self.processed += 1
 				self.skip(tsk)
 				self.add_more_tasks(tsk)
+			elif st == Task.CANCEL_ME:
+				# A dependency problem has occurred, and the
+				# build is most likely run with `waf -k`
+				if Logs.verbose > 1:
+					self.error.append(tsk)
+				self.processed += 1
+				self.cancel(tsk)
 
 		# self.count represents the tasks that have been made available to the consumer threads
 		# collect all the tasks after an error else the message may be incomplete
 		while self.error and self.count:
 			self.get_out()
 
-		#print loop
-		assert (self.count == 0 or self.stop)
+		self.ready.put(None)
+		if not self.stop:
+			assert not self.count
+			assert not self.postponed
+			assert not self.incomplete
+
+	def prio_and_split(self, tasks):
+		"""
+		Label input tasks with priority values, and return a pair containing
+		the tasks that are ready to run and the tasks that are necessarily
+		waiting for other tasks to complete.
+
+		The priority system is really meant as an optional layer for optimization:
+		dependency cycles are found quickly, and builds should be more efficient.
+		A high priority number means that a task is processed first.
+
+		This method can be overridden to disable the priority system::
+
+			def prio_and_split(self, tasks):
+				return tasks, []
 
-		# free the task pool, if any
-		self.free_task_pool()
+		:return: A pair of task lists
+		:rtype: tuple
+		"""
+		# to disable:
+		#return tasks, []
+		for x in tasks:
+			x.visited = 0
+
+		reverse = self.revdeps
+
+		groups_done = set()
+		for x in tasks:
+			for k in x.run_after:
+				if isinstance(k, Task.TaskGroup):
+					if k not in groups_done:
+						groups_done.add(k)
+						for j in k.prev:
+							reverse[j].add(k)
+				else:
+					reverse[k].add(x)
+
+		# the priority number is not the tree depth
+		def visit(n):
+			if isinstance(n, Task.TaskGroup):
+				return sum(visit(k) for k in n.next)
+
+			if n.visited == 0:
+				n.visited = 1
+
+				if n in reverse:
+					rev = reverse[n]
+					n.prio_order = n.tree_weight + len(rev) + sum(visit(k) for k in rev)
+				else:
+					n.prio_order = n.tree_weight
+
+				n.visited = 2
+			elif n.visited == 1:
+				raise Errors.WafError('Dependency cycle found!')
+			return n.prio_order
+
+		for x in tasks:
+			if x.visited != 0:
+				# must visit all to detect cycles
+				continue
+			try:
+				visit(x)
+			except Errors.WafError:
+				self.debug_cycles(tasks, reverse)
+
+		ready = []
+		waiting = []
+		for x in tasks:
+			for k in x.run_after:
+				if not k.hasrun:
+					waiting.append(x)
+					break
+			else:
+				ready.append(x)
+		return (ready, waiting)
+
+	def debug_cycles(self, tasks, reverse):
+		tmp = {}
+		for x in tasks:
+			tmp[x] = 0
+
+		def visit(n, acc):
+			if isinstance(n, Task.TaskGroup):
+				for k in n.next:
+					visit(k, acc)
+				return
+			if tmp[n] == 0:
+				tmp[n] = 1
+				for k in reverse.get(n, []):
+					visit(k, [n] + acc)
+				tmp[n] = 2
+			elif tmp[n] == 1:
+				lst = []
+				for tsk in acc:
+					lst.append(repr(tsk))
+					if tsk is n:
+						# exclude prior nodes, we want the minimum cycle
+						break
+				raise Errors.WafError('Task dependency cycle in "run_after" constraints: %s' % ''.join(lst))
+		for x in tasks:
+			visit(x, [])
 
diff --git a/waflib/Scripting.py b/waflib/Scripting.py
index f912d510..749d4f2e 100644
--- a/waflib/Scripting.py
+++ b/waflib/Scripting.py
@@ -1,9 +1,11 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 "Module called for configuring, compiling and installing targets"
 
+from __future__ import with_statement
+
 import os, shlex, shutil, traceback, errno, sys, stat
 from waflib import Utils, Configure, Logs, Options, ConfigSet, Context, Errors, Build, Node
 
@@ -24,73 +26,66 @@ def waf_entry_point(current_directory, version, wafdir):
 	:param wafdir: absolute path representing the directory of the waf library
 	:type wafdir: string
 	"""
-
 	Logs.init_log()
 
 	if Context.WAFVERSION != version:
-		Logs.error('Waf script %r and library %r do not match (directory %r)' % (version, Context.WAFVERSION, wafdir))
+		Logs.error('Waf script %r and library %r do not match (directory %r)', version, Context.WAFVERSION, wafdir)
 		sys.exit(1)
 
-	if '--version' in sys.argv:
-		Context.run_dir = current_directory
-		ctx = Context.create_context('options')
-		ctx.curdir = current_directory
-		ctx.parse_args()
-		sys.exit(0)
+	# Store current directory before any chdir
+	Context.waf_dir = wafdir
+	Context.run_dir = Context.launch_dir = current_directory
+	start_dir = current_directory
+	no_climb = os.environ.get('NOCLIMB')
 
 	if len(sys.argv) > 1:
-		# os.path.join handles absolute paths in sys.argv[1] accordingly (it discards the previous ones)
+		# os.path.join handles absolute paths
 		# if sys.argv[1] is not an absolute path, then it is relative to the current working directory
 		potential_wscript = os.path.join(current_directory, sys.argv[1])
-		# maybe check if the file is executable
-		# perhaps extract 'wscript' as a constant
-		if os.path.basename(potential_wscript) == 'wscript' and os.path.isfile(potential_wscript):
+		if os.path.basename(potential_wscript) == Context.WSCRIPT_FILE and os.path.isfile(potential_wscript):
 			# need to explicitly normalize the path, as it may contain extra '/.'
-			# TODO abspath?
-			current_directory = os.path.normpath(os.path.dirname(potential_wscript))
+			path = os.path.normpath(os.path.dirname(potential_wscript))
+			start_dir = os.path.abspath(path)
+			no_climb = True
 			sys.argv.pop(1)
 
-	Context.waf_dir = wafdir
-	Context.launch_dir = current_directory
+	ctx = Context.create_context('options')
+	(options, commands, env) = ctx.parse_cmd_args(allow_unknown=True)
+	if options.top:
+		start_dir = Context.run_dir = Context.top_dir = options.top
+		no_climb = True
+	if options.out:
+		Context.out_dir = options.out
 
 	# if 'configure' is in the commands, do not search any further
-	no_climb = os.environ.get('NOCLIMB', None)
 	if not no_climb:
 		for k in no_climb_commands:
-			for y in sys.argv:
+			for y in commands:
 				if y.startswith(k):
 					no_climb = True
 					break
 
-	# if --top is provided assume the build started in the top directory
-	for i, x in enumerate(sys.argv):
-		# WARNING: this modifies sys.argv
-		if x.startswith('--top='):
-			Context.run_dir = Context.top_dir = Utils.sane_path(x[6:])
-			sys.argv[i] = '--top=' + Context.run_dir
-		if x.startswith('--out='):
-			Context.out_dir = Utils.sane_path(x[6:])
-			sys.argv[i] = '--out=' + Context.out_dir
-
 	# try to find a lock file (if the project was configured)
 	# at the same time, store the first wscript file seen
-	cur = current_directory
-	while cur and not Context.top_dir:
+	cur = start_dir
+	while cur:
 		try:
 			lst = os.listdir(cur)
 		except OSError:
 			lst = []
-			Logs.error('Directory %r is unreadable!' % cur)
+			Logs.error('Directory %r is unreadable!', cur)
 		if Options.lockfile in lst:
 			env = ConfigSet.ConfigSet()
 			try:
 				env.load(os.path.join(cur, Options.lockfile))
 				ino = os.stat(cur)[stat.ST_INO]
-			except Exception:
+			except EnvironmentError:
 				pass
 			else:
 				# check if the folder was not moved
 				for x in (env.run_dir, env.top_dir, env.out_dir):
+					if not x:
+						continue
 					if Utils.is_win32:
 						if cur == x:
 							load = True
@@ -106,7 +101,7 @@ def waf_entry_point(current_directory, version, wafdir):
 								load = True
 								break
 				else:
-					Logs.warn('invalid lock file in %s' % cur)
+					Logs.warn('invalid lock file in %s', cur)
 					load = False
 
 				if load:
@@ -127,56 +122,62 @@ def waf_entry_point(current_directory, version, wafdir):
 		if no_climb:
 			break
 
-	if not Context.run_dir:
-		if '-h' in sys.argv or '--help' in sys.argv:
-			Logs.warn('No wscript file found: the help message may be incomplete')
-			Context.run_dir = current_directory
-			ctx = Context.create_context('options')
-			ctx.curdir = current_directory
-			ctx.parse_args()
+	wscript = os.path.normpath(os.path.join(Context.run_dir, Context.WSCRIPT_FILE))
+	if not os.path.exists(wscript):
+		if options.whelp:
+			Logs.warn('These are the generic options (no wscript/project found)')
+			ctx.parser.print_help()
 			sys.exit(0)
-		Logs.error('Waf: Run from a directory containing a file named %r' % Context.WSCRIPT_FILE)
+		Logs.error('Waf: Run from a folder containing a %r file (or try -h for the generic options)', Context.WSCRIPT_FILE)
 		sys.exit(1)
 
 	try:
 		os.chdir(Context.run_dir)
 	except OSError:
-		Logs.error('Waf: The folder %r is unreadable' % Context.run_dir)
+		Logs.error('Waf: The folder %r is unreadable', Context.run_dir)
 		sys.exit(1)
 
 	try:
-		set_main_module(os.path.normpath(os.path.join(Context.run_dir, Context.WSCRIPT_FILE)))
+		set_main_module(wscript)
 	except Errors.WafError as e:
 		Logs.pprint('RED', e.verbose_msg)
 		Logs.error(str(e))
 		sys.exit(1)
 	except Exception as e:
-		Logs.error('Waf: The wscript in %r is unreadable' % Context.run_dir, e)
+		Logs.error('Waf: The wscript in %r is unreadable', Context.run_dir)
 		traceback.print_exc(file=sys.stdout)
 		sys.exit(2)
 
-	"""
-	import cProfile, pstats
-	cProfile.runctx("from waflib import Scripting; Scripting.run_commands()", {}, {}, 'profi.txt')
-	p = pstats.Stats('profi.txt')
-	p.sort_stats('time').print_stats(75) # or 'cumulative'
-	"""
-	try:
-		run_commands()
-	except Errors.WafError as e:
-		if Logs.verbose > 1:
-			Logs.pprint('RED', e.verbose_msg)
-		Logs.error(e.msg)
-		sys.exit(1)
-	except SystemExit:
-		raise
-	except Exception as e:
-		traceback.print_exc(file=sys.stdout)
-		sys.exit(2)
-	except KeyboardInterrupt:
-		Logs.pprint('RED', 'Interrupted')
-		sys.exit(68)
-	#"""
+	if options.profile:
+		import cProfile, pstats
+		cProfile.runctx('from waflib import Scripting; Scripting.run_commands()', {}, {}, 'profi.txt')
+		p = pstats.Stats('profi.txt')
+		p.sort_stats('time').print_stats(75) # or 'cumulative'
+	else:
+		try:
+			try:
+				run_commands()
+			except:
+				if options.pdb:
+					import pdb
+					type, value, tb = sys.exc_info()
+					traceback.print_exc()
+					pdb.post_mortem(tb)
+				else:
+					raise
+		except Errors.WafError as e:
+			if Logs.verbose > 1:
+				Logs.pprint('RED', e.verbose_msg)
+			Logs.error(e.msg)
+			sys.exit(1)
+		except SystemExit:
+			raise
+		except Exception as e:
+			traceback.print_exc(file=sys.stdout)
+			sys.exit(2)
+		except KeyboardInterrupt:
+			Logs.pprint('RED', 'Interrupted')
+			sys.exit(68)
 
 def set_main_module(file_path):
 	"""
@@ -197,7 +198,7 @@ def set_main_module(file_path):
 		name = obj.__name__
 		if not name in Context.g_module.__dict__:
 			setattr(Context.g_module, name, obj)
-	for k in (update, dist, distclean, distcheck):
+	for k in (dist, distclean, distcheck):
 		set_def(k)
 	# add dummy init and shutdown functions if they're not defined
 	if not 'init' in Context.g_module.__dict__:
@@ -209,36 +210,20 @@ def set_main_module(file_path):
 
 def parse_options():
 	"""
-	Parse the command-line options and initialize the logging system.
+	Parses the command-line options and initialize the logging system.
 	Called by :py:func:`waflib.Scripting.waf_entry_point` during the initialization.
 	"""
-	Context.create_context('options').execute()
-
-	for var in Options.envvars:
-		(name, value) = var.split('=', 1)
-		os.environ[name.strip()] = value
-
+	ctx = Context.create_context('options')
+	ctx.execute()
 	if not Options.commands:
-		Options.commands = [default_cmd]
-	Options.commands = [x for x in Options.commands if x != 'options'] # issue 1076
-
-	# process some internal Waf options
-	Logs.verbose = Options.options.verbose
-	#Logs.init_log()
-
-	if Options.options.zones:
-		Logs.zones = Options.options.zones.split(',')
-		if not Logs.verbose:
-			Logs.verbose = 1
-	elif Logs.verbose > 0:
-		Logs.zones = ['runner']
-
-	if Logs.verbose > 2:
-		Logs.zones = ['*']
+		Options.commands.append(default_cmd)
+	if Options.options.whelp:
+		ctx.parser.print_help()
+		sys.exit(0)
 
 def run_command(cmd_name):
 	"""
-	Execute a single command. Called by :py:func:`waflib.Scripting.run_commands`.
+	Executes a single Waf command. Called by :py:func:`waflib.Scripting.run_commands`.
 
 	:param cmd_name: command to execute, like ``build``
 	:type cmd_name: string
@@ -256,7 +241,7 @@ def run_command(cmd_name):
 
 def run_commands():
 	"""
-	Execute the commands that were given on the command-line, and the other options
+	Execute the Waf commands that were given on the command-line, and the other options
 	Called by :py:func:`waflib.Scripting.waf_entry_point` during the initialization, and executed
 	after :py:func:`waflib.Scripting.parse_options`.
 	"""
@@ -265,18 +250,11 @@ def run_commands():
 	while Options.commands:
 		cmd_name = Options.commands.pop(0)
 		ctx = run_command(cmd_name)
-		Logs.info('%r finished successfully (%s)' % (cmd_name, str(ctx.log_timer)))
+		Logs.info('%r finished successfully (%s)', cmd_name, ctx.log_timer)
 	run_command('shutdown')
 
 ###########################################################################################
 
-def _can_distclean(name):
-	# WARNING: this method may disappear anytime
-	for k in '.o .moc .exe'.split():
-		if name.endswith(k):
-			return True
-	return False
-
 def distclean_dir(dirname):
 	"""
 	Distclean function called in the particular case when::
@@ -288,12 +266,12 @@ def distclean_dir(dirname):
 	"""
 	for (root, dirs, files) in os.walk(dirname):
 		for f in files:
-			if _can_distclean(f):
+			if f.endswith(('.o', '.moc', '.exe')):
 				fname = os.path.join(root, f)
 				try:
 					os.remove(fname)
 				except OSError:
-					Logs.warn('Could not remove %r' % fname)
+					Logs.warn('Could not remove %r', fname)
 
 	for x in (Context.DBFILE, 'config.log'):
 		try:
@@ -307,40 +285,53 @@ def distclean_dir(dirname):
 		pass
 
 def distclean(ctx):
-	'''removes the build directory'''
-	lst = os.listdir('.')
-	for f in lst:
-		if f == Options.lockfile:
-			try:
-				proj = ConfigSet.ConfigSet(f)
-			except IOError:
-				Logs.warn('Could not read %r' % f)
-				continue
+	'''removes build folders and data'''
 
-			if proj['out_dir'] != proj['top_dir']:
-				try:
-					shutil.rmtree(proj['out_dir'])
-				except IOError:
-					pass
-				except OSError as e:
-					if e.errno != errno.ENOENT:
-						Logs.warn('Could not remove %r' % proj['out_dir'])
-			else:
-				distclean_dir(proj['out_dir'])
+	def remove_and_log(k, fun):
+		try:
+			fun(k)
+		except EnvironmentError as e:
+			if e.errno != errno.ENOENT:
+				Logs.warn('Could not remove %r', k)
 
-			for k in (proj['out_dir'], proj['top_dir'], proj['run_dir']):
-				p = os.path.join(k, Options.lockfile)
-				try:
-					os.remove(p)
-				except OSError as e:
-					if e.errno != errno.ENOENT:
-						Logs.warn('Could not remove %r' % p)
+	# remove waf cache folders on the top-level
+	if not Options.commands:
+		for k in os.listdir('.'):
+			for x in '.waf-2 waf-2 .waf3-2 waf3-2'.split():
+				if k.startswith(x):
+					remove_and_log(k, shutil.rmtree)
+
+	# remove a build folder, if any
+	cur = '.'
+	if ctx.options.no_lock_in_top:
+		cur = ctx.options.out
+
+	try:
+		lst = os.listdir(cur)
+	except OSError:
+		Logs.warn('Could not read %r', cur)
+		return
+
+	if Options.lockfile in lst:
+		f = os.path.join(cur, Options.lockfile)
+		try:
+			env = ConfigSet.ConfigSet(f)
+		except EnvironmentError:
+			Logs.warn('Could not read %r', f)
+			return
+
+		if not env.out_dir or not env.top_dir:
+			Logs.warn('Invalid lock file %r', f)
+			return
+
+		if env.out_dir == env.top_dir:
+			distclean_dir(env.out_dir)
+		else:
+			remove_and_log(env.out_dir, shutil.rmtree)
 
-		# remove local waf cache folders
-		if not Options.commands:
-			for x in '.waf-1. waf-1. .waf3-1. waf3-1.'.split():
-				if f.startswith(x):
-					shutil.rmtree(f, ignore_errors=True)
+		for k in (env.out_dir, env.top_dir, env.run_dir):
+			p = os.path.join(k, Options.lockfile)
+			remove_and_log(p, os.remove)
 
 class Dist(Context.Context):
 	'''creates an archive containing the project source code'''
@@ -358,7 +349,7 @@ class Dist(Context.Context):
 
 	def archive(self):
 		"""
-		Create the archive.
+		Creates the source archive.
 		"""
 		import tarfile
 
@@ -378,14 +369,14 @@ class Dist(Context.Context):
 		files = self.get_files()
 
 		if self.algo.startswith('tar.'):
-			tar = tarfile.open(arch_name, 'w:' + self.algo.replace('tar.', ''))
+			tar = tarfile.open(node.abspath(), 'w:' + self.algo.replace('tar.', ''))
 
 			for x in files:
 				self.add_tar_file(x, tar)
 			tar.close()
 		elif self.algo == 'zip':
 			import zipfile
-			zip = zipfile.ZipFile(arch_name, 'w', compression=zipfile.ZIP_DEFLATED)
+			zip = zipfile.ZipFile(node.abspath(), 'w', compression=zipfile.ZIP_DEFLATED)
 
 			for x in files:
 				archive_name = self.get_base_name() + '/' + x.path_from(self.base_path)
@@ -395,26 +386,30 @@ class Dist(Context.Context):
 			self.fatal('Valid algo types are tar.bz2, tar.gz, tar.xz or zip')
 
 		try:
-			from hashlib import sha1 as sha
+			from hashlib import sha256
 		except ImportError:
-			from sha import sha
-		try:
-			digest = " (sha=%r)" % sha(node.read()).hexdigest()
-		except Exception:
 			digest = ''
+		else:
+			digest = ' (sha256=%r)' % sha256(node.read(flags='rb')).hexdigest()
 
-		Logs.info('New archive created: %s%s' % (self.arch_name, digest))
+		Logs.info('New archive created: %s%s', self.arch_name, digest)
 
 	def get_tar_path(self, node):
 		"""
-		return the path to use for a node in the tar archive, the purpose of this
+		Return the path to use for a node in the tar archive, the purpose of this
 		is to let subclases resolve symbolic links or to change file names
+
+		:return: absolute path
+		:rtype: string
 		"""
 		return node.abspath()
 
 	def add_tar_file(self, x, tar):
 		"""
-		Add a file to the tar archive. Transform symlinks into files if the files lie out of the project tree.
+		Adds a file to the tar archive. Symlinks are not verified.
+
+		:param x: file path
+		:param tar: tar file object
 		"""
 		p = self.get_tar_path(x)
 		tinfo = tar.gettarinfo(name=p, arcname=self.get_tar_prefix() + '/' + x.path_from(self.base_path))
@@ -423,15 +418,18 @@ class Dist(Context.Context):
 		tinfo.uname = 'root'
 		tinfo.gname = 'root'
 
-		fu = None
-		try:
-			fu = open(p, 'rb')
-			tar.addfile(tinfo, fileobj=fu)
-		finally:
-			if fu:
-				fu.close()
+		if os.path.isfile(p):
+			with open(p, 'rb') as f:
+				tar.addfile(tinfo, fileobj=f)
+		else:
+			tar.addfile(tinfo)
 
 	def get_tar_prefix(self):
+		"""
+		Returns the base path for files added into the archive tar file
+
+		:rtype: string
+		"""
 		try:
 			return self.tar_prefix
 		except AttributeError:
@@ -439,7 +437,8 @@ class Dist(Context.Context):
 
 	def get_arch_name(self):
 		"""
-		Return the name of the archive to create. Change the default value by setting *arch_name*::
+		Returns the archive file name.
+		Set the attribute *arch_name* to change the default value::
 
 			def dist(ctx):
 				ctx.arch_name = 'ctx.tar.bz2'
@@ -454,7 +453,7 @@ class Dist(Context.Context):
 
 	def get_base_name(self):
 		"""
-		Return the default name of the main directory in the archive, which is set to *appname-version*.
+		Returns the default name of the main directory in the archive, which is set to *appname-version*.
 		Set the attribute *base_name* to change the default value::
 
 			def dist(ctx):
@@ -472,8 +471,8 @@ class Dist(Context.Context):
 
 	def get_excl(self):
 		"""
-		Return the patterns to exclude for finding the files in the top-level directory. Set the attribute *excl*
-		to change the default value::
+		Returns the patterns to exclude for finding the files in the top-level directory.
+		Set the attribute *excl* to change the default value::
 
 			def dist(ctx):
 				ctx.excl = 'build **/*.o **/*.class'
@@ -483,7 +482,7 @@ class Dist(Context.Context):
 		try:
 			return self.excl
 		except AttributeError:
-			self.excl = Node.exclude_regs + ' **/waf-1.8.* **/.waf-1.8* **/waf3-1.8.* **/.waf3-1.8* **/*~ **/*.rej **/*.orig **/*.pyc **/*.pyo **/*.bak **/*.swp **/.lock-w*'
+			self.excl = Node.exclude_regs + ' **/waf-2.* **/.waf-2.* **/waf3-2.* **/.waf3-2.* **/*~ **/*.rej **/*.orig **/*.pyc **/*.pyo **/*.bak **/*.swp **/.lock-w*'
 			if Context.out_dir:
 				nd = self.root.find_node(Context.out_dir)
 				if nd:
@@ -492,13 +491,13 @@ class Dist(Context.Context):
 
 	def get_files(self):
 		"""
-		The files to package are searched automatically by :py:func:`waflib.Node.Node.ant_glob`. Set
-		*files* to prevent this behaviour::
+		Files to package are searched automatically by :py:func:`waflib.Node.Node.ant_glob`.
+		Set *files* to prevent this behaviour::
 
 			def dist(ctx):
 				ctx.files = ctx.path.find_node('wscript')
 
-		The files are searched from the directory 'base_path', to change it, set::
+		Files are also searched from the directory 'base_path', to change it, set::
 
 			def dist(ctx):
 				ctx.base_path = path
@@ -511,18 +510,12 @@ class Dist(Context.Context):
 			files = self.base_path.ant_glob('**/*', excl=self.get_excl())
 		return files
 
-
 def dist(ctx):
 	'''makes a tarball for redistributing the sources'''
 	pass
 
 class DistCheck(Dist):
-	"""
-	Create an archive of the project, and try to build the project in a temporary directory::
-
-		$ waf distcheck
-	"""
-
+	"""creates an archive with dist, then tries to build it"""
 	fun = 'distcheck'
 	cmd = 'distcheck'
 
@@ -534,32 +527,30 @@ class DistCheck(Dist):
 		self.archive()
 		self.check()
 
+	def make_distcheck_cmd(self, tmpdir):
+		cfg = []
+		if Options.options.distcheck_args:
+			cfg = shlex.split(Options.options.distcheck_args)
+		else:
+			cfg = [x for x in sys.argv if x.startswith('-')]
+		cmd = [sys.executable, sys.argv[0], 'configure', 'build', 'install', 'uninstall', '--destdir=' + tmpdir] + cfg
+		return cmd
+
 	def check(self):
 		"""
-		Create the archive, uncompress it and try to build the project
+		Creates the archive, uncompresses it and tries to build the project
 		"""
 		import tempfile, tarfile
 
-		t = None
-		try:
-			t = tarfile.open(self.get_arch_name())
+		with tarfile.open(self.get_arch_name()) as t:
 			for x in t:
 				t.extract(x)
-		finally:
-			if t:
-				t.close()
-
-		cfg = []
-
-		if Options.options.distcheck_args:
-			cfg = shlex.split(Options.options.distcheck_args)
-		else:
-			cfg = [x for x in sys.argv if x.startswith('-')]
 
 		instdir = tempfile.mkdtemp('.inst', self.get_base_name())
-		ret = Utils.subprocess.Popen([sys.executable, sys.argv[0], 'configure', 'install', 'uninstall', '--destdir=' + instdir] + cfg, cwd=self.get_base_name()).wait()
+		cmd = self.make_distcheck_cmd(instdir)
+		ret = Utils.subprocess.Popen(cmd, cwd=self.get_base_name()).wait()
 		if ret:
-			raise Errors.WafError('distcheck failed with code %i' % ret)
+			raise Errors.WafError('distcheck failed with code %r' % ret)
 
 		if os.path.exists(instdir):
 			raise Errors.WafError('distcheck succeeded, but files were left in %s' % instdir)
@@ -571,33 +562,14 @@ def distcheck(ctx):
 	'''checks if the project compiles (tarball from 'dist')'''
 	pass
 
-def update(ctx):
-	lst = Options.options.files
-	if lst:
-		lst = lst.split(',')
-	else:
-		path = os.path.join(Context.waf_dir, 'waflib', 'extras')
-		lst = [x for x in Utils.listdir(path) if x.endswith('.py')]
-	for x in lst:
-		tool = x.replace('.py', '')
-		if not tool:
-			continue
-		try:
-			dl = Configure.download_tool
-		except AttributeError:
-			ctx.fatal('The command "update" is dangerous; include the tool "use_config" in your project!')
-		try:
-			dl(tool, force=True, ctx=ctx)
-		except Errors.WafError:
-			Logs.error('Could not find the tool %r in the remote repository' % x)
-		else:
-			Logs.warn('Updated %r' % tool)
-
 def autoconfigure(execute_method):
 	"""
-	Decorator used to set the commands that can be configured automatically
+	Decorator that enables context commands to run *configure* as needed.
 	"""
 	def execute(self):
+		"""
+		Wraps :py:func:`waflib.Context.Context.execute` on the context class
+		"""
 		if not Configure.autoconfig:
 			return execute_method(self)
 
@@ -605,7 +577,7 @@ def autoconfigure(execute_method):
 		do_config = False
 		try:
 			env.load(os.path.join(Context.top_dir, Options.lockfile))
-		except Exception:
+		except EnvironmentError:
 			Logs.warn('Configuring the project')
 			do_config = True
 		else:
@@ -613,15 +585,21 @@ def autoconfigure(execute_method):
 				do_config = True
 			else:
 				h = 0
-				for f in env['files']:
-					h = Utils.h_list((h, Utils.readf(f, 'rb')))
-				do_config = h != env.hash
+				for f in env.files:
+					try:
+						h = Utils.h_list((h, Utils.readf(f, 'rb')))
+					except EnvironmentError:
+						do_config = True
+						break
+				else:
+					do_config = h != env.hash
 
 		if do_config:
-			cmd = env['config_cmd'] or 'configure'
+			cmd = env.config_cmd or 'configure'
 			if Configure.autoconfig == 'clobber':
 				tmp = Options.options.__dict__
-				Options.options.__dict__ = env.options
+				if env.options:
+					Options.options.__dict__ = env.options
 				try:
 					run_command(cmd)
 				finally:
diff --git a/waflib/Task.py b/waflib/Task.py
index b61b50b2..0fc449d4 100644
--- a/waflib/Task.py
+++ b/waflib/Task.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Tasks represent atomic operations such as processes.
 """
 
-import os, re, sys
+import os, re, sys, tempfile, traceback
 from waflib import Utils, Logs, Errors
 
 # task states
@@ -20,7 +20,10 @@ CRASHED = 2
 """The task execution returned a non-zero exit status"""
 
 EXCEPTION = 3
-"""An exception occured in the task execution"""
+"""An exception occurred in the task execution"""
+
+CANCELED = 4
+"""A dependency for the task is missing so it was cancelled"""
 
 SKIPPED = 8
 """The task did not have to be executed"""
@@ -37,18 +40,18 @@ SKIP_ME = -2
 RUN_ME = -3
 """The task must be executed"""
 
-# To save some memory during the build, consider discarding tsk.last_cmd in the two templates below
+CANCEL_ME = -4
+"""The task cannot be executed because of a dependency problem"""
 
 COMPILE_TEMPLATE_SHELL = '''
 def f(tsk):
 	env = tsk.env
 	gen = tsk.generator
 	bld = gen.bld
-	cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
-	wd = getattr(tsk, 'cwd', None)
+	cwdx = tsk.get_cwd()
 	p = env.get_flat
 	tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
-	return tsk.exec_command(cmd, cwd=wd, env=env.env or None)
+	return tsk.exec_command(cmd, cwd=cwdx, env=env.env or None)
 '''
 
 COMPILE_TEMPLATE_NOSHELL = '''
@@ -56,35 +59,53 @@ def f(tsk):
 	env = tsk.env
 	gen = tsk.generator
 	bld = gen.bld
-	cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
-	wd = getattr(tsk, 'cwd', None)
+	cwdx = tsk.get_cwd()
 	def to_list(xx):
 		if isinstance(xx, str): return [xx]
 		return xx
-	tsk.last_cmd = lst = []
+	def merge(lst1, lst2):
+		if lst1 and lst2:
+			return lst1[:-1] + [lst1[-1] + lst2[0]] + lst2[1:]
+		return lst1 + lst2
+	lst = []
 	%s
-	lst = [x for x in lst if x]
-	return tsk.exec_command(lst, cwd=wd, env=env.env or None)
+	if '' in lst:
+		lst = [x for x in lst if x]
+	tsk.last_cmd = lst
+	return tsk.exec_command(lst, cwd=cwdx, env=env.env or None)
+'''
+
+COMPILE_TEMPLATE_SIG_VARS = '''
+def f(tsk):
+	super(tsk.__class__, tsk).sig_vars()
+	env = tsk.env
+	gen = tsk.generator
+	bld = gen.bld
+	cwdx = tsk.get_cwd()
+	p = env.get_flat
+	buf = []
+	%s
+	tsk.m.update(repr(buf).encode())
 '''
 
 classes = {}
-"Class tasks created by user scripts or Waf tools (maps names to class objects). Task classes defined in Waf tools are registered here through the metaclass :py:class:`waflib.Task.store_task_type`."
+"""
+The metaclass :py:class:`waflib.Task.store_task_type` stores all class tasks
+created by user scripts or Waf tools to this dict. It maps class names to class objects.
+"""
 
 class store_task_type(type):
 	"""
-	Metaclass: store the task classes into :py:const:`waflib.Task.classes`, or to the dict pointed
-	by the class attribute 'register'.
-	The attribute 'run_str' will be processed to compute a method 'run' on the task class
-	The decorator :py:func:`waflib.Task.cache_outputs` is also applied to the class
+	Metaclass: store the task classes into the dict pointed by the
+	class attribute 'register' which defaults to :py:const:`waflib.Task.classes`,
+
+	The attribute 'run_str' is compiled into a method 'run' bound to the task class.
 	"""
 	def __init__(cls, name, bases, dict):
 		super(store_task_type, cls).__init__(name, bases, dict)
 		name = cls.__name__
 
-		if name.endswith('_task'):
-			name = name.replace('_task', '')
-		if name != 'evil' and name != 'TaskBase':
-			global classes
+		if name != 'evil' and name != 'Task':
 			if getattr(cls, 'run_str', None):
 				# if a string is provided, convert it to a method
 				(f, dvars) = compile_fun(cls.run_str, cls.shell)
@@ -93,8 +114,13 @@ class store_task_type(type):
 				# change the name of run_str or it is impossible to subclass with a function
 				cls.run_str = None
 				cls.run = f
+				# process variables
 				cls.vars = list(set(cls.vars + dvars))
 				cls.vars.sort()
+				if cls.vars:
+					fun = compile_sig_vars(cls.vars)
+					if fun:
+						cls.sig_vars = fun
 			elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__:
 				# getattr(cls, 'hcode') would look in the upper classes
 				cls.hcode = Utils.h_cmd(cls.run)
@@ -105,31 +131,32 @@ class store_task_type(type):
 evil = store_task_type('evil', (object,), {})
 "Base class provided to avoid writing a metaclass, so the code can run in python 2.6 and 3.x unmodified"
 
-class TaskBase(evil):
+class Task(evil):
 	"""
-	Base class for all Waf tasks, which should be seen as an interface.
-	For illustration purposes, instances of this class will execute the attribute
-	'fun' in :py:meth:`waflib.Task.TaskBase.run`. When in doubt, create
-	subclasses of :py:class:`waflib.Task.Task` instead.
+	Task objects represents actions to perform such as commands to execute by calling the `run` method.
 
-	Subclasses should override these methods:
+	Detecting when to execute a task occurs in the method :py:meth:`waflib.Task.Task.runnable_status`.
 
-	#. __str__: string to display to the user
-	#. runnable_status: ask the task if it should be run, skipped, or if we have to ask later
-	#. run: let threads execute the task
-	#. post_run: let threads update the data regarding the task (cache)
-
-	.. warning:: For backward compatibility reasons, the suffix "_task" is truncated in derived class names. This limitation will be removed in Waf 1.9.
+	Detecting which tasks to execute is performed through a hash value returned by
+	:py:meth:`waflib.Task.Task.signature`. The task signature is persistent from build to build.
 	"""
+	vars = []
+	"""ConfigSet variables that should trigger a rebuild (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)"""
+
+	always_run = False
+	"""Specify whether task instances must always be executed or not (class attribute)"""
+
+	shell = False
+	"""Execute the command with the shell (class attribute)"""
 
 	color = 'GREEN'
 	"""Color for the console display, see :py:const:`waflib.Logs.colors_lst`"""
 
 	ext_in = []
-	"""File extensions that objects of this task class might use"""
+	"""File extensions that objects of this task class may use"""
 
 	ext_out = []
-	"""File extensions that objects of this task class might create"""
+	"""File extensions that objects of this task class may create"""
 
 	before = []
 	"""List of task class names to execute before instances of this class"""
@@ -137,128 +164,207 @@ class TaskBase(evil):
 	after = []
 	"""List of task class names to execute after instances of this class"""
 
-	hcode = ''
+	hcode = Utils.SIG_NIL
 	"""String representing an additional hash for the class representation"""
 
+	keep_last_cmd = False
+	"""Whether to keep the last command executed on the instance after execution.
+	This may be useful for certain extensions but it can a lot of memory.
+	"""
+
+	weight = 0
+	"""Optional weight to tune the priority for task instances.
+	The higher, the earlier. The weight only applies to single task objects."""
+
+	tree_weight = 0
+	"""Optional weight to tune the priority of task instances and whole subtrees.
+	The higher, the earlier."""
+
+	prio_order = 0
+	"""Priority order set by the scheduler on instances during the build phase.
+	You most likely do not need to set it.
+	"""
+
+	__slots__ = ('hasrun', 'generator', 'env', 'inputs', 'outputs', 'dep_nodes', 'run_after')
+
 	def __init__(self, *k, **kw):
-		"""
-		The base task class requires a task generator, which will be itself if missing
-		"""
 		self.hasrun = NOT_RUN
 		try:
 			self.generator = kw['generator']
 		except KeyError:
 			self.generator = self
 
-	def __repr__(self):
-		"for debugging purposes"
-		return '\n\t{task %r: %s %s}' % (self.__class__.__name__, id(self), str(getattr(self, 'fun', '')))
+		self.env = kw['env']
+		""":py:class:`waflib.ConfigSet.ConfigSet` object (make sure to provide one)"""
 
-	def __str__(self):
-		"string to display to the user"
-		if hasattr(self, 'fun'):
-			return self.fun.__name__
-		return self.__class__.__name__
+		self.inputs  = []
+		"""List of input nodes, which represent the files used by the task instance"""
 
-	def __hash__(self):
-		"Very fast hashing scheme but not persistent (replace/implement in subclasses and see :py:meth:`waflib.Task.Task.uid`)"
-		return id(self)
+		self.outputs = []
+		"""List of output nodes, which represent the files created by the task instance"""
 
-	def keyword(self):
-		if hasattr(self, 'fun'):
-			return 'Function'
-		return 'Processing'
+		self.dep_nodes = []
+		"""List of additional nodes to depend on"""
 
-	def exec_command(self, cmd, **kw):
-		"""
-		Wrapper for :py:meth:`waflib.Context.Context.exec_command` which sets a current working directory to ``build.variant_dir``
+		self.run_after = set()
+		"""Set of tasks that must be executed before this one"""
 
-		:return: the return code
-		:rtype: int
+	def __lt__(self, other):
+		return self.priority() > other.priority()
+	def __le__(self, other):
+		return self.priority() >= other.priority()
+	def __gt__(self, other):
+		return self.priority() < other.priority()
+	def __ge__(self, other):
+		return self.priority() <= other.priority()
+
+	def get_cwd(self):
+		"""
+		:return: current working directory
+		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		bld = self.generator.bld
-		try:
-			if not kw.get('cwd', None):
-				kw['cwd'] = bld.cwd
-		except AttributeError:
-			bld.cwd = kw['cwd'] = bld.variant_dir
-		return bld.exec_command(cmd, **kw)
+		ret = getattr(self, 'cwd', None) or getattr(bld, 'cwd', bld.bldnode)
+		if isinstance(ret, str):
+			if os.path.isabs(ret):
+				ret = bld.root.make_node(ret)
+			else:
+				ret = self.generator.path.make_node(ret)
+		return ret
 
-	def runnable_status(self):
+	def quote_flag(self, x):
+		"""
+		Surround a process argument by quotes so that a list of arguments can be written to a file
+
+		:param x: flag
+		:type x: string
+		:return: quoted flag
+		:rtype: string
+		"""
+		old = x
+		if '\\' in x:
+			x = x.replace('\\', '\\\\')
+		if '"' in x:
+			x = x.replace('"', '\\"')
+		if old != x or ' ' in x or '\t' in x or "'" in x:
+			x = '"%s"' % x
+		return x
+
+	def priority(self):
+		"""
+		Priority of execution; the higher, the earlier
+
+		:return: the priority value
+		:rtype: a tuple of numeric values
 		"""
-		State of the task
+		return (self.weight + self.prio_order, - getattr(self.generator, 'tg_idx_count', 0))
 
-		:return: a task state in :py:const:`waflib.Task.RUN_ME`, :py:const:`waflib.Task.SKIP_ME` or :py:const:`waflib.Task.ASK_LATER`.
+	def split_argfile(self, cmd):
+		"""
+		Splits a list of process commands into the executable part and its list of arguments
+
+		:return: a tuple containing the executable first and then the rest of arguments
+		:rtype: tuple
+		"""
+		return ([cmd[0]], [self.quote_flag(x) for x in cmd[1:]])
+
+	def exec_command(self, cmd, **kw):
+		"""
+		Wrapper for :py:meth:`waflib.Context.Context.exec_command`.
+		This version set the current working directory (``build.variant_dir``),
+		applies PATH settings (if self.env.PATH is provided), and can run long
+		commands through a temporary ``@argfile``.
+
+		:param cmd: process command to execute
+		:type cmd: list of string (best) or string (process will use a shell)
+		:return: the return code
 		:rtype: int
+
+		Optional parameters:
+
+		#. cwd: current working directory (Node or string)
+		#. stdout: set to None to prevent waf from capturing the process standard output
+		#. stderr: set to None to prevent waf from capturing the process standard error
+		#. timeout: timeout value (Python 3)
 		"""
-		return RUN_ME
+		if not 'cwd' in kw:
+			kw['cwd'] = self.get_cwd()
+
+		if hasattr(self, 'timeout'):
+			kw['timeout'] = self.timeout
+
+		if self.env.PATH:
+			env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ)
+			env['PATH'] = self.env.PATH if isinstance(self.env.PATH, str) else os.pathsep.join(self.env.PATH)
+
+		if hasattr(self, 'stdout'):
+			kw['stdout'] = self.stdout
+		if hasattr(self, 'stderr'):
+			kw['stderr'] = self.stderr
+
+		# workaround for command line length limit:
+		# http://support.microsoft.com/kb/830473
+		if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000):
+			cmd, args = self.split_argfile(cmd)
+			try:
+				(fd, tmp) = tempfile.mkstemp()
+				os.write(fd, '\r\n'.join(args).encode())
+				os.close(fd)
+				if Logs.verbose:
+					Logs.debug('argfile: @%r -> %r', tmp, args)
+				return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw)
+			finally:
+				try:
+					os.remove(tmp)
+				except OSError:
+					# anti-virus and indexers can keep files open -_-
+					pass
+		else:
+			return self.generator.bld.exec_command(cmd, **kw)
 
 	def process(self):
 		"""
-		Assume that the task has had a new attribute ``master`` which is an instance of :py:class:`waflib.Runner.Parallel`.
-		Execute the task and then put it back in the queue :py:attr:`waflib.Runner.Parallel.out` (may be replaced by subclassing).
-		"""
-		m = self.master
-		if m.stop:
-			m.out.put(self)
-			return
+		Runs the task and handles errors
 
+		:return: 0 or None if everything is fine
+		:rtype: integer
+		"""
 		# remove the task signature immediately before it is executed
-		# in case of failure the task will be executed again
+		# so that the task will be executed again in case of failure
 		try:
-			# TODO waf 1.9 - this breaks encapsulation
 			del self.generator.bld.task_sigs[self.uid()]
 		except KeyError:
 			pass
 
 		try:
-			self.generator.bld.returned_tasks.append(self)
-			self.log_display(self.generator.bld)
 			ret = self.run()
 		except Exception:
-			self.err_msg = Utils.ex_stack()
+			self.err_msg = traceback.format_exc()
 			self.hasrun = EXCEPTION
-
-			# TODO cleanup
-			m.error_handler(self)
-			m.out.put(self)
-			return
-
-		if ret:
-			self.err_code = ret
-			self.hasrun = CRASHED
 		else:
+			if ret:
+				self.err_code = ret
+				self.hasrun = CRASHED
+			else:
+				try:
+					self.post_run()
+				except Errors.WafError:
+					pass
+				except Exception:
+					self.err_msg = traceback.format_exc()
+					self.hasrun = EXCEPTION
+				else:
+					self.hasrun = SUCCESS
+
+		if self.hasrun != SUCCESS and self.scan:
+			# rescan dependencies on next run
 			try:
-				self.post_run()
-			except Errors.WafError:
+				del self.generator.bld.imp_sigs[self.uid()]
+			except KeyError:
 				pass
-			except Exception:
-				self.err_msg = Utils.ex_stack()
-				self.hasrun = EXCEPTION
-			else:
-				self.hasrun = SUCCESS
-		if self.hasrun != SUCCESS:
-			m.error_handler(self)
-
-		m.out.put(self)
-
-	def run(self):
-		"""
-		Called by threads to execute the tasks. The default is empty and meant to be overridden in subclasses.
-		It is a bad idea to create nodes in this method (so, no node.ant_glob)
-
-		:rtype: int
-		"""
-		if hasattr(self, 'fun'):
-			return self.fun(self)
-		return 0
-
-	def post_run(self):
-		"Update the cache files (executed by threads). Override in subclasses."
-		pass
 
 	def log_display(self, bld):
-		"Write the execution status on the context logger"
+		"Writes the execution status on the context logger"
 		if self.generator.bld.progress_bar == 3:
 			return
 
@@ -278,20 +384,17 @@ class TaskBase(evil):
 
 	def display(self):
 		"""
-		Return an execution status for the console, the progress bar, or the IDE output.
+		Returns an execution status for the console, the progress bar, or the IDE output.
 
 		:rtype: string
 		"""
 		col1 = Logs.colors(self.color)
 		col2 = Logs.colors.NORMAL
-		master = self.master
+		master = self.generator.bld.producer
 
 		def cur():
 			# the current task position, computed as late as possible
-			tmp = -1
-			if hasattr(master, 'ready'):
-				tmp -= master.ready.qsize()
-			return master.processed + tmp
+			return master.processed - master.ready.qsize()
 
 		if self.generator.bld.progress_bar == 1:
 			return self.generator.bld.progress_line(cur(), master.total, col1, col2)
@@ -320,37 +423,25 @@ class TaskBase(evil):
 			kw += ' '
 		return fs % (cur(), total, kw, col1, s, col2)
 
-	def attr(self, att, default=None):
-		"""
-		Retrieve an attribute from the instance or from the class.
-
-		:param att: variable name
-		:type att: string
-		:param default: default value
-		"""
-		ret = getattr(self, att, self)
-		if ret is self: return getattr(self.__class__, att, default)
-		return ret
-
 	def hash_constraints(self):
 		"""
-		Identify a task type for all the constraints relevant for the scheduler: precedence, file production
+		Identifies a task type for all the constraints relevant for the scheduler: precedence, file production
 
 		:return: a hash value
 		:rtype: string
 		"""
-		cls = self.__class__
-		tup = (str(cls.before), str(cls.after), str(cls.ext_in), str(cls.ext_out), cls.__name__, cls.hcode)
-		h = hash(tup)
-		return h
+		return (tuple(self.before), tuple(self.after), tuple(self.ext_in), tuple(self.ext_out), self.__class__.__name__, self.hcode)
 
 	def format_error(self):
 		"""
-		Error message to display to the user when a build fails
+		Returns an error message to display the build failure reasons
 
 		:rtype: string
 		"""
-		msg = getattr(self, 'last_cmd', '')
+		if Logs.verbose:
+			msg = ': %r\n%r' % (self, getattr(self, 'last_cmd', ''))
+		else:
+			msg = ' (run with -v to display more information)'
 		name = getattr(self.generator, 'name', '')
 		if getattr(self, "err_msg", None):
 			return self.err_msg
@@ -358,27 +449,29 @@ class TaskBase(evil):
 			return 'task in %r was not executed for some reason: %r' % (name, self)
 		elif self.hasrun == CRASHED:
 			try:
-				return ' -> task in %r failed (exit status %r): %r\n%r' % (name, self.err_code, self, msg)
+				return ' -> task in %r failed with exit status %r%s' % (name, self.err_code, msg)
 			except AttributeError:
-				return ' -> task in %r failed: %r\n%r' % (name, self, msg)
+				return ' -> task in %r failed%s' % (name, msg)
 		elif self.hasrun == MISSING:
-			return ' -> missing files in %r: %r\n%r' % (name, self, msg)
+			return ' -> missing files in %r%s' % (name, msg)
+		elif self.hasrun == CANCELED:
+			return ' -> %r canceled because of missing dependencies' % name
 		else:
 			return 'invalid status for task in %r: %r' % (name, self.hasrun)
 
 	def colon(self, var1, var2):
 		"""
-		Support code for scriptlet expressions such as ${FOO_ST:FOO}
+		Enable scriptlet expressions of the form ${FOO_ST:FOO}
 		If the first variable (FOO_ST) is empty, then an empty list is returned
 
 		The results will be slightly different if FOO_ST is a list, for example::
 
-			env.FOO_ST = ['-a', '-b']
+			env.FOO    = ['p1', 'p2']
 			env.FOO_ST = '-I%s'
 			# ${FOO_ST:FOO} returns
 			['-Ip1', '-Ip2']
 
-			env.FOO    = ['p1', 'p2']
+			env.FOO_ST = ['-a', '-b']
 			# ${FOO_ST:FOO} returns
 			['-a', '-b', 'p1', '-a', '-b', 'p2']
 		"""
@@ -399,47 +492,11 @@ class TaskBase(evil):
 				lst.append(y)
 			return lst
 
-class Task(TaskBase):
-	"""
-	This class deals with the filesystem (:py:class:`waflib.Node.Node`). The method :py:class:`waflib.Task.Task.runnable_status`
-	uses a hash value (from :py:class:`waflib.Task.Task.signature`) which is persistent from build to build. When the value changes,
-	the task has to be executed. The method :py:class:`waflib.Task.Task.post_run` will assign the task signature to the output
-	nodes (if present).
-
-	.. warning:: For backward compatibility reasons, the suffix "_task" is truncated in derived class names. This limitation will be removed in Waf 1.9.
-	"""
-	vars = []
-	"""Variables to depend on (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)"""
-
-	shell = False
-	"""Execute the command with the shell (class attribute)"""
-
-	def __init__(self, *k, **kw):
-		TaskBase.__init__(self, *k, **kw)
-
-		self.env = kw['env']
-		"""ConfigSet object (make sure to provide one)"""
-
-		self.inputs  = []
-		"""List of input nodes, which represent the files used by the task instance"""
-
-		self.outputs = []
-		"""List of output nodes, which represent the files created by the task instance"""
-
-		self.dep_nodes = []
-		"""List of additional nodes to depend on"""
-
-		self.run_after = set([])
-		"""Set of tasks that must be executed before this one"""
-
-		# Additionally, you may define the following
-		#self.dep_vars  = 'PREFIX DATADIR'
-
 	def __str__(self):
 		"string to display to the user"
 		name = self.__class__.__name__
 		if self.outputs:
-			if (name.endswith('lib') or name.endswith('program')) or not self.inputs:
+			if name.endswith(('lib', 'program')) or not self.inputs:
 				node = self.outputs[0]
 				return node.path_from(node.ctx.launch_node())
 		if not (self.inputs or self.outputs):
@@ -450,13 +507,16 @@ class Task(TaskBase):
 
 		src_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.inputs])
 		tgt_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.outputs])
-		if self.outputs: sep = ' -> '
-		else: sep = ''
-		return '%s: %s%s%s' % (self.__class__.__name__.replace('_task', ''), src_str, sep, tgt_str)
+		if self.outputs:
+			sep = ' -> '
+		else:
+			sep = ''
+		return '%s: %s%s%s' % (self.__class__.__name__, src_str, sep, tgt_str)
 
 	def keyword(self):
+		"Display keyword used to prettify the console outputs"
 		name = self.__class__.__name__
-		if name.endswith('lib') or name.endswith('program'):
+		if name.endswith(('lib', 'program')):
 			return 'Linking'
 		if len(self.inputs) == 1 and len(self.outputs) == 1:
 			return 'Compiling'
@@ -479,10 +539,10 @@ class Task(TaskBase):
 
 	def uid(self):
 		"""
-		Return an identifier used to determine if tasks are up-to-date. Since the
+		Returns an identifier used to determine if tasks are up-to-date. Since the
 		identifier will be stored between executions, it must be:
 
-			- unique: no two tasks return the same value (for a given build context)
+			- unique for a task: no two tasks return the same value (for a given build context)
 			- the same for a given task instance
 
 		By default, the node paths, the class name, and the function are used
@@ -497,44 +557,48 @@ class Task(TaskBase):
 		try:
 			return self.uid_
 		except AttributeError:
-			m = Utils.md5()
+			m = Utils.md5(self.__class__.__name__)
 			up = m.update
-			up(self.__class__.__name__)
 			for x in self.inputs + self.outputs:
 				up(x.abspath())
 			self.uid_ = m.digest()
 			return self.uid_
 
-
 	def set_inputs(self, inp):
 		"""
-		Append the nodes to the *inputs*
+		Appends the nodes to the *inputs* list
 
 		:param inp: input nodes
 		:type inp: node or list of nodes
 		"""
-		if isinstance(inp, list): self.inputs += inp
-		else: self.inputs.append(inp)
+		if isinstance(inp, list):
+			self.inputs += inp
+		else:
+			self.inputs.append(inp)
 
 	def set_outputs(self, out):
 		"""
-		Append the nodes to the *outputs*
+		Appends the nodes to the *outputs* list
 
 		:param out: output nodes
 		:type out: node or list of nodes
 		"""
-		if isinstance(out, list): self.outputs += out
-		else: self.outputs.append(out)
+		if isinstance(out, list):
+			self.outputs += out
+		else:
+			self.outputs.append(out)
 
 	def set_run_after(self, task):
 		"""
-		Run this task only after *task*. Affect :py:meth:`waflib.Task.runnable_status`
-		You probably want to use tsk.run_after.add(task) directly
+		Run this task only after the given *task*.
+
+		Calling this method from :py:meth:`waflib.Task.Task.runnable_status` may cause
+		build deadlocks; see :py:meth:`waflib.Tools.fc.fc.runnable_status` for details.
 
 		:param task: task
 		:type task: :py:class:`waflib.Task.Task`
 		"""
-		assert isinstance(task, TaskBase)
+		assert isinstance(task, Task)
 		self.run_after.add(task)
 
 	def signature(self):
@@ -544,7 +608,7 @@ class Task(TaskBase):
 
 		* explicit dependencies: files listed in the inputs (list of node objects) :py:meth:`waflib.Task.Task.sig_explicit_deps`
 		* implicit dependencies: list of nodes returned by scanner methods (when present) :py:meth:`waflib.Task.Task.sig_implicit_deps`
-		* hashed data: variables/values read from task.__class__.vars/task.env :py:meth:`waflib.Task.Task.sig_vars`
+		* hashed data: variables/values read from task.vars/task.env :py:meth:`waflib.Task.Task.sig_vars`
 
 		If the signature is expected to give a different result, clear the cache kept in ``self.cache_sig``::
 
@@ -554,12 +618,16 @@ class Task(TaskBase):
 					sig = super(Task.Task, self).signature()
 					delattr(self, 'cache_sig')
 					return super(Task.Task, self).signature()
+
+		:return: the signature value
+		:rtype: string or bytes
 		"""
-		try: return self.cache_sig
-		except AttributeError: pass
+		try:
+			return self.cache_sig
+		except AttributeError:
+			pass
 
-		self.m = Utils.md5()
-		self.m.update(self.hcode)
+		self.m = Utils.md5(self.hcode)
 
 		# explicit deps
 		self.sig_explicit_deps()
@@ -579,16 +647,22 @@ class Task(TaskBase):
 
 	def runnable_status(self):
 		"""
-		Override :py:meth:`waflib.Task.TaskBase.runnable_status` to determine if the task is ready
-		to be run (:py:attr:`waflib.Task.Task.run_after`)
+		Returns the Task status
+
+		:return: a task state in :py:const:`waflib.Task.RUN_ME`,
+			:py:const:`waflib.Task.SKIP_ME`, :py:const:`waflib.Task.CANCEL_ME` or :py:const:`waflib.Task.ASK_LATER`.
+		:rtype: int
 		"""
-		#return 0 # benchmarking
+		bld = self.generator.bld
+		if bld.is_install < 0:
+			return SKIP_ME
 
 		for t in self.run_after:
 			if not t.hasrun:
 				return ASK_LATER
-
-		bld = self.generator.bld
+			elif t.hasrun < SKIPPED:
+				# a dependency has an error
+				return CANCEL_ME
 
 		# first compute the signature
 		try:
@@ -601,105 +675,110 @@ class Task(TaskBase):
 		try:
 			prev_sig = bld.task_sigs[key]
 		except KeyError:
-			Logs.debug("task: task %r must run as it was never run before or the task code changed" % self)
+			Logs.debug('task: task %r must run: it was never run before or the task code changed', self)
+			return RUN_ME
+
+		if new_sig != prev_sig:
+			Logs.debug('task: task %r must run: the task signature changed', self)
 			return RUN_ME
 
 		# compare the signatures of the outputs
 		for node in self.outputs:
-			try:
-				if node.sig != new_sig:
-					return RUN_ME
-			except AttributeError:
-				Logs.debug("task: task %r must run as the output nodes do not exist" % self)
+			sig = bld.node_sigs.get(node)
+			if not sig:
+				Logs.debug('task: task %r must run: an output node has no signature', self)
+				return RUN_ME
+			if sig != key:
+				Logs.debug('task: task %r must run: an output node was produced by another task', self)
+				return RUN_ME
+			if not node.exists():
+				Logs.debug('task: task %r must run: an output node does not exist', self)
 				return RUN_ME
 
-		if new_sig != prev_sig:
-			return RUN_ME
-		return SKIP_ME
+		return (self.always_run and RUN_ME) or SKIP_ME
 
 	def post_run(self):
 		"""
-		Called after successful execution to update the cache data :py:class:`waflib.Node.Node` sigs
-		and :py:attr:`waflib.Build.BuildContext.task_sigs`.
-
-		The node signature is obtained from the task signature, but the output nodes may also get the signature
-		of their contents. See the class decorator :py:func:`waflib.Task.update_outputs` if you need this behaviour.
+		Called after successful execution to record that the task has run by
+		updating the entry in :py:attr:`waflib.Build.BuildContext.task_sigs`.
 		"""
 		bld = self.generator.bld
-		sig = self.signature()
-
 		for node in self.outputs:
-			# check if the node exists ..
-			try:
-				os.stat(node.abspath())
-			except OSError:
+			if not node.exists():
 				self.hasrun = MISSING
 				self.err_msg = '-> missing file: %r' % node.abspath()
 				raise Errors.WafError(self.err_msg)
-
-			# important, store the signature for the next run
-			node.sig = node.cache_sig = sig
-
-		bld.task_sigs[self.uid()] = self.cache_sig
+			bld.node_sigs[node] = self.uid() # make sure this task produced the files in question
+		bld.task_sigs[self.uid()] = self.signature()
+		if not self.keep_last_cmd:
+			try:
+				del self.last_cmd
+			except AttributeError:
+				pass
 
 	def sig_explicit_deps(self):
 		"""
-		Used by :py:meth:`waflib.Task.Task.signature`, hash :py:attr:`waflib.Task.Task.inputs`
+		Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.inputs`
 		and :py:attr:`waflib.Task.Task.dep_nodes` signatures.
-
-		:rtype: hash value
 		"""
 		bld = self.generator.bld
 		upd = self.m.update
 
 		# the inputs
 		for x in self.inputs + self.dep_nodes:
-			try:
-				upd(x.get_bld_sig())
-			except (AttributeError, TypeError):
-				raise Errors.WafError('Missing node signature for %r (required by %r)' % (x, self))
+			upd(x.get_bld_sig())
 
 		# manual dependencies, they can slow down the builds
 		if bld.deps_man:
 			additional_deps = bld.deps_man
 			for x in self.inputs + self.outputs:
 				try:
-					d = additional_deps[id(x)]
+					d = additional_deps[x]
 				except KeyError:
 					continue
 
 				for v in d:
-					if isinstance(v, bld.root.__class__):
-						try:
-							v = v.get_bld_sig()
-						except AttributeError:
-							raise Errors.WafError('Missing node signature for %r (required by %r)' % (v, self))
-					elif hasattr(v, '__call__'):
-						v = v() # dependency is a function, call it
+					try:
+						v = v.get_bld_sig()
+					except AttributeError:
+						if hasattr(v, '__call__'):
+							v = v() # dependency is a function, call it
 					upd(v)
 
-		return self.m.digest()
-
-	def sig_vars(self):
+	def sig_deep_inputs(self):
 		"""
-		Used by :py:meth:`waflib.Task.Task.signature`, hash :py:attr:`waflib.Task.Task.env` variables/values
+		Enable rebuilds on input files task signatures. Not used by default.
 
-		:rtype: hash value
-		"""
-		bld = self.generator.bld
-		env = self.env
-		upd = self.m.update
+		Example: hashes of output programs can be unchanged after being re-linked,
+		despite the libraries being different. This method can thus prevent stale unit test
+		results (waf_unit_test.py).
 
-		# dependencies on the environment vars
-		act_sig = bld.hash_env_vars(env, self.__class__.vars)
-		upd(act_sig)
+		Hashing input file timestamps is another possibility for the implementation.
+		This may cause unnecessary rebuilds when input tasks are frequently executed.
+		Here is an implementation example::
 
-		# additional variable dependencies, if provided
-		dep_vars = getattr(self, 'dep_vars', None)
-		if dep_vars:
-			upd(bld.hash_env_vars(env, dep_vars))
+			lst = []
+			for node in self.inputs + self.dep_nodes:
+				st = os.stat(node.abspath())
+				lst.append(st.st_mtime)
+				lst.append(st.st_size)
+			self.m.update(Utils.h_list(lst))
+
+		The downside of the implementation is that it absolutely requires all build directory
+		files to be declared within the current build.
+		"""
+		bld = self.generator.bld
+		lst = [bld.task_sigs[bld.node_sigs[node]] for node in (self.inputs + self.dep_nodes) if node.is_bld()]
+		self.m.update(Utils.h_list(lst))
 
-		return self.m.digest()
+	def sig_vars(self):
+		"""
+		Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.env` variables/values
+		When overriding this method, and if scriptlet expressions are used, make sure to follow
+		the code in :py:meth:`waflib.Task.Task.compile_sig_vars` to enable dependencies on scriptlet results.
+		"""
+		sig = self.generator.bld.hash_env_vars(self.env, self.vars)
+		self.m.update(sig)
 
 	scan = None
 	"""
@@ -713,29 +792,26 @@ class Task(TaskBase):
 		from waflib.Task import Task
 		class mytask(Task):
 			def scan(self, node):
-				return ((), ())
+				return ([], [])
 
-	The first and second lists are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and
+	The first and second lists in the tuple are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and
 	:py:attr:`waflib.Build.BuildContext.raw_deps` respectively.
 	"""
 
 	def sig_implicit_deps(self):
 		"""
-		Used by :py:meth:`waflib.Task.Task.signature` hashes node signatures obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`).
+		Used by :py:meth:`waflib.Task.Task.signature`; it hashes node signatures
+		obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`).
 
 		The exception :py:class:`waflib.Errors.TaskRescan` is thrown
-		when a file has changed. When this occurs, :py:meth:`waflib.Task.Task.signature` is called
-		once again, and this method will be executed once again, this time calling :py:meth:`waflib.Task.Task.scan`
-		for searching the dependencies.
-
-		:rtype: hash value
+		when a file has changed. In this case, the method :py:meth:`waflib.Task.Task.signature` is called
+		once again, and return here to call :py:meth:`waflib.Task.Task.scan` and searching for dependencies.
 		"""
-
 		bld = self.generator.bld
 
 		# get the task signatures from previous runs
 		key = self.uid()
-		prev = bld.task_sigs.get((key, 'imp'), [])
+		prev = bld.imp_sigs.get(key, [])
 
 		# for issue #379
 		if prev:
@@ -745,74 +821,56 @@ class Task(TaskBase):
 			except Errors.TaskNotReady:
 				raise
 			except EnvironmentError:
-				# when a file was renamed (IOError usually), remove the stale nodes (headers in folders without source files)
+				# when a file was renamed, remove the stale nodes (headers in folders without source files)
 				# this will break the order calculation for headers created during the build in the source directory (should be uncommon)
 				# the behaviour will differ when top != out
 				for x in bld.node_deps.get(self.uid(), []):
-					if not x.is_bld():
+					if not x.is_bld() and not x.exists():
 						try:
-							os.stat(x.abspath())
-						except OSError:
-							try:
-								del x.parent.children[x.name]
-							except KeyError:
-								pass
-			del bld.task_sigs[(key, 'imp')]
+							del x.parent.children[x.name]
+						except KeyError:
+							pass
+			del bld.imp_sigs[key]
 			raise Errors.TaskRescan('rescan')
 
 		# no previous run or the signature of the dependencies has changed, rescan the dependencies
-		(nodes, names) = self.scan()
+		(bld.node_deps[key], bld.raw_deps[key]) = self.scan()
 		if Logs.verbose:
-			Logs.debug('deps: scanner for %s returned %s %s' % (str(self), str(nodes), str(names)))
-
-		# store the dependencies in the cache
-		bld.node_deps[key] = nodes
-		bld.raw_deps[key] = names
-
-		# might happen
-		self.are_implicit_nodes_ready()
+			Logs.debug('deps: scanner for %s: %r; unresolved: %r', self, bld.node_deps[key], bld.raw_deps[key])
 
 		# recompute the signature and return it
 		try:
-			bld.task_sigs[(key, 'imp')] = sig = self.compute_sig_implicit_deps()
-		except Exception:
-			if Logs.verbose:
-				for k in bld.node_deps.get(self.uid(), []):
-					try:
-						k.get_bld_sig()
-					except Exception:
-						Logs.warn('Missing signature for node %r (may cause rebuilds)' % k)
-		else:
-			return sig
+			bld.imp_sigs[key] = self.compute_sig_implicit_deps()
+		except EnvironmentError:
+			for k in bld.node_deps.get(self.uid(), []):
+				if not k.exists():
+					Logs.warn('Dependency %r for %r is missing: check the task declaration and the build order!', k, self)
+			raise
 
 	def compute_sig_implicit_deps(self):
 		"""
 		Used by :py:meth:`waflib.Task.Task.sig_implicit_deps` for computing the actual hash of the
 		:py:class:`waflib.Node.Node` returned by the scanner.
 
-		:return: hash value
-		:rtype: string
+		:return: a hash value for the implicit dependencies
+		:rtype: string or bytes
 		"""
-
 		upd = self.m.update
-
-		bld = self.generator.bld
-
 		self.are_implicit_nodes_ready()
 
 		# scanner returns a node that does not have a signature
 		# just *ignore* the error and let them figure out from the compiler output
 		# waf -k behaviour
-		for k in bld.node_deps.get(self.uid(), []):
+		for k in self.generator.bld.node_deps.get(self.uid(), []):
 			upd(k.get_bld_sig())
 		return self.m.digest()
 
 	def are_implicit_nodes_ready(self):
 		"""
-		For each node returned by the scanner, see if there is a task behind it, and force the build order
+		For each node returned by the scanner, see if there is a task that creates it,
+		and infer the build order
 
-		The performance impact on null builds is nearly invisible (1.66s->1.86s), but this is due to
-		agressive caching (1.86s->28s)
+		This has a low performance impact on null builds (1.86s->1.66s) thanks to caching (28s->1.86s)
 		"""
 		bld = self.generator.bld
 		try:
@@ -820,10 +878,11 @@ class Task(TaskBase):
 		except AttributeError:
 			bld.dct_implicit_nodes = cache = {}
 
+		# one cache per build group
 		try:
-			dct = cache[bld.cur]
+			dct = cache[bld.current_group]
 		except KeyError:
-			dct = cache[bld.cur] = {}
+			dct = cache[bld.current_group] = {}
 			for tsk in bld.cur_tasks:
 				for x in tsk.outputs:
 					dct[x] = tsk
@@ -844,11 +903,10 @@ if sys.hexversion > 0x3000000:
 		try:
 			return self.uid_
 		except AttributeError:
-			m = Utils.md5()
+			m = Utils.md5(self.__class__.__name__.encode('latin-1', 'xmlcharrefreplace'))
 			up = m.update
-			up(self.__class__.__name__.encode('iso8859-1', 'xmlcharrefreplace'))
 			for x in self.inputs + self.outputs:
-				up(x.abspath().encode('iso8859-1', 'xmlcharrefreplace'))
+				up(x.abspath().encode('latin-1', 'xmlcharrefreplace'))
 			self.uid_ = m.digest()
 			return self.uid_
 	uid.__doc__ = Task.uid.__doc__
@@ -856,7 +914,7 @@ if sys.hexversion > 0x3000000:
 
 def is_before(t1, t2):
 	"""
-	Return a non-zero value if task t1 is to be executed before task t2::
+	Returns a non-zero value if task t1 is to be executed before task t2::
 
 		t1.ext_out = '.h'
 		t2.ext_in = '.h'
@@ -864,10 +922,10 @@ def is_before(t1, t2):
 		t1.before = ['t2']
 		waflib.Task.is_before(t1, t2) # True
 
-	:param t1: task
-	:type t1: :py:class:`waflib.Task.TaskBase`
-	:param t2: task
-	:type t2: :py:class:`waflib.Task.TaskBase`
+	:param t1: Task object
+	:type t1: :py:class:`waflib.Task.Task`
+	:param t2: Task object
+	:type t2: :py:class:`waflib.Task.Task`
 	"""
 	to_list = Utils.to_list
 	for k in to_list(t2.ext_in):
@@ -884,30 +942,53 @@ def is_before(t1, t2):
 
 def set_file_constraints(tasks):
 	"""
-	Adds tasks to the task 'run_after' attribute based on the task inputs and outputs
+	Updates the ``run_after`` attribute of all tasks based on the task inputs and outputs
 
 	:param tasks: tasks
-	:type tasks: list of :py:class:`waflib.Task.TaskBase`
+	:type tasks: list of :py:class:`waflib.Task.Task`
 	"""
 	ins = Utils.defaultdict(set)
 	outs = Utils.defaultdict(set)
 	for x in tasks:
-		for a in getattr(x, 'inputs', []) + getattr(x, 'dep_nodes', []):
-			ins[id(a)].add(x)
-		for a in getattr(x, 'outputs', []):
-			outs[id(a)].add(x)
+		for a in x.inputs:
+			ins[a].add(x)
+		for a in x.dep_nodes:
+			ins[a].add(x)
+		for a in x.outputs:
+			outs[a].add(x)
 
 	links = set(ins.keys()).intersection(outs.keys())
 	for k in links:
 		for a in ins[k]:
 			a.run_after.update(outs[k])
 
+
+class TaskGroup(object):
+	"""
+	Wrap nxm task order constraints into a single object
+	to prevent the creation of large list/set objects
+
+	This is an optimization
+	"""
+	def __init__(self, prev, next):
+		self.prev = prev
+		self.next = next
+		self.done = False
+
+	def get_hasrun(self):
+		for k in self.prev:
+			if not k.hasrun:
+				return NOT_RUN
+		return SUCCESS
+
+	hasrun = property(get_hasrun, None)
+
 def set_precedence_constraints(tasks):
 	"""
-	Add tasks to the task 'run_after' attribute based on the after/before/ext_out/ext_in attributes
+	Updates the ``run_after`` attribute of all tasks based on the after/before/ext_out/ext_in attributes
 
 	:param tasks: tasks
-	:type tasks: list of :py:class:`waflib.Task.TaskBase`
+	:type tasks: list of :py:class:`waflib.Task.Task`
 	"""
 	cstr_groups = Utils.defaultdict(list)
 	for x in tasks:
@@ -933,13 +1014,20 @@ def set_precedence_constraints(tasks):
 			else:
 				continue
 
-			aval = set(cstr_groups[keys[a]])
-			for x in cstr_groups[keys[b]]:
-				x.run_after.update(aval)
+			a = cstr_groups[keys[a]]
+			b = cstr_groups[keys[b]]
+
+			if len(a) < 2 or len(b) < 2:
+				for x in b:
+					x.run_after.update(a)
+			else:
+				group = TaskGroup(set(a), set(b))
+				for x in b:
+					x.run_after.add(group)
 
 def funex(c):
 	"""
-	Compile a function by 'exec'
+	Compiles a scriptlet expression into a Python function
 
 	:param c: function to compile
 	:type c: string
@@ -950,38 +1038,57 @@ def funex(c):
 	exec(c, dc)
 	return dc['f']
 
-re_novar = re.compile(r"^(SRC|TGT)\W+.*?$")
-reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})", re.M)
+re_cond = re.compile('(?P<var>\w+)|(?P<or>\|)|(?P<and>&)')
+re_novar = re.compile(r'^(SRC|TGT)\W+.*?$')
+reg_act = re.compile(r'(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})', re.M)
 def compile_fun_shell(line):
 	"""
-	Create a compiled function to execute a process with the shell
-	WARNING: this method may disappear anytime, so use compile_fun instead
+	Creates a compiled function to execute a process through a sub-shell
 	"""
-
 	extr = []
 	def repl(match):
 		g = match.group
-		if g('dollar'): return "$"
-		elif g('backslash'): return '\\\\'
-		elif g('subst'): extr.append((g('var'), g('code'))); return "%s"
+		if g('dollar'):
+			return "$"
+		elif g('backslash'):
+			return '\\\\'
+		elif g('subst'):
+			extr.append((g('var'), g('code')))
+			return "%s"
 		return None
-
 	line = reg_act.sub(repl, line) or line
+	dvars = []
+	def add_dvar(x):
+		if x not in dvars:
+			dvars.append(x)
+
+	def replc(m):
+		# performs substitutions and populates dvars
+		if m.group('and'):
+			return ' and '
+		elif m.group('or'):
+			return ' or '
+		else:
+			x = m.group('var')
+			add_dvar(x)
+			return 'env[%r]' % x
 
 	parm = []
-	dvars = []
 	app = parm.append
 	for (var, meth) in extr:
 		if var == 'SRC':
-			if meth: app('tsk.inputs%s' % meth)
-			else: app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
+			if meth:
+				app('tsk.inputs%s' % meth)
+			else:
+				app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
 		elif var == 'TGT':
-			if meth: app('tsk.outputs%s' % meth)
-			else: app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
+			if meth:
+				app('tsk.outputs%s' % meth)
+			else:
+				app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
 		elif meth:
 			if meth.startswith(':'):
-				if var not in dvars:
-					dvars.append(var)
+				add_dvar(var)
 				m = meth[1:]
 				if m == 'SRC':
 					m = '[a.path_from(cwdx) for a in tsk.inputs]'
@@ -991,91 +1098,123 @@ def compile_fun_shell(line):
 					m = '[tsk.inputs%s]' % m[3:]
 				elif re_novar.match(m):
 					m = '[tsk.outputs%s]' % m[3:]
-				elif m[:3] not in ('tsk', 'gen', 'bld'):
-					dvars.append(meth[1:])
-					m = '%r' % m
+				else:
+					add_dvar(m)
+					if m[:3] not in ('tsk', 'gen', 'bld'):
+						m = '%r' % m
 				app('" ".join(tsk.colon(%r, %s))' % (var, m))
+			elif meth.startswith('?'):
+				# In A?B|C output env.A if one of env.B or env.C is non-empty
+				expr = re_cond.sub(replc, meth[1:])
+				app('p(%r) if (%s) else ""' % (var, expr))
 			else:
-				app('%s%s' % (var, meth))
+				call = '%s%s' % (var, meth)
+				add_dvar(call)
+				app(call)
 		else:
-			if var not in dvars:
-				dvars.append(var)
+			add_dvar(var)
 			app("p('%s')" % var)
-	if parm: parm = "%% (%s) " % (',\n\t\t'.join(parm))
-	else: parm = ''
+	if parm:
+		parm = "%% (%s) " % (',\n\t\t'.join(parm))
+	else:
+		parm = ''
 
 	c = COMPILE_TEMPLATE_SHELL % (line, parm)
-	Logs.debug('action: %s' % c.strip().splitlines())
+	Logs.debug('action: %s', c.strip().splitlines())
 	return (funex(c), dvars)
 
+reg_act_noshell = re.compile(r"(?P<space>\s+)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})|(?P<text>([^$ \t\n\r\f\v]|\$\$)+)", re.M)
 def compile_fun_noshell(line):
 	"""
-	Create a compiled function to execute a process without the shell
-	WARNING: this method may disappear anytime, so use compile_fun instead
+	Creates a compiled function to execute a process without a sub-shell
 	"""
-	extr = []
-	def repl(match):
-		g = match.group
-		if g('dollar'): return "$"
-		elif g('backslash'): return '\\'
-		elif g('subst'): extr.append((g('var'), g('code'))); return "<<|@|>>"
-		return None
-
-	line2 = reg_act.sub(repl, line)
-	params = line2.split('<<|@|>>')
-	assert(extr)
-
 	buf = []
 	dvars = []
+	merge = False
 	app = buf.append
-	for x in range(len(extr)):
-		params[x] = params[x].strip()
-		if params[x]:
-			app("lst.extend(%r)" % params[x].split())
-		(var, meth) = extr[x]
-		if var == 'SRC':
-			if meth: app('lst.append(tsk.inputs%s)' % meth)
-			else: app("lst.extend([a.path_from(cwdx) for a in tsk.inputs])")
-		elif var == 'TGT':
-			if meth: app('lst.append(tsk.outputs%s)' % meth)
-			else: app("lst.extend([a.path_from(cwdx) for a in tsk.outputs])")
-		elif meth:
-			if meth.startswith(':'):
-				if not var in dvars:
-					dvars.append(var)
-				m = meth[1:]
-				if m == 'SRC':
-					m = '[a.path_from(cwdx) for a in tsk.inputs]'
-				elif m == 'TGT':
-					m = '[a.path_from(cwdx) for a in tsk.outputs]'
-				elif re_novar.match(m):
-					m = '[tsk.inputs%s]' % m[3:]
-				elif re_novar.match(m):
-					m = '[tsk.outputs%s]' % m[3:]
-				elif m[:3] not in ('tsk', 'gen', 'bld'):
-					dvars.append(m)
-					m = '%r' % m
-				app('lst.extend(tsk.colon(%r, %s))' % (var, m))
-			else:
-				app('lst.extend(gen.to_list(%s%s))' % (var, meth))
-		else:
-			app('lst.extend(to_list(env[%r]))' % var)
-			if not var in dvars:
-				dvars.append(var)
 
-	if extr:
-		if params[-1]:
-			app("lst.extend(%r)" % params[-1].split())
+	def add_dvar(x):
+		if x not in dvars:
+			dvars.append(x)
+
+	def replc(m):
+		# performs substitutions and populates dvars
+		if m.group('and'):
+			return ' and '
+		elif m.group('or'):
+			return ' or '
+		else:
+			x = m.group('var')
+			add_dvar(x)
+			return 'env[%r]' % x
+
+	for m in reg_act_noshell.finditer(line):
+		if m.group('space'):
+			merge = False
+			continue
+		elif m.group('text'):
+			app('[%r]' % m.group('text').replace('$$', '$'))
+		elif m.group('subst'):
+			var = m.group('var')
+			code = m.group('code')
+			if var == 'SRC':
+				if code:
+					app('[tsk.inputs%s]' % code)
+				else:
+					app('[a.path_from(cwdx) for a in tsk.inputs]')
+			elif var == 'TGT':
+				if code:
+					app('[tsk.outputs%s]' % code)
+				else:
+					app('[a.path_from(cwdx) for a in tsk.outputs]')
+			elif code:
+				if code.startswith(':'):
+					# a composed variable ${FOO:OUT}
+					add_dvar(var)
+					m = code[1:]
+					if m == 'SRC':
+						m = '[a.path_from(cwdx) for a in tsk.inputs]'
+					elif m == 'TGT':
+						m = '[a.path_from(cwdx) for a in tsk.outputs]'
+					elif re_novar.match(m):
+						m = '[tsk.inputs%s]' % m[3:]
+					elif re_novar.match(m):
+						m = '[tsk.outputs%s]' % m[3:]
+					else:
+						add_dvar(m)
+						if m[:3] not in ('tsk', 'gen', 'bld'):
+							m = '%r' % m
+					app('tsk.colon(%r, %s)' % (var, m))
+				elif code.startswith('?'):
+					# In A?B|C output env.A if one of env.B or env.C is non-empty
+					expr = re_cond.sub(replc, code[1:])
+					app('to_list(env[%r] if (%s) else [])' % (var, expr))
+				else:
+					# plain code such as ${tsk.inputs[0].abspath()}
+					call = '%s%s' % (var, code)
+					add_dvar(call)
+					app('gen.to_list(%s)' % call)
+			else:
+				# a plain variable such as # a plain variable like ${AR}
+				app('to_list(env[%r])' % var)
+				add_dvar(var)
+		if merge:
+			tmp = 'merge(%s, %s)' % (buf[-2], buf[-1])
+			del buf[-1]
+			buf[-1] = tmp
+		merge = True # next turn
+
+	buf = ['lst.extend(%s)' % x for x in buf]
 	fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf)
-	Logs.debug('action: %s' % fun.strip().splitlines())
+	Logs.debug('action: %s', fun.strip().splitlines())
 	return (funex(fun), dvars)
 
 def compile_fun(line, shell=False):
 	"""
-	Parse a string expression such as "${CC} ${SRC} -o ${TGT}" and return a pair containing:
+	Parses a string expression such as '${CC} ${SRC} -o ${TGT}' and returns a pair containing:
 
-	* the function created (compiled) for use as :py:meth:`waflib.Task.TaskBase.run`
-	* the list of variables that imply a dependency from self.env
+	* The function created (compiled) for use as :py:meth:`waflib.Task.Task.run`
+	* The list of variables that must cause rebuilds when *env* data is modified
 
 	for example::
 
@@ -1085,8 +1224,8 @@ def compile_fun(line, shell=False):
 		def build(bld):
 			bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"')
 
-	The env variables (CXX, ..) on the task must not hold dicts (order)
-	The reserved keywords *TGT* and *SRC* represent the task input and output nodes
+	The env variables (CXX, ..) on the task must not hold dicts so as to preserve a consistent order.
+	The reserved keywords ``TGT`` and ``SRC`` represent the task input and output nodes
 
 	"""
 	if isinstance(line, str):
@@ -1109,12 +1248,42 @@ def compile_fun(line, shell=False):
 				if ret:
 					return ret
 			return None
-		return composed_fun, dvars
+		return composed_fun, dvars_lst
 	if shell:
 		return compile_fun_shell(line)
 	else:
 		return compile_fun_noshell(line)
 
+def compile_sig_vars(vars):
+	"""
+	This method produces a sig_vars method suitable for subclasses that provide
+	scriptlet code in their run_str code.
+	If no such method can be created, this method returns None.
+
+	The purpose of the sig_vars method returned is to ensures
+	that rebuilds occur whenever the contents of the expression changes.
+	This is the case B below::
+
+		import time
+		# case A: regular variables
+		tg = bld(rule='echo ${FOO}')
+		tg.env.FOO = '%s' % time.time()
+		# case B
+		bld(rule='echo ${gen.foo}', foo='%s' % time.time())
+
+	:param vars: env variables such as CXXFLAGS or gen.foo
+	:type vars: list of string
+	:return: A sig_vars method relevant for dependencies if adequate, else None
+	:rtype: A function, or None in most cases
+	"""
+	buf = []
+	for x in sorted(vars):
+		if x[:3] in ('tsk', 'gen', 'bld'):
+			buf.append('buf.append(%s)' % x)
+	if buf:
+		return funex(COMPILE_TEMPLATE_SIG_VARS % '\n\t'.join(buf))
+	return None
+
 def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[], before=[], after=[], shell=False, scan=None):
 	"""
 	Returns a new task subclass with the function ``run`` compiled from the line given.
@@ -1136,10 +1305,6 @@ def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[
 		'vars': vars or [], # function arguments are static, and this one may be modified by the class
 		'color': color,
 		'name': name,
-		'ext_in': Utils.to_list(ext_in),
-		'ext_out': Utils.to_list(ext_out),
-		'before': Utils.to_list(before),
-		'after': Utils.to_list(after),
 		'shell': shell,
 		'scan': scan,
 	}
@@ -1150,80 +1315,80 @@ def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[
 		params['run'] = func
 
 	cls = type(Task)(name, (Task,), params)
-	global classes
 	classes[name] = cls
-	return cls
 
+	if ext_in:
+		cls.ext_in = Utils.to_list(ext_in)
+	if ext_out:
+		cls.ext_out = Utils.to_list(ext_out)
+	if before:
+		cls.before = Utils.to_list(before)
+	if after:
+		cls.after = Utils.to_list(after)
 
-def always_run(cls):
-	"""
-	Task class decorator
+	return cls
 
-	Set all task instances of this class to be executed whenever a build is started
-	The task signature is calculated, but the result of the comparation between
-	task signatures is bypassed
+def deep_inputs(cls):
 	"""
-	old = cls.runnable_status
-	def always(self):
-		ret = old(self)
-		if ret == SKIP_ME:
-			ret = RUN_ME
-		return ret
-	cls.runnable_status = always
+	Task class decorator to enable rebuilds on input files task signatures
+	"""
+	def sig_explicit_deps(self):
+		Task.sig_explicit_deps(self)
+		Task.sig_deep_inputs(self)
+	cls.sig_explicit_deps = sig_explicit_deps
 	return cls
 
-def update_outputs(cls):
+TaskBase = Task
+"Provided for compatibility reasons, TaskBase should not be used"
+
+class TaskSemaphore(object):
 	"""
-	Task class decorator
+	Task semaphores provide a simple and efficient way of throttling the amount of
+	a particular task to run concurrently. The throttling value is capped
+	by the amount of maximum jobs, so for example, a `TaskSemaphore(10)`
+	has no effect in a `-j2` build.
 
-	If you want to create files in the source directory. For example, to keep *foo.txt* in the source
-	directory, create it first and declare::
+	Task semaphores are typically specified on the task class level::
 
-		def build(bld):
-			bld(rule='cp ${SRC} ${TGT}', source='wscript', target='foo.txt', update_outputs=True)
+		class compile(waflib.Task.Task):
+			semaphore = waflib.Task.TaskSemaphore(2)
+			run_str = 'touch ${TGT}'
+
+	Task semaphores are meant to be used by the build scheduler in the main
+	thread, so there are no guarantees of thread safety.
 	"""
-	old_post_run = cls.post_run
-	def post_run(self):
-		old_post_run(self)
-		for node in self.outputs:
-			node.sig = node.cache_sig = Utils.h_file(node.abspath())
-			self.generator.bld.task_sigs[node.abspath()] = self.uid() # issue #1017
-	cls.post_run = post_run
+	def __init__(self, num):
+		"""
+		:param num: maximum value of concurrent tasks
+		:type num: int
+		"""
+		self.num = num
+		self.locking = set()
+		self.waiting = set()
 
+	def is_locked(self):
+		"""Returns True if this semaphore cannot be acquired by more tasks"""
+		return len(self.locking) >= self.num
 
-	old_runnable_status = cls.runnable_status
-	def runnable_status(self):
-		status = old_runnable_status(self)
-		if status != RUN_ME:
-			return status
+	def acquire(self, tsk):
+		"""
+		Mark the semaphore as used by the given task (not re-entrant).
 
-		try:
-			# by default, we check that the output nodes have the signature of the task
-			# perform a second check, returning 'SKIP_ME' as we are expecting that
-			# the signatures do not match
-			bld = self.generator.bld
-			prev_sig = bld.task_sigs[self.uid()]
-			if prev_sig == self.signature():
-				for x in self.outputs:
-					if not x.is_child_of(bld.bldnode):
-						# special case of files created in the source directory
-						# hash them here for convenience -_-
-						x.sig = Utils.h_file(x.abspath())
-					if not x.sig or bld.task_sigs[x.abspath()] != self.uid():
-						return RUN_ME
-				return SKIP_ME
-		except OSError:
-			pass
-		except IOError:
-			pass
-		except KeyError:
-			pass
-		except IndexError:
-			pass
-		except AttributeError:
-			pass
-		return RUN_ME
-	cls.runnable_status = runnable_status
+		:param tsk: task object
+		:type tsk: :py:class:`waflib.Task.Task`
+		:raises: :py:class:`IndexError` in case the resource is already acquired
+		"""
+		if self.is_locked():
+			raise IndexError('Cannot lock more %r' % self.locking)
+		self.locking.add(tsk)
 
-	return cls
+	def release(self, tsk):
+		"""
+		Mark the semaphore as unused by the given task.
+
+		:param tsk: task object
+		:type tsk: :py:class:`waflib.Task.Task`
+		:raises: :py:class:`KeyError` in case the resource is not acquired by the task
+		"""
+		self.locking.remove(tsk)
 
diff --git a/waflib/TaskGen.py b/waflib/TaskGen.py
index 34035949..a74e6431 100644
--- a/waflib/TaskGen.py
+++ b/waflib/TaskGen.py
@@ -1,18 +1,16 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Task generators
 
 The class :py:class:`waflib.TaskGen.task_gen` encapsulates the creation of task objects (low-level code)
 The instances can have various parameters, but the creation of task nodes (Task.py)
-is always postponed. To achieve this, various methods are called from the method "apply"
-
-
+is deferred. To achieve this, various methods are called from the method "apply"
 """
 
-import copy, re, os
+import copy, re, os, functools
 from waflib import Task, Utils, Logs, Errors, ConfigSet, Node
 
 feats = Utils.defaultdict(set)
@@ -22,7 +20,7 @@ HEADER_EXTS = ['.h', '.hpp', '.hxx', '.hh']
 
 class task_gen(object):
 	"""
-	Instances of this class create :py:class:`waflib.Task.TaskBase` when
+	Instances of this class create :py:class:`waflib.Task.Task` when
 	calling the method :py:meth:`waflib.TaskGen.task_gen.post` from the main thread.
 	A few notes:
 
@@ -34,42 +32,28 @@ class task_gen(object):
 	"""
 
 	mappings = Utils.ordered_iter_dict()
-	"""Mappings are global file extension mappings, they are retrieved in the order of definition"""
+	"""Mappings are global file extension mappings that are retrieved in the order of definition"""
 
-	prec = Utils.defaultdict(list)
-	"""Dict holding the precedence rules for task generator methods"""
+	prec = Utils.defaultdict(set)
+	"""Dict that holds the precedence execution rules for task generator methods"""
 
 	def __init__(self, *k, **kw):
 		"""
-		The task generator objects predefine various attributes (source, target) for possible
+		Task generator objects predefine various attributes (source, target) for possible
 		processing by process_rule (make-like rules) or process_source (extensions, misc methods)
 
-		The tasks are stored on the attribute 'tasks'. They are created by calling methods
-		listed in self.meths *or* referenced in the attribute features
-		A topological sort is performed to ease the method re-use.
+		Tasks are stored on the attribute 'tasks'. They are created by calling methods
+		listed in ``self.meths`` or referenced in the attribute ``features``
+		A topological sort is performed to execute the methods in correct order.
 
-		The extra key/value elements passed in kw are set as attributes
+		The extra key/value elements passed in ``kw`` are set as attributes
 		"""
-
-		# so we will have to play with directed acyclic graphs
-		# detect cycles, etc
-		self.source = ''
+		self.source = []
 		self.target = ''
 
 		self.meths = []
 		"""
-		List of method names to execute (it is usually a good idea to avoid touching this)
-		"""
-
-		self.prec = Utils.defaultdict(list)
-		"""
-		Precedence table for sorting the methods in self.meths
-		"""
-
-		self.mappings = {}
-		"""
-		List of mappings {extension -> function} for processing files by extension
-		This is very rarely used, so we do not use an ordered dict here
+		List of method names to execute (internal)
 		"""
 
 		self.features = []
@@ -79,7 +63,7 @@ class task_gen(object):
 
 		self.tasks = []
 		"""
-		List of tasks created.
+		Tasks created are added to this list
 		"""
 
 		if not 'bld' in kw:
@@ -92,31 +76,50 @@ class task_gen(object):
 			self.env = self.bld.env.derive()
 			self.path = self.bld.path # emulate chdir when reading scripts
 
-			# provide a unique id
+			# Provide a unique index per folder
+			# This is part of a measure to prevent output file name collisions
+			path = self.path.abspath()
 			try:
-				self.idx = self.bld.idx[id(self.path)] = self.bld.idx.get(id(self.path), 0) + 1
+				self.idx = self.bld.idx[path] = self.bld.idx.get(path, 0) + 1
 			except AttributeError:
 				self.bld.idx = {}
-				self.idx = self.bld.idx[id(self.path)] = 1
+				self.idx = self.bld.idx[path] = 1
+
+			# Record the global task generator count
+			try:
+				self.tg_idx_count = self.bld.tg_idx_count = self.bld.tg_idx_count + 1
+			except AttributeError:
+				self.tg_idx_count = self.bld.tg_idx_count = 1
 
 		for key, val in kw.items():
 			setattr(self, key, val)
 
 	def __str__(self):
-		"""for debugging purposes"""
+		"""Debugging helper"""
 		return "<task_gen %r declared in %s>" % (self.name, self.path.abspath())
 
 	def __repr__(self):
-		"""for debugging purposes"""
+		"""Debugging helper"""
 		lst = []
-		for x in self.__dict__.keys():
+		for x in self.__dict__:
 			if x not in ('env', 'bld', 'compiled_tasks', 'tasks'):
 				lst.append("%s=%s" % (x, repr(getattr(self, x))))
 		return "bld(%s) in %s" % (", ".join(lst), self.path.abspath())
 
+	def get_cwd(self):
+		"""
+		Current working directory for the task generator, defaults to the build directory.
+		This is still used in a few places but it should disappear at some point as the classes
+		define their own working directory.
+
+		:rtype: :py:class:`waflib.Node.Node`
+		"""
+		return self.bld.bldnode
+
 	def get_name(self):
 		"""
-		If not set, the name is computed from the target name::
+		If the attribute ``name`` is not set on the instance,
+		the name is computed from the target name::
 
 			def build(bld):
 				x = bld(name='foo')
@@ -143,18 +146,20 @@ class task_gen(object):
 
 	def to_list(self, val):
 		"""
-		Ensure that a parameter is a list
+		Ensures that a parameter is a list, see :py:func:`waflib.Utils.to_list`
 
 		:type val: string or list of string
 		:param val: input to return as a list
 		:rtype: list
 		"""
-		if isinstance(val, str): return val.split()
-		else: return val
+		if isinstance(val, str):
+			return val.split()
+		else:
+			return val
 
 	def post(self):
 		"""
-		Create task objects. The following operations are performed:
+		Creates tasks for this task generators. The following operations are performed:
 
 		#. The body of this method is called only once and sets the attribute ``posted``
 		#. The attribute ``features`` is used to add more methods in ``self.meths``
@@ -162,27 +167,25 @@ class task_gen(object):
 		#. The methods are then executed in order
 		#. The tasks created are added to :py:attr:`waflib.TaskGen.task_gen.tasks`
 		"""
-
-		# we could add a decorator to let the task run once, but then python 2.3 will be difficult to support
 		if getattr(self, 'posted', None):
-			#error("OBJECT ALREADY POSTED" + str( self))
 			return False
 		self.posted = True
 
 		keys = set(self.meths)
+		keys.update(feats['*'])
 
 		# add the methods listed in the features
 		self.features = Utils.to_list(self.features)
-		for x in self.features + ['*']:
+		for x in self.features:
 			st = feats[x]
-			if not st:
-				if not x in Task.classes:
-					Logs.warn('feature %r does not exist - bind at least one method to it' % x)
-			keys.update(list(st)) # ironpython 2.7 wants the cast to list
+			if st:
+				keys.update(st)
+			elif not x in Task.classes:
+				Logs.warn('feature %r does not exist - bind at least one method to it?', x)
 
 		# copy the precedence table
 		prec = {}
-		prec_tbl = self.prec or task_gen.prec
+		prec_tbl = self.prec
 		for x in prec_tbl:
 			if x in keys:
 				prec[x] = prec_tbl[x]
@@ -191,17 +194,19 @@ class task_gen(object):
 		tmp = []
 		for a in keys:
 			for x in prec.values():
-				if a in x: break
+				if a in x:
+					break
 			else:
 				tmp.append(a)
 
-		tmp.sort()
+		tmp.sort(reverse=True)
 
 		# topological sort
 		out = []
 		while tmp:
 			e = tmp.pop()
-			if e in keys: out.append(e)
+			if e in keys:
+				out.append(e)
 			try:
 				nlst = prec[e]
 			except KeyError:
@@ -214,47 +219,52 @@ class task_gen(object):
 							break
 					else:
 						tmp.append(x)
+						tmp.sort(reverse=True)
 
 		if prec:
-			txt = '\n'.join(['- %s after %s' % (k, repr(v)) for k, v in prec.items()])
-			raise Errors.WafError('Cycle detected in the method execution\n%s' % txt)
-		out.reverse()
+			buf = ['Cycle detected in the method execution:']
+			for k, v in prec.items():
+				buf.append('- %s after %s' % (k, [x for x in v if x in prec]))
+			raise Errors.WafError('\n'.join(buf))
 		self.meths = out
 
 		# then we run the methods in order
-		Logs.debug('task_gen: posting %s %d' % (self, id(self)))
+		Logs.debug('task_gen: posting %s %d', self, id(self))
 		for x in out:
 			try:
 				v = getattr(self, x)
 			except AttributeError:
 				raise Errors.WafError('%r is not a valid task generator method' % x)
-			Logs.debug('task_gen: -> %s (%d)' % (x, id(self)))
+			Logs.debug('task_gen: -> %s (%d)', x, id(self))
 			v()
 
-		Logs.debug('task_gen: posted %s' % self.name)
+		Logs.debug('task_gen: posted %s', self.name)
 		return True
 
 	def get_hook(self, node):
 		"""
+		Returns the ``@extension`` method to call for a Node of a particular extension.
+
 		:param node: Input file to process
 		:type node: :py:class:`waflib.Tools.Node.Node`
 		:return: A method able to process the input node by looking at the extension
 		:rtype: function
 		"""
 		name = node.name
-		if self.mappings:
-			for k in self.mappings:
+		for k in self.mappings:
+			try:
 				if name.endswith(k):
 					return self.mappings[k]
-		for k in task_gen.mappings:
-			if name.endswith(k):
-				return task_gen.mappings[k]
-		raise Errors.WafError("File %r has no mapping in %r (have you forgotten to load a waf tool?)" % (node, task_gen.mappings.keys()))
+			except TypeError:
+				# regexps objects
+				if k.match(name):
+					return self.mappings[k]
+		keys = list(self.mappings.keys())
+		raise Errors.WafError("File %r has no mapping in %r (load a waf tool?)" % (node, keys))
 
 	def create_task(self, name, src=None, tgt=None, **kw):
 		"""
-		Wrapper for creating task instances. The classes are retrieved from the
-		context class if possible, then from the global dict Task.classes.
+		Creates task instances.
 
 		:param name: task class name
 		:type name: string
@@ -263,7 +273,7 @@ class task_gen(object):
 		:param tgt: output nodes
 		:type tgt: list of :py:class:`waflib.Tools.Node.Node`
 		:return: A task object
-		:rtype: :py:class:`waflib.Task.TaskBase`
+		:rtype: :py:class:`waflib.Task.Task`
 		"""
 		task = Task.classes[name](env=self.env.derive(), generator=self)
 		if src:
@@ -276,7 +286,7 @@ class task_gen(object):
 
 	def clone(self, env):
 		"""
-		Make a copy of a task generator. Once the copy is made, it is necessary to ensure that the
+		Makes a copy of a task generator. Once the copy is made, it is necessary to ensure that the
 		it does not create the same output files as the original, or the same files may
 		be compiled several times.
 
@@ -305,7 +315,7 @@ class task_gen(object):
 def declare_chain(name='', rule=None, reentrant=None, color='BLUE',
 	ext_in=[], ext_out=[], before=[], after=[], decider=None, scan=None, install_path=None, shell=False):
 	"""
-	Create a new mapping and a task class for processing files by extension.
+	Creates a new mapping and a task class for processing files by extension.
 	See Tools/flex.py for an example.
 
 	:param name: name for the task class
@@ -324,7 +334,7 @@ def declare_chain(name='', rule=None, reentrant=None, color='BLUE',
 	:type before: list of string
 	:param after: execute instances of this task after classes of the given names
 	:type after: list of string
-	:param decider: if present, use it to create the output nodes for the task
+	:param decider: if present, function that returns a list of output file extensions (overrides ext_out for output files, but not for the build order)
 	:type decider: function
 	:param scan: scanner function for the task
 	:type scan: function
@@ -338,14 +348,13 @@ def declare_chain(name='', rule=None, reentrant=None, color='BLUE',
 	cls = Task.task_factory(name, rule, color=color, ext_in=ext_in, ext_out=ext_out, before=before, after=after, scan=scan, shell=shell)
 
 	def x_file(self, node):
-		ext = decider and decider(self, node) or cls.ext_out
 		if ext_in:
 			_ext_in = ext_in[0]
 
 		tsk = self.create_task(name, node)
 		cnt = 0
 
-		keys = set(self.mappings.keys()) | set(self.__class__.mappings.keys())
+		ext = decider(self, node) if decider else cls.ext_out
 		for x in ext:
 			k = node.change_ext(x, ext_in=_ext_in)
 			tsk.outputs.append(k)
@@ -355,14 +364,14 @@ def declare_chain(name='', rule=None, reentrant=None, color='BLUE',
 					self.source.append(k)
 			else:
 				# reinject downstream files into the build
-				for y in keys: # ~ nfile * nextensions :-/
+				for y in self.mappings: # ~ nfile * nextensions :-/
 					if k.name.endswith(y):
 						self.source.append(k)
 						break
 			cnt += 1
 
 		if install_path:
-			self.bld.install_files(install_path, tsk.outputs)
+			self.install_task = self.add_install_files(install_to=install_path, install_from=tsk.outputs)
 		return tsk
 
 	for x in cls.ext_in:
@@ -371,7 +380,7 @@ def declare_chain(name='', rule=None, reentrant=None, color='BLUE',
 
 def taskgen_method(func):
 	"""
-	Decorator: register a method as a task generator method.
+	Decorator that registers method as a task generator method.
 	The function must accept a task generator as first parameter::
 
 		from waflib.TaskGen import taskgen_method
@@ -388,8 +397,8 @@ def taskgen_method(func):
 
 def feature(*k):
 	"""
-	Decorator: register a task generator method that will be executed when the
-	object attribute 'feature' contains the corresponding key(s)::
+	Decorator that registers a task generator method that will be executed when the
+	object attribute ``feature`` contains the corresponding key(s)::
 
 		from waflib.Task import feature
 		@feature('myfeature')
@@ -410,7 +419,7 @@ def feature(*k):
 
 def before_method(*k):
 	"""
-	Decorator: register a task generator method which will be executed
+	Decorator that registera task generator method which will be executed
 	before the functions of given name(s)::
 
 		from waflib.TaskGen import feature, before
@@ -430,16 +439,14 @@ def before_method(*k):
 	def deco(func):
 		setattr(task_gen, func.__name__, func)
 		for fun_name in k:
-			if not func.__name__ in task_gen.prec[fun_name]:
-				task_gen.prec[fun_name].append(func.__name__)
-				#task_gen.prec[fun_name].sort()
+			task_gen.prec[func.__name__].add(fun_name)
 		return func
 	return deco
 before = before_method
 
 def after_method(*k):
 	"""
-	Decorator: register a task generator method which will be executed
+	Decorator that registers a task generator method which will be executed
 	after the functions of given name(s)::
 
 		from waflib.TaskGen import feature, after
@@ -459,16 +466,14 @@ def after_method(*k):
 	def deco(func):
 		setattr(task_gen, func.__name__, func)
 		for fun_name in k:
-			if not fun_name in task_gen.prec[func.__name__]:
-				task_gen.prec[func.__name__].append(fun_name)
-				#task_gen.prec[func.__name__].sort()
+			task_gen.prec[fun_name].add(func.__name__)
 		return func
 	return deco
 after = after_method
 
 def extension(*k):
 	"""
-	Decorator: register a task generator method which will be invoked during
+	Decorator that registers a task generator method which will be invoked during
 	the processing of source files for the extension given::
 
 		from waflib import Task
@@ -487,14 +492,11 @@ def extension(*k):
 		return func
 	return deco
 
-# ---------------------------------------------------------------
-# The following methods are task generator methods commonly used
-# they are almost examples, the rest of waf core does not depend on them
-
 @taskgen_method
 def to_nodes(self, lst, path=None):
 	"""
-	Convert the input list into a list of nodes.
+	Flatten the input list of string/nodes/lists into a list of nodes.
+
 	It is used by :py:func:`waflib.TaskGen.process_source` and :py:func:`waflib.TaskGen.process_rule`.
 	It is designed for source files, for folders, see :py:func:`waflib.Tools.ccroot.to_incnodes`:
 
@@ -511,21 +513,23 @@ def to_nodes(self, lst, path=None):
 	if isinstance(lst, Node.Node):
 		lst = [lst]
 
-	# either a list or a string, convert to a list of nodes
 	for x in Utils.to_list(lst):
 		if isinstance(x, str):
 			node = find(x)
-		else:
+		elif hasattr(x, 'name'):
 			node = x
+		else:
+			tmp.extend(self.to_nodes(x))
+			continue
 		if not node:
-			raise Errors.WafError("source not found: %r in %r" % (x, self))
+			raise Errors.WafError('source not found: %r in %r' % (x, self))
 		tmp.append(node)
 	return tmp
 
 @feature('*')
 def process_source(self):
 	"""
-	Process each element in the attribute ``source`` by extension.
+	Processes each element in the attribute ``source`` by extension.
 
 	#. The *source* list is converted through :py:meth:`waflib.TaskGen.to_nodes` to a list of :py:class:`waflib.Node.Node` first.
 	#. File extensions are mapped to methods having the signature: ``def meth(self, node)`` by :py:meth:`waflib.TaskGen.extension`
@@ -541,10 +545,29 @@ def process_source(self):
 @before_method('process_source')
 def process_rule(self):
 	"""
-	Process the attribute ``rule``. When present, :py:meth:`waflib.TaskGen.process_source` is disabled::
+	Processes the attribute ``rule``. When present, :py:meth:`waflib.TaskGen.process_source` is disabled::
 
 		def build(bld):
 			bld(rule='cp ${SRC} ${TGT}', source='wscript', target='bar.txt')
+
+	Main attributes processed:
+
+	* rule: command to execute, it can be a tuple of strings for multiple commands
+	* chmod: permissions for the resulting files (integer value such as Utils.O755)
+	* shell: set to False to execute the command directly (default is True to use a shell)
+	* scan: scanner function
+	* vars: list of variables to trigger rebuilds, such as CFLAGS
+	* cls_str: string to display when executing the task
+	* cls_keyword: label to display when executing the task
+	* cache_rule: by default, try to re-use similar classes, set to False to disable
+	* source: list of Node or string objects representing the source files required by this task
+	* target: list of Node or string objects representing the files that this task creates
+	* cwd: current working directory (Node or string)
+	* stdout: standard output, set to None to prevent waf from capturing the text
+	* stderr: standard error, set to None to prevent waf from capturing the text
+	* timeout: timeout for command execution (Python 3)
+	* always: whether to always run the command (False by default)
+	* deep_inputs: whether the task must depend on the input file tasks too (False by default)
 	"""
 	if not getattr(self, 'rule', None):
 		return
@@ -558,28 +581,55 @@ def process_rule(self):
 	except AttributeError:
 		cache = self.bld.cache_rule_attr = {}
 
+	chmod = getattr(self, 'chmod', None)
+	shell = getattr(self, 'shell', True)
+	color = getattr(self, 'color', 'BLUE')
+	scan = getattr(self, 'scan', None)
+	_vars = getattr(self, 'vars', [])
+	cls_str = getattr(self, 'cls_str', None)
+	cls_keyword = getattr(self, 'cls_keyword', None)
+	use_cache = getattr(self, 'cache_rule', 'True')
+	deep_inputs = getattr(self, 'deep_inputs', False)
+
+	scan_val = has_deps = hasattr(self, 'deps')
+	if scan:
+		scan_val = id(scan)
+
+	key = Utils.h_list((name, self.rule, chmod, shell, color, cls_str, cls_keyword, scan_val, _vars, deep_inputs))
+
 	cls = None
-	if getattr(self, 'cache_rule', 'True'):
+	if use_cache:
 		try:
-			cls = cache[(name, self.rule)]
+			cls = cache[key]
 		except KeyError:
 			pass
 	if not cls:
-
 		rule = self.rule
-		if hasattr(self, 'chmod'):
+		if chmod is not None:
 			def chmod_fun(tsk):
 				for x in tsk.outputs:
-					os.chmod(x.abspath(), self.chmod)
-			rule = (self.rule, chmod_fun)
-
-		cls = Task.task_factory(name, rule,
-			getattr(self, 'vars', []),
-			shell=getattr(self, 'shell', True), color=getattr(self, 'color', 'BLUE'),
-			scan = getattr(self, 'scan', None))
-		if getattr(self, 'scan', None):
+					os.chmod(x.abspath(), tsk.generator.chmod)
+			if isinstance(rule, tuple):
+				rule = list(rule)
+				rule.append(chmod_fun)
+				rule = tuple(rule)
+			else:
+				rule = (rule, chmod_fun)
+
+		cls = Task.task_factory(name, rule, _vars, shell=shell, color=color)
+
+		if cls_str:
+			setattr(cls, '__str__', self.cls_str)
+
+		if cls_keyword:
+			setattr(cls, 'keyword', self.cls_keyword)
+
+		if deep_inputs:
+			Task.deep_inputs(cls)
+
+		if scan:
 			cls.scan = self.scan
-		elif getattr(self, 'deps', None):
+		elif has_deps:
 			def scan(self):
 				nodes = []
 				for x in self.generator.to_list(getattr(self.generator, 'deps', None)):
@@ -590,26 +640,26 @@ def process_rule(self):
 				return [nodes, []]
 			cls.scan = scan
 
-		if getattr(self, 'update_outputs', None):
-			Task.update_outputs(cls)
+		if use_cache:
+			cache[key] = cls
 
-		if getattr(self, 'always', None):
-			Task.always_run(cls)
+	# now create one instance
+	tsk = self.create_task(name)
 
-		for x in ('after', 'before', 'ext_in', 'ext_out'):
-			setattr(cls, x, getattr(self, x, []))
+	for x in ('after', 'before', 'ext_in', 'ext_out'):
+		setattr(tsk, x, getattr(self, x, []))
 
-		if getattr(self, 'cache_rule', 'True'):
-			cache[(name, self.rule)] = cls
+	if hasattr(self, 'stdout'):
+		tsk.stdout = self.stdout
 
-		if getattr(self, 'cls_str', None):
-			setattr(cls, '__str__', self.cls_str)
+	if hasattr(self, 'stderr'):
+		tsk.stderr = self.stderr
 
-		if getattr(self, 'cls_keyword', None):
-			setattr(cls, 'keyword', self.cls_keyword)
+	if getattr(self, 'timeout', None):
+		tsk.timeout = self.timeout
 
-	# now create one instance
-	tsk = self.create_task(name)
+	if getattr(self, 'always', None):
+		tsk.always_run = True
 
 	if getattr(self, 'target', None):
 		if isinstance(self.target, str):
@@ -623,7 +673,8 @@ def process_rule(self):
 				x.parent.mkdir() # if a node was given, create the required folders
 				tsk.outputs.append(x)
 		if getattr(self, 'install_path', None):
-			self.bld.install_files(self.install_path, tsk.outputs, chmod=getattr(self, 'chmod', Utils.O644))
+			self.install_task = self.add_install_files(install_to=self.install_path,
+				install_from=tsk.outputs, chmod=getattr(self, 'chmod', Utils.O644))
 
 	if getattr(self, 'source', None):
 		tsk.inputs = self.to_nodes(self.source)
@@ -633,10 +684,16 @@ def process_rule(self):
 	if getattr(self, 'cwd', None):
 		tsk.cwd = self.cwd
 
+	if isinstance(tsk.run, functools.partial):
+		# Python documentation says: "partial objects defined in classes
+		# behave like static methods and do not transform into bound
+		# methods during instance attribute look-up."
+		tsk.run = functools.partial(tsk.run, tsk)
+
 @feature('seq')
 def sequence_order(self):
 	"""
-	Add a strict sequential constraint between the tasks generated by task generators.
+	Adds a strict sequential constraint between the tasks generated by task generators.
 	It works because task generators are posted in order.
 	It will not post objects which belong to other folders.
 
@@ -674,7 +731,7 @@ re_m4 = re.compile('@(\w+)@', re.M)
 
 class subst_pc(Task.Task):
 	"""
-	Create *.pc* files from *.pc.in*. The task is executed whenever an input variable used
+	Creates *.pc* files from *.pc.in*. The task is executed whenever an input variable used
 	in the substitution changes.
 	"""
 
@@ -690,6 +747,8 @@ class subst_pc(Task.Task):
 		if getattr(self.generator, 'is_copy', None):
 			for i, x in enumerate(self.outputs):
 				x.write(self.inputs[i].read('rb'), 'wb')
+				stat = os.stat(self.inputs[i].abspath()) # Preserve mtime of the copy
+				os.utime(self.outputs[i].abspath(), (stat.st_atime, stat.st_mtime))
 			self.force_permissions()
 			return None
 
@@ -699,11 +758,11 @@ class subst_pc(Task.Task):
 				self.force_permissions()
 			return ret
 
-		code = self.inputs[0].read(encoding=getattr(self.generator, 'encoding', 'ISO8859-1'))
+		code = self.inputs[0].read(encoding=getattr(self.generator, 'encoding', 'latin-1'))
 		if getattr(self.generator, 'subst_fun', None):
 			code = self.generator.subst_fun(self, code)
 			if code is not None:
-				self.outputs[0].write(code, encoding=getattr(self.generator, 'encoding', 'ISO8859-1'))
+				self.outputs[0].write(code, encoding=getattr(self.generator, 'encoding', 'latin-1'))
 			self.force_permissions()
 			return None
 
@@ -718,7 +777,6 @@ class subst_pc(Task.Task):
 				lst.append(g(1))
 				return "%%(%s)s" % g(1)
 			return ''
-		global re_m4
 		code = getattr(self.generator, 're_m4', re_m4).sub(repl, code)
 
 		try:
@@ -734,12 +792,14 @@ class subst_pc(Task.Task):
 				d[x] = tmp
 
 		code = code % d
-		self.outputs[0].write(code, encoding=getattr(self.generator, 'encoding', 'ISO8859-1'))
-		self.generator.bld.raw_deps[self.uid()] = self.dep_vars = lst
+		self.outputs[0].write(code, encoding=getattr(self.generator, 'encoding', 'latin-1'))
+		self.generator.bld.raw_deps[self.uid()] = lst
 
 		# make sure the signature is updated
-		try: delattr(self, 'cache_sig')
-		except AttributeError: pass
+		try:
+			delattr(self, 'cache_sig')
+		except AttributeError:
+			pass
 
 		self.force_permissions()
 
@@ -771,13 +831,14 @@ class subst_pc(Task.Task):
 @extension('.pc.in')
 def add_pcfile(self, node):
 	"""
-	Process *.pc.in* files to *.pc*. Install the results to ``${PREFIX}/lib/pkgconfig/``
+	Processes *.pc.in* files to *.pc*. Installs the results to ``${PREFIX}/lib/pkgconfig/`` by default
 
 		def build(bld):
 			bld(source='foo.pc.in', install_path='${LIBDIR}/pkgconfig/')
 	"""
 	tsk = self.create_task('subst_pc', node, node.change_ext('.pc', '.pc.in'))
-	self.bld.install_files(getattr(self, 'install_path', '${LIBDIR}/pkgconfig/'), tsk.outputs)
+	self.install_task = self.add_install_files(
+		install_to=getattr(self, 'install_path', '${LIBDIR}/pkgconfig/'), install_from=tsk.outputs)
 
 class subst(subst_pc):
 	pass
@@ -786,7 +847,7 @@ class subst(subst_pc):
 @before_method('process_source', 'process_rule')
 def process_subst(self):
 	"""
-	Define a transformation that substitutes the contents of *source* files to *target* files::
+	Defines a transformation that substitutes the contents of *source* files to *target* files::
 
 		def build(bld):
 			bld(
@@ -821,7 +882,6 @@ def process_subst(self):
 			a = self.path.find_node(x)
 			b = self.path.get_bld().make_node(y)
 			if not os.path.isfile(b.abspath()):
-				b.sig = None
 				b.parent.mkdir()
 		else:
 			if isinstance(x, str):
@@ -836,25 +896,22 @@ def process_subst(self):
 		if not a:
 			raise Errors.WafError('could not find %r for %r' % (x, self))
 
-		has_constraints = False
 		tsk = self.create_task('subst', a, b)
 		for k in ('after', 'before', 'ext_in', 'ext_out'):
 			val = getattr(self, k, None)
 			if val:
-				has_constraints = True
 				setattr(tsk, k, val)
 
 		# paranoid safety measure for the general case foo.in->foo.h with ambiguous dependencies
-		if not has_constraints:
-			global HEADER_EXTS
-			for xt in HEADER_EXTS:
-				if b.name.endswith(xt):
-					tsk.before = [k for k in ('c', 'cxx') if k in Task.classes]
-					break
+		for xt in HEADER_EXTS:
+			if b.name.endswith(xt):
+				tsk.ext_in = tsk.ext_in + ['.h']
+				break
 
 		inst_to = getattr(self, 'install_path', None)
 		if inst_to:
-			self.bld.install_files(inst_to, b, chmod=getattr(self, 'chmod', Utils.O644))
+			self.install_task = self.add_install_files(install_to=inst_to,
+				install_from=b, chmod=getattr(self, 'chmod', Utils.O644))
 
 	self.source = []
 
diff --git a/waflib/Tools/__init__.py b/waflib/Tools/__init__.py
index c8a3c349..079df358 100644
--- a/waflib/Tools/__init__.py
+++ b/waflib/Tools/__init__.py
@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
diff --git a/waflib/Tools/ar.py b/waflib/Tools/ar.py
index aac39c0c..b39b6459 100644
--- a/waflib/Tools/ar.py
+++ b/waflib/Tools/ar.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 
 """
@@ -16,7 +16,7 @@ def find_ar(conf):
 	conf.load('ar')
 
 def configure(conf):
-	"""Find the ar program and set the default flags in ``conf.env.ARFLAGS``"""
+	"""Finds the ar program and sets the default flags in ``conf.env.ARFLAGS``"""
 	conf.find_program('ar', var='AR')
 	conf.add_os_flags('ARFLAGS')
 	if not conf.env.ARFLAGS:
diff --git a/waflib/Tools/c.py b/waflib/Tools/c.py
index 0b06a7fa..effd6b6e 100644
--- a/waflib/Tools/c.py
+++ b/waflib/Tools/c.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 
 "Base for c programs/libraries"
 
@@ -10,30 +10,30 @@ from waflib.Tools.ccroot import link_task, stlink_task
 
 @TaskGen.extension('.c')
 def c_hook(self, node):
-	"Bind the c file extension to the creation of a :py:class:`waflib.Tools.c.c` instance"
+	"Binds the c file extensions create :py:class:`waflib.Tools.c.c` instances"
 	if not self.env.CC and self.env.CXX:
 		return self.create_compiled_task('cxx', node)
 	return self.create_compiled_task('c', node)
 
 class c(Task.Task):
-	"Compile C files into object files"
-	run_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} ${CC_SRC_F}${SRC} ${CC_TGT_F}${TGT[0].abspath()}'
+	"Compiles C files into object files"
+	run_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} ${CC_SRC_F}${SRC} ${CC_TGT_F}${TGT[0].abspath()} ${CPPFLAGS}'
 	vars    = ['CCDEPS'] # unused variable to depend on, just in case
 	ext_in  = ['.h'] # set the build order easily by using ext_out=['.h']
 	scan    = c_preproc.scan
 
 class cprogram(link_task):
-	"Link object files into a c program"
+	"Links object files into c programs"
 	run_str = '${LINK_CC} ${LINKFLAGS} ${CCLNK_SRC_F}${SRC} ${CCLNK_TGT_F}${TGT[0].abspath()} ${RPATH_ST:RPATH} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${FRAMEWORK_ST:FRAMEWORK} ${ARCH_ST:ARCH} ${STLIB_MARKER} ${STLIBPATH_ST:STLIBPATH} ${STLIB_ST:STLIB} ${SHLIB_MARKER} ${LIBPATH_ST:LIBPATH} ${LIB_ST:LIB} ${LDFLAGS}'
 	ext_out = ['.bin']
 	vars    = ['LINKDEPS']
 	inst_to = '${BINDIR}'
 
 class cshlib(cprogram):
-	"Link object files into a c shared library"
+	"Links object files into c shared libraries"
 	inst_to = '${LIBDIR}'
 
 class cstlib(stlink_task):
-	"Link object files into a c static library"
+	"Links object files into a c static libraries"
 	pass # do not remove
 
diff --git a/waflib/Tools/c_aliases.py b/waflib/Tools/c_aliases.py
index 0747abf5..c9d53692 100644
--- a/waflib/Tools/c_aliases.py
+++ b/waflib/Tools/c_aliases.py
@@ -9,6 +9,8 @@ from waflib.Configure import conf
 
 def get_extensions(lst):
 	"""
+	Returns the file extensions for the list of files given as input
+
 	:param lst: files to process
 	:list lst: list of string or :py:class:`waflib.Node.Node`
 	:return: list of file extensions
@@ -16,17 +18,15 @@ def get_extensions(lst):
 	"""
 	ret = []
 	for x in Utils.to_list(lst):
-		try:
-			if not isinstance(x, str):
-				x = x.name
-			ret.append(x[x.rfind('.') + 1:])
-		except Exception:
-			pass
+		if not isinstance(x, str):
+			x = x.name
+		ret.append(x[x.rfind('.') + 1:])
 	return ret
 
 def sniff_features(**kw):
 	"""
-	Look at the source files and return the features for a task generator (mainly cc and cxx)::
+	Computes and returns the features required for a task generator by
+	looking at the file extensions. This aimed for C/C++ mainly::
 
 		snif_features(source=['foo.c', 'foo.cxx'], type='shlib')
 		# returns  ['cxx', 'c', 'cxxshlib', 'cshlib']
@@ -39,7 +39,7 @@ def sniff_features(**kw):
 	:rtype: list of string
 	"""
 	exts = get_extensions(kw['source'])
-	type = kw['_type']
+	typ = kw['typ']
 	feats = []
 
 	# watch the order, cxx will have the precedence
@@ -63,18 +63,27 @@ def sniff_features(**kw):
 		feats.append('java')
 		return 'java'
 
-	if type in ('program', 'shlib', 'stlib'):
+	if typ in ('program', 'shlib', 'stlib'):
 		will_link = False
 		for x in feats:
 			if x in ('cxx', 'd', 'fc', 'c'):
-				feats.append(x + type)
+				feats.append(x + typ)
 				will_link = True
 		if not will_link and not kw.get('features', []):
 			raise Errors.WafError('Cannot link from %r, try passing eg: features="c cprogram"?' % kw)
 	return feats
 
-def set_features(kw, _type):
-	kw['_type'] = _type
+def set_features(kw, typ):
+	"""
+	Inserts data in the input dict *kw* based on existing data and on the type of target
+	required (typ).
+
+	:param kw: task generator parameters
+	:type kw: dict
+	:param typ: type of target
+	:type typ: string
+	"""
+	kw['typ'] = typ
 	kw['features'] = Utils.to_list(kw.get('features', [])) + Utils.to_list(sniff_features(**kw))
 
 @conf
diff --git a/waflib/Tools/c_config.py b/waflib/Tools/c_config.py
index 09550f36..d2b3c0d8 100644
--- a/waflib/Tools/c_config.py
+++ b/waflib/Tools/c_config.py
@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 C/C++/D configuration helpers
 """
 
+from __future__ import with_statement
+
 import os, re, shlex
 from waflib import Build, Utils, Task, Options, Logs, Errors, Runner
 from waflib.TaskGen import after_method, feature
@@ -17,32 +19,6 @@ WAF_CONFIG_H   = 'config.h'
 DEFKEYS = 'define_key'
 INCKEYS = 'include_key'
 
-cfg_ver = {
-	'atleast-version': '>=',
-	'exact-version': '==',
-	'max-version': '<=',
-}
-
-SNIP_FUNCTION = '''
-int main(int argc, char **argv) {
-	void (*p)();
-	(void)argc; (void)argv;
-	p=(void(*)())(%s);
-	return !p;
-}
-'''
-"""Code template for checking for functions"""
-
-SNIP_TYPE = '''
-int main(int argc, char **argv) {
-	(void)argc; (void)argv;
-	if ((%(type_name)s *) 0) return 0;
-	if (sizeof (%(type_name)s)) return 0;
-	return 1;
-}
-'''
-"""Code template for checking for types"""
-
 SNIP_EMPTY_PROGRAM = '''
 int main(int argc, char **argv) {
 	(void)argc; (void)argv;
@@ -50,15 +26,6 @@ int main(int argc, char **argv) {
 }
 '''
 
-SNIP_FIELD = '''
-int main(int argc, char **argv) {
-	char *off;
-	(void)argc; (void)argv;
-	off = (char*) &((%(type_name)s*)0)->%(field_name)s;
-	return (size_t) off < sizeof(%(type_name)s);
-}
-'''
-
 MACRO_TO_DESTOS = {
 '__linux__'                                      : 'linux',
 '__GNU__'                                        : 'gnu', # hurd
@@ -75,7 +42,7 @@ MACRO_TO_DESTOS = {
 '_WIN64'                                         : 'win32',
 '_WIN32'                                         : 'win32',
 # Note about darwin: this is also tested with 'defined __APPLE__ && defined __MACH__' somewhere below in this file.
-'__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__'  : 'darwin', 
+'__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__'  : 'darwin',
 '__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__' : 'darwin', # iphone
 '__QNX__'                                        : 'qnx',
 '__native_client__'                              : 'nacl' # google native client platform
@@ -106,7 +73,7 @@ MACRO_TO_DEST_CPU = {
 @conf
 def parse_flags(self, line, uselib_store, env=None, force_static=False, posix=None):
 	"""
-	Parse the flags from the input lines, and add them to the relevant use variables::
+	Parses flags from the input lines, and adds them to the relevant use variables::
 
 		def configure(conf):
 			conf.parse_flags('-O3', 'FOO')
@@ -138,9 +105,11 @@ def parse_flags(self, line, uselib_store, env=None, force_static=False, posix=No
 
 	# append_unique is not always possible
 	# for example, apple flags may require both -arch i386 and -arch ppc
-	app = env.append_value
-	appu = env.append_unique
 	uselib = uselib_store
+	def app(var, val):
+		env.append_value('%s_%s' % (var, uselib), val)
+	def appu(var, val):
+		env.append_unique('%s_%s' % (var, uselib), val)
 	static = False
 	while lst:
 		x = lst.pop(0)
@@ -148,74 +117,79 @@ def parse_flags(self, line, uselib_store, env=None, force_static=False, posix=No
 		ot = x[2:]
 
 		if st == '-I' or st == '/I':
-			if not ot: ot = lst.pop(0)
-			appu('INCLUDES_' + uselib, [ot])
+			if not ot:
+				ot = lst.pop(0)
+			appu('INCLUDES', ot)
 		elif st == '-i':
 			tmp = [x, lst.pop(0)]
 			app('CFLAGS', tmp)
 			app('CXXFLAGS', tmp)
 		elif st == '-D' or (env.CXX_NAME == 'msvc' and st == '/D'): # not perfect but..
-			if not ot: ot = lst.pop(0)
-			app('DEFINES_' + uselib, [ot])
+			if not ot:
+				ot = lst.pop(0)
+			app('DEFINES', ot)
 		elif st == '-l':
-			if not ot: ot = lst.pop(0)
-			prefix = (force_static or static) and 'STLIB_' or 'LIB_'
-			appu(prefix + uselib, [ot])
+			if not ot:
+				ot = lst.pop(0)
+			prefix = 'STLIB' if (force_static or static) else 'LIB'
+			app(prefix, ot)
 		elif st == '-L':
-			if not ot: ot = lst.pop(0)
-			prefix = (force_static or static) and 'STLIBPATH_' or 'LIBPATH_'
-			appu(prefix + uselib, [ot])
+			if not ot:
+				ot = lst.pop(0)
+			prefix = 'STLIBPATH' if (force_static or static) else 'LIBPATH'
+			appu(prefix, ot)
 		elif x.startswith('/LIBPATH:'):
-			prefix = (force_static or static) and 'STLIBPATH_' or 'LIBPATH_'
-			appu(prefix + uselib, [x.replace('/LIBPATH:', '')])
+			prefix = 'STLIBPATH' if (force_static or static) else 'LIBPATH'
+			appu(prefix, x.replace('/LIBPATH:', ''))
 		elif x.startswith('-std='):
-			if '++' in x:
-				app('CXXFLAGS_' + uselib, [x])
-			else:
-				app('CFLAGS_' + uselib, [x])
-		elif x == '-pthread' or x.startswith('+'):
-			app('CFLAGS_' + uselib, [x])
-			app('CXXFLAGS_' + uselib, [x])
-			app('LINKFLAGS_' + uselib, [x])
+			prefix = 'CXXFLAGS' if '++' in x else 'CFLAGS'
+			app(prefix, x)
+		elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie'):
+			app('CFLAGS', x)
+			app('CXXFLAGS', x)
+			app('LINKFLAGS', x)
 		elif x == '-framework':
-			appu('FRAMEWORK_' + uselib, [lst.pop(0)])
+			appu('FRAMEWORK', lst.pop(0))
 		elif x.startswith('-F'):
-			appu('FRAMEWORKPATH_' + uselib, [x[2:]])
+			appu('FRAMEWORKPATH', x[2:])
 		elif x == '-Wl,-rpath' or x == '-Wl,-R':
-			app('RPATH_' + uselib, lst.pop(0).lstrip('-Wl,'))
+			app('RPATH', lst.pop(0).lstrip('-Wl,'))
 		elif x.startswith('-Wl,-R,'):
-			app('RPATH_' + uselib, x[7:])
+			app('RPATH', x[7:])
 		elif x.startswith('-Wl,-R'):
-			app('RPATH_' + uselib, x[6:])
+			app('RPATH', x[6:])
 		elif x.startswith('-Wl,-rpath,'):
-			app('RPATH_' + uselib, x[11:])
+			app('RPATH', x[11:])
 		elif x == '-Wl,-Bstatic' or x == '-Bstatic':
 			static = True
 		elif x == '-Wl,-Bdynamic' or x == '-Bdynamic':
 			static = False
-		elif x.startswith('-Wl'):
-			app('LINKFLAGS_' + uselib, [x])
-		elif x.startswith('-m') or x.startswith('-f') or x.startswith('-dynamic'):
-			app('CFLAGS_' + uselib, [x])
-			app('CXXFLAGS_' + uselib, [x])
+		elif x.startswith('-Wl') or x in ('-rdynamic', '-pie'):
+			app('LINKFLAGS', x)
+		elif x.startswith(('-m', '-f', '-dynamic', '-O', '-g')):
+			# Adding the -W option breaks python builds on Openindiana
+			app('CFLAGS', x)
+			app('CXXFLAGS', x)
 		elif x.startswith('-bundle'):
-			app('LINKFLAGS_' + uselib, [x])
-		elif x.startswith('-undefined') or x.startswith('-Xlinker'):
+			app('LINKFLAGS', x)
+		elif x.startswith(('-undefined', '-Xlinker')):
 			arg = lst.pop(0)
-			app('LINKFLAGS_' + uselib, [x, arg])
-		elif x.startswith('-arch') or x.startswith('-isysroot'):
+			app('LINKFLAGS', [x, arg])
+		elif x.startswith(('-arch', '-isysroot')):
 			tmp = [x, lst.pop(0)]
-			app('CFLAGS_' + uselib, tmp)
-			app('CXXFLAGS_' + uselib, tmp)
-			app('LINKFLAGS_' + uselib, tmp)
-		elif x.endswith('.a') or x.endswith('.so') or x.endswith('.dylib') or x.endswith('.lib'):
-			appu('LINKFLAGS_' + uselib, [x]) # not cool, #762
+			app('CFLAGS', tmp)
+			app('CXXFLAGS', tmp)
+			app('LINKFLAGS', tmp)
+		elif x.endswith(('.a', '.so', '.dylib', '.lib')):
+			appu('LINKFLAGS', x) # not cool, #762
+		else:
+			self.to_log('Unhandled flag %r' % x)
 
 @conf
 def validate_cfg(self, kw):
 	"""
-	Search for the program *pkg-config* if missing, and validate the parameters to pass to
-	:py:func:`waflib.Tools.c_config.exec_cfg`.
+	Searches for the program *pkg-config* if missing, and validates the
+	parameters to pass to :py:func:`waflib.Tools.c_config.exec_cfg`.
 
 	:param path: the **-config program to use** (default is *pkg-config*)
 	:type path: list of string
@@ -231,47 +205,42 @@ def validate_cfg(self, kw):
 			self.find_program('pkg-config', var='PKGCONFIG')
 		kw['path'] = self.env.PKGCONFIG
 
-	# pkg-config version
-	if 'atleast_pkgconfig_version' in kw:
-		if not 'msg' in kw:
+	# verify that exactly one action is requested
+	s = ('atleast_pkgconfig_version' in kw) + ('modversion' in kw) + ('package' in kw)
+	if s != 1:
+		raise ValueError('exactly one of atleast_pkgconfig_version, modversion and package must be set')
+	if not 'msg' in kw:
+		if 'atleast_pkgconfig_version' in kw:
 			kw['msg'] = 'Checking for pkg-config version >= %r' % kw['atleast_pkgconfig_version']
-		return
+		elif 'modversion' in kw:
+			kw['msg'] = 'Checking for %r version' % kw['modversion']
+		else:
+			kw['msg'] = 'Checking for %r' %(kw['package'])
 
-	if not 'okmsg' in kw:
+	# let the modversion check set the okmsg to the detected version
+	if not 'okmsg' in kw and not 'modversion' in kw:
 		kw['okmsg'] = 'yes'
 	if not 'errmsg' in kw:
 		kw['errmsg'] = 'not found'
 
-	if 'modversion' in kw:
-		if not 'msg' in kw:
-			kw['msg'] = 'Checking for %r version' % kw['modversion']
-		return
-
-	# checking for the version of a module, for the moment, one thing at a time
-	for x in cfg_ver.keys():
-		y = x.replace('-', '_')
-		if y in kw:
-			if not 'package' in kw:
-				raise ValueError('%s requires a package' % x)
-
-			if not 'msg' in kw:
-				kw['msg'] = 'Checking for %r %s %s' % (kw['package'], cfg_ver[x], kw[y])
-			return
-
-	if not 'define_name' in kw:
-		pkgname = kw.get('uselib_store', kw['package'].upper())
-		kw['define_name'] = self.have_define(pkgname)
-
-	if not 'uselib_store' in kw:
-		self.undefine(kw['define_name'])
-
-	if not 'msg' in kw:
-		kw['msg'] = 'Checking for %r' % (kw['package'] or kw['path'])
+	# pkg-config version
+	if 'atleast_pkgconfig_version' in kw:
+		pass
+	elif 'modversion' in kw:
+		if not 'uselib_store' in kw:
+			kw['uselib_store'] = kw['modversion']
+		if not 'define_name' in kw:
+			kw['define_name'] = '%s_VERSION' % Utils.quote_define_name(kw['uselib_store'])
+	else:
+		if not 'uselib_store' in kw:
+			kw['uselib_store'] = Utils.to_list(kw['package'])[0].upper()
+		if not 'define_name' in kw:
+			kw['define_name'] = self.have_define(kw['uselib_store'])
 
 @conf
 def exec_cfg(self, kw):
 	"""
-	Execute the program *pkg-config*:
+	Executes ``pkg-config`` or other ``-config`` applications to collect configuration flags:
 
 	* if atleast_pkgconfig_version is given, check that pkg-config has the version n and return
 	* if modversion is given, then return the module version
@@ -295,42 +264,39 @@ def exec_cfg(self, kw):
 
 	path = Utils.to_list(kw['path'])
 	env = self.env.env or None
+	if kw.get('pkg_config_path'):
+		if not env:
+			env = dict(self.environ)
+		env['PKG_CONFIG_PATH'] = kw['pkg_config_path']
+
 	def define_it():
-		pkgname = kw.get('uselib_store', kw['package'].upper())
-		if kw.get('global_define'):
-			# compatibility, replace by pkgname in WAF 1.9?
-			self.define(self.have_define(kw['package']), 1, False)
+		define_name = kw['define_name']
+		# by default, add HAVE_X to the config.h, else provide DEFINES_X for use=X
+		if kw.get('global_define', 1):
+			self.define(define_name, 1, False)
 		else:
-			self.env.append_unique('DEFINES_%s' % pkgname, "%s=1" % self.have_define(pkgname))
-		self.env[self.have_define(pkgname)] = 1
+			self.env.append_unique('DEFINES_%s' % kw['uselib_store'], "%s=1" % define_name)
+
+		if kw.get('add_have_to_env', 1):
+			self.env[define_name] = 1
 
 	# pkg-config version
 	if 'atleast_pkgconfig_version' in kw:
 		cmd = path + ['--atleast-pkgconfig-version=%s' % kw['atleast_pkgconfig_version']]
 		self.cmd_and_log(cmd, env=env)
-		if not 'okmsg' in kw:
-			kw['okmsg'] = 'yes'
 		return
 
-	# checking for the version of a module
-	for x in cfg_ver:
-		y = x.replace('-', '_')
-		if y in kw:
-			self.cmd_and_log(path + ['--%s=%s' % (x, kw[y]), kw['package']], env=env)
-			if not 'okmsg' in kw:
-				kw['okmsg'] = 'yes'
-			define_it()
-			break
-
-	# retrieving the version of a module
+	# single version for a module
 	if 'modversion' in kw:
 		version = self.cmd_and_log(path + ['--modversion', kw['modversion']], env=env).strip()
-		self.define('%s_VERSION' % Utils.quote_define_name(kw.get('uselib_store', kw['modversion'])), version)
+		if not 'okmsg' in kw:
+			kw['okmsg'] = version
+		self.define(kw['define_name'], version)
 		return version
 
 	lst = [] + path
 
-	defi = kw.get('define_variable', None)
+	defi = kw.get('define_variable')
 	if not defi:
 		defi = self.env.PKG_CONFIG_DEFINES or {}
 	for key, val in defi.items():
@@ -349,38 +315,31 @@ def exec_cfg(self, kw):
 	# retrieving variables of a module
 	if 'variables' in kw:
 		v_env = kw.get('env', self.env)
-		uselib = kw.get('uselib_store', kw['package'].upper())
 		vars = Utils.to_list(kw['variables'])
 		for v in vars:
 			val = self.cmd_and_log(lst + ['--variable=' + v], env=env).strip()
-			var = '%s_%s' % (uselib, v)
+			var = '%s_%s' % (kw['uselib_store'], v)
 			v_env[var] = val
-		if not 'okmsg' in kw:
-			kw['okmsg'] = 'yes'
 		return
 
 	# so we assume the command-line will output flags to be parsed afterwards
 	ret = self.cmd_and_log(lst, env=env)
-	if not 'okmsg' in kw:
-		kw['okmsg'] = 'yes'
 
 	define_it()
-	self.parse_flags(ret, kw.get('uselib_store', kw['package'].upper()), kw.get('env', self.env), force_static=static, posix=kw.get('posix', None))
+	self.parse_flags(ret, kw['uselib_store'], kw.get('env', self.env), force_static=static, posix=kw.get('posix'))
 	return ret
 
 @conf
 def check_cfg(self, *k, **kw):
 	"""
-	Check for configuration flags using a **-config**-like program (pkg-config, sdl-config, etc).
-	Encapsulate the calls to :py:func:`waflib.Tools.c_config.validate_cfg` and :py:func:`waflib.Tools.c_config.exec_cfg`
+	Checks for configuration flags using a **-config**-like program (pkg-config, sdl-config, etc).
+	This wraps internal calls to :py:func:`waflib.Tools.c_config.validate_cfg` and :py:func:`waflib.Tools.c_config.exec_cfg`
 
 	A few examples::
 
 		def configure(conf):
 			conf.load('compiler_c')
 			conf.check_cfg(package='glib-2.0', args='--libs --cflags')
-			conf.check_cfg(package='glib-2.0', uselib_store='GLIB', atleast_version='2.10.0',
-				args='--cflags --libs')
 			conf.check_cfg(package='pango')
 			conf.check_cfg(package='pango', uselib_store='MYPANGO', args=['--cflags', '--libs'])
 			conf.check_cfg(package='pango',
@@ -393,24 +352,18 @@ def check_cfg(self, *k, **kw):
 			conf.check_cfg(package='gtk+-2.0', variables=['includedir', 'prefix'], uselib_store='FOO')
 			print(conf.env.FOO_includedir)
 	"""
-	if k:
-		lst = k[0].split()
-		kw['package'] = lst[0]
-		kw['args'] = ' '.join(lst[1:])
-
 	self.validate_cfg(kw)
 	if 'msg' in kw:
 		self.start_msg(kw['msg'], **kw)
 	ret = None
 	try:
 		ret = self.exec_cfg(kw)
-	except self.errors.WafError:
+	except self.errors.WafError as e:
 		if 'errmsg' in kw:
 			self.end_msg(kw['errmsg'], 'YELLOW', **kw)
 		if Logs.verbose > 1:
-			raise
-		else:
-			self.fatal('The configuration failed')
+			self.to_log('Command failure: %s' % e)
+		self.fatal('The configuration failed')
 	else:
 		if not ret:
 			ret = True
@@ -421,6 +374,9 @@ def check_cfg(self, *k, **kw):
 	return ret
 
 def build_fun(bld):
+	"""
+	Build function that is used for running configuration tests with ``conf.check()``
+	"""
 	if bld.kw['compile_filename']:
 		node = bld.srcnode.make_node(bld.kw['compile_filename'])
 		node.write(bld.kw['code'])
@@ -430,13 +386,13 @@ def build_fun(bld):
 	for k, v in bld.kw.items():
 		setattr(o, k, v)
 
-	if not bld.kw.get('quiet', None):
+	if not bld.kw.get('quiet'):
 		bld.conf.to_log("==>\n%s\n<==" % bld.kw['code'])
 
 @conf
 def validate_c(self, kw):
 	"""
-	pre-check the parameters that will be given to :py:func:`waflib.Configure.run_build`
+	Pre-checks the parameters that will be given to :py:func:`waflib.Configure.run_build`
 
 	:param compiler: c or cxx (tries to guess what is best)
 	:type compiler: string
@@ -461,6 +417,9 @@ def validate_c(self, kw):
 	:param auto_add_header_name: if header_name was set, add the headers in env.INCKEYS so the next tests will include these headers
 	:type auto_add_header_name: bool
 	"""
+	for x in ('type_name', 'field_name', 'function_name'):
+		if x in kw:
+			Logs.warn('Invalid argument %r in test' % x)
 
 	if not 'build_fun' in kw:
 		kw['build_fun'] = build_fun
@@ -471,17 +430,17 @@ def validate_c(self, kw):
 
 	if not 'compiler' in kw and not 'features' in kw:
 		kw['compiler'] = 'c'
-		if env['CXX_NAME'] and Task.classes.get('cxx', None):
+		if env.CXX_NAME and Task.classes.get('cxx'):
 			kw['compiler'] = 'cxx'
-			if not self.env['CXX']:
+			if not self.env.CXX:
 				self.fatal('a c++ compiler is required')
 		else:
-			if not self.env['CC']:
+			if not self.env.CC:
 				self.fatal('a c compiler is required')
 
 	if not 'compile_mode' in kw:
 		kw['compile_mode'] = 'c'
-		if 'cxx' in Utils.to_list(kw.get('features',[])) or kw.get('compiler', '') == 'cxx':
+		if 'cxx' in Utils.to_list(kw.get('features', [])) or kw.get('compiler') == 'cxx':
 			kw['compile_mode'] = 'cxx'
 
 	if not 'type' in kw:
@@ -498,71 +457,36 @@ def validate_c(self, kw):
 	if not 'compile_filename' in kw:
 		kw['compile_filename'] = 'test.c' + ((kw['compile_mode'] == 'cxx') and 'pp' or '')
 
-
 	def to_header(dct):
 		if 'header_name' in dct:
 			dct = Utils.to_list(dct['header_name'])
 			return ''.join(['#include <%s>\n' % x for x in dct])
 		return ''
 
-	#OSX
 	if 'framework_name' in kw:
+		# OSX, not sure this is used anywhere
 		fwkname = kw['framework_name']
 		if not 'uselib_store' in kw:
 			kw['uselib_store'] = fwkname.upper()
-
-		if not kw.get('no_header', False):
-			if not 'header_name' in kw:
-				kw['header_name'] = []
+		if not kw.get('no_header'):
 			fwk = '%s/%s.h' % (fwkname, fwkname)
-			if kw.get('remove_dot_h', None):
+			if kw.get('remove_dot_h'):
 				fwk = fwk[:-2]
-			kw['header_name'] = Utils.to_list(kw['header_name']) + [fwk]
-
+			val = kw.get('header_name', [])
+			kw['header_name'] = Utils.to_list(val) + [fwk]
 		kw['msg'] = 'Checking for framework %s' % fwkname
 		kw['framework'] = fwkname
-		#kw['frameworkpath'] = set it yourself
-
-	if 'function_name' in kw:
-		fu = kw['function_name']
-		if not 'msg' in kw:
-			kw['msg'] = 'Checking for function %s' % fu
-		kw['code'] = to_header(kw) + SNIP_FUNCTION % fu
-		if not 'uselib_store' in kw:
-			kw['uselib_store'] = fu.upper()
-		if not 'define_name' in kw:
-			kw['define_name'] = self.have_define(fu)
-
-	elif 'type_name' in kw:
-		tu = kw['type_name']
-		if not 'header_name' in kw:
-			kw['header_name'] = 'stdint.h'
-		if 'field_name' in kw:
-			field = kw['field_name']
-			kw['code'] = to_header(kw) + SNIP_FIELD % {'type_name' : tu, 'field_name' : field}
-			if not 'msg' in kw:
-				kw['msg'] = 'Checking for field %s in %s' % (field, tu)
-			if not 'define_name' in kw:
-				kw['define_name'] = self.have_define((tu + '_' + field).upper())
-		else:
-			kw['code'] = to_header(kw) + SNIP_TYPE % {'type_name' : tu}
-			if not 'msg' in kw:
-				kw['msg'] = 'Checking for type %s' % tu
-			if not 'define_name' in kw:
-				kw['define_name'] = self.have_define(tu.upper())
 
 	elif 'header_name' in kw:
 		if not 'msg' in kw:
 			kw['msg'] = 'Checking for header %s' % kw['header_name']
 
 		l = Utils.to_list(kw['header_name'])
-		assert len(l)>0, 'list of headers in header_name is empty'
+		assert len(l), 'list of headers in header_name is empty'
 
 		kw['code'] = to_header(kw) + SNIP_EMPTY_PROGRAM
-
 		if not 'uselib_store' in kw:
 			kw['uselib_store'] = l[0].upper()
-
 		if not 'define_name' in kw:
 			kw['define_name'] = self.have_define(l[0])
 
@@ -598,7 +522,7 @@ def validate_c(self, kw):
 		kw['execute'] = False
 	if kw['execute']:
 		kw['features'].append('test_exec')
-		kw['chmod'] = 493
+		kw['chmod'] = Utils.O755
 
 	if not 'errmsg' in kw:
 		kw['errmsg'] = 'not found'
@@ -614,11 +538,12 @@ def validate_c(self, kw):
 		kw['code'] = '\n'.join(['#include <%s>' % x for x in self.env[INCKEYS]]) + '\n' + kw['code']
 
 	# in case defines lead to very long command-lines
-	if kw.get('merge_config_header', False) or env.merge_config_header:
+	if kw.get('merge_config_header') or env.merge_config_header:
 		kw['code'] = '%s\n\n%s' % (self.get_config_header(), kw['code'])
 		env.DEFINES = [] # modify the copy
 
-	if not kw.get('success'): kw['success'] = None
+	if not kw.get('success'):
+		kw['success'] = None
 
 	if 'define_name' in kw:
 		self.undefine(kw['define_name'])
@@ -627,65 +552,76 @@ def validate_c(self, kw):
 
 @conf
 def post_check(self, *k, **kw):
-	"Set the variables after a test executed in :py:func:`waflib.Tools.c_config.check` was run successfully"
-
+	"""
+	Sets the variables after a test executed in
+	:py:func:`waflib.Tools.c_config.check` was run successfully
+	"""
 	is_success = 0
 	if kw['execute']:
 		if kw['success'] is not None:
-			if kw.get('define_ret', False):
+			if kw.get('define_ret'):
 				is_success = kw['success']
 			else:
 				is_success = (kw['success'] == 0)
 	else:
 		is_success = (kw['success'] == 0)
 
-	if 'define_name' in kw:
-		# TODO simplify!
+	if kw.get('define_name'):
 		comment = kw.get('comment', '')
 		define_name = kw['define_name']
-		if 'header_name' in kw or 'function_name' in kw or 'type_name' in kw or 'fragment' in kw:
-			if kw['execute'] and kw.get('define_ret', None) and isinstance(is_success, str):
+		if kw['execute'] and kw.get('define_ret') and isinstance(is_success, str):
+			if kw.get('global_define', 1):
 				self.define(define_name, is_success, quote=kw.get('quote', 1), comment=comment)
 			else:
-				self.define_cond(define_name, is_success, comment=comment)
+				if kw.get('quote', 1):
+					succ = '"%s"' % is_success
+				else:
+					succ = int(is_success)
+				val = '%s=%s' % (define_name, succ)
+				var = 'DEFINES_%s' % kw['uselib_store']
+				self.env.append_value(var, val)
 		else:
-			self.define_cond(define_name, is_success, comment=comment)
-
-		# consistency with check_cfg
-		if kw.get('global_define', None):
-			self.env[kw['define_name']] = is_success
+			if kw.get('global_define', 1):
+				self.define_cond(define_name, is_success, comment=comment)
+			else:
+				var = 'DEFINES_%s' % kw['uselib_store']
+				self.env.append_value(var, '%s=%s' % (define_name, int(is_success)))
+
+		# define conf.env.HAVE_X to 1
+		if kw.get('add_have_to_env', 1):
+			if kw.get('uselib_store'):
+				self.env[self.have_define(kw['uselib_store'])] = 1
+			elif kw['execute'] and kw.get('define_ret'):
+				self.env[define_name] = is_success
+			else:
+				self.env[define_name] = int(is_success)
 
 	if 'header_name' in kw:
-		if kw.get('auto_add_header_name', False):
+		if kw.get('auto_add_header_name'):
 			self.env.append_value(INCKEYS, Utils.to_list(kw['header_name']))
 
 	if is_success and 'uselib_store' in kw:
 		from waflib.Tools import ccroot
-
-		# TODO see get_uselib_vars from ccroot.py
-		_vars = set([])
+		# See get_uselib_vars in ccroot.py
+		_vars = set()
 		for x in kw['features']:
 			if x in ccroot.USELIB_VARS:
 				_vars |= ccroot.USELIB_VARS[x]
 
 		for k in _vars:
-			lk = k.lower()
-			if lk in kw:
-				val = kw[lk]
-				# remove trailing slash
-				if isinstance(val, str):
-					val = val.rstrip(os.path.sep)
-				self.env.append_unique(k + '_' + kw['uselib_store'], Utils.to_list(val))
+			x = k.lower()
+			if x in kw:
+				self.env.append_value(k + '_' + kw['uselib_store'], kw[x])
 	return is_success
 
 @conf
 def check(self, *k, **kw):
 	"""
-	Perform a configuration test by calling :py:func:`waflib.Configure.run_build`.
+	Performs a configuration test by calling :py:func:`waflib.Configure.run_build`.
 	For the complete list of parameters, see :py:func:`waflib.Tools.c_config.validate_c`.
-	To force a specific compiler, pass "compiler='c'" or "compiler='cxx'" in the arguments
+	To force a specific compiler, pass ``compiler='c'`` or ``compiler='cxx'`` to the list of arguments
 
-	Besides build targets, complete builds can be given though a build function. All files will
+	Besides build targets, complete builds can be given through a build function. All files will
 	be written to a temporary directory::
 
 		def build(bld):
@@ -719,7 +655,7 @@ def check(self, *k, **kw):
 
 class test_exec(Task.Task):
 	"""
-	A task for executing a programs after they are built. See :py:func:`waflib.Tools.c_config.test_exec_fun`.
+	A task that runs programs after they are built. See :py:func:`waflib.Tools.c_config.test_exec_fun`.
 	"""
 	color = 'PINK'
 	def run(self):
@@ -754,19 +690,32 @@ def test_exec_fun(self):
 
 @conf
 def check_cxx(self, *k, **kw):
-	# DO NOT USE
+	"""
+	Runs a test with a task generator of the form::
+
+		conf.check(features='cxx cxxprogram', ...)
+	"""
 	kw['compiler'] = 'cxx'
 	return self.check(*k, **kw)
 
 @conf
 def check_cc(self, *k, **kw):
-	# DO NOT USE
+	"""
+	Runs a test with a task generator of the form::
+
+		conf.check(features='c cprogram', ...)
+	"""
 	kw['compiler'] = 'c'
 	return self.check(*k, **kw)
 
 @conf
 def set_define_comment(self, key, comment):
-	# comments that appear in get_config_header
+	"""
+	Sets a comment that will appear in the configuration header
+
+	:type key: string
+	:type comment: string
+	"""
 	coms = self.env.DEFINE_COMMENTS
 	if not coms:
 		coms = self.env.DEFINE_COMMENTS = {}
@@ -774,13 +723,18 @@ def set_define_comment(self, key, comment):
 
 @conf
 def get_define_comment(self, key):
+	"""
+	Returns the comment associated to a define
+
+	:type key: string
+	"""
 	coms = self.env.DEFINE_COMMENTS or {}
 	return coms.get(key, '')
 
 @conf
 def define(self, key, val, quote=True, comment=''):
 	"""
-	Store a single define and its state into conf.env.DEFINES. If the value is True, False or None it is cast to 1 or 0.
+	Stores a single define and its state into ``conf.env.DEFINES``. The value is cast to an integer (0/1).
 
 	:param key: define name
 	:type key: string
@@ -789,8 +743,9 @@ def define(self, key, val, quote=True, comment=''):
 	:param quote: enclose strings in quotes (yes by default)
 	:type quote: bool
 	"""
-	assert key and isinstance(key, str)
-
+	assert isinstance(key, str)
+	if not key:
+		return
 	if val is True:
 		val = 1
 	elif val in (False, None):
@@ -803,7 +758,7 @@ def define(self, key, val, quote=True, comment=''):
 	app = s % (key, str(val))
 
 	ban = key + '='
-	lst = self.env['DEFINES']
+	lst = self.env.DEFINES
 	for x in lst:
 		if x.startswith(ban):
 			lst[lst.index(x)] = app
@@ -817,23 +772,24 @@ def define(self, key, val, quote=True, comment=''):
 @conf
 def undefine(self, key, comment=''):
 	"""
-	Remove a define from conf.env.DEFINES
+	Removes a global define from ``conf.env.DEFINES``
 
 	:param key: define name
 	:type key: string
 	"""
-	assert key and isinstance(key, str)
-
+	assert isinstance(key, str)
+	if not key:
+		return
 	ban = key + '='
-	lst = [x for x in self.env['DEFINES'] if not x.startswith(ban)]
-	self.env['DEFINES'] = lst
+	lst = [x for x in self.env.DEFINES if not x.startswith(ban)]
+	self.env.DEFINES = lst
 	self.env.append_unique(DEFKEYS, key)
 	self.set_define_comment(key, comment)
 
 @conf
 def define_cond(self, key, val, comment=''):
 	"""
-	Conditionally define a name::
+	Conditionally defines a name::
 
 		def configure(conf):
 			conf.define_cond('A', True)
@@ -846,8 +802,9 @@ def define_cond(self, key, val, comment=''):
 	:param val: value
 	:type val: int or string
 	"""
-	assert key and isinstance(key, str)
-
+	assert isinstance(key, str)
+	if not key:
+		return
 	if val:
 		self.define(key, 1, comment=comment)
 	else:
@@ -856,6 +813,8 @@ def define_cond(self, key, val, comment=''):
 @conf
 def is_defined(self, key):
 	"""
+	Indicates whether a particular define is globally set in ``conf.env.DEFINES``.
+
 	:param key: define name
 	:type key: string
 	:return: True if the define is set
@@ -864,7 +823,7 @@ def is_defined(self, key):
 	assert key and isinstance(key, str)
 
 	ban = key + '='
-	for x in self.env['DEFINES']:
+	for x in self.env.DEFINES:
 		if x.startswith(ban):
 			return True
 	return False
@@ -872,14 +831,16 @@ def is_defined(self, key):
 @conf
 def get_define(self, key):
 	"""
+	Returns the value of an existing define, or None if not found
+
 	:param key: define name
 	:type key: string
-	:return: the value of a previously stored define or None if it is not set
+	:rtype: string
 	"""
 	assert key and isinstance(key, str)
 
 	ban = key + '='
-	for x in self.env['DEFINES']:
+	for x in self.env.DEFINES:
 		if x.startswith(ban):
 			return x[len(ban):]
 	return None
@@ -887,6 +848,9 @@ def get_define(self, key):
 @conf
 def have_define(self, key):
 	"""
+	Returns a variable suitable for command-line or header use by removing invalid characters
+	and prefixing it with ``HAVE_``
+
 	:param key: define name
 	:type key: string
 	:return: the input key prefixed by *HAVE_* and substitute any invalid characters.
@@ -897,7 +861,7 @@ def have_define(self, key):
 @conf
 def write_config_header(self, configfile='', guard='', top=False, defines=True, headers=False, remove=True, define_prefix=''):
 	"""
-	Write a configuration header containing defines and includes::
+	Writes a configuration header containing defines and includes::
 
 		def configure(cnf):
 			cnf.define('A', 1)
@@ -921,7 +885,8 @@ def write_config_header(self, configfile='', guard='', top=False, defines=True,
 	:type define_prefix: string
 	:param define_prefix: prefix all the defines in the file with a particular prefix
 	"""
-	if not configfile: configfile = WAF_CONFIG_H
+	if not configfile:
+		configfile = WAF_CONFIG_H
 	waf_guard = guard or 'W_%s_WAF' % Utils.quote_define_name(configfile)
 
 	node = top and self.bldnode or self.path.get_bld()
@@ -946,7 +911,7 @@ def write_config_header(self, configfile='', guard='', top=False, defines=True,
 @conf
 def get_config_header(self, defines=True, headers=False, define_prefix=''):
 	"""
-	Create the contents of a ``config.h`` file from the defines and includes
+	Creates the contents of a ``config.h`` file from the defines and includes
 	set in conf.env.define_key / conf.env.include_key. No include guards are added.
 
 	A prelude will be added from the variable env.WAF_CONFIG_H_PRELUDE if provided. This
@@ -976,7 +941,7 @@ def get_config_header(self, defines=True, headers=False, define_prefix=''):
 
 	if defines:
 		tbl = {}
-		for k in self.env['DEFINES']:
+		for k in self.env.DEFINES:
 			a, _, b = k.partition('=')
 			tbl[a] = b
 
@@ -994,7 +959,7 @@ def get_config_header(self, defines=True, headers=False, define_prefix=''):
 @conf
 def cc_add_flags(conf):
 	"""
-	Add CFLAGS / CPPFLAGS from os.environ to conf.env
+	Adds CFLAGS / CPPFLAGS from os.environ to conf.env
 	"""
 	conf.add_os_flags('CPPFLAGS', dup=False)
 	conf.add_os_flags('CFLAGS', dup=False)
@@ -1002,7 +967,7 @@ def cc_add_flags(conf):
 @conf
 def cxx_add_flags(conf):
 	"""
-	Add CXXFLAGS / CPPFLAGS from os.environ to conf.env
+	Adds CXXFLAGS / CPPFLAGS from os.environ to conf.env
 	"""
 	conf.add_os_flags('CPPFLAGS', dup=False)
 	conf.add_os_flags('CXXFLAGS', dup=False)
@@ -1010,7 +975,7 @@ def cxx_add_flags(conf):
 @conf
 def link_add_flags(conf):
 	"""
-	Add LINKFLAGS / LDFLAGS from os.environ to conf.env
+	Adds LINKFLAGS / LDFLAGS from os.environ to conf.env
 	"""
 	conf.add_os_flags('LINKFLAGS', dup=False)
 	conf.add_os_flags('LDFLAGS', dup=False)
@@ -1018,7 +983,7 @@ def link_add_flags(conf):
 @conf
 def cc_load_tools(conf):
 	"""
-	Load the c tool
+	Loads the Waf c extensions
 	"""
 	if not conf.env.DEST_OS:
 		conf.env.DEST_OS = Utils.unversioned_sys_platform()
@@ -1027,7 +992,7 @@ def cc_load_tools(conf):
 @conf
 def cxx_load_tools(conf):
 	"""
-	Load the cxx tool
+	Loads the Waf c++ extensions
 	"""
 	if not conf.env.DEST_OS:
 		conf.env.DEST_OS = Utils.unversioned_sys_platform()
@@ -1036,15 +1001,17 @@ def cxx_load_tools(conf):
 @conf
 def get_cc_version(conf, cc, gcc=False, icc=False, clang=False):
 	"""
-	Run the preprocessor to determine the compiler version
+	Runs the preprocessor to determine the gcc/icc/clang version
 
 	The variables CC_VERSION, DEST_OS, DEST_BINFMT and DEST_CPU will be set in *conf.env*
+
+	:raise: :py:class:`waflib.Errors.ConfigurationError`
 	"""
 	cmd = cc + ['-dM', '-E', '-']
 	env = conf.env.env or None
 	try:
 		out, err = conf.cmd_and_log(cmd, output=0, input='\n'.encode(), env=env)
-	except Exception:
+	except Errors.WafError:
 		conf.fatal('Could not determine the compiler version %r' % cmd)
 
 	if gcc:
@@ -1092,6 +1059,8 @@ def get_cc_version(conf, cc, gcc=False, icc=False, clang=False):
 			conf.env.DEST_BINFMT = 'elf'
 		elif isD('__WINNT__') or isD('__CYGWIN__') or isD('_WIN32'):
 			conf.env.DEST_BINFMT = 'pe'
+			if not conf.env.IMPLIBDIR:
+				conf.env.IMPLIBDIR = conf.env.LIBDIR # for .lib or .dll.a files
 			conf.env.LIBDIR = conf.env.BINDIR
 		elif isD('__APPLE__'):
 			conf.env.DEST_BINFMT = 'mac-o'
@@ -1108,19 +1077,22 @@ def get_cc_version(conf, cc, gcc=False, icc=False, clang=False):
 		Logs.debug('ccroot: dest platform: ' + ' '.join([conf.env[x] or '?' for x in ('DEST_OS', 'DEST_BINFMT', 'DEST_CPU')]))
 		if icc:
 			ver = k['__INTEL_COMPILER']
-			conf.env['CC_VERSION'] = (ver[:-2], ver[-2], ver[-1])
+			conf.env.CC_VERSION = (ver[:-2], ver[-2], ver[-1])
 		else:
 			if isD('__clang__') and isD('__clang_major__'):
-				conf.env['CC_VERSION'] = (k['__clang_major__'], k['__clang_minor__'], k['__clang_patchlevel__'])
+				conf.env.CC_VERSION = (k['__clang_major__'], k['__clang_minor__'], k['__clang_patchlevel__'])
 			else:
 				# older clang versions and gcc
-				conf.env['CC_VERSION'] = (k['__GNUC__'], k['__GNUC_MINOR__'], k.get('__GNUC_PATCHLEVEL__', '0'))
+				conf.env.CC_VERSION = (k['__GNUC__'], k['__GNUC_MINOR__'], k.get('__GNUC_PATCHLEVEL__', '0'))
 	return k
 
 @conf
 def get_xlc_version(conf, cc):
-	"""Get the compiler version"""
+	"""
+	Returns the Aix compiler version
 
+	:raise: :py:class:`waflib.Errors.ConfigurationError`
+	"""
 	cmd = cc + ['-qversion']
 	try:
 		out, err = conf.cmd_and_log(cmd, output=0)
@@ -1133,15 +1105,18 @@ def get_xlc_version(conf, cc):
 		match = version_re(out or err)
 		if match:
 			k = match.groupdict()
-			conf.env['CC_VERSION'] = (k['major'], k['minor'])
+			conf.env.CC_VERSION = (k['major'], k['minor'])
 			break
 	else:
 		conf.fatal('Could not determine the XLC version.')
 
 @conf
 def get_suncc_version(conf, cc):
-	"""Get the compiler version"""
+	"""
+	Returns the Sun compiler version
 
+	:raise: :py:class:`waflib.Errors.ConfigurationError`
+	"""
 	cmd = cc + ['-V']
 	try:
 		out, err = conf.cmd_and_log(cmd, output=0)
@@ -1162,7 +1137,7 @@ def get_suncc_version(conf, cc):
 	match = version_re(version)
 	if match:
 		k = match.groupdict()
-		conf.env['CC_VERSION'] = (k['major'], k['minor'])
+		conf.env.CC_VERSION = (k['major'], k['minor'])
 	else:
 		conf.fatal('Could not determine the suncc version.')
 
@@ -1171,7 +1146,7 @@ def get_suncc_version(conf, cc):
 @conf
 def add_as_needed(self):
 	"""
-	Add ``--as-needed`` to the *LINKFLAGS*
+	Adds ``--as-needed`` to the *LINKFLAGS*
 	On some platforms, it is a default flag.  In some cases (e.g., in NS-3) it is necessary to explicitly disable this feature with `-Wl,--no-as-needed` flag.
 	"""
 	if self.env.DEST_BINFMT == 'elf' and 'gcc' in (self.env.CXX_NAME, self.env.CC_NAME):
@@ -1179,22 +1154,31 @@ def add_as_needed(self):
 
 # ============ parallel configuration
 
-class cfgtask(Task.TaskBase):
+class cfgtask(Task.Task):
 	"""
-	A task that executes configuration tests
-	make sure that the checks write to conf.env in a thread-safe manner
+	A task that executes build configuration tests (calls conf.check)
 
-	for the moment it only executes conf.check
+	Make sure to use locks if concurrent access to the same conf.env data is necessary.
 	"""
+	def __init__(self, *k, **kw):
+		Task.Task.__init__(self, *k, **kw)
+		self.run_after = set()
+
 	def display(self):
 		return ''
 
 	def runnable_status(self):
+		for x in self.run_after:
+			if not x.hasrun:
+				return Task.ASK_LATER
 		return Task.RUN_ME
 
 	def uid(self):
 		return Utils.SIG_NIL
 
+	def signature(self):
+		return Utils.SIG_NIL
+
 	def run(self):
 		conf = self.conf
 		bld = Build.BuildContext(top_dir=conf.srcnode.abspath(), out_dir=conf.bldnode.abspath())
@@ -1202,22 +1186,73 @@ class cfgtask(Task.TaskBase):
 		bld.init_dirs()
 		bld.in_msg = 1 # suppress top-level start_msg
 		bld.logger = self.logger
+		bld.multicheck_task = self
+		args = self.args
 		try:
-			bld.check(**self.args)
+			if 'func' in args:
+				bld.test(build_fun=args['func'],
+					msg=args.get('msg', ''),
+					okmsg=args.get('okmsg', ''),
+					errmsg=args.get('errmsg', ''),
+					)
+			else:
+				args['multicheck_mandatory'] = args.get('mandatory', True)
+				args['mandatory'] = True
+				try:
+					bld.check(**args)
+				finally:
+					args['mandatory'] = args['multicheck_mandatory']
 		except Exception:
 			return 1
 
+	def process(self):
+		Task.Task.process(self)
+		if 'msg' in self.args:
+			with self.generator.bld.multicheck_lock:
+				self.conf.start_msg(self.args['msg'])
+				if self.hasrun == Task.NOT_RUN:
+					self.conf.end_msg('test cancelled', 'YELLOW')
+				elif self.hasrun != Task.SUCCESS:
+					self.conf.end_msg(self.args.get('errmsg', 'no'), 'YELLOW')
+				else:
+					self.conf.end_msg(self.args.get('okmsg', 'yes'), 'GREEN')
+
 @conf
 def multicheck(self, *k, **kw):
 	"""
-	Use tuples to perform parallel configuration tests
+	Runs configuration tests in parallel; results are printed sequentially at the end of the build
+	but each test must provide its own msg value to display a line::
+
+		def test_build(ctx):
+			ctx.in_msg = True # suppress console outputs
+			ctx.check_large_file(mandatory=False)
+
+		conf.multicheck(
+			{'header_name':'stdio.h', 'msg':'... stdio', 'uselib_store':'STDIO', 'global_define':False},
+			{'header_name':'xyztabcd.h', 'msg':'... optional xyztabcd.h', 'mandatory': False},
+			{'header_name':'stdlib.h', 'msg':'... stdlib', 'okmsg': 'aye', 'errmsg': 'nope'},
+			{'func': test_build, 'msg':'... testing an arbitrary build function', 'okmsg':'ok'},
+			msg       = 'Checking for headers in parallel',
+			mandatory = True, # mandatory tests raise an error at the end
+			run_all_tests = True, # try running all tests
+		)
+
+	The configuration tests may modify the values in conf.env in any order, and the define
+	values can affect configuration tests being executed. It is hence recommended
+	to provide `uselib_store` values with `global_define=False` to prevent such issues.
 	"""
 	self.start_msg(kw.get('msg', 'Executing %d configuration tests' % len(k)), **kw)
 
+	# Force a copy so that threads append to the same list at least
+	# no order is guaranteed, but the values should not disappear at least
+	for var in ('DEFINES', DEFKEYS):
+		self.env.append_value(var, [])
+	self.env.DEFINE_COMMENTS = self.env.DEFINE_COMMENTS or {}
+
+	# define a task object that will execute our tests
 	class par(object):
 		def __init__(self):
 			self.keep = False
-			self.returned_tasks = []
 			self.task_sigs = {}
 			self.progress_bar = 0
 		def total(self):
@@ -1226,9 +1261,13 @@ def multicheck(self, *k, **kw):
 			return
 
 	bld = par()
+	bld.keep = kw.get('run_all_tests', True)
+	bld.imp_sigs = {}
 	tasks = []
+
+	id_to_task = {}
 	for dct in k:
-		x = cfgtask(bld=bld)
+		x = Task.classes['cfgtask'](bld=bld, env=None)
 		tasks.append(x)
 		x.args = dct
 		x.bld = bld
@@ -1238,18 +1277,38 @@ def multicheck(self, *k, **kw):
 		# bind a logger that will keep the info in memory
 		x.logger = Logs.make_mem_logger(str(id(x)), self.logger)
 
+		if 'id' in dct:
+			id_to_task[dct['id']] = x
+
+	# second pass to set dependencies with after_test/before_test
+	for x in tasks:
+		for key in Utils.to_list(x.args.get('before_tests', [])):
+			tsk = id_to_task[key]
+			if not tsk:
+				raise ValueError('No test named %r' % key)
+			tsk.run_after.add(x)
+		for key in Utils.to_list(x.args.get('after_tests', [])):
+			tsk = id_to_task[key]
+			if not tsk:
+				raise ValueError('No test named %r' % key)
+			x.run_after.add(tsk)
+
 	def it():
 		yield tasks
 		while 1:
 			yield []
-	p = Runner.Parallel(bld, Options.options.jobs)
+	bld.producer = p = Runner.Parallel(bld, Options.options.jobs)
+	bld.multicheck_lock = Utils.threading.Lock()
 	p.biter = it()
+
+	self.end_msg('started')
 	p.start()
 
 	# flush the logs in order into the config.log
 	for x in tasks:
 		x.logger.memhandler.flush()
 
+	self.start_msg('-> processing test results')
 	if p.error:
 		for x in p.error:
 			if getattr(x, 'err_msg', None):
@@ -1257,10 +1316,36 @@ def multicheck(self, *k, **kw):
 				self.end_msg('fail', color='RED')
 				raise Errors.WafError('There is an error in the library, read config.log for more information')
 
+	failure_count = 0
+	for x in tasks:
+		if x.hasrun not in (Task.SUCCESS, Task.NOT_RUN):
+			failure_count += 1
+
+	if failure_count:
+		self.end_msg(kw.get('errmsg', '%s test failed' % failure_count), color='YELLOW', **kw)
+	else:
+		self.end_msg('all ok', **kw)
+
 	for x in tasks:
 		if x.hasrun != Task.SUCCESS:
-			self.end_msg(kw.get('errmsg', 'no'), color='YELLOW', **kw)
-			self.fatal(kw.get('fatalmsg', None) or 'One of the tests has failed, read config.log for more information')
+			if x.args.get('mandatory', True):
+				self.fatal(kw.get('fatalmsg') or 'One of the tests has failed, read config.log for more information')
 
-	self.end_msg('ok', **kw)
+@conf
+def check_gcc_o_space(self, mode='c'):
+	if int(self.env.CC_VERSION[0]) > 4:
+		# this is for old compilers
+		return
+	self.env.stash()
+	if mode == 'c':
+		self.env.CCLNK_TGT_F = ['-o', '']
+	elif mode == 'cxx':
+		self.env.CXXLNK_TGT_F = ['-o', '']
+	features = '%s %sshlib' % (mode, mode)
+	try:
+		self.check(msg='Checking if the -o link must be split from arguments', fragment=SNIP_EMPTY_PROGRAM, features=features)
+	except self.errors.ConfigurationError:
+		self.env.revert()
+	else:
+		self.env.commit()
 
diff --git a/waflib/Tools/c_osx.py b/waflib/Tools/c_osx.py
index 4337cc68..f70b128b 100644
--- a/waflib/Tools/c_osx.py
+++ b/waflib/Tools/c_osx.py
@@ -1,13 +1,13 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy 2008-2010
+# Thomas Nagy 2008-2018 (ita)
 
 """
 MacOSX related tools
 """
 
 import os, shutil, platform
-from waflib import Task, Utils, Errors
+from waflib import Task, Utils
 from waflib.TaskGen import taskgen_method, feature, after_method, before_method
 
 app_info = '''
@@ -37,8 +37,8 @@ def set_macosx_deployment_target(self):
 	"""
 	see WAF issue 285 and also and also http://trac.macports.org/ticket/17059
 	"""
-	if self.env['MACOSX_DEPLOYMENT_TARGET']:
-		os.environ['MACOSX_DEPLOYMENT_TARGET'] = self.env['MACOSX_DEPLOYMENT_TARGET']
+	if self.env.MACOSX_DEPLOYMENT_TARGET:
+		os.environ['MACOSX_DEPLOYMENT_TARGET'] = self.env.MACOSX_DEPLOYMENT_TARGET
 	elif 'MACOSX_DEPLOYMENT_TARGET' not in os.environ:
 		if Utils.unversioned_sys_platform() == 'darwin':
 			os.environ['MACOSX_DEPLOYMENT_TARGET'] = '.'.join(platform.mac_ver()[0].split('.')[:2])
@@ -46,7 +46,7 @@ def set_macosx_deployment_target(self):
 @taskgen_method
 def create_bundle_dirs(self, name, out):
 	"""
-	Create bundle folders, used by :py:func:`create_task_macplist` and :py:func:`create_task_macapp`
+	Creates bundle folders, used by :py:func:`create_task_macplist` and :py:func:`create_task_macapp`
 	"""
 	dir = out.parent.find_or_declare(name)
 	dir.mkdir()
@@ -78,7 +78,7 @@ def create_task_macapp(self):
 			bld.env.MACAPP = True
 			bld.shlib(source='a.c', target='foo')
 	"""
-	if self.env['MACAPP'] or getattr(self, 'mac_app', False):
+	if self.env.MACAPP or getattr(self, 'mac_app', False):
 		out = self.link_task.outputs[0]
 
 		name = bundle_name_for_output(out)
@@ -88,7 +88,7 @@ def create_task_macapp(self):
 
 		self.apptask = self.create_task('macapp', self.link_task.outputs, n1)
 		inst_to = getattr(self, 'install_path', '/Applications') + '/%s/Contents/MacOS/' % name
-		self.bld.install_files(inst_to, n1, chmod=Utils.O755)
+		self.add_install_files(install_to=inst_to, install_from=n1, chmod=Utils.O755)
 
 		if getattr(self, 'mac_files', None):
 			# this only accepts files; they will be installed as seen from mac_files_root
@@ -102,38 +102,19 @@ def create_task_macapp(self):
 			for node in self.to_nodes(self.mac_files):
 				relpath = node.path_from(mac_files_root or node.parent)
 				self.create_task('macapp', node, res_dir.make_node(relpath))
-				self.bld.install_as(os.path.join(inst_to, relpath), node)
-
-		if getattr(self, 'mac_resources', None):
-			# TODO remove in waf 1.9
-			res_dir = n1.parent.parent.make_node('Resources')
-			inst_to = getattr(self, 'install_path', '/Applications') + '/%s/Resources' % name
-			for x in self.to_list(self.mac_resources):
-				node = self.path.find_node(x)
-				if not node:
-					raise Errors.WafError('Missing mac_resource %r in %r' % (x, self))
-
-				parent = node.parent
-				if os.path.isdir(node.abspath()):
-					nodes = node.ant_glob('**')
-				else:
-					nodes = [node]
-				for node in nodes:
-					rel = node.path_from(parent)
-					self.create_task('macapp', node, res_dir.make_node(rel))
-					self.bld.install_as(inst_to + '/%s' % rel, node)
+				self.add_install_as(install_to=os.path.join(inst_to, relpath), install_from=node)
 
 		if getattr(self.bld, 'is_install', None):
-			# disable the normal binary installation
+			# disable regular binary installation
 			self.install_task.hasrun = Task.SKIP_ME
 
 @feature('cprogram', 'cxxprogram')
 @after_method('apply_link')
 def create_task_macplist(self):
 	"""
-	Create a :py:class:`waflib.Tools.c_osx.macplist` instance.
+	Creates a :py:class:`waflib.Tools.c_osx.macplist` instance.
 	"""
-	if  self.env['MACAPP'] or getattr(self, 'mac_app', False):
+	if  self.env.MACAPP or getattr(self, 'mac_app', False):
 		out = self.link_task.outputs[0]
 
 		name = bundle_name_for_output(out)
@@ -160,7 +141,7 @@ def create_task_macplist(self):
 			plisttask.code = app_info
 
 		inst_to = getattr(self, 'install_path', '/Applications') + '/%s/Contents/' % name
-		self.bld.install_files(inst_to, n1)
+		self.add_install_files(install_to=inst_to, install_from=n1)
 
 @feature('cshlib', 'cxxshlib')
 @before_method('apply_link', 'propagate_uselib_vars')
@@ -177,9 +158,9 @@ def apply_bundle(self):
 			bld.env.MACBUNDLE = True
 			bld.shlib(source='a.c', target='foo')
 	"""
-	if self.env['MACBUNDLE'] or getattr(self, 'mac_bundle', False):
-		self.env['LINKFLAGS_cshlib'] = self.env['LINKFLAGS_cxxshlib'] = [] # disable the '-dynamiclib' flag
-		self.env['cshlib_PATTERN'] = self.env['cxxshlib_PATTERN'] = self.env['macbundle_PATTERN']
+	if self.env.MACBUNDLE or getattr(self, 'mac_bundle', False):
+		self.env.LINKFLAGS_cshlib = self.env.LINKFLAGS_cxxshlib = [] # disable the '-dynamiclib' flag
+		self.env.cshlib_PATTERN = self.env.cxxshlib_PATTERN = self.env.macbundle_PATTERN
 		use = self.use = self.to_list(getattr(self, 'use', []))
 		if not 'MACBUNDLE' in use:
 			use.append('MACBUNDLE')
@@ -188,7 +169,7 @@ app_dirs = ['Contents', 'Contents/MacOS', 'Contents/Resources']
 
 class macapp(Task.Task):
 	"""
-	Create mac applications
+	Creates mac applications
 	"""
 	color = 'PINK'
 	def run(self):
@@ -197,7 +178,7 @@ class macapp(Task.Task):
 
 class macplist(Task.Task):
 	"""
-	Create plist files
+	Creates plist files
 	"""
 	color = 'PINK'
 	ext_in = ['.bin']
@@ -209,3 +190,4 @@ class macplist(Task.Task):
 		context = getattr(self, 'context', {})
 		txt = txt.format(**context)
 		self.outputs[0].write(txt)
+
diff --git a/waflib/Tools/c_preproc.py b/waflib/Tools/c_preproc.py
index 07e80097..7e04b4a7 100644
--- a/waflib/Tools/c_preproc.py
+++ b/waflib/Tools/c_preproc.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 
 """
 C/C++ preprocessor for finding dependencies
@@ -28,11 +28,13 @@ A dumb preprocessor is also available in the tool *c_dumbpreproc*
 
 import re, string, traceback
 from waflib import Logs, Utils, Errors
-from waflib.Logs import debug, error
 
 class PreprocError(Errors.WafError):
 	pass
 
+FILE_CACHE_SIZE = 100000
+LINE_CACHE_SIZE = 100000
+
 POPFILE = '-'
 "Constant representing a special token used in :py:meth:`waflib.Tools.c_preproc.c_parser.start` iteration to switch to a header read previously"
 
@@ -42,15 +44,15 @@ recursion_limit = 150
 go_absolute = False
 "Set to True to track headers on files in /usr/include, else absolute paths are ignored (but it becomes very slow)"
 
-standard_includes = ['/usr/include']
+standard_includes = ['/usr/local/include', '/usr/include']
 if Utils.is_win32:
 	standard_includes = []
 
 use_trigraphs = 0
 """Apply trigraph rules (False by default)"""
 
+# obsolete, do not use
 strict_quotes = 0
-"""Reserve the "#include <>" quotes for system includes (do not search for those includes). False by default."""
 
 g_optrans = {
 'not':'!',
@@ -69,7 +71,7 @@ g_optrans = {
 
 # ignore #warning and #error
 re_lines = re.compile(
-	'^[ \t]*(#|%:)[ \t]*(ifdef|ifndef|if|else|elif|endif|include|import|define|undef|pragma)[ \t]*(.*)\r*$',
+	'^[ \t]*(?:#|%:)[ \t]*(ifdef|ifndef|if|else|elif|endif|include|import|define|undef|pragma)[ \t]*(.*)\r*$',
 	re.IGNORECASE | re.MULTILINE)
 """Match #include lines"""
 
@@ -137,54 +139,22 @@ skipped   = 's'
 
 def repl(m):
 	"""Replace function used with :py:attr:`waflib.Tools.c_preproc.re_cpp`"""
-	s = m.group(0)
-	if s.startswith('/'):
+	s = m.group()
+	if s[0] == '/':
 		return ' '
 	return s
 
-def filter_comments(filename):
-	"""
-	Filter the comments from a c/h file, and return the preprocessor lines.
-	The regexps :py:attr:`waflib.Tools.c_preproc.re_cpp`, :py:attr:`waflib.Tools.c_preproc.re_nl` and :py:attr:`waflib.Tools.c_preproc.re_lines` are used internally.
-
-	:return: the preprocessor directives as a list of (keyword, line)
-	:rtype: a list of string pairs
-	"""
-	# return a list of tuples : keyword, line
-	code = Utils.readf(filename)
-	if use_trigraphs:
-		for (a, b) in trig_def: code = code.split(a).join(b)
-	code = re_nl.sub('', code)
-	code = re_cpp.sub(repl, code)
-	return [(m.group(2), m.group(3)) for m in re.finditer(re_lines, code)]
-
 prec = {}
 """
-Operator precendence rules required for parsing expressions of the form::
+Operator precedence rules required for parsing expressions of the form::
 
 	#if 1 && 2 != 0
 """
 ops = ['* / %', '+ -', '<< >>', '< <= >= >', '== !=', '& | ^', '&& ||', ',']
-for x in range(len(ops)):
-	syms = ops[x]
+for x, syms in enumerate(ops):
 	for u in syms.split():
 		prec[u] = x
 
-def trimquotes(s):
-	"""
-	Remove the single quotes around an expression::
-
-		trimquotes("'test'") == "test"
-
-	:param s: expression to transform
-	:type s: string
-	:rtype: string
-	"""
-	if not s: return ''
-	s = s.rstrip()
-	if s[0] == "'" and s[-1] == "'": return s[1:-1]
-	return s
-
 def reduce_nums(val_1, val_2, val_op):
 	"""
 	Apply arithmetic rules to compute a result
@@ -200,32 +170,56 @@ def reduce_nums(val_1, val_2, val_op):
 	#print val_1, val_2, val_op
 
 	# now perform the operation, make certain a and b are numeric
-	try:    a = 0 + val_1
-	except TypeError: a = int(val_1)
-	try:    b = 0 + val_2
-	except TypeError: b = int(val_2)
+	try:
+		a = 0 + val_1
+	except TypeError:
+		a = int(val_1)
+	try:
+		b = 0 + val_2
+	except TypeError:
+		b = int(val_2)
 
 	d = val_op
-	if d == '%':  c = a%b
-	elif d=='+':  c = a+b
-	elif d=='-':  c = a-b
-	elif d=='*':  c = a*b
-	elif d=='/':  c = a/b
-	elif d=='^':  c = a^b
-	elif d=='==': c = int(a == b)
-	elif d=='|'  or d == 'bitor':  c = a|b
-	elif d=='||' or d == 'or' : c = int(a or b)
-	elif d=='&'  or d == 'bitand':  c = a&b
-	elif d=='&&' or d == 'and': c = int(a and b)
-	elif d=='!=' or d == 'not_eq': c = int(a != b)
-	elif d=='^'  or d == 'xor':  c = int(a^b)
-	elif d=='<=': c = int(a <= b)
-	elif d=='<':  c = int(a < b)
-	elif d=='>':  c = int(a > b)
-	elif d=='>=': c = int(a >= b)
-	elif d=='<<': c = a<<b
-	elif d=='>>': c = a>>b
-	else: c = 0
+	if d == '%':
+		c = a % b
+	elif d=='+':
+		c = a + b
+	elif d=='-':
+		c = a - b
+	elif d=='*':
+		c = a * b
+	elif d=='/':
+		c = a / b
+	elif d=='^':
+		c = a ^ b
+	elif d=='==':
+		c = int(a == b)
+	elif d=='|'  or d == 'bitor':
+		c = a | b
+	elif d=='||' or d == 'or' :
+		c = int(a or b)
+	elif d=='&'  or d == 'bitand':
+		c = a & b
+	elif d=='&&' or d == 'and':
+		c = int(a and b)
+	elif d=='!=' or d == 'not_eq':
+		c = int(a != b)
+	elif d=='^'  or d == 'xor':
+		c = int(a^b)
+	elif d=='<=':
+		c = int(a <= b)
+	elif d=='<':
+		c = int(a < b)
+	elif d=='>':
+		c = int(a > b)
+	elif d=='>=':
+		c = int(a >= b)
+	elif d=='<<':
+		c = a << b
+	elif d=='>>':
+		c = a >> b
+	else:
+		c = 0
 	return c
 
 def get_num(lst):
@@ -237,7 +231,8 @@ def get_num(lst):
 	:return: a pair containing the number and the rest of the list
 	:rtype: tuple(value, list)
 	"""
-	if not lst: raise PreprocError("empty list for get_num")
+	if not lst:
+		raise PreprocError('empty list for get_num')
 	(p, v) = lst[0]
 	if p == OP:
 		if v == '(':
@@ -255,7 +250,7 @@ def get_num(lst):
 						count_par += 1
 				i += 1
 			else:
-				raise PreprocError("rparen expected %r" % lst)
+				raise PreprocError('rparen expected %r' % lst)
 
 			(num, _) = get_term(lst[1:i])
 			return (num, lst[i+1:])
@@ -272,14 +267,14 @@ def get_num(lst):
 			num, lst = get_num(lst[1:])
 			return (~ int(num), lst)
 		else:
-			raise PreprocError("Invalid op token %r for get_num" % lst)
+			raise PreprocError('Invalid op token %r for get_num' % lst)
 	elif p == NUM:
 		return v, lst[1:]
 	elif p == IDENT:
 		# all macros should have been replaced, remaining identifiers eval to 0
 		return 0, lst[1:]
 	else:
-		raise PreprocError("Invalid token %r for get_num" % lst)
+		raise PreprocError('Invalid token %r for get_num' % lst)
 
 def get_term(lst):
 	"""
@@ -293,7 +288,8 @@ def get_term(lst):
 	:rtype: value, list
 	"""
 
-	if not lst: raise PreprocError("empty list for get_term")
+	if not lst:
+		raise PreprocError('empty list for get_term')
 	num, lst = get_num(lst)
 	if not lst:
 		return (num, [])
@@ -318,7 +314,7 @@ def get_term(lst):
 							break
 				i += 1
 			else:
-				raise PreprocError("rparen expected %r" % lst)
+				raise PreprocError('rparen expected %r' % lst)
 
 			if int(num):
 				return get_term(lst[1:i])
@@ -336,7 +332,7 @@ def get_term(lst):
 			# operator precedence
 			p2, v2 = lst[0]
 			if p2 != OP:
-				raise PreprocError("op expected %r" % lst)
+				raise PreprocError('op expected %r' % lst)
 
 			if prec[v2] >= prec[v]:
 				num2 = reduce_nums(num, num2, v)
@@ -347,7 +343,7 @@ def get_term(lst):
 				return get_term([(NUM, num), (p, v), (NUM, num3)] + lst)
 
 
-	raise PreprocError("cannot reduce %r" % lst)
+	raise PreprocError('cannot reduce %r' % lst)
 
 def reduce_eval(lst):
 	"""
@@ -432,7 +428,7 @@ def reduce_tokens(lst, defs, ban=[]):
 					else:
 						lst[i] = (NUM, 0)
 				else:
-					raise PreprocError("Invalid define expression %r" % lst)
+					raise PreprocError('Invalid define expression %r' % lst)
 
 		elif p == IDENT and v in defs:
 
@@ -447,8 +443,8 @@ def reduce_tokens(lst, defs, ban=[]):
 				del lst[i]
 				accu = to_add[:]
 				reduce_tokens(accu, defs, ban+[v])
-				for x in range(len(accu)):
-					lst.insert(i, accu[x])
+				for tmp in accu:
+					lst.insert(i, tmp)
 					i += 1
 			else:
 				# collect the arguments for the funcall
@@ -457,11 +453,11 @@ def reduce_tokens(lst, defs, ban=[]):
 				del lst[i]
 
 				if i >= len(lst):
-					raise PreprocError("expected '(' after %r (got nothing)" % v)
+					raise PreprocError('expected ( after %r (got nothing)' % v)
 
 				(p2, v2) = lst[i]
 				if p2 != OP or v2 != '(':
-					raise PreprocError("expected '(' after %r" % v)
+					raise PreprocError('expected ( after %r' % v)
 
 				del lst[i]
 
@@ -476,18 +472,22 @@ def reduce_tokens(lst, defs, ban=[]):
 							one_param.append((p2, v2))
 							count_paren += 1
 						elif v2 == ')':
-							if one_param: args.append(one_param)
+							if one_param:
+								args.append(one_param)
 							break
 						elif v2 == ',':
-							if not one_param: raise PreprocError("empty param in funcall %s" % v)
+							if not one_param:
+								raise PreprocError('empty param in funcall %r' % v)
 							args.append(one_param)
 							one_param = []
 						else:
 							one_param.append((p2, v2))
 					else:
 						one_param.append((p2, v2))
-						if   v2 == '(': count_paren += 1
-						elif v2 == ')': count_paren -= 1
+						if   v2 == '(':
+							count_paren += 1
+						elif v2 == ')':
+							count_paren -= 1
 				else:
 					raise PreprocError('malformed macro')
 
@@ -524,7 +524,6 @@ def reduce_tokens(lst, defs, ban=[]):
 									accu.append((p2, v2))
 									accu.extend(toks)
 							elif to_add[j+1][0] == IDENT and to_add[j+1][1] == '__VA_ARGS__':
-								# TODO not sure
 								# first collect the tokens
 								va_toks = []
 								st = len(macro_def[0])
@@ -532,7 +531,8 @@ def reduce_tokens(lst, defs, ban=[]):
 								for x in args[pt-st+1:]:
 									va_toks.extend(x)
 									va_toks.append((OP, ','))
-								if va_toks: va_toks.pop() # extra comma
+								if va_toks:
+									va_toks.pop() # extra comma
 								if len(accu)>1:
 									(p3, v3) = accu[-1]
 									(p4, v4) = accu[-2]
@@ -580,8 +580,15 @@ def eval_macro(lst, defs):
 	:rtype: int
 	"""
 	reduce_tokens(lst, defs, [])
-	if not lst: raise PreprocError("missing tokens to evaluate")
-	(p, v) = reduce_eval(lst)
+	if not lst:
+		raise PreprocError('missing tokens to evaluate')
+
+	if lst:
+		p, v = lst[0]
+		if p == IDENT and v not in defs:
+			raise PreprocError('missing macro %r' % lst)
+
+	p, v = reduce_eval(lst)
 	return int(v) != 0
 
 def extract_macro(txt):
@@ -601,7 +608,8 @@ def extract_macro(txt):
 		p, name = t[0]
 
 		p, v = t[1]
-		if p != OP: raise PreprocError("expected open parenthesis")
+		if p != OP:
+			raise PreprocError('expected (')
 
 		i = 1
 		pindex = 0
@@ -620,27 +628,27 @@ def extract_macro(txt):
 				elif p == OP and v == ')':
 					break
 				else:
-					raise PreprocError("unexpected token (3)")
+					raise PreprocError('unexpected token (3)')
 			elif prev == IDENT:
 				if p == OP and v == ',':
 					prev = v
 				elif p == OP and v == ')':
 					break
 				else:
-					raise PreprocError("comma or ... expected")
+					raise PreprocError('comma or ... expected')
 			elif prev == ',':
 				if p == IDENT:
 					params[v] = pindex
 					pindex += 1
 					prev = p
 				elif p == OP and v == '...':
-					raise PreprocError("not implemented (1)")
+					raise PreprocError('not implemented (1)')
 				else:
-					raise PreprocError("comma or ... expected (2)")
+					raise PreprocError('comma or ... expected (2)')
 			elif prev == '...':
-				raise PreprocError("not implemented (2)")
+				raise PreprocError('not implemented (2)')
 			else:
-				raise PreprocError("unexpected else")
+				raise PreprocError('unexpected else')
 
 		#~ print (name, [params, t[i+1:]])
 		return (name, [params, t[i+1:]])
@@ -652,7 +660,7 @@ def extract_macro(txt):
 			# empty define, assign an empty token
 			return (v, [[], [('T','')]])
 
-re_include = re.compile('^\s*(<(?P<a>.*)>|"(?P<b>.*)")')
+re_include = re.compile('^\s*(<(?:.*)>|"(?:.*)")')
 def extract_include(txt, defs):
 	"""
 	Process a line in the form::
@@ -668,15 +676,15 @@ def extract_include(txt, defs):
 	"""
 	m = re_include.search(txt)
 	if m:
-		if m.group('a'): return '<', m.group('a')
-		if m.group('b'): return '"', m.group('b')
+		txt = m.group(1)
+		return txt[0], txt[1:-1]
 
 	# perform preprocessing and look at the result, it must match an include
 	toks = tokenize(txt)
 	reduce_tokens(toks, defs, ['waf_include'])
 
 	if not toks:
-		raise PreprocError("could not parse include %s" % txt)
+		raise PreprocError('could not parse include %r' % txt)
 
 	if len(toks) == 1:
 		if toks[0][0] == STR:
@@ -686,7 +694,7 @@ def extract_include(txt, defs):
 			ret = '<', stringize(toks).lstrip('<').rstrip('>')
 			return ret
 
-	raise PreprocError("could not parse include %s." % txt)
+	raise PreprocError('could not parse include %r' % txt)
 
 def parse_char(txt):
 	"""
@@ -698,21 +706,26 @@ def parse_char(txt):
 	:rtype: string
 	"""
 
-	if not txt: raise PreprocError("attempted to parse a null char")
+	if not txt:
+		raise PreprocError('attempted to parse a null char')
 	if txt[0] != '\\':
 		return ord(txt)
 	c = txt[1]
 	if c == 'x':
-		if len(txt) == 4 and txt[3] in string.hexdigits: return int(txt[2:], 16)
+		if len(txt) == 4 and txt[3] in string.hexdigits:
+			return int(txt[2:], 16)
 		return int(txt[2:], 16)
 	elif c.isdigit():
-		if c == '0' and len(txt)==2: return 0
+		if c == '0' and len(txt)==2:
+			return 0
 		for i in 3, 2, 1:
 			if len(txt) > i and txt[1:1+i].isdigit():
 				return (1+i, int(txt[1:1+i], 8))
 	else:
-		try: return chr_esc[c]
-		except KeyError: raise PreprocError("could not parse char literal '%s'" % txt)
+		try:
+			return chr_esc[c]
+		except KeyError:
+			raise PreprocError('could not parse char literal %r' % txt)
 
 def tokenize(s):
 	"""
@@ -725,7 +738,6 @@ def tokenize(s):
 	"""
 	return tokenize_private(s)[:] # force a copy of the results
 
-@Utils.run_once
 def tokenize_private(s):
 	ret = []
 	for match in re_clexer.finditer(s):
@@ -734,28 +746,32 @@ def tokenize_private(s):
 			v = m(name)
 			if v:
 				if name == IDENT:
-					try:
-						g_optrans[v]
+					if v in g_optrans:
 						name = OP
-					except KeyError:
-						# c++ specific
-						if v.lower() == "true":
-							v = 1
-							name = NUM
-						elif v.lower() == "false":
-							v = 0
-							name = NUM
+					elif v.lower() == "true":
+						v = 1
+						name = NUM
+					elif v.lower() == "false":
+						v = 0
+						name = NUM
 				elif name == NUM:
-					if m('oct'): v = int(v, 8)
-					elif m('hex'): v = int(m('hex'), 16)
-					elif m('n0'): v = m('n0')
+					if m('oct'):
+						v = int(v, 8)
+					elif m('hex'):
+						v = int(m('hex'), 16)
+					elif m('n0'):
+						v = m('n0')
 					else:
 						v = m('char')
-						if v: v = parse_char(v)
-						else: v = m('n2') or m('n4')
+						if v:
+							v = parse_char(v)
+						else:
+							v = m('n2') or m('n4')
 				elif name == OP:
-					if v == '%:': v = '#'
-					elif v == '%:%:': v = '##'
+					if v == '%:':
+						v = '#'
+					elif v == '%:%:':
+						v = '##'
 				elif name == STR:
 					# remove the quotes around the string
 					v = v[1:-1]
@@ -763,15 +779,20 @@ def tokenize_private(s):
 				break
 	return ret
 
-@Utils.run_once
-def define_name(line):
-	"""
-	:param line: define line
-	:type line: string
-	:rtype: string
-	:return: the define name
-	"""
-	return re_mac.match(line).group(0)
+def format_defines(lst):
+	ret = []
+	for y in lst:
+		if y:
+			pos = y.find('=')
+			if pos == -1:
+				# "-DFOO" should give "#define FOO 1"
+				ret.append(y)
+			elif pos > 0:
+				# all others are assumed to be -DX=Y
+				ret.append('%s %s' % (y[:pos], y[pos+1:]))
+			else:
+				raise ValueError('Invalid define expression %r' % y)
+	return ret
 
 class c_parser(object):
 	"""
@@ -803,9 +824,12 @@ class c_parser(object):
 		self.curfile = ''
 		"""Current file"""
 
-		self.ban_includes = set([])
+		self.ban_includes = set()
 		"""Includes that must not be read (#pragma once)"""
 
+		self.listed = set()
+		"""Include nodes/names already listed to avoid duplicates in self.nodes/self.names"""
+
 	def cached_find_resource(self, node, filename):
 		"""
 		Find a file from the input directory
@@ -818,13 +842,13 @@ class c_parser(object):
 		:rtype: :py:class:`waflib.Node.Node`
 		"""
 		try:
-			nd = node.ctx.cache_nd
+			cache = node.ctx.preproc_cache_node
 		except AttributeError:
-			nd = node.ctx.cache_nd = {}
+			cache = node.ctx.preproc_cache_node = Utils.lru_cache(FILE_CACHE_SIZE)
 
-		tup = (node, filename)
+		key = (node, filename)
 		try:
-			return nd[tup]
+			return cache[key]
 		except KeyError:
 			ret = node.find_resource(filename)
 			if ret:
@@ -834,10 +858,10 @@ class c_parser(object):
 					tmp = node.ctx.srcnode.search_node(ret.path_from(node.ctx.bldnode))
 					if tmp and getattr(tmp, 'children', None):
 						ret = None
-			nd[tup] = ret
+			cache[key] = ret
 			return ret
 
-	def tryfind(self, filename):
+	def tryfind(self, filename, kind='"', env=None):
 		"""
 		Try to obtain a node from the filename based from the include paths. Will add
 		the node found to :py:attr:`waflib.Tools.c_preproc.c_parser.nodes` or the file name to
@@ -851,29 +875,70 @@ class c_parser(object):
 		"""
 		if filename.endswith('.moc'):
 			# we could let the qt4 module use a subclass, but then the function "scan" below must be duplicated
-			# in the qt4 and in the qt5 classes. So we have two lines here and it is sufficient. TODO waf 1.9
+			# in the qt4 and in the qt5 classes. So we have two lines here and it is sufficient.
 			self.names.append(filename)
 			return None
 
 		self.curfile = filename
 
-		# for msvc it should be a for loop over the whole stack
-		found = self.cached_find_resource(self.currentnode_stack[-1], filename)
+		found = None
+		if kind == '"':
+			if env.MSVC_VERSION:
+				for n in reversed(self.currentnode_stack):
+					found = self.cached_find_resource(n, filename)
+					if found:
+						break
+			else:
+				found = self.cached_find_resource(self.currentnode_stack[-1], filename)
 
-		for n in self.nodepaths:
-			if found:
-				break
-			found = self.cached_find_resource(n, filename)
+		if not found:
+			for n in self.nodepaths:
+				found = self.cached_find_resource(n, filename)
+				if found:
+					break
 
+		listed = self.listed
 		if found and not found in self.ban_includes:
-			# TODO duplicates do not increase the no-op build times too much, but they may be worth removing
-			self.nodes.append(found)
+			if found not in listed:
+				listed.add(found)
+				self.nodes.append(found)
 			self.addlines(found)
 		else:
-			if not filename in self.names:
+			if filename not in listed:
+				listed.add(filename)
 				self.names.append(filename)
 		return found
 
+	def filter_comments(self, node):
+		"""
+		Filter the comments from a c/h file, and return the preprocessor lines.
+		The regexps :py:attr:`waflib.Tools.c_preproc.re_cpp`, :py:attr:`waflib.Tools.c_preproc.re_nl` and :py:attr:`waflib.Tools.c_preproc.re_lines` are used internally.
+
+		:return: the preprocessor directives as a list of (keyword, line)
+		:rtype: a list of string pairs
+		"""
+		# return a list of tuples : keyword, line
+		code = node.read()
+		if use_trigraphs:
+			for (a, b) in trig_def:
+				code = code.split(a).join(b)
+		code = re_nl.sub('', code)
+		code = re_cpp.sub(repl, code)
+		return re_lines.findall(code)
+
+	def parse_lines(self, node):
+		try:
+			cache = node.ctx.preproc_cache_lines
+		except AttributeError:
+			cache = node.ctx.preproc_cache_lines = Utils.lru_cache(LINE_CACHE_SIZE)
+		try:
+			return cache[node]
+		except KeyError:
+			cache[node] = lines = self.filter_comments(node)
+			lines.append((POPFILE, ''))
+			lines.reverse()
+			return lines
+
 	def addlines(self, node):
 		"""
 		Add the lines from a header in the list of preprocessor lines to parse
@@ -883,34 +948,23 @@ class c_parser(object):
 		"""
 
 		self.currentnode_stack.append(node.parent)
-		filepath = node.abspath()
 
 		self.count_files += 1
 		if self.count_files > recursion_limit:
 			# issue #812
-			raise PreprocError("recursion limit exceeded")
-		pc = self.parse_cache
-		debug('preproc: reading file %r', filepath)
-		try:
-			lns = pc[filepath]
-		except KeyError:
-			pass
-		else:
-			self.lines.extend(lns)
-			return
+			raise PreprocError('recursion limit exceeded')
 
+		if Logs.verbose:
+			Logs.debug('preproc: reading file %r', node)
 		try:
-			lines = filter_comments(filepath)
-			lines.append((POPFILE, ''))
-			lines.reverse()
-			pc[filepath] = lines # cache the lines filtered
-			self.lines.extend(lines)
-		except IOError:
-			raise PreprocError("could not read the file %s" % filepath)
+			lines = self.parse_lines(node)
+		except EnvironmentError:
+			raise PreprocError('could not read the file %r' % node)
 		except Exception:
 			if Logs.verbose > 0:
-				error("parsing %s failed" % filepath)
-				traceback.print_exc()
+				Logs.error('parsing %r failed %s', node, traceback.format_exc())
+		else:
+			self.lines.extend(lines)
 
 	def start(self, node, env):
 		"""
@@ -922,27 +976,16 @@ class c_parser(object):
 		:param env: config set containing additional defines to take into account
 		:type env: :py:class:`waflib.ConfigSet.ConfigSet`
 		"""
-
-		debug('preproc: scanning %s (in %s)', node.name, node.parent.name)
-
-		bld = node.ctx
-		try:
-			self.parse_cache = bld.parse_cache
-		except AttributeError:
-			self.parse_cache = bld.parse_cache = {}
+		Logs.debug('preproc: scanning %s (in %s)', node.name, node.parent.name)
 
 		self.current_file = node
 		self.addlines(node)
 
 		# macros may be defined on the command-line, so they must be parsed as if they were part of the file
-		if env['DEFINES']:
-			try:
-				lst = ['%s %s' % (x[0], trimquotes('='.join(x[1:]))) for x in [y.split('=') for y in env['DEFINES']]]
-				lst.reverse()
-				self.lines.extend([('define', x) for x in lst])
-			except AttributeError:
-				# if the defines are invalid the compiler will tell the user
-				pass
+		if env.DEFINES:
+			lst = format_defines(env.DEFINES)
+			lst.reverse()
+			self.lines.extend([('define', x) for x in lst])
 
 		while self.lines:
 			(token, line) = self.lines.pop()
@@ -952,8 +995,6 @@ class c_parser(object):
 				continue
 
 			try:
-				ve = Logs.verbose
-				if ve: debug('preproc: line is %s - %s state is %s', token, line, self.state)
 				state = self.state
 
 				# make certain we define the state if we are about to enter in an if block
@@ -969,23 +1010,27 @@ class c_parser(object):
 
 				if token == 'if':
 					ret = eval_macro(tokenize(line), self.defs)
-					if ret: state[-1] = accepted
-					else: state[-1] = ignored
+					if ret:
+						state[-1] = accepted
+					else:
+						state[-1] = ignored
 				elif token == 'ifdef':
 					m = re_mac.match(line)
-					if m and m.group(0) in self.defs: state[-1] = accepted
-					else: state[-1] = ignored
+					if m and m.group() in self.defs:
+						state[-1] = accepted
+					else:
+						state[-1] = ignored
 				elif token == 'ifndef':
 					m = re_mac.match(line)
-					if m and m.group(0) in self.defs: state[-1] = ignored
-					else: state[-1] = accepted
+					if m and m.group() in self.defs:
+						state[-1] = ignored
+					else:
+						state[-1] = accepted
 				elif token == 'include' or token == 'import':
 					(kind, inc) = extract_include(line, self.defs)
-					if ve: debug('preproc: include found %s    (%s) ', inc, kind)
-					if kind == '"' or not strict_quotes:
-						self.current_file = self.tryfind(inc)
-						if token == 'import':
-							self.ban_includes.add(self.current_file)
+					self.current_file = self.tryfind(inc, kind, env)
+					if token == 'import':
+						self.ban_includes.add(self.current_file)
 				elif token == 'elif':
 					if state[-1] == accepted:
 						state[-1] = skipped
@@ -993,24 +1038,35 @@ class c_parser(object):
 						if eval_macro(tokenize(line), self.defs):
 							state[-1] = accepted
 				elif token == 'else':
-					if state[-1] == accepted: state[-1] = skipped
-					elif state[-1] == ignored: state[-1] = accepted
+					if state[-1] == accepted:
+						state[-1] = skipped
+					elif state[-1] == ignored:
+						state[-1] = accepted
 				elif token == 'define':
 					try:
-						self.defs[define_name(line)] = line
-					except Exception:
-						raise PreprocError("Invalid define line %s" % line)
+						self.defs[self.define_name(line)] = line
+					except AttributeError:
+						raise PreprocError('Invalid define line %r' % line)
 				elif token == 'undef':
 					m = re_mac.match(line)
-					if m and m.group(0) in self.defs:
-						self.defs.__delitem__(m.group(0))
+					if m and m.group() in self.defs:
+						self.defs.__delitem__(m.group())
 						#print "undef %s" % name
 				elif token == 'pragma':
 					if re_pragma_once.match(line.lower()):
 						self.ban_includes.add(self.current_file)
 			except Exception as e:
 				if Logs.verbose:
-					debug('preproc: line parsing failed (%s): %s %s', e, line, Utils.ex_stack())
+					Logs.debug('preproc: line parsing failed (%s): %s %s', e, line, traceback.format_exc())
+
+	def define_name(self, line):
+		"""
+		:param line: define line
+		:type line: string
+		:rtype: string
+		:return: the define name
+		"""
+		return re_mac.match(line).group()
 
 def scan(task):
 	"""
@@ -1020,9 +1076,6 @@ def scan(task):
 
 	This function is bound as a task method on :py:class:`waflib.Tools.c.c` and :py:class:`waflib.Tools.cxx.cxx` for example
 	"""
-
-	global go_absolute
-
 	try:
 		incn = task.generator.includes_nodes
 	except AttributeError:
@@ -1035,7 +1088,4 @@ def scan(task):
 
 	tmp = c_parser(nodepaths)
 	tmp.start(task.inputs[0], task.env)
-	if Logs.verbose:
-		debug('deps: deps for %r: %r; unresolved %r' % (task.inputs, tmp.nodes, tmp.names))
 	return (tmp.nodes, tmp.names)
-
diff --git a/waflib/Tools/c_tests.py b/waflib/Tools/c_tests.py
index 3b37f54c..f858df57 100644
--- a/waflib/Tools/c_tests.py
+++ b/waflib/Tools/c_tests.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2010 (ita)
+# Thomas Nagy, 2016-2018 (ita)
 
 """
 Various configuration tests.
@@ -58,7 +58,7 @@ def link_lib_test_fun(self):
 @conf
 def check_library(self, mode=None, test_exec=True):
 	"""
-	Check if libraries can be linked with the current linker. Uses :py:func:`waflib.Tools.c_tests.link_lib_test_fun`.
+	Checks if libraries can be linked with the current linker. Uses :py:func:`waflib.Tools.c_tests.link_lib_test_fun`.
 
 	:param mode: c or cxx or d
 	:type mode: string
@@ -72,8 +72,7 @@ def check_library(self, mode=None, test_exec=True):
 		features = 'link_lib_test',
 		msg = 'Checking for libraries',
 		mode = mode,
-		test_exec = test_exec,
-		)
+		test_exec = test_exec)
 
 ########################################################################################
 
@@ -89,7 +88,7 @@ INLINE_VALUES = ['inline', '__inline__', '__inline']
 @conf
 def check_inline(self, **kw):
 	"""
-	Check for the right value for inline macro.
+	Checks for the right value for inline macro.
 	Define INLINE_MACRO to 1 if the define is found.
 	If the inline macro is not 'inline', add a define to the ``config.h`` (#define inline __inline__)
 
@@ -98,7 +97,6 @@ def check_inline(self, **kw):
 	:param features: by default *c* or *cxx* depending on the compiler present
 	:type features: list of string
 	"""
-
 	self.start_msg('Checking for inline')
 
 	if not 'define_name' in kw:
@@ -135,7 +133,7 @@ int main(int argc, char **argv) {
 @conf
 def check_large_file(self, **kw):
 	"""
-	Check for large file support and define the macro HAVE_LARGEFILE
+	Checks for large file support and define the macro HAVE_LARGEFILE
 	The test is skipped on win32 systems (DEST_BINFMT == pe).
 
 	:param define_name: define to set, by default *HAVE_LARGEFILE*
@@ -143,7 +141,6 @@ def check_large_file(self, **kw):
 	:param execute: execute the test (yes by default)
 	:type execute: bool
 	"""
-
 	if not 'define_name' in kw:
 		kw['define_name'] = 'HAVE_LARGEFILE'
 	if not 'execute' in kw:
@@ -197,9 +194,12 @@ extern int foo;
 '''
 
 class grep_for_endianness(Task.Task):
+	"""
+	Task that reads a binary and tries to determine the endianness
+	"""
 	color = 'PINK'
 	def run(self):
-		txt = self.inputs[0].read(flags='rb').decode('iso8859-1')
+		txt = self.inputs[0].read(flags='rb').decode('latin-1')
 		if txt.find('LiTTleEnDian') > -1:
 			self.generator.tmp.append('little')
 		elif txt.find('BIGenDianSyS') > -1:
@@ -211,18 +211,19 @@ class grep_for_endianness(Task.Task):
 @after_method('process_source')
 def grep_for_endianness_fun(self):
 	"""
-	Used by the endiannes configuration test
+	Used by the endianness configuration test
 	"""
 	self.create_task('grep_for_endianness', self.compiled_tasks[0].outputs[0])
 
 @conf
 def check_endianness(self):
 	"""
-	Execute a configuration test to determine the endianness
+	Executes a configuration test to determine the endianness
 	"""
 	tmp = []
 	def check_msg(self):
 		return tmp[0]
-	self.check(fragment=ENDIAN_FRAGMENT, features='c grep_for_endianness', msg="Checking for endianness", define='ENDIANNESS', tmp=tmp, okmsg=check_msg)
+	self.check(fragment=ENDIAN_FRAGMENT, features='c grep_for_endianness',
+		msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, okmsg=check_msg)
 	return tmp[0]
 
diff --git a/waflib/Tools/ccroot.py b/waflib/Tools/ccroot.py
index 7fb53bba..cfef8bf5 100644
--- a/waflib/Tools/ccroot.py
+++ b/waflib/Tools/ccroot.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Classes and methods shared by tools providing support for C-like language such
@@ -8,7 +8,7 @@ as C/C++/D/Assembly/Go (this support module is almost never used alone).
 """
 
 import os, re
-from waflib import Task, Utils, Node, Errors
+from waflib import Task, Utils, Node, Errors, Logs
 from waflib.TaskGen import after_method, before_method, feature, taskgen_method, extension
 from waflib.Tools import c_aliases, c_preproc, c_config, c_osx, c_tests
 from waflib.Configure import conf
@@ -77,7 +77,7 @@ def to_incnodes(self, inlst):
 	:return: list of include folders as nodes
 	"""
 	lst = []
-	seen = set([])
+	seen = set()
 	for x in self.to_list(inlst):
 		if x in seen or not x:
 			continue
@@ -118,9 +118,10 @@ def apply_incpaths(self):
 	and the list of include paths in ``tg.env.INCLUDES``.
 	"""
 
-	lst = self.to_incnodes(self.to_list(getattr(self, 'includes', [])) + self.env['INCLUDES'])
+	lst = self.to_incnodes(self.to_list(getattr(self, 'includes', [])) + self.env.INCLUDES)
 	self.includes_nodes = lst
-	self.env['INCPATHS'] = [x.abspath() for x in lst]
+	cwd = self.get_cwd()
+	self.env.INCPATHS = [x.path_from(cwd) for x in lst]
 
 class link_task(Task.Task):
 	"""
@@ -130,6 +131,9 @@ class link_task(Task.Task):
 	"""
 	color   = 'YELLOW'
 
+	weight  = 3
+	"""Try to process link tasks as early as possible"""
+
 	inst_to = None
 	"""Default installation path for the link task outputs, or None to disable"""
 
@@ -142,6 +146,12 @@ class link_task(Task.Task):
 		The settings are retrieved from ``env.clsname_PATTERN``
 		"""
 		if isinstance(target, str):
+			base = self.generator.path
+			if target.startswith('#'):
+				# for those who like flat structures
+				target = target[1:]
+				base = self.generator.bld.bldnode
+
 			pattern = self.env[self.__class__.__name__ + '_PATTERN']
 			if not pattern:
 				pattern = '%s'
@@ -151,7 +161,7 @@ class link_task(Task.Task):
 				nums = self.generator.vnum.split('.')
 				if self.env.DEST_BINFMT == 'pe':
 					# include the version in the dll file name,
-					# the import lib file name stays unversionned.
+					# the import lib file name stays unversioned.
 					name = name + '-' + nums[0]
 				elif self.env.DEST_OS == 'openbsd':
 					pattern = '%s.%s' % (pattern, nums[0])
@@ -162,9 +172,51 @@ class link_task(Task.Task):
 				tmp = folder + os.sep + pattern % name
 			else:
 				tmp = pattern % name
-			target = self.generator.path.find_or_declare(tmp)
+			target = base.find_or_declare(tmp)
 		self.set_outputs(target)
 
+	def exec_command(self, *k, **kw):
+		ret = super(link_task, self).exec_command(*k, **kw)
+		if not ret and self.env.DO_MANIFEST:
+			ret = self.exec_mf()
+		return ret
+
+	def exec_mf(self):
+		"""
+		Create manifest files for VS-like compilers (msvc, ifort, ...)
+		"""
+		if not self.env.MT:
+			return 0
+
+		manifest = None
+		for out_node in self.outputs:
+			if out_node.name.endswith('.manifest'):
+				manifest = out_node.abspath()
+				break
+		else:
+			# Should never get here.  If we do, it means the manifest file was
+			# never added to the outputs list, thus we don't have a manifest file
+			# to embed, so we just return.
+			return 0
+
+		# embedding mode. Different for EXE's and DLL's.
+		# see: http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx
+		mode = ''
+		for x in Utils.to_list(self.generator.features):
+			if x in ('cprogram', 'cxxprogram', 'fcprogram', 'fcprogram_test'):
+				mode = 1
+			elif x in ('cshlib', 'cxxshlib', 'fcshlib'):
+				mode = 2
+
+		Logs.debug('msvc: embedding manifest in mode %r', mode)
+
+		lst = [] + self.env.MT
+		lst.extend(Utils.to_list(self.env.MTFLAGS))
+		lst.extend(['-manifest', manifest])
+		lst.append('-outputresource:%s;%s' % (self.outputs[0].abspath(), mode))
+
+		return super(link_task, self).exec_command(lst)
+
 class stlink_task(link_task):
 	"""
 	Base for static link tasks, which use *ar* most of the time.
@@ -178,8 +230,10 @@ class stlink_task(link_task):
 def rm_tgt(cls):
 	old = cls.run
 	def wrap(self):
-		try: os.remove(self.outputs[0].abspath())
-		except OSError: pass
+		try:
+			os.remove(self.outputs[0].abspath())
+		except OSError:
+			pass
 		return old(self)
 	setattr(cls, 'run', wrap)
 rm_tgt(stlink_task)
@@ -219,10 +273,12 @@ def apply_link(self):
 	try:
 		inst_to = self.install_path
 	except AttributeError:
-		inst_to = self.link_task.__class__.inst_to
+		inst_to = self.link_task.inst_to
 	if inst_to:
 		# install a copy of the node list we have at this moment (implib not added)
-		self.install_task = self.bld.install_files(inst_to, self.link_task.outputs[:], env=self.env, chmod=self.link_task.chmod, task=self.link_task)
+		self.install_task = self.add_install_files(
+			install_to=inst_to, install_from=self.link_task.outputs[:],
+			chmod=self.link_task.chmod, task=self.link_task)
 
 @taskgen_method
 def use_rec(self, name, **kw):
@@ -282,7 +338,7 @@ def process_use(self):
 	See :py:func:`waflib.Tools.ccroot.use_rec`.
 	"""
 
-	use_not = self.tmp_use_not = set([])
+	use_not = self.tmp_use_not = set()
 	self.tmp_use_seen = [] # we would like an ordered set
 	use_prec = self.tmp_use_prec = {}
 	self.uselib = self.to_list(getattr(self, 'uselib', []))
@@ -297,7 +353,7 @@ def process_use(self):
 			del use_prec[x]
 
 	# topological sort
-	out = []
+	out = self.tmp_use_sorted = []
 	tmp = []
 	for x in self.tmp_use_seen:
 		for k in use_prec.values():
@@ -333,14 +389,15 @@ def process_use(self):
 			if var == 'LIB' or y.tmp_use_stlib or x in names:
 				self.env.append_value(var, [y.target[y.target.rfind(os.sep) + 1:]])
 				self.link_task.dep_nodes.extend(y.link_task.outputs)
-				tmp_path = y.link_task.outputs[0].parent.path_from(self.bld.bldnode)
+				tmp_path = y.link_task.outputs[0].parent.path_from(self.get_cwd())
 				self.env.append_unique(var + 'PATH', [tmp_path])
 		else:
 			if y.tmp_use_objects:
 				self.add_objects_from_tgen(y)
 
 		if getattr(y, 'export_includes', None):
-			self.includes.extend(y.to_incnodes(y.export_includes))
+			# self.includes may come from a global variable #2035
+			self.includes = self.includes + y.to_incnodes(y.export_includes)
 
 		if getattr(y, 'export_defines', None):
 			self.env.append_value('DEFINES', self.to_list(y.export_defines))
@@ -390,7 +447,7 @@ def get_uselib_vars(self):
 	:return: the *uselib* variables associated to the *features* attribute (see :py:attr:`waflib.Tools.ccroot.USELIB_VARS`)
 	:rtype: list of string
 	"""
-	_vars = set([])
+	_vars = set()
 	for x in self.features:
 		if x in USELIB_VARS:
 			_vars |= USELIB_VARS[x]
@@ -405,7 +462,7 @@ def propagate_uselib_vars(self):
 		def build(bld):
 			bld.env.AFLAGS_aaa = ['bar']
 			from waflib.Tools.ccroot import USELIB_VARS
-			USELIB_VARS['aaa'] = set('AFLAGS')
+			USELIB_VARS['aaa'] = ['AFLAGS']
 
 			tg = bld(features='aaa', aflags='test')
 
@@ -447,20 +504,20 @@ def apply_implib(self):
 		name = self.target.name
 	else:
 		name = os.path.split(self.target)[1]
-	implib = self.env['implib_PATTERN'] % name
+	implib = self.env.implib_PATTERN % name
 	implib = dll.parent.find_or_declare(implib)
-	self.env.append_value('LINKFLAGS', self.env['IMPLIB_ST'] % implib.bldpath())
+	self.env.append_value('LINKFLAGS', self.env.IMPLIB_ST % implib.bldpath())
 	self.link_task.outputs.append(implib)
 
 	if getattr(self, 'defs', None) and self.env.DEST_BINFMT == 'pe':
 		node = self.path.find_resource(self.defs)
 		if not node:
 			raise Errors.WafError('invalid def file %r' % self.defs)
-		if 'msvc' in (self.env.CC_NAME, self.env.CXX_NAME):
-			self.env.append_value('LINKFLAGS', '/def:%s' % node.path_from(self.bld.bldnode))
+		if self.env.def_PATTERN:
+			self.env.append_value('LINKFLAGS', self.env.def_PATTERN % node.path_from(self.get_cwd()))
 			self.link_task.dep_nodes.append(node)
 		else:
-			#gcc for windows takes *.def file a an input without any special flag
+			# gcc for windows takes *.def file as input without any special flag
 			self.link_task.inputs.append(node)
 
 	# where to put the import library
@@ -475,10 +532,11 @@ def apply_implib(self):
 			except AttributeError:
 				# else, put the library in BINDIR and the import library in LIBDIR
 				inst_to = '${IMPLIBDIR}'
-				self.install_task.dest = '${BINDIR}'
+				self.install_task.install_to = '${BINDIR}'
 				if not self.env.IMPLIBDIR:
 					self.env.IMPLIBDIR = self.env.LIBDIR
-		self.implib_install_task = self.bld.install_files(inst_to, implib, env=self.env, chmod=self.link_task.chmod, task=self.link_task)
+		self.implib_install_task = self.add_install_files(install_to=inst_to, install_from=implib,
+			chmod=self.link_task.chmod, task=self.link_task)
 
 # ============ the code above must not know anything about vnum processing on unix platforms =========
 
@@ -535,34 +593,34 @@ def apply_vnum(self):
 
 	# the following task is just to enable execution from the build dir :-/
 	if self.env.DEST_OS != 'openbsd':
-		outs = [node.parent.find_or_declare(name3)]
+		outs = [node.parent.make_node(name3)]
 		if name2 != name3:
-			outs.append(node.parent.find_or_declare(name2))
+			outs.append(node.parent.make_node(name2))
 		self.create_task('vnum', node, outs)
 
 	if getattr(self, 'install_task', None):
-		self.install_task.hasrun = Task.SKIP_ME
-		bld = self.bld
-		path = self.install_task.dest
+		self.install_task.hasrun = Task.SKIPPED
+		self.install_task.no_errcheck_out = True
+		path = self.install_task.install_to
 		if self.env.DEST_OS == 'openbsd':
 			libname = self.link_task.outputs[0].name
-			t1 = bld.install_as('%s%s%s' % (path, os.sep, libname), node, env=self.env, chmod=self.link_task.chmod)
+			t1 = self.add_install_as(install_to='%s/%s' % (path, libname), install_from=node, chmod=self.link_task.chmod)
 			self.vnum_install_task = (t1,)
 		else:
-			t1 = bld.install_as(path + os.sep + name3, node, env=self.env, chmod=self.link_task.chmod)
-			t3 = bld.symlink_as(path + os.sep + libname, name3)
+			t1 = self.add_install_as(install_to=path + os.sep + name3, install_from=node, chmod=self.link_task.chmod)
+			t3 = self.add_symlink_as(install_to=path + os.sep + libname, install_from=name3)
 			if name2 != name3:
-				t2 = bld.symlink_as(path + os.sep + name2, name3)
+				t2 = self.add_symlink_as(install_to=path + os.sep + name2, install_from=name3)
 				self.vnum_install_task = (t1, t2, t3)
 			else:
 				self.vnum_install_task = (t1, t3)
 
-	if '-dynamiclib' in self.env['LINKFLAGS']:
+	if '-dynamiclib' in self.env.LINKFLAGS:
 		# this requires after(propagate_uselib_vars)
 		try:
 			inst_to = self.install_path
 		except AttributeError:
-			inst_to = self.link_task.__class__.inst_to
+			inst_to = self.link_task.inst_to
 		if inst_to:
 			p = Utils.subst_vars(inst_to, self.env)
 			path = os.path.join(p, name2)
@@ -575,7 +633,6 @@ class vnum(Task.Task):
 	Create the symbolic links for a versioned shared library. Instances are created by :py:func:`waflib.Tools.ccroot.apply_vnum`
 	"""
 	color = 'CYAN'
-	quient = True
 	ext_in = ['.bin']
 	def keyword(self):
 		return 'Symlinking'
@@ -600,9 +657,6 @@ class fake_shlib(link_task):
 		for t in self.run_after:
 			if not t.hasrun:
 				return Task.ASK_LATER
-
-		for x in self.outputs:
-			x.sig = Utils.h_file(x.abspath())
 		return Task.SKIP_ME
 
 class fake_stlib(stlink_task):
@@ -613,9 +667,6 @@ class fake_stlib(stlink_task):
 		for t in self.run_after:
 			if not t.hasrun:
 				return Task.ASK_LATER
-
-		for x in self.outputs:
-			x.sig = Utils.h_file(x.abspath())
 		return Task.SKIP_ME
 
 @conf
@@ -658,7 +709,10 @@ def process_lib(self):
 		for y in names:
 			node = x.find_node(y)
 			if node:
-				node.sig = Utils.h_file(node.abspath())
+				try:
+					Utils.h_file(node.abspath())
+				except EnvironmentError:
+					raise ValueError('Could not read %r' % y)
 				break
 		else:
 			continue
diff --git a/waflib/Tools/clang.py b/waflib/Tools/clang.py
index 2259c5df..3828e391 100644
--- a/waflib/Tools/clang.py
+++ b/waflib/Tools/clang.py
@@ -12,7 +12,7 @@ from waflib.Configure import conf
 @conf
 def find_clang(conf):
 	"""
-	Find the program clang and execute it to ensure it really is clang
+	Finds the program clang and executes it to ensure it really is clang
 	"""
 	cc = conf.find_program('clang', var='CC')
 	conf.get_cc_version(cc, clang=True)
diff --git a/waflib/Tools/clangxx.py b/waflib/Tools/clangxx.py
index b539b287..152013ce 100644
--- a/waflib/Tools/clangxx.py
+++ b/waflib/Tools/clangxx.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy 2009-2010 (ita)
+# Thomas Nagy 2009-2018 (ita)
 
 """
 Detect the Clang++ C++ compiler
@@ -12,7 +12,7 @@ from waflib.Configure import conf
 @conf
 def find_clangxx(conf):
 	"""
-	Find the program clang++, and execute it to ensure it really is clang++
+	Finds the program clang++, and executes it to ensure it really is clang++
 	"""
 	cxx = conf.find_program('clang++', var='CXX')
 	conf.get_cc_version(cxx, clang=True)
diff --git a/waflib/Tools/compiler_c.py b/waflib/Tools/compiler_c.py
index 7d4e22ca..2dba3f82 100644
--- a/waflib/Tools/compiler_c.py
+++ b/waflib/Tools/compiler_c.py
@@ -47,10 +47,10 @@ c_compiler = {
 'osf1V':  ['gcc'],
 'gnu':    ['gcc', 'clang'],
 'java':   ['gcc', 'msvc', 'clang', 'icc'],
-'default':['gcc', 'clang'],
+'default':['clang', 'gcc'],
 }
 """
-Dict mapping the platform names to Waf tools finding specific C compilers::
+Dict mapping platform names to Waf tools finding specific C compilers::
 
 	from waflib.Tools.compiler_c import c_compiler
 	c_compiler['linux'] = ['gcc', 'icc', 'suncc']
@@ -63,10 +63,14 @@ def default_compilers():
 
 def configure(conf):
 	"""
-	Try to find a suitable C compiler or raise a :py:class:`waflib.Errors.ConfigurationError`.
+	Detects a suitable C compiler
+
+	:raises: :py:class:`waflib.Errors.ConfigurationError` when no suitable compiler is found
 	"""
-	try: test_for_compiler = conf.options.check_c_compiler or default_compilers()
-	except AttributeError: conf.fatal("Add options(opt): opt.load('compiler_c')")
+	try:
+		test_for_compiler = conf.options.check_c_compiler or default_compilers()
+	except AttributeError:
+		conf.fatal("Add options(opt): opt.load('compiler_c')")
 
 	for compiler in re.split('[ ,]+', test_for_compiler):
 		conf.env.stash()
@@ -76,19 +80,21 @@ def configure(conf):
 		except conf.errors.ConfigurationError as e:
 			conf.env.revert()
 			conf.end_msg(False)
-			debug('compiler_c: %r' % e)
+			debug('compiler_c: %r', e)
 		else:
-			if conf.env['CC']:
+			if conf.env.CC:
 				conf.end_msg(conf.env.get_flat('CC'))
-				conf.env['COMPILER_CC'] = compiler
+				conf.env.COMPILER_CC = compiler
+				conf.env.commit()
 				break
+			conf.env.revert()
 			conf.end_msg(False)
 	else:
 		conf.fatal('could not configure a C compiler!')
 
 def options(opt):
 	"""
-	Restrict the compiler detection from the command-line::
+	This is how to provide compiler preferences on the command-line::
 
 		$ waf configure --check-c-compiler=gcc
 	"""
diff --git a/waflib/Tools/compiler_cxx.py b/waflib/Tools/compiler_cxx.py
index 6f0ea9ea..1af65a22 100644
--- a/waflib/Tools/compiler_cxx.py
+++ b/waflib/Tools/compiler_cxx.py
@@ -48,7 +48,7 @@ cxx_compiler = {
 'osf1V':  ['g++'],
 'gnu':    ['g++', 'clang++'],
 'java':   ['g++', 'msvc', 'clang++', 'icpc'],
-'default': ['g++', 'clang++']
+'default': ['clang++', 'g++']
 }
 """
 Dict mapping the platform names to Waf tools finding specific C++ compilers::
@@ -64,10 +64,14 @@ def default_compilers():
 
 def configure(conf):
 	"""
-	Try to find a suitable C++ compiler or raise a :py:class:`waflib.Errors.ConfigurationError`.
+	Detects a suitable C++ compiler
+
+	:raises: :py:class:`waflib.Errors.ConfigurationError` when no suitable compiler is found
 	"""
-	try: test_for_compiler = conf.options.check_cxx_compiler or default_compilers()
-	except AttributeError: conf.fatal("Add options(opt): opt.load('compiler_cxx')")
+	try:
+		test_for_compiler = conf.options.check_cxx_compiler or default_compilers()
+	except AttributeError:
+		conf.fatal("Add options(opt): opt.load('compiler_cxx')")
 
 	for compiler in re.split('[ ,]+', test_for_compiler):
 		conf.env.stash()
@@ -77,19 +81,21 @@ def configure(conf):
 		except conf.errors.ConfigurationError as e:
 			conf.env.revert()
 			conf.end_msg(False)
-			debug('compiler_cxx: %r' % e)
+			debug('compiler_cxx: %r', e)
 		else:
-			if conf.env['CXX']:
+			if conf.env.CXX:
 				conf.end_msg(conf.env.get_flat('CXX'))
-				conf.env['COMPILER_CXX'] = compiler
+				conf.env.COMPILER_CXX = compiler
+				conf.env.commit()
 				break
+			conf.env.revert()
 			conf.end_msg(False)
 	else:
 		conf.fatal('could not configure a C++ compiler!')
 
 def options(opt):
 	"""
-	Restrict the compiler detection from the command-line::
+	This is how to provide compiler preferences on the command-line::
 
 		$ waf configure --check-cxx-compiler=gxx
 	"""
diff --git a/waflib/Tools/cxx.py b/waflib/Tools/cxx.py
index 311c1e52..194fad74 100644
--- a/waflib/Tools/cxx.py
+++ b/waflib/Tools/cxx.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 "Base for c++ programs and libraries"
 
@@ -10,31 +10,31 @@ from waflib.Tools.ccroot import link_task, stlink_task
 
 @TaskGen.extension('.cpp','.cc','.cxx','.C','.c++')
 def cxx_hook(self, node):
-	"Bind the c++ file extensions to the creation of a :py:class:`waflib.Tools.cxx.cxx` instance"
+	"Binds c++ file extensions to create :py:class:`waflib.Tools.cxx.cxx` instances"
 	return self.create_compiled_task('cxx', node)
 
 if not '.c' in TaskGen.task_gen.mappings:
 	TaskGen.task_gen.mappings['.c'] = TaskGen.task_gen.mappings['.cpp']
 
 class cxx(Task.Task):
-	"Compile C++ files into object files"
-	run_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} ${CXX_SRC_F}${SRC} ${CXX_TGT_F}${TGT[0].abspath()}'
+	"Compiles C++ files into object files"
+	run_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} ${CXX_SRC_F}${SRC} ${CXX_TGT_F}${TGT[0].abspath()} ${CPPFLAGS}'
 	vars    = ['CXXDEPS'] # unused variable to depend on, just in case
 	ext_in  = ['.h'] # set the build order easily by using ext_out=['.h']
 	scan    = c_preproc.scan
 
 class cxxprogram(link_task):
-	"Link object files into a c++ program"
+	"Links object files into c++ programs"
 	run_str = '${LINK_CXX} ${LINKFLAGS} ${CXXLNK_SRC_F}${SRC} ${CXXLNK_TGT_F}${TGT[0].abspath()} ${RPATH_ST:RPATH} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${FRAMEWORK_ST:FRAMEWORK} ${ARCH_ST:ARCH} ${STLIB_MARKER} ${STLIBPATH_ST:STLIBPATH} ${STLIB_ST:STLIB} ${SHLIB_MARKER} ${LIBPATH_ST:LIBPATH} ${LIB_ST:LIB} ${LDFLAGS}'
 	vars    = ['LINKDEPS']
 	ext_out = ['.bin']
 	inst_to = '${BINDIR}'
 
 class cxxshlib(cxxprogram):
-	"Link object files into a c++ shared library"
+	"Links object files into c++ shared libraries"
 	inst_to = '${LIBDIR}'
 
 class cxxstlib(stlink_task):
-	"Link object files into a c++ static library"
+	"Links object files into c++ static libraries"
 	pass # do not remove
 
diff --git a/waflib/Tools/errcheck.py b/waflib/Tools/errcheck.py
index 1f548aca..de8d75a4 100644
--- a/waflib/Tools/errcheck.py
+++ b/waflib/Tools/errcheck.py
@@ -3,9 +3,9 @@
 # Thomas Nagy, 2011 (ita)
 
 """
-errcheck: highlight common mistakes
+Common mistakes highlighting.
 
-There is a performance hit, so this tool is only loaded when running "waf -v"
+There is a performance impact, so this tool is only loaded when running ``waf -v``
 """
 
 typos = {
@@ -18,13 +18,14 @@ typos = {
 'importpath':'includes',
 'installpath':'install_path',
 'iscopy':'is_copy',
+'uses':'use',
 }
 
 meths_typos = ['__call__', 'program', 'shlib', 'stlib', 'objects']
 
 import sys
 from waflib import Logs, Build, Node, Task, TaskGen, ConfigSet, Errors, Utils
-import waflib.Tools.ccroot
+from waflib.Tools import ccroot
 
 def check_same_targets(self):
 	mp = Utils.defaultdict(list)
@@ -33,6 +34,8 @@ def check_same_targets(self):
 	def check_task(tsk):
 		if not isinstance(tsk, Task.Task):
 			return
+		if hasattr(tsk, 'no_errcheck_out'):
+			return
 
 		for node in tsk.outputs:
 			mp[node].append(tsk)
@@ -58,30 +61,34 @@ def check_same_targets(self):
 			Logs.error(msg)
 			for x in v:
 				if Logs.verbose > 1:
-					Logs.error('  %d. %r' % (1 + v.index(x), x.generator))
+					Logs.error('  %d. %r', 1 + v.index(x), x.generator)
 				else:
-					Logs.error('  %d. %r in %r' % (1 + v.index(x), x.generator.name, getattr(x.generator, 'path', None)))
+					Logs.error('  %d. %r in %r', 1 + v.index(x), x.generator.name, getattr(x.generator, 'path', None))
+			Logs.error('If you think that this is an error, set no_errcheck_out on the task instance')
 
 	if not dupe:
 		for (k, v) in uids.items():
 			if len(v) > 1:
 				Logs.error('* Several tasks use the same identifier. Please check the information on\n   https://waf.io/apidocs/Task.html?highlight=uid#waflib.Task.Task.uid')
+				tg_details = tsk.generator.name
+				if Logs.verbose > 2:
+					tg_details = tsk.generator
 				for tsk in v:
-					Logs.error('  - object %r (%r) defined in %r' % (tsk.__class__.__name__, tsk, tsk.generator))
+					Logs.error('  - object %r (%r) defined in %r', tsk.__class__.__name__, tsk, tg_details)
 
 def check_invalid_constraints(self):
-	feat = set([])
+	feat = set()
 	for x in list(TaskGen.feats.values()):
 		feat.union(set(x))
 	for (x, y) in TaskGen.task_gen.prec.items():
 		feat.add(x)
 		feat.union(set(y))
-	ext = set([])
+	ext = set()
 	for x in TaskGen.task_gen.mappings.values():
 		ext.add(x.__name__)
 	invalid = ext & feat
 	if invalid:
-		Logs.error('The methods %r have invalid annotations:  @extension <-> @feature/@before_method/@after_method' % list(invalid))
+		Logs.error('The methods %r have invalid annotations:  @extension <-> @feature/@before_method/@after_method', list(invalid))
 
 	# the build scripts have been read, so we can check for invalid after/before attributes on task classes
 	for cls in list(Task.classes.values()):
@@ -90,15 +97,15 @@ def check_invalid_constraints(self):
 
 		for x in ('before', 'after'):
 			for y in Utils.to_list(getattr(cls, x, [])):
-				if not Task.classes.get(y, None):
-					Logs.error('Erroneous order constraint %r=%r on task class %r' % (x, y, cls.__name__))
+				if not Task.classes.get(y):
+					Logs.error('Erroneous order constraint %r=%r on task class %r', x, y, cls.__name__)
 		if getattr(cls, 'rule', None):
-			Logs.error('Erroneous attribute "rule" on task class %r (rename to "run_str")' % cls.__name__)
+			Logs.error('Erroneous attribute "rule" on task class %r (rename to "run_str")', cls.__name__)
 
 def replace(m):
 	"""
-	We could add properties, but they would not work in some cases:
-	bld.program(...) requires 'source' in the attributes
+	Replaces existing BuildContext methods to verify parameter names,
+	for example ``bld(source=)`` has no ending *s*
 	"""
 	oldcall = getattr(Build.BuildContext, m)
 	def call(self, *k, **kw):
@@ -107,13 +114,13 @@ def replace(m):
 			if x in kw:
 				if x == 'iscopy' and 'subst' in getattr(self, 'features', ''):
 					continue
-				Logs.error('Fix the typo %r -> %r on %r' % (x, typos[x], ret))
+				Logs.error('Fix the typo %r -> %r on %r', x, typos[x], ret)
 		return ret
 	setattr(Build.BuildContext, m, call)
 
 def enhance_lib():
 	"""
-	modify existing classes and methods
+	Modifies existing classes and methods to enable error verification
 	"""
 	for m in meths_typos:
 		replace(m)
@@ -121,26 +128,36 @@ def enhance_lib():
 	# catch '..' in ant_glob patterns
 	def ant_glob(self, *k, **kw):
 		if k:
-			lst=Utils.to_list(k[0])
+			lst = Utils.to_list(k[0])
 			for pat in lst:
-				if '..' in pat.split('/'):
-					Logs.error("In ant_glob pattern %r: '..' means 'two dots', not 'parent directory'" % k[0])
-		if kw.get('remove', True):
-			try:
-				if self.is_child_of(self.ctx.bldnode) and not kw.get('quiet', False):
-					Logs.error('Using ant_glob on the build folder (%r) is dangerous (quiet=True to disable this warning)' % self)
-			except AttributeError:
-				pass
+				sp = pat.split('/')
+				if '..' in sp:
+					Logs.error("In ant_glob pattern %r: '..' means 'two dots', not 'parent directory'", k[0])
+				if '.' in sp:
+					Logs.error("In ant_glob pattern %r: '.' means 'one dot', not 'current directory'", k[0])
 		return self.old_ant_glob(*k, **kw)
 	Node.Node.old_ant_glob = Node.Node.ant_glob
 	Node.Node.ant_glob = ant_glob
 
+	# catch ant_glob on build folders
+	def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remove=True, quiet=False):
+		if remove:
+			try:
+				if self.is_child_of(self.ctx.bldnode) and not quiet:
+					quiet = True
+					Logs.error('Calling ant_glob on build folders (%r) is dangerous: add quiet=True / remove=False', self)
+			except AttributeError:
+				pass
+		return self.old_ant_iter(accept, maxdepth, pats, dir, src, remove, quiet)
+	Node.Node.old_ant_iter = Node.Node.ant_iter
+	Node.Node.ant_iter = ant_iter
+
 	# catch conflicting ext_in/ext_out/before/after declarations
 	old = Task.is_before
 	def is_before(t1, t2):
 		ret = old(t1, t2)
 		if ret and old(t2, t1):
-			Logs.error('Contradictory order constraints in classes %r %r' % (t1, t2))
+			Logs.error('Contradictory order constraints in classes %r %r', t1, t2)
 		return ret
 	Task.is_before = is_before
 
@@ -152,7 +169,7 @@ def enhance_lib():
 			Logs.error('feature shlib -> cshlib, dshlib or cxxshlib')
 		for x in ('c', 'cxx', 'd', 'fc'):
 			if not x in lst and lst and lst[0] in [x+y for y in ('program', 'shlib', 'stlib')]:
-				Logs.error('%r features is probably missing %r' % (self, x))
+				Logs.error('%r features is probably missing %r', self, x)
 	TaskGen.feature('*')(check_err_features)
 
 	# check for erroneous order constraints
@@ -160,12 +177,12 @@ def enhance_lib():
 		if not hasattr(self, 'rule') and not 'subst' in Utils.to_list(self.features):
 			for x in ('before', 'after', 'ext_in', 'ext_out'):
 				if hasattr(self, x):
-					Logs.warn('Erroneous order constraint %r on non-rule based task generator %r' % (x, self))
+					Logs.warn('Erroneous order constraint %r on non-rule based task generator %r', x, self)
 		else:
 			for x in ('before', 'after'):
 				for y in self.to_list(getattr(self, x, [])):
-					if not Task.classes.get(y, None):
-						Logs.error('Erroneous order constraint %s=%r on %r (no such class)' % (x, y, self))
+					if not Task.classes.get(y):
+						Logs.error('Erroneous order constraint %s=%r on %r (no such class)', x, y, self)
 	TaskGen.feature('*')(check_err_order)
 
 	# check for @extension used with @feature/@before_method/@after_method
@@ -200,24 +217,21 @@ def enhance_lib():
 	TaskGen.task_gen.use_rec = use_rec
 
 	# check for env.append
-	def getattri(self, name, default=None):
+	def _getattr(self, name, default=None):
 		if name == 'append' or name == 'add':
 			raise Errors.WafError('env.append and env.add do not exist: use env.append_value/env.append_unique')
 		elif name == 'prepend':
 			raise Errors.WafError('env.prepend does not exist: use env.prepend_value')
 		if name in self.__slots__:
-			return object.__getattr__(self, name, default)
+			return super(ConfigSet.ConfigSet, self).__getattr__(name, default)
 		else:
 			return self[name]
-	ConfigSet.ConfigSet.__getattr__ = getattri
+	ConfigSet.ConfigSet.__getattr__ = _getattr
 
 
 def options(opt):
 	"""
-	Add a few methods
+	Error verification can be enabled by default (not just on ``waf -v``) by adding to the user script options
 	"""
 	enhance_lib()
 
-def configure(conf):
-	pass
-
diff --git a/waflib/Tools/gcc.py b/waflib/Tools/gcc.py
index 0b897c7d..acdd473a 100644
--- a/waflib/Tools/gcc.py
+++ b/waflib/Tools/gcc.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 # Yinon Ehrlich, 2009
 
@@ -27,54 +27,51 @@ def gcc_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CC_SRC_F']            = []
-	v['CC_TGT_F']            = ['-c', '-o']
+	v.CC_SRC_F            = []
+	v.CC_TGT_F            = ['-c', '-o']
 
-	# linker
-	if not v['LINK_CC']: v['LINK_CC'] = v['CC']
-	v['CCLNK_SRC_F']         = []
-	v['CCLNK_TGT_F']         = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CC:
+		v.LINK_CC = v.CC
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
-	v['RPATH_ST']            = '-Wl,-rpath,%s'
+	v.CCLNK_SRC_F         = []
+	v.CCLNK_TGT_F         = ['-o']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = '-Wl,-h,%s'
-	v['SHLIB_MARKER']        = '-Wl,-Bdynamic'
-	v['STLIB_MARKER']        = '-Wl,-Bstatic'
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
+	v.RPATH_ST            = '-Wl,-rpath,%s'
 
-	# program
-	v['cprogram_PATTERN']    = '%s'
+	v.SONAME_ST           = '-Wl,-h,%s'
+	v.SHLIB_MARKER        = '-Wl,-Bdynamic'
+	v.STLIB_MARKER        = '-Wl,-Bstatic'
 
-	# shared librar
-	v['CFLAGS_cshlib']       = ['-fPIC']
-	v['LINKFLAGS_cshlib']    = ['-shared']
-	v['cshlib_PATTERN']      = 'lib%s.so'
+	v.cprogram_PATTERN    = '%s'
 
-	# static lib
-	v['LINKFLAGS_cstlib']    = ['-Wl,-Bstatic']
-	v['cstlib_PATTERN']      = 'lib%s.a'
+	v.CFLAGS_cshlib       = ['-fPIC']
+	v.LINKFLAGS_cshlib    = ['-shared']
+	v.cshlib_PATTERN      = 'lib%s.so'
 
-	# osx stuff
-	v['LINKFLAGS_MACBUNDLE'] = ['-bundle', '-undefined', 'dynamic_lookup']
-	v['CFLAGS_MACBUNDLE']    = ['-fPIC']
-	v['macbundle_PATTERN']   = '%s.bundle'
+	v.LINKFLAGS_cstlib    = ['-Wl,-Bstatic']
+	v.cstlib_PATTERN      = 'lib%s.a'
+
+	v.LINKFLAGS_MACBUNDLE = ['-bundle', '-undefined', 'dynamic_lookup']
+	v.CFLAGS_MACBUNDLE    = ['-fPIC']
+	v.macbundle_PATTERN   = '%s.bundle'
 
 @conf
 def gcc_modifier_win32(conf):
 	"""Configuration flags for executing gcc on Windows"""
 	v = conf.env
-	v['cprogram_PATTERN']    = '%s.exe'
+	v.cprogram_PATTERN    = '%s.exe'
 
-	v['cshlib_PATTERN']      = '%s.dll'
-	v['implib_PATTERN']      = 'lib%s.dll.a'
-	v['IMPLIB_ST']           = '-Wl,--out-implib,%s'
+	v.cshlib_PATTERN      = '%s.dll'
+	v.implib_PATTERN      = '%s.dll.a'
+	v.IMPLIB_ST           = '-Wl,--out-implib,%s'
 
-	v['CFLAGS_cshlib']       = []
+	v.CFLAGS_cshlib       = []
 
 	# Auto-import is enabled by default even without this option,
 	# but enabling it explicitly has the nice effect of suppressing the rather boring, debug-level messages
@@ -86,42 +83,42 @@ def gcc_modifier_cygwin(conf):
 	"""Configuration flags for executing gcc on Cygwin"""
 	gcc_modifier_win32(conf)
 	v = conf.env
-	v['cshlib_PATTERN'] = 'cyg%s.dll'
+	v.cshlib_PATTERN = 'cyg%s.dll'
 	v.append_value('LINKFLAGS_cshlib', ['-Wl,--enable-auto-image-base'])
-	v['CFLAGS_cshlib'] = []
+	v.CFLAGS_cshlib = []
 
 @conf
 def gcc_modifier_darwin(conf):
 	"""Configuration flags for executing gcc on MacOS"""
 	v = conf.env
-	v['CFLAGS_cshlib']       = ['-fPIC']
-	v['LINKFLAGS_cshlib']    = ['-dynamiclib']
-	v['cshlib_PATTERN']      = 'lib%s.dylib'
-	v['FRAMEWORKPATH_ST']    = '-F%s'
-	v['FRAMEWORK_ST']        = ['-framework']
-	v['ARCH_ST']             = ['-arch']
+	v.CFLAGS_cshlib       = ['-fPIC']
+	v.LINKFLAGS_cshlib    = ['-dynamiclib']
+	v.cshlib_PATTERN      = 'lib%s.dylib'
+	v.FRAMEWORKPATH_ST    = '-F%s'
+	v.FRAMEWORK_ST        = ['-framework']
+	v.ARCH_ST             = ['-arch']
 
-	v['LINKFLAGS_cstlib']    = []
+	v.LINKFLAGS_cstlib    = []
 
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['SONAME_ST']           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.SONAME_ST           = []
 
 @conf
 def gcc_modifier_aix(conf):
 	"""Configuration flags for executing gcc on AIX"""
 	v = conf.env
-	v['LINKFLAGS_cprogram']  = ['-Wl,-brtl']
-	v['LINKFLAGS_cshlib']    = ['-shared','-Wl,-brtl,-bexpfull']
-	v['SHLIB_MARKER']        = []
+	v.LINKFLAGS_cprogram  = ['-Wl,-brtl']
+	v.LINKFLAGS_cshlib    = ['-shared','-Wl,-brtl,-bexpfull']
+	v.SHLIB_MARKER        = []
 
 @conf
 def gcc_modifier_hpux(conf):
 	v = conf.env
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['CFLAGS_cshlib']       = ['-fPIC','-DPIC']
-	v['cshlib_PATTERN']      = 'lib%s.sl'
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.CFLAGS_cshlib       = ['-fPIC','-DPIC']
+	v.cshlib_PATTERN      = 'lib%s.sl'
 
 @conf
 def gcc_modifier_openbsd(conf):
@@ -130,9 +127,9 @@ def gcc_modifier_openbsd(conf):
 @conf
 def gcc_modifier_osf1V(conf):
 	v = conf.env
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['SONAME_ST']           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.SONAME_ST           = []
 
 @conf
 def gcc_modifier_platform(conf):
@@ -155,5 +152,5 @@ def configure(conf):
 	conf.cc_load_tools()
 	conf.cc_add_flags()
 	conf.link_add_flags()
-
+	conf.check_gcc_o_space()
 
diff --git a/waflib/Tools/gxx.py b/waflib/Tools/gxx.py
index 9cf52070..22c5d26f 100644
--- a/waflib/Tools/gxx.py
+++ b/waflib/Tools/gxx.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 # Yinon Ehrlich, 2009
 
@@ -14,7 +14,7 @@ from waflib.Configure import conf
 @conf
 def find_gxx(conf):
 	"""
-	Find the program g++, and if present, try to detect its version number
+	Finds the program g++, and if present, try to detect its version number
 	"""
 	cxx = conf.find_program(['g++', 'c++'], var='CXX')
 	conf.get_cc_version(cxx, gcc=True)
@@ -27,54 +27,51 @@ def gxx_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CXX_SRC_F']           = []
-	v['CXX_TGT_F']           = ['-c', '-o']
+	v.CXX_SRC_F           = []
+	v.CXX_TGT_F           = ['-c', '-o']
 
-	# linker
-	if not v['LINK_CXX']: v['LINK_CXX'] = v['CXX']
-	v['CXXLNK_SRC_F']        = []
-	v['CXXLNK_TGT_F']        = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CXX:
+		v.LINK_CXX = v.CXX
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
-	v['RPATH_ST']            = '-Wl,-rpath,%s'
+	v.CXXLNK_SRC_F        = []
+	v.CXXLNK_TGT_F        = ['-o']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = '-Wl,-h,%s'
-	v['SHLIB_MARKER']        = '-Wl,-Bdynamic'
-	v['STLIB_MARKER']        = '-Wl,-Bstatic'
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
+	v.RPATH_ST            = '-Wl,-rpath,%s'
 
-	# program
-	v['cxxprogram_PATTERN']  = '%s'
+	v.SONAME_ST           = '-Wl,-h,%s'
+	v.SHLIB_MARKER        = '-Wl,-Bdynamic'
+	v.STLIB_MARKER        = '-Wl,-Bstatic'
 
-	# shared library
-	v['CXXFLAGS_cxxshlib']   = ['-fPIC']
-	v['LINKFLAGS_cxxshlib']  = ['-shared']
-	v['cxxshlib_PATTERN']    = 'lib%s.so'
+	v.cxxprogram_PATTERN  = '%s'
 
-	# static lib
-	v['LINKFLAGS_cxxstlib']  = ['-Wl,-Bstatic']
-	v['cxxstlib_PATTERN']    = 'lib%s.a'
+	v.CXXFLAGS_cxxshlib   = ['-fPIC']
+	v.LINKFLAGS_cxxshlib  = ['-shared']
+	v.cxxshlib_PATTERN    = 'lib%s.so'
 
-	# osx stuff
-	v['LINKFLAGS_MACBUNDLE'] = ['-bundle', '-undefined', 'dynamic_lookup']
-	v['CXXFLAGS_MACBUNDLE']  = ['-fPIC']
-	v['macbundle_PATTERN']   = '%s.bundle'
+	v.LINKFLAGS_cxxstlib  = ['-Wl,-Bstatic']
+	v.cxxstlib_PATTERN    = 'lib%s.a'
+
+	v.LINKFLAGS_MACBUNDLE = ['-bundle', '-undefined', 'dynamic_lookup']
+	v.CXXFLAGS_MACBUNDLE  = ['-fPIC']
+	v.macbundle_PATTERN   = '%s.bundle'
 
 @conf
 def gxx_modifier_win32(conf):
 	"""Configuration flags for executing gcc on Windows"""
 	v = conf.env
-	v['cxxprogram_PATTERN']  = '%s.exe'
+	v.cxxprogram_PATTERN  = '%s.exe'
 
-	v['cxxshlib_PATTERN']    = '%s.dll'
-	v['implib_PATTERN']      = 'lib%s.dll.a'
-	v['IMPLIB_ST']           = '-Wl,--out-implib,%s'
+	v.cxxshlib_PATTERN    = '%s.dll'
+	v.implib_PATTERN      = '%s.dll.a'
+	v.IMPLIB_ST           = '-Wl,--out-implib,%s'
 
-	v['CXXFLAGS_cxxshlib']   = []
+	v.CXXFLAGS_cxxshlib   = []
 
 	# Auto-import is enabled by default even without this option,
 	# but enabling it explicitly has the nice effect of suppressing the rather boring, debug-level messages
@@ -86,43 +83,43 @@ def gxx_modifier_cygwin(conf):
 	"""Configuration flags for executing g++ on Cygwin"""
 	gxx_modifier_win32(conf)
 	v = conf.env
-	v['cxxshlib_PATTERN']    = 'cyg%s.dll'
+	v.cxxshlib_PATTERN    = 'cyg%s.dll'
 	v.append_value('LINKFLAGS_cxxshlib', ['-Wl,--enable-auto-image-base'])
-	v['CXXFLAGS_cxxshlib']   = []
+	v.CXXFLAGS_cxxshlib   = []
 
 @conf
 def gxx_modifier_darwin(conf):
 	"""Configuration flags for executing g++ on MacOS"""
 	v = conf.env
-	v['CXXFLAGS_cxxshlib']   = ['-fPIC']
-	v['LINKFLAGS_cxxshlib']  = ['-dynamiclib']
-	v['cxxshlib_PATTERN']    = 'lib%s.dylib'
-	v['FRAMEWORKPATH_ST']    = '-F%s'
-	v['FRAMEWORK_ST']        = ['-framework']
-	v['ARCH_ST']             = ['-arch']
+	v.CXXFLAGS_cxxshlib   = ['-fPIC']
+	v.LINKFLAGS_cxxshlib  = ['-dynamiclib']
+	v.cxxshlib_PATTERN    = 'lib%s.dylib'
+	v.FRAMEWORKPATH_ST    = '-F%s'
+	v.FRAMEWORK_ST        = ['-framework']
+	v.ARCH_ST             = ['-arch']
 
-	v['LINKFLAGS_cxxstlib']  = []
+	v.LINKFLAGS_cxxstlib  = []
 
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['SONAME_ST']           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.SONAME_ST           = []
 
 @conf
 def gxx_modifier_aix(conf):
 	"""Configuration flags for executing g++ on AIX"""
 	v = conf.env
-	v['LINKFLAGS_cxxprogram']= ['-Wl,-brtl']
+	v.LINKFLAGS_cxxprogram= ['-Wl,-brtl']
 
-	v['LINKFLAGS_cxxshlib']  = ['-shared', '-Wl,-brtl,-bexpfull']
-	v['SHLIB_MARKER']        = []
+	v.LINKFLAGS_cxxshlib  = ['-shared', '-Wl,-brtl,-bexpfull']
+	v.SHLIB_MARKER        = []
 
 @conf
 def gxx_modifier_hpux(conf):
 	v = conf.env
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['CFLAGS_cxxshlib']     = ['-fPIC','-DPIC']
-	v['cxxshlib_PATTERN']    = 'lib%s.sl'
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.CFLAGS_cxxshlib     = ['-fPIC','-DPIC']
+	v.cxxshlib_PATTERN    = 'lib%s.sl'
 
 @conf
 def gxx_modifier_openbsd(conf):
@@ -131,9 +128,9 @@ def gxx_modifier_openbsd(conf):
 @conf
 def gcc_modifier_osf1V(conf):
 	v = conf.env
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
-	v['SONAME_ST']           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
+	v.SONAME_ST           = []
 
 @conf
 def gxx_modifier_platform(conf):
@@ -156,4 +153,5 @@ def configure(conf):
 	conf.cxx_load_tools()
 	conf.cxx_add_flags()
 	conf.link_add_flags()
+	conf.check_gcc_o_space('cxx')
 
diff --git a/waflib/Tools/icc.py b/waflib/Tools/icc.py
index f3395030..b6492c8e 100644
--- a/waflib/Tools/icc.py
+++ b/waflib/Tools/icc.py
@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # encoding: utf-8
 # Stian Selnes 2008
-# Thomas Nagy 2009-2010 (ita)
+# Thomas Nagy 2009-2018 (ita)
 
 """
-Detect the Intel C compiler
+Detects the Intel C compiler
 """
 
 import sys
@@ -14,11 +14,8 @@ from waflib.Configure import conf
 @conf
 def find_icc(conf):
 	"""
-	Find the program icc and execute it to ensure it really is icc
+	Finds the program icc and execute it to ensure it really is icc
 	"""
-	if sys.platform == 'cygwin':
-		conf.fatal('The Intel compiler does not work on Cygwin')
-
 	cc = conf.find_program(['icc', 'ICL'], var='CC')
 	conf.get_cc_version(cc, icc=True)
 	conf.env.CC_NAME = 'icc'
diff --git a/waflib/Tools/icpc.py b/waflib/Tools/icpc.py
index 03603d44..8a6cc6c4 100644
--- a/waflib/Tools/icpc.py
+++ b/waflib/Tools/icpc.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy 2009-2010 (ita)
+# Thomas Nagy 2009-2018 (ita)
 
 """
-Detect the Intel C++ compiler
+Detects the Intel C++ compiler
 """
 
 import sys
@@ -13,11 +13,8 @@ from waflib.Configure import conf
 @conf
 def find_icpc(conf):
 	"""
-	Find the program icpc, and execute it to ensure it really is icpc
+	Finds the program icpc, and execute it to ensure it really is icpc
 	"""
-	if sys.platform == 'cygwin':
-		conf.fatal('The Intel compiler does not work on Cygwin')
-
 	cxx = conf.find_program('icpc', var='CXX')
 	conf.get_cc_version(cxx, icc=True)
 	conf.env.CXX_NAME = 'icc'
diff --git a/waflib/Tools/irixcc.py b/waflib/Tools/irixcc.py
index 55eb70be..c3ae1ac9 100644
--- a/waflib/Tools/irixcc.py
+++ b/waflib/Tools/irixcc.py
@@ -1,11 +1,12 @@
 #! /usr/bin/env python
+# encoding: utf-8
 # imported from samba
 
 """
-compiler definition for irix/MIPSpro cc compiler
-based on suncc.py from waf
+Compiler definition for irix/MIPSpro cc compiler
 """
 
+from waflib import Errors
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 
@@ -13,41 +14,46 @@ from waflib.Configure import conf
 def find_irixcc(conf):
 	v = conf.env
 	cc = None
-	if v['CC']: cc = v['CC']
-	elif 'CC' in conf.environ: cc = conf.environ['CC']
-	if not cc: cc = conf.find_program('cc', var='CC')
-	if not cc: conf.fatal('irixcc was not found')
+	if v.CC:
+		cc = v.CC
+	elif 'CC' in conf.environ:
+		cc = conf.environ['CC']
+	if not cc:
+		cc = conf.find_program('cc', var='CC')
+	if not cc:
+		conf.fatal('irixcc was not found')
 
 	try:
 		conf.cmd_and_log(cc + ['-version'])
-	except Exception:
+	except Errors.WafError:
 		conf.fatal('%r -version could not be executed' % cc)
 
-	v['CC']  = cc
-	v['CC_NAME'] = 'irix'
+	v.CC = cc
+	v.CC_NAME = 'irix'
 
 @conf
 def irixcc_common_flags(conf):
 	v = conf.env
 
-	v['CC_SRC_F']            = ''
-	v['CC_TGT_F']            = ['-c', '-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	v.CC_SRC_F            = ''
+	v.CC_TGT_F            = ['-c', '-o']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	# linker
-	if not v['LINK_CC']: v['LINK_CC'] = v['CC']
-	v['CCLNK_SRC_F']         = ''
-	v['CCLNK_TGT_F']         = ['-o']
+	if not v.LINK_CC:
+		v.LINK_CC = v.CC
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
+	v.CCLNK_SRC_F         = ''
+	v.CCLNK_TGT_F         = ['-o']
 
-	v['cprogram_PATTERN']     = '%s'
-	v['cshlib_PATTERN']       = 'lib%s.so'
-	v['cstlib_PATTERN']      = 'lib%s.a'
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
+
+	v.cprogram_PATTERN    = '%s'
+	v.cshlib_PATTERN      = 'lib%s.so'
+	v.cstlib_PATTERN      = 'lib%s.a'
 
 def configure(conf):
 	conf.find_irixcc()
diff --git a/waflib/Tools/msvc.py b/waflib/Tools/msvc.py
index d98432e0..17b347d4 100644
--- a/waflib/Tools/msvc.py
+++ b/waflib/Tools/msvc.py
@@ -8,6 +8,12 @@
 """
 Microsoft Visual C++/Intel C++ compiler support
 
+If you get detection problems, first try any of the following::
+
+	chcp 65001
+	set PYTHONIOENCODING=...
+	set PYTHONLEGACYWINDOWSSTDIO=1
+
 Usage::
 
 	$ waf configure --msvc_version="msvc 10.0,msvc 9.0" --msvc_target="x64"
@@ -15,8 +21,8 @@ Usage::
 or::
 
 	def configure(conf):
-		conf.env['MSVC_VERSIONS'] = ['msvc 10.0', 'msvc 9.0', 'msvc 8.0', 'msvc 7.1', 'msvc 7.0', 'msvc 6.0', 'wsdk 7.0', 'intel 11', 'PocketPC 9.0', 'Smartphone 8.0']
-		conf.env['MSVC_TARGETS'] = ['x64']
+		conf.env.MSVC_VERSIONS = ['msvc 10.0', 'msvc 9.0', 'msvc 8.0', 'msvc 7.1', 'msvc 7.0', 'msvc 6.0', 'wsdk 7.0', 'intel 11', 'PocketPC 9.0', 'Smartphone 8.0']
+		conf.env.MSVC_TARGETS = ['x64']
 		conf.load('msvc')
 
 or::
@@ -31,14 +37,14 @@ or::
 Platforms and targets will be tested in the order they appear;
 the first good configuration will be used.
 
-To skip testing all the configurations that are not used, use the ``--msvc_lazy_autodetect`` option
-or set ``conf.env['MSVC_LAZY_AUTODETECT']=True``.
+To force testing all the configurations that are not used, use the ``--no-msvc-lazy`` option
+or set ``conf.env.MSVC_LAZY_AUTODETECT=False``.
 
 Supported platforms: ia64, x64, x86, x86_amd64, x86_ia64, x86_arm, amd64_x86, amd64_arm
 
 Compilers supported:
 
-* msvc       => Visual Studio, versions 6.0 (VC 98, VC .NET 2002) to 12.0 (Visual Studio 2013)
+* msvc       => Visual Studio, versions 6.0 (VC 98, VC .NET 2002) to 15 (Visual Studio 2017)
 * wsdk       => Windows SDK, versions 6.0, 6.1, 7.0, 7.1, 8.0
 * icl        => Intel compiler, versions 9, 10, 11, 13
 * winphone   => Visual Studio to target Windows Phone 8 native (version 8.0 for now)
@@ -52,13 +58,12 @@ cmd.exe  /C  "chcp 1252 & set PYTHONUNBUFFERED=true && set && waf  configure"
 Setting PYTHONUNBUFFERED gives the unbuffered output.
 """
 
-import os, sys, re, tempfile
-from waflib import Utils, Task, Logs, Options, Errors
-from waflib.Logs import debug, warn
+import os, sys, re, traceback
+from waflib import Utils, Logs, Options, Errors
 from waflib.TaskGen import after_method, feature
 
 from waflib.Configure import conf
-from waflib.Tools import ccroot, c, cxx, ar, winres
+from waflib.Tools import ccroot, c, cxx, ar
 
 g_msvc_systemlibs = '''
 aclui activeds ad1 adptif adsiid advapi32 asycfilt authz bhsupp bits bufferoverflowu cabinet
@@ -82,7 +87,9 @@ wintrust wldap32 wmiutils wow32 ws2_32 wsnmp32 wsock32 wst wtsapi32 xaswitch xol
 '''.split()
 """importlibs provided by MSVC/Platform SDK. Do NOT search them"""
 
-all_msvc_platforms = [ ('x64', 'amd64'), ('x86', 'x86'), ('ia64', 'ia64'), ('x86_amd64', 'amd64'), ('x86_ia64', 'ia64'), ('x86_arm', 'arm'), ('amd64_x86', 'x86'), ('amd64_arm', 'arm') ]
+all_msvc_platforms = [	('x64', 'amd64'), ('x86', 'x86'), ('ia64', 'ia64'),
+						('x86_amd64', 'amd64'), ('x86_ia64', 'ia64'), ('x86_arm', 'arm'), ('x86_arm64', 'arm64'),
+						('amd64_x86', 'x86'), ('amd64_arm', 'arm'), ('amd64_arm64', 'arm64') ]
 """List of msvc platforms"""
 
 all_wince_platforms = [ ('armv4', 'arm'), ('armv4i', 'arm'), ('mipsii', 'mips'), ('mipsii_fp', 'mips'), ('mipsiv', 'mips'), ('mipsiv_fp', 'mips'), ('sh4', 'sh'), ('x86', 'cex86') ]
@@ -94,45 +101,63 @@ all_icl_platforms = [ ('intel64', 'amd64'), ('em64t', 'amd64'), ('ia32', 'x86'),
 def options(opt):
 	opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default='')
 	opt.add_option('--msvc_targets', type='string', help = 'msvc targets, eg: "x64,arm"', default='')
-	opt.add_option('--msvc_lazy_autodetect', action='store_true', help = 'lazily check msvc target environments')
+	opt.add_option('--no-msvc-lazy', action='store_false', help = 'lazily check msvc target environments', default=True, dest='msvc_lazy')
 
-def setup_msvc(conf, versions, arch = False):
+@conf
+def setup_msvc(conf, versiondict):
 	"""
 	Checks installed compilers and targets and returns the first combination from the user's
 	options, env, or the global supported lists that checks.
 
-	:param versions: A list of tuples of all installed compilers and available targets.
-	:param arch: Whether to return the target architecture.
-	:return: the compiler, revision, path, include dirs, library paths, and (optionally) target architecture
+	:param versiondict: dict(platform -> dict(architecture -> configuration))
+	:type versiondict: dict(string -> dict(string -> target_compiler)
+	:return: the compiler, revision, path, include dirs, library paths and target architecture
 	:rtype: tuple of strings
 	"""
 	platforms = getattr(Options.options, 'msvc_targets', '').split(',')
 	if platforms == ['']:
-		platforms=Utils.to_list(conf.env['MSVC_TARGETS']) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms]
+		platforms=Utils.to_list(conf.env.MSVC_TARGETS) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms]
 	desired_versions = getattr(Options.options, 'msvc_version', '').split(',')
 	if desired_versions == ['']:
-		desired_versions = conf.env['MSVC_VERSIONS'] or [v for v,_ in versions][::-1]
-	versiondict = dict(versions)
+		desired_versions = conf.env.MSVC_VERSIONS or list(reversed(sorted(versiondict.keys())))
+
+	# Override lazy detection by evaluating after the fact.
+	lazy_detect = getattr(Options.options, 'msvc_lazy', True)
+	if conf.env.MSVC_LAZY_AUTODETECT is False:
+		lazy_detect = False
+
+	if not lazy_detect:
+		for val in versiondict.values():
+			for arch in list(val.keys()):
+				cfg = val[arch]
+				cfg.evaluate()
+				if not cfg.is_valid:
+					del val[arch]
+		conf.env.MSVC_INSTALLED_VERSIONS = versiondict
 
 	for version in desired_versions:
+		Logs.debug('msvc: detecting %r - %r', version, desired_versions)
 		try:
-			targets = dict(versiondict[version])
-			for target in platforms:
-				try:
-					try:
-						realtarget,(p1,p2,p3) = targets[target]
-					except conf.errors.ConfigurationError:
-						# lazytup target evaluation errors
-						del(targets[target])
-					else:
-						compiler,revision = version.rsplit(' ', 1)
-						if arch:
-							return compiler,revision,p1,p2,p3,realtarget
-						else:
-							return compiler,revision,p1,p2,p3
-				except KeyError: continue
-		except KeyError: continue
-	conf.fatal('msvc: Impossible to find a valid architecture for building (in setup_msvc)')
+			targets = versiondict[version]
+		except KeyError:
+			continue
+
+		seen = set()
+		for arch in platforms:
+			if arch in seen:
+				continue
+			else:
+				seen.add(arch)
+			try:
+				cfg = targets[arch]
+			except KeyError:
+				continue
+
+			cfg.evaluate()
+			if cfg.is_valid:
+				compiler,revision = version.rsplit(' ', 1)
+				return compiler,revision,cfg.bindirs,cfg.incdirs,cfg.libdirs,cfg.cpu
+	conf.fatal('msvc: Impossible to find a valid architecture for building %r - %r' % (desired_versions, list(versiondict.keys())))
 
 @conf
 def get_msvc_version(conf, compiler, version, target, vcvars):
@@ -147,7 +172,7 @@ def get_msvc_version(conf, compiler, version, target, vcvars):
 	:return: the location of the compiler executable, the location of include dirs, and the library paths
 	:rtype: tuple of strings
 	"""
-	debug('msvc: get_msvc_version: %r %r %r', compiler, version, target)
+	Logs.debug('msvc: get_msvc_version: %r %r %r', compiler, version, target)
 
 	try:
 		conf.msvc_cnt += 1
@@ -187,67 +212,27 @@ echo LIB=%%LIB%%;%%LIBPATH%%
 	compiler_name, linker_name, lib_name = _get_prog_names(conf, compiler)
 	cxx = conf.find_program(compiler_name, path_list=MSVC_PATH)
 
-	# delete CL if exists. because it could contain parameters wich can change cl's behaviour rather catastrophically.
+	# delete CL if exists. because it could contain parameters which can change cl's behaviour rather catastrophically.
 	if 'CL' in env:
 		del(env['CL'])
 
 	try:
-		try:
-			conf.cmd_and_log(cxx + ['/help'], env=env)
-		except UnicodeError:
-			st = Utils.ex_stack()
-			if conf.logger:
-				conf.logger.error(st)
-			conf.fatal('msvc: Unicode error - check the code page?')
-		except Exception as e:
-			debug('msvc: get_msvc_version: %r %r %r -> failure %s' % (compiler, version, target, str(e)))
-			conf.fatal('msvc: cannot run the compiler in get_msvc_version (run with -v to display errors)')
-		else:
-			debug('msvc: get_msvc_version: %r %r %r -> OK', compiler, version, target)
+		conf.cmd_and_log(cxx + ['/help'], env=env)
+	except UnicodeError:
+		st = traceback.format_exc()
+		if conf.logger:
+			conf.logger.error(st)
+		conf.fatal('msvc: Unicode error - check the code page?')
+	except Exception as e:
+		Logs.debug('msvc: get_msvc_version: %r %r %r -> failure %s', compiler, version, target, str(e))
+		conf.fatal('msvc: cannot run the compiler in get_msvc_version (run with -v to display errors)')
+	else:
+		Logs.debug('msvc: get_msvc_version: %r %r %r -> OK', compiler, version, target)
 	finally:
 		conf.env[compiler_name] = ''
 
 	return (MSVC_PATH, MSVC_INCDIR, MSVC_LIBDIR)
 
-@conf
-def gather_wsdk_versions(conf, versions):
-	"""
-	Use winreg to add the msvc versions to the input list
-
-	:param versions: list to modify
-	:type versions: list
-	"""
-	version_pattern = re.compile('^v..?.?\...?.?')
-	try:
-		all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Microsoft SDKs\\Windows')
-	except WindowsError:
-		try:
-			all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows')
-		except WindowsError:
-			return
-	index = 0
-	while 1:
-		try:
-			version = Utils.winreg.EnumKey(all_versions, index)
-		except WindowsError:
-			break
-		index = index + 1
-		if not version_pattern.match(version):
-			continue
-		try:
-			msvc_version = Utils.winreg.OpenKey(all_versions, version)
-			path,type = Utils.winreg.QueryValueEx(msvc_version,'InstallationFolder')
-		except WindowsError:
-			continue
-		if path and os.path.isfile(os.path.join(path, 'bin', 'SetEnv.cmd')):
-			targets = []
-			for target,arch in all_msvc_platforms:
-				try:
-					targets.append((target, (arch, get_compiler_env(conf, 'wsdk', version, '/'+target, os.path.join(path, 'bin', 'SetEnv.cmd')))))
-				except conf.errors.ConfigurationError:
-					pass
-			versions.append(('wsdk ' + version[1:], targets))
-
 def gather_wince_supported_platforms():
 	"""
 	Checks SmartPhones SDKs
@@ -258,31 +243,31 @@ def gather_wince_supported_platforms():
 	supported_wince_platforms = []
 	try:
 		ce_sdk = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Windows CE Tools\\SDKs')
-	except WindowsError:
+	except OSError:
 		try:
 			ce_sdk = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows CE Tools\\SDKs')
-		except WindowsError:
+		except OSError:
 			ce_sdk = ''
 	if not ce_sdk:
 		return supported_wince_platforms
 
-	ce_index = 0
+	index = 0
 	while 1:
 		try:
-			sdk_device = Utils.winreg.EnumKey(ce_sdk, ce_index)
-		except WindowsError:
+			sdk_device = Utils.winreg.EnumKey(ce_sdk, index)
+			sdk = Utils.winreg.OpenKey(ce_sdk, sdk_device)
+		except OSError:
 			break
-		ce_index = ce_index + 1
-		sdk = Utils.winreg.OpenKey(ce_sdk, sdk_device)
+		index += 1
 		try:
 			path,type = Utils.winreg.QueryValueEx(sdk, 'SDKRootDir')
-		except WindowsError:
+		except OSError:
 			try:
 				path,type = Utils.winreg.QueryValueEx(sdk,'SDKInformation')
-				path,xml = os.path.split(path)
-			except WindowsError:
+			except OSError:
 				continue
-		path=str(path)
+			path,xml = os.path.split(path)
+		path = str(path)
 		path,device = os.path.split(path)
 		if not device:
 			path,device = os.path.split(path)
@@ -299,122 +284,140 @@ def gather_msvc_detected_versions():
 	version_pattern = re.compile('^(\d\d?\.\d\d?)(Exp)?$')
 	detected_versions = []
 	for vcver,vcvar in (('VCExpress','Exp'), ('VisualStudio','')):
+		prefix = 'SOFTWARE\\Wow6432node\\Microsoft\\' + vcver
 		try:
-			prefix = 'SOFTWARE\\Wow6432node\\Microsoft\\'+vcver
 			all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, prefix)
-		except WindowsError:
+		except OSError:
+			prefix = 'SOFTWARE\\Microsoft\\' + vcver
 			try:
-				prefix = 'SOFTWARE\\Microsoft\\'+vcver
 				all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, prefix)
-			except WindowsError:
+			except OSError:
 				continue
 
 		index = 0
 		while 1:
 			try:
 				version = Utils.winreg.EnumKey(all_versions, index)
-			except WindowsError:
+			except OSError:
 				break
-			index = index + 1
+			index += 1
 			match = version_pattern.match(version)
-			if not match:
-				continue
-			else:
+			if match:
 				versionnumber = float(match.group(1))
-			detected_versions.append((versionnumber, version+vcvar, prefix+"\\"+version))
+			else:
+				continue
+			detected_versions.append((versionnumber, version+vcvar, prefix+'\\'+version))
 	def fun(tup):
 		return tup[0]
 
 	detected_versions.sort(key = fun)
 	return detected_versions
 
-def get_compiler_env(conf, compiler, version, bat_target, bat, select=None):
-	"""
-	Gets the compiler environment variables as a tuple. Evaluation is eager by default.
-	If set to lazy with ``--msvc_lazy_autodetect`` or ``env.MSVC_LAZY_AUTODETECT``
-	the environment is evaluated when the tuple is destructured or iterated. This means
-	destructuring can throw :py:class:`conf.errors.ConfigurationError`.
-
-	:param conf: configuration context to use to eventually get the version environment
-	:param compiler: compiler name
-	:param version: compiler version number
-	:param bat: path to the batch file to run
-	:param select: optional function to take the realized environment variables tup and map it (e.g. to combine other constant paths)
-	"""
-	lazy = getattr(Options.options, 'msvc_lazy_autodetect', False) or conf.env['MSVC_LAZY_AUTODETECT']
+class target_compiler(object):
+	"""
+	Wrap a compiler configuration; call evaluate() to determine
+	whether the configuration is usable.
+	"""
+	def __init__(self, ctx, compiler, cpu, version, bat_target, bat, callback=None):
+		"""
+		:param ctx: configuration context to use to eventually get the version environment
+		:param compiler: compiler name
+		:param cpu: target cpu
+		:param version: compiler version number
+		:param bat_target: ?
+		:param bat: path to the batch file to run
+		"""
+		self.conf = ctx
+		self.name = None
+		self.is_valid = False
+		self.is_done = False
+
+		self.compiler = compiler
+		self.cpu = cpu
+		self.version = version
+		self.bat_target = bat_target
+		self.bat = bat
+		self.callback = callback
 
-	def msvc_thunk():
-		vs = conf.get_msvc_version(compiler, version, bat_target, bat)
-		if select:
-			return select(vs)
-		else:
-			return vs
-	return lazytup(msvc_thunk, lazy, ([], [], []))
+	def evaluate(self):
+		if self.is_done:
+			return
+		self.is_done = True
+		try:
+			vs = self.conf.get_msvc_version(self.compiler, self.version, self.bat_target, self.bat)
+		except Errors.ConfigurationError:
+			self.is_valid = False
+			return
+		if self.callback:
+			vs = self.callback(self, vs)
+		self.is_valid = True
+		(self.bindirs, self.incdirs, self.libdirs) = vs
+
+	def __str__(self):
+		return str((self.compiler, self.cpu, self.version, self.bat_target, self.bat))
 
-class lazytup(object):
+	def __repr__(self):
+		return repr((self.compiler, self.cpu, self.version, self.bat_target, self.bat))
+
+@conf
+def gather_wsdk_versions(conf, versions):
 	"""
-	A tuple that evaluates its elements from a function when iterated or destructured.
+	Use winreg to add the msvc versions to the input list
 
-	:param fn: thunk to evaluate the tuple on demand
-	:param lazy: whether to delay evaluation or evaluate in the constructor
-	:param default: optional default for :py:func:`repr` if it should not evaluate
+	:param versions: list to modify
+	:type versions: list
 	"""
-	def __init__(self, fn, lazy=True, default=None):
-		self.fn = fn
-		self.default = default
-		if not lazy:
-			self.evaluate()
-	def __len__(self):
-		self.evaluate()
-		return len(self.value)
-	def __iter__(self):
-		self.evaluate()
-		for i, v in enumerate(self.value):
-			yield v
-	def __getitem__(self, i):
-		self.evaluate()
-		return self.value[i]
-	def __repr__(self):
-		if hasattr(self, 'value'):
-			return repr(self.value)
-		elif self.default:
-			return repr(self.default)
-		else:
-			self.evaluate()
-			return repr(self.value)
-	def evaluate(self):
-		if hasattr(self, 'value'):
+	version_pattern = re.compile('^v..?.?\...?.?')
+	try:
+		all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Microsoft SDKs\\Windows')
+	except OSError:
+		try:
+			all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows')
+		except OSError:
 			return
-		self.value = self.fn()
+	index = 0
+	while 1:
+		try:
+			version = Utils.winreg.EnumKey(all_versions, index)
+		except OSError:
+			break
+		index += 1
+		if not version_pattern.match(version):
+			continue
+		try:
+			msvc_version = Utils.winreg.OpenKey(all_versions, version)
+			path,type = Utils.winreg.QueryValueEx(msvc_version,'InstallationFolder')
+		except OSError:
+			continue
+		if path and os.path.isfile(os.path.join(path, 'bin', 'SetEnv.cmd')):
+			targets = {}
+			for target,arch in all_msvc_platforms:
+				targets[target] = target_compiler(conf, 'wsdk', arch, version, '/'+target, os.path.join(path, 'bin', 'SetEnv.cmd'))
+			versions['wsdk ' + version[1:]] = targets
 
 @conf
 def gather_msvc_targets(conf, versions, version, vc_path):
 	#Looking for normal MSVC compilers!
-	targets = []
-	if os.path.isfile(os.path.join(vc_path, 'vcvarsall.bat')):
+	targets = {}
+
+	if os.path.isfile(os.path.join(vc_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat')):
 		for target,realtarget in all_msvc_platforms[::-1]:
-			try:
-				targets.append((target, (realtarget, get_compiler_env(conf, 'msvc', version, target, os.path.join(vc_path, 'vcvarsall.bat')))))
-			except conf.errors.ConfigurationError:
-				pass
+			targets[target] = target_compiler(conf, 'msvc', realtarget, version, target, os.path.join(vc_path, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat'))
+	elif os.path.isfile(os.path.join(vc_path, 'vcvarsall.bat')):
+		for target,realtarget in all_msvc_platforms[::-1]:
+			targets[target] = target_compiler(conf, 'msvc', realtarget, version, target, os.path.join(vc_path, 'vcvarsall.bat'))
 	elif os.path.isfile(os.path.join(vc_path, 'Common7', 'Tools', 'vsvars32.bat')):
-		try:
-			targets.append(('x86', ('x86', get_compiler_env(conf, 'msvc', version, 'x86', os.path.join(vc_path, 'Common7', 'Tools', 'vsvars32.bat')))))
-		except conf.errors.ConfigurationError:
-			pass
+		targets['x86'] = target_compiler(conf, 'msvc', 'x86', version, 'x86', os.path.join(vc_path, 'Common7', 'Tools', 'vsvars32.bat'))
 	elif os.path.isfile(os.path.join(vc_path, 'Bin', 'vcvars32.bat')):
-		try:
-			targets.append(('x86', ('x86', get_compiler_env(conf, 'msvc', version, '', os.path.join(vc_path, 'Bin', 'vcvars32.bat')))))
-		except conf.errors.ConfigurationError:
-			pass
+		targets['x86'] = target_compiler(conf, 'msvc', 'x86', version, '', os.path.join(vc_path, 'Bin', 'vcvars32.bat'))
 	if targets:
-		versions.append(('msvc '+ version, targets))
+		versions['msvc %s' % version] = targets
 
 @conf
 def gather_wince_targets(conf, versions, version, vc_path, vsvars, supported_platforms):
 	#Looking for Win CE compilers!
 	for device,platforms in supported_platforms:
-		cetargets = []
+		targets = {}
 		for platform,compiler,include,lib in platforms:
 			winCEpath = os.path.join(vc_path, 'ce')
 			if not os.path.isdir(winCEpath):
@@ -424,27 +427,52 @@ def gather_wince_targets(conf, versions, version, vc_path, vsvars, supported_pla
 				bindirs = [os.path.join(winCEpath, 'bin', compiler), os.path.join(winCEpath, 'bin', 'x86_'+compiler)]
 				incdirs = [os.path.join(winCEpath, 'include'), os.path.join(winCEpath, 'atlmfc', 'include'), include]
 				libdirs = [os.path.join(winCEpath, 'lib', platform), os.path.join(winCEpath, 'atlmfc', 'lib', platform), lib]
-				def combine_common(compiler_env):
+				def combine_common(obj, compiler_env):
+					# TODO this is likely broken, remove in waf 2.1
 					(common_bindirs,_1,_2) = compiler_env
 					return (bindirs + common_bindirs, incdirs, libdirs)
-				try:
-					cetargets.append((platform, (platform, get_compiler_env(conf, 'msvc', version, 'x86', vsvars, combine_common))))
-				except conf.errors.ConfigurationError:
-					continue
-		if cetargets:
-			versions.append((device + ' ' + version, cetargets))
+				targets[platform] = target_compiler(conf, 'msvc', platform, version, 'x86', vsvars, combine_common)
+		if targets:
+			versions[device + ' ' + version] = targets
 
 @conf
 def gather_winphone_targets(conf, versions, version, vc_path, vsvars):
 	#Looking for WinPhone compilers
-	targets = []
+	targets = {}
 	for target,realtarget in all_msvc_platforms[::-1]:
-		try:
-			targets.append((target, (realtarget, get_compiler_env(conf, 'winphone', version, target, vsvars))))
-		except conf.errors.ConfigurationError:
-			pass
+		targets[target] = target_compiler(conf, 'winphone', realtarget, version, target, vsvars)
 	if targets:
-		versions.append(('winphone '+ version, targets))
+		versions['winphone ' + version] = targets
+
+@conf
+def gather_vswhere_versions(conf, versions):
+	try:
+		import json
+	except ImportError:
+		Logs.error('Visual Studio 2017 detection requires Python 2.6')
+		return
+
+	prg_path = os.environ.get('ProgramFiles(x86)', os.environ.get('ProgramFiles', 'C:\\Program Files (x86)'))
+
+	vswhere = os.path.join(prg_path, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe')
+	args = [vswhere, '-products', '*', '-legacy', '-format', 'json']
+	try:
+		txt = conf.cmd_and_log(args)
+	except Errors.WafError as e:
+		Logs.debug('msvc: vswhere.exe failed %s', e)
+		return
+
+	if sys.version_info[0] < 3:
+		txt = txt.decode(Utils.console_encoding())
+
+	arr = json.loads(txt)
+	arr.sort(key=lambda x: x['installationVersion'])
+	for entry in arr:
+		ver = entry['installationVersion']
+		ver = str('.'.join(ver.split('.')[:2]))
+		path = str(os.path.abspath(entry['installationPath']))
+		if os.path.exists(path) and ('msvc %s' % ver) not in versions:
+			conf.gather_msvc_targets(versions, ver, path)
 
 @conf
 def gather_msvc_versions(conf, versions):
@@ -453,12 +481,20 @@ def gather_msvc_versions(conf, versions):
 		try:
 			try:
 				msvc_version = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, reg + "\\Setup\\VC")
-			except WindowsError:
+			except OSError:
 				msvc_version = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, reg + "\\Setup\\Microsoft Visual C++")
 			path,type = Utils.winreg.QueryValueEx(msvc_version, 'ProductDir')
-			vc_paths.append((version, os.path.abspath(str(path))))
-		except WindowsError:
+		except OSError:
+			try:
+				msvc_version = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, "SOFTWARE\\Wow6432node\\Microsoft\\VisualStudio\\SxS\\VS7")
+				path,type = Utils.winreg.QueryValueEx(msvc_version, version)
+			except OSError:
+				continue
+			else:
+				vc_paths.append((version, os.path.abspath(str(path))))
 			continue
+		else:
+			vc_paths.append((version, os.path.abspath(str(path))))
 
 	wince_supported_platforms = gather_wince_supported_platforms()
 
@@ -492,50 +528,48 @@ def gather_icl_versions(conf, versions):
 	version_pattern = re.compile('^...?.?\....?.?')
 	try:
 		all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Compilers\\C++')
-	except WindowsError:
+	except OSError:
 		try:
 			all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Intel\\Compilers\\C++')
-		except WindowsError:
+		except OSError:
 			return
 	index = 0
 	while 1:
 		try:
 			version = Utils.winreg.EnumKey(all_versions, index)
-		except WindowsError:
+		except OSError:
 			break
-		index = index + 1
+		index += 1
 		if not version_pattern.match(version):
 			continue
-		targets = []
+		targets = {}
 		for target,arch in all_icl_platforms:
+			if target=='intel64':
+				targetDir='EM64T_NATIVE'
+			else:
+				targetDir=target
 			try:
-				if target=='intel64': targetDir='EM64T_NATIVE'
-				else: targetDir=target
 				Utils.winreg.OpenKey(all_versions,version+'\\'+targetDir)
 				icl_version=Utils.winreg.OpenKey(all_versions,version)
 				path,type=Utils.winreg.QueryValueEx(icl_version,'ProductDir')
+			except OSError:
+				pass
+			else:
 				batch_file=os.path.join(path,'bin','iclvars.bat')
 				if os.path.isfile(batch_file):
-					try:
-						targets.append((target,(arch,get_compiler_env(conf,'intel',version,target,batch_file))))
-					except conf.errors.ConfigurationError:
-						pass
-			except WindowsError:
-				pass
+					targets[target] = target_compiler(conf, 'intel', arch, version, target, batch_file)
 		for target,arch in all_icl_platforms:
 			try:
 				icl_version = Utils.winreg.OpenKey(all_versions, version+'\\'+target)
 				path,type = Utils.winreg.QueryValueEx(icl_version,'ProductDir')
+			except OSError:
+				continue
+			else:
 				batch_file=os.path.join(path,'bin','iclvars.bat')
 				if os.path.isfile(batch_file):
-					try:
-						targets.append((target, (arch, get_compiler_env(conf, 'intel', version, target, batch_file))))
-					except conf.errors.ConfigurationError:
-						pass
-			except WindowsError:
-				continue
+					targets[target] = target_compiler(conf, 'intel', arch, version, target, batch_file)
 		major = version[0:2]
-		versions.append(('intel ' + major, targets))
+		versions['intel ' + major] = targets
 
 @conf
 def gather_intel_composer_versions(conf, versions):
@@ -548,42 +582,44 @@ def gather_intel_composer_versions(conf, versions):
 	version_pattern = re.compile('^...?.?\...?.?.?')
 	try:
 		all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Suites')
-	except WindowsError:
+	except OSError:
 		try:
 			all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Intel\\Suites')
-		except WindowsError:
+		except OSError:
 			return
 	index = 0
 	while 1:
 		try:
 			version = Utils.winreg.EnumKey(all_versions, index)
-		except WindowsError:
+		except OSError:
 			break
-		index = index + 1
+		index += 1
 		if not version_pattern.match(version):
 			continue
-		targets = []
+		targets = {}
 		for target,arch in all_icl_platforms:
+			if target=='intel64':
+				targetDir='EM64T_NATIVE'
+			else:
+				targetDir=target
 			try:
-				if target=='intel64': targetDir='EM64T_NATIVE'
-				else: targetDir=target
 				try:
 					defaults = Utils.winreg.OpenKey(all_versions,version+'\\Defaults\\C++\\'+targetDir)
-				except WindowsError:
-					if targetDir=='EM64T_NATIVE':
+				except OSError:
+					if targetDir == 'EM64T_NATIVE':
 						defaults = Utils.winreg.OpenKey(all_versions,version+'\\Defaults\\C++\\EM64T')
 					else:
-						raise WindowsError
+						raise
 				uid,type = Utils.winreg.QueryValueEx(defaults, 'SubKey')
 				Utils.winreg.OpenKey(all_versions,version+'\\'+uid+'\\C++\\'+targetDir)
 				icl_version=Utils.winreg.OpenKey(all_versions,version+'\\'+uid+'\\C++')
 				path,type=Utils.winreg.QueryValueEx(icl_version,'ProductDir')
+			except OSError:
+				pass
+			else:
 				batch_file=os.path.join(path,'bin','iclvars.bat')
 				if os.path.isfile(batch_file):
-					try:
-						targets.append((target,(arch,get_compiler_env(conf,'intel',version,target,batch_file))))
-					except conf.errors.ConfigurationError:
-						pass
+					targets[target] = target_compiler(conf, 'intel', arch, version, target, batch_file)
 				# The intel compilervar_arch.bat is broken when used with Visual Studio Express 2012
 				# http://software.intel.com/en-us/forums/topic/328487
 				compilervars_warning_attr = '_compilervars_warning_key'
@@ -601,72 +637,41 @@ def gather_intel_composer_versions(conf, versions):
 								'(VSWinExpress.exe) but it does not seem to be installed at %r. '
 								'The intel command line set up will fail to configure unless the file %r'
 								'is patched. See: %s') % (vs_express_path, compilervars_arch, patch_url))
-			except WindowsError:
-				pass
 		major = version[0:2]
-		versions.append(('intel ' + major, targets))
+		versions['intel ' + major] = targets
 
 @conf
-def get_msvc_versions(conf, eval_and_save=True):
-	"""
-	:return: list of compilers installed
-	:rtype: list of string
-	"""
-	if conf.env['MSVC_INSTALLED_VERSIONS']:
-		return conf.env['MSVC_INSTALLED_VERSIONS']
-
-	# Gather all the compiler versions and targets. This phase can be lazy
-	# per lazy detection settings.
-	lst = []
-	conf.gather_icl_versions(lst)
-	conf.gather_intel_composer_versions(lst)
-	conf.gather_wsdk_versions(lst)
-	conf.gather_msvc_versions(lst)
-
-	# Override lazy detection by evaluating after the fact.
-	if eval_and_save:
-		def checked_target(t):
-			target,(arch,paths) = t
-			try:
-				paths.evaluate()
-			except conf.errors.ConfigurationError:
-				return None
-			else:
-				return t
-		lst = [(version, list(filter(checked_target, targets))) for version, targets in lst]
-		conf.env['MSVC_INSTALLED_VERSIONS'] = lst
-
-	return lst
+def detect_msvc(self):
+	return self.setup_msvc(self.get_msvc_versions())
 
 @conf
-def print_all_msvc_detected(conf):
+def get_msvc_versions(self):
 	"""
-	Print the contents of *conf.env.MSVC_INSTALLED_VERSIONS*
+	:return: platform to compiler configurations
+	:rtype: dict
 	"""
-	for version,targets in conf.env['MSVC_INSTALLED_VERSIONS']:
-		Logs.info(version)
-		for target,l in targets:
-			Logs.info("\t"+target)
-
-@conf
-def detect_msvc(conf, arch = False):
-	# Save installed versions only if lazy detection is disabled.
-	lazy_detect = getattr(Options.options, 'msvc_lazy_autodetect', False) or conf.env['MSVC_LAZY_AUTODETECT']
-	versions = get_msvc_versions(conf, not lazy_detect)
-	return setup_msvc(conf, versions, arch)
+	dct = Utils.ordered_iter_dict()
+	self.gather_icl_versions(dct)
+	self.gather_intel_composer_versions(dct)
+	self.gather_wsdk_versions(dct)
+	self.gather_msvc_versions(dct)
+	self.gather_vswhere_versions(dct)
+	Logs.debug('msvc: detected versions %r', list(dct.keys()))
+	return dct
 
 @conf
 def find_lt_names_msvc(self, libname, is_static=False):
 	"""
 	Win32/MSVC specific code to glean out information from libtool la files.
-	this function is not attached to the task_gen class
+	this function is not attached to the task_gen class. Returns a triplet:
+	(library absolute path, library name without extension, whether the library is static)
 	"""
 	lt_names=[
 		'lib%s.la' % libname,
 		'%s.la' % libname,
 	]
 
-	for path in self.env['LIBPATH']:
+	for path in self.env.LIBPATH:
 		for la in lt_names:
 			laf=os.path.join(path,la)
 			dll=None
@@ -708,14 +713,14 @@ def libname_msvc(self, libname, is_static=False):
 	(lt_path, lt_libname, lt_static) = self.find_lt_names_msvc(lib, is_static)
 
 	if lt_path != None and lt_libname != None:
-		if lt_static == True:
-			# file existance check has been made by find_lt_names
+		if lt_static:
+			# file existence check has been made by find_lt_names
 			return os.path.join(lt_path,lt_libname)
 
 	if lt_path != None:
-		_libpaths=[lt_path] + self.env['LIBPATH']
+		_libpaths = [lt_path] + self.env.LIBPATH
 	else:
-		_libpaths=self.env['LIBPATH']
+		_libpaths = self.env.LIBPATH
 
 	static_libs=[
 		'lib%ss.lib' % lib,
@@ -741,11 +746,11 @@ def libname_msvc(self, libname, is_static=False):
 	for path in _libpaths:
 		for libn in libnames:
 			if os.path.exists(os.path.join(path, libn)):
-				debug('msvc: lib found: %s' % os.path.join(path,libn))
+				Logs.debug('msvc: lib found: %s', os.path.join(path,libn))
 				return re.sub('\.lib$', '',libn)
 
 	#if no lib can be found, just return the libname as msvc expects it
-	self.fatal("The library %r could not be found" % libname)
+	self.fatal('The library %r could not be found' % libname)
 	return re.sub('\.lib$', '', libname)
 
 @conf
@@ -753,7 +758,7 @@ def check_lib_msvc(self, libname, is_static=False, uselib_store=None):
 	"""
 	Ideally we should be able to place the lib in the right env var, either STLIB or LIB,
 	but we don't distinguish static libs from shared libs.
-	This is ok since msvc doesn't have any special linker flag to select static libs (no env['STLIB_MARKER'])
+	This is ok since msvc doesn't have any special linker flag to select static libs (no env.STLIB_MARKER)
 	"""
 	libn = self.libname_msvc(libname, is_static)
 
@@ -790,27 +795,26 @@ def no_autodetect(conf):
 	configure(conf)
 
 @conf
-def autodetect(conf, arch = False):
+def autodetect(conf, arch=False):
 	v = conf.env
 	if v.NO_MSVC_DETECT:
 		return
+
+	compiler, version, path, includes, libdirs, cpu = conf.detect_msvc()
 	if arch:
-		compiler, version, path, includes, libdirs, arch = conf.detect_msvc(True)
-		v['DEST_CPU'] = arch
-	else:
-		compiler, version, path, includes, libdirs = conf.detect_msvc()
+		v.DEST_CPU = cpu
 
-	v['PATH'] = path
-	v['INCLUDES'] = includes
-	v['LIBPATH'] = libdirs
-	v['MSVC_COMPILER'] = compiler
+	v.PATH = path
+	v.INCLUDES = includes
+	v.LIBPATH = libdirs
+	v.MSVC_COMPILER = compiler
 	try:
-		v['MSVC_VERSION'] = float(version)
-	except Exception:
-		v['MSVC_VERSION'] = float(version[:-3])
+		v.MSVC_VERSION = float(version)
+	except ValueError:
+		v.MSVC_VERSION = float(version[:-3])
 
 def _get_prog_names(conf, compiler):
-	if compiler=='intel':
+	if compiler == 'intel':
 		compiler_name = 'ICL'
 		linker_name = 'XILINK'
 		lib_name = 'XILIB'
@@ -829,9 +833,9 @@ def find_msvc(conf):
 
 	# the autodetection is supposed to be performed before entering in this method
 	v = conf.env
-	path = v['PATH']
-	compiler = v['MSVC_COMPILER']
-	version = v['MSVC_VERSION']
+	path = v.PATH
+	compiler = v.MSVC_COMPILER
+	version = v.MSVC_VERSION
 
 	compiler_name, linker_name, lib_name = _get_prog_names(conf, compiler)
 	v.MSVC_MANIFEST = (compiler == 'msvc' and version >= 8) or (compiler == 'wsdk' and version >= 6) or (compiler == 'intel' and version >= 11)
@@ -841,48 +845,47 @@ def find_msvc(conf):
 
 	# before setting anything, check if the compiler is really msvc
 	env = dict(conf.environ)
-	if path: env.update(PATH = ';'.join(path))
+	if path:
+		env.update(PATH = ';'.join(path))
 	if not conf.cmd_and_log(cxx + ['/nologo', '/help'], env=env):
 		conf.fatal('the msvc compiler could not be identified')
 
 	# c/c++ compiler
-	v['CC'] = v['CXX'] = cxx
-	v['CC_NAME'] = v['CXX_NAME'] = 'msvc'
+	v.CC = v.CXX = cxx
+	v.CC_NAME = v.CXX_NAME = 'msvc'
 
 	# linker
-	if not v['LINK_CXX']:
-		link = conf.find_program(linker_name, path_list=path)
-		if link: v['LINK_CXX'] = link
-		else: conf.fatal('%s was not found (linker)' % linker_name)
-		v['LINK'] = link
+	if not v.LINK_CXX:
+		conf.find_program(linker_name, path_list=path, errmsg='%s was not found (linker)' % linker_name, var='LINK_CXX')
 
-	if not v['LINK_CC']:
-		v['LINK_CC'] = v['LINK_CXX']
+	if not v.LINK_CC:
+		v.LINK_CC = v.LINK_CXX
 
 	# staticlib linker
-	if not v['AR']:
+	if not v.AR:
 		stliblink = conf.find_program(lib_name, path_list=path, var='AR')
-		if not stliblink: return
-		v['ARFLAGS'] = ['/NOLOGO']
+		if not stliblink:
+			return
+		v.ARFLAGS = ['/nologo']
 
 	# manifest tool. Not required for VS 2003 and below. Must have for VS 2005 and later
 	if v.MSVC_MANIFEST:
 		conf.find_program('MT', path_list=path, var='MT')
-		v['MTFLAGS'] = ['/NOLOGO']
+		v.MTFLAGS = ['/nologo']
 
 	try:
 		conf.load('winres')
-	except Errors.WafError:
-		warn('Resource compiler not found. Compiling resource file is disabled')
+	except Errors.ConfigurationError:
+		Logs.warn('Resource compiler not found. Compiling resource file is disabled')
 
 @conf
 def visual_studio_add_flags(self):
 	"""visual studio flags found in the system environment"""
 	v = self.env
-	try: v.prepend_value('INCLUDES', [x for x in self.environ['INCLUDE'].split(';') if x]) # notice the 'S'
-	except Exception: pass
-	try: v.prepend_value('LIBPATH', [x for x in self.environ['LIB'].split(';') if x])
-	except Exception: pass
+	if self.environ.get('INCLUDE'):
+		v.prepend_value('INCLUDES', [x for x in self.environ['INCLUDE'].split(';') if x]) # notice the 'S'
+	if self.environ.get('LIB'):
+		v.prepend_value('LIBPATH', [x for x in self.environ['LIB'].split(';') if x])
 
 @conf
 def msvc_common_flags(conf):
@@ -891,62 +894,53 @@ def msvc_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['DEST_BINFMT'] = 'pe'
+	v.DEST_BINFMT = 'pe'
 	v.append_value('CFLAGS', ['/nologo'])
 	v.append_value('CXXFLAGS', ['/nologo'])
-	v['DEFINES_ST']     = '/D%s'
+	v.append_value('LINKFLAGS', ['/nologo'])
+	v.DEFINES_ST   = '/D%s'
 
-	v['CC_SRC_F']     = ''
-	v['CC_TGT_F']     = ['/c', '/Fo']
-	v['CXX_SRC_F']    = ''
-	v['CXX_TGT_F']    = ['/c', '/Fo']
+	v.CC_SRC_F     = ''
+	v.CC_TGT_F     = ['/c', '/Fo']
+	v.CXX_SRC_F    = ''
+	v.CXX_TGT_F    = ['/c', '/Fo']
 
 	if (v.MSVC_COMPILER == 'msvc' and v.MSVC_VERSION >= 8) or (v.MSVC_COMPILER == 'wsdk' and v.MSVC_VERSION >= 6):
-		v['CC_TGT_F']= ['/FC'] + v['CC_TGT_F']
-		v['CXX_TGT_F']= ['/FC'] + v['CXX_TGT_F']
-
-	v['CPPPATH_ST']   = '/I%s' # template for adding include paths
+		v.CC_TGT_F = ['/FC'] + v.CC_TGT_F
+		v.CXX_TGT_F = ['/FC'] + v.CXX_TGT_F
 
-	v['AR_TGT_F'] = v['CCLNK_TGT_F'] = v['CXXLNK_TGT_F'] = '/OUT:'
+	v.CPPPATH_ST = '/I%s' # template for adding include paths
 
-	# Subsystem specific flags
-	v['CFLAGS_CONSOLE']   = v['CXXFLAGS_CONSOLE']   = ['/SUBSYSTEM:CONSOLE']
-	v['CFLAGS_NATIVE']    = v['CXXFLAGS_NATIVE']    = ['/SUBSYSTEM:NATIVE']
-	v['CFLAGS_POSIX']     = v['CXXFLAGS_POSIX']     = ['/SUBSYSTEM:POSIX']
-	v['CFLAGS_WINDOWS']   = v['CXXFLAGS_WINDOWS']   = ['/SUBSYSTEM:WINDOWS']
-	v['CFLAGS_WINDOWSCE'] = v['CXXFLAGS_WINDOWSCE'] = ['/SUBSYSTEM:WINDOWSCE']
+	v.AR_TGT_F = v.CCLNK_TGT_F = v.CXXLNK_TGT_F = '/OUT:'
 
 	# CRT specific flags
-	v['CFLAGS_CRT_MULTITHREADED']     = v['CXXFLAGS_CRT_MULTITHREADED']     = ['/MT']
-	v['CFLAGS_CRT_MULTITHREADED_DLL'] = v['CXXFLAGS_CRT_MULTITHREADED_DLL'] = ['/MD']
+	v.CFLAGS_CRT_MULTITHREADED     = v.CXXFLAGS_CRT_MULTITHREADED     = ['/MT']
+	v.CFLAGS_CRT_MULTITHREADED_DLL = v.CXXFLAGS_CRT_MULTITHREADED_DLL = ['/MD']
 
-	v['CFLAGS_CRT_MULTITHREADED_DBG']     = v['CXXFLAGS_CRT_MULTITHREADED_DBG']     = ['/MTd']
-	v['CFLAGS_CRT_MULTITHREADED_DLL_DBG'] = v['CXXFLAGS_CRT_MULTITHREADED_DLL_DBG'] = ['/MDd']
+	v.CFLAGS_CRT_MULTITHREADED_DBG     = v.CXXFLAGS_CRT_MULTITHREADED_DBG     = ['/MTd']
+	v.CFLAGS_CRT_MULTITHREADED_DLL_DBG = v.CXXFLAGS_CRT_MULTITHREADED_DLL_DBG = ['/MDd']
 
-	# linker
-	v['LIB_ST']            = '%s.lib' # template for adding shared libs
-	v['LIBPATH_ST']        = '/LIBPATH:%s' # template for adding libpaths
-	v['STLIB_ST']          = '%s.lib'
-	v['STLIBPATH_ST']      = '/LIBPATH:%s'
+	v.LIB_ST            = '%s.lib'
+	v.LIBPATH_ST        = '/LIBPATH:%s'
+	v.STLIB_ST          = '%s.lib'
+	v.STLIBPATH_ST      = '/LIBPATH:%s'
 
-	v.append_value('LINKFLAGS', ['/NOLOGO'])
-	if v['MSVC_MANIFEST']:
+	if v.MSVC_MANIFEST:
 		v.append_value('LINKFLAGS', ['/MANIFEST'])
 
-	# shared library
-	v['CFLAGS_cshlib']     = []
-	v['CXXFLAGS_cxxshlib'] = []
-	v['LINKFLAGS_cshlib']  = v['LINKFLAGS_cxxshlib'] = ['/DLL']
-	v['cshlib_PATTERN']    = v['cxxshlib_PATTERN'] = '%s.dll'
-	v['implib_PATTERN']    = '%s.lib'
-	v['IMPLIB_ST']         = '/IMPLIB:%s'
+	v.CFLAGS_cshlib     = []
+	v.CXXFLAGS_cxxshlib = []
+	v.LINKFLAGS_cshlib  = v.LINKFLAGS_cxxshlib = ['/DLL']
+	v.cshlib_PATTERN    = v.cxxshlib_PATTERN = '%s.dll'
+	v.implib_PATTERN    = '%s.lib'
+	v.IMPLIB_ST         = '/IMPLIB:%s'
+
+	v.LINKFLAGS_cstlib  = []
+	v.cstlib_PATTERN    = v.cxxstlib_PATTERN = '%s.lib'
 
-	# static library
-	v['LINKFLAGS_cstlib']  = []
-	v['cstlib_PATTERN']    = v['cxxstlib_PATTERN'] = '%s.lib'
+	v.cprogram_PATTERN  = v.cxxprogram_PATTERN = '%s.exe'
 
-	# program
-	v['cprogram_PATTERN']  = v['cxxprogram_PATTERN']    = '%s.exe'
+	v.def_PATTERN       = '/def:%s'
 
 
 #######################################################################################################
@@ -980,12 +974,10 @@ def apply_flags_msvc(self):
 				self.link_task.outputs.append(pdbnode)
 
 				if getattr(self, 'install_task', None):
-					self.pdb_install_task = self.bld.install_files(self.install_task.dest, pdbnode, env=self.env)
-
+					self.pdb_install_task = self.add_install_files(
+						install_to=self.install_task.install_to, install_from=pdbnode)
 				break
 
-# split the manifest file processing from the link task, like for the rc processing
-
 @feature('cprogram', 'cshlib', 'cxxprogram', 'cxxshlib')
 @after_method('apply_link')
 def apply_manifest(self):
@@ -995,161 +987,16 @@ def apply_manifest(self):
 	the manifest file, the binaries are unusable.
 	See: http://msdn2.microsoft.com/en-us/library/ms235542(VS.80).aspx
 	"""
-
 	if self.env.CC_NAME == 'msvc' and self.env.MSVC_MANIFEST and getattr(self, 'link_task', None):
 		out_node = self.link_task.outputs[0]
 		man_node = out_node.parent.find_or_declare(out_node.name + '.manifest')
 		self.link_task.outputs.append(man_node)
-		self.link_task.do_manifest = True
-
-def exec_mf(self):
-	"""
-	Create the manifest file
-	"""
-	env = self.env
-	mtool = env['MT']
-	if not mtool:
-		return 0
-
-	self.do_manifest = False
-
-	outfile = self.outputs[0].abspath()
-
-	manifest = None
-	for out_node in self.outputs:
-		if out_node.name.endswith('.manifest'):
-			manifest = out_node.abspath()
-			break
-	if manifest is None:
-		# Should never get here.  If we do, it means the manifest file was
-		# never added to the outputs list, thus we don't have a manifest file
-		# to embed, so we just return.
-		return 0
-
-	# embedding mode. Different for EXE's and DLL's.
-	# see: http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx
-	mode = ''
-	if 'cprogram' in self.generator.features or 'cxxprogram' in self.generator.features:
-		mode = '1'
-	elif 'cshlib' in self.generator.features or 'cxxshlib' in self.generator.features:
-		mode = '2'
-
-	debug('msvc: embedding manifest in mode %r' % mode)
-
-	lst = [] + mtool
-	lst.extend(Utils.to_list(env['MTFLAGS']))
-	lst.extend(['-manifest', manifest])
-	lst.append('-outputresource:%s;%s' % (outfile, mode))
-
-	return self.exec_command(lst)
-
-def quote_response_command(self, flag):
-	if flag.find(' ') > -1:
-		for x in ('/LIBPATH:', '/IMPLIB:', '/OUT:', '/I'):
-			if flag.startswith(x):
-				flag = '%s"%s"' % (x, flag[len(x):])
-				break
-		else:
-			flag = '"%s"' % flag
-	return flag
-
-def exec_response_command(self, cmd, **kw):
-	# not public yet
-	try:
-		tmp = None
-		if sys.platform.startswith('win') and isinstance(cmd, list) and len(' '.join(cmd)) >= 8192:
-			program = cmd[0] #unquoted program name, otherwise exec_command will fail
-			cmd = [self.quote_response_command(x) for x in cmd]
-			(fd, tmp) = tempfile.mkstemp()
-			os.write(fd, '\r\n'.join(i.replace('\\', '\\\\') for i in cmd[1:]).encode())
-			os.close(fd)
-			cmd = [program, '@' + tmp]
-		# no return here, that's on purpose
-		ret = self.generator.bld.exec_command(cmd, **kw)
-	finally:
-		if tmp:
-			try:
-				os.remove(tmp)
-			except OSError:
-				pass # anti-virus and indexers can keep the files open -_-
-	return ret
-
-########## stupid evil command modification: concatenate the tokens /Fx, /doc, and /x: with the next token
-
-def exec_command_msvc(self, *k, **kw):
-	"""
-	Change the command-line execution for msvc programs.
-	Instead of quoting all the paths and keep using the shell, we can just join the options msvc is interested in
-	"""
-	if isinstance(k[0], list):
-		lst = []
-		carry = ''
-		for a in k[0]:
-			if a == '/Fo' or a == '/doc' or a[-1] == ':':
-				carry = a
-			else:
-				lst.append(carry + a)
-				carry = ''
-		k = [lst]
-
-	if self.env['PATH']:
-		env = dict(self.env.env or os.environ)
-		env.update(PATH = ';'.join(self.env['PATH']))
-		kw['env'] = env
-
-	bld = self.generator.bld
-	try:
-		if not kw.get('cwd', None):
-			kw['cwd'] = bld.cwd
-	except AttributeError:
-		bld.cwd = kw['cwd'] = bld.variant_dir
-
-	ret = self.exec_response_command(k[0], **kw)
-	if not ret and getattr(self, 'do_manifest', None):
-		ret = self.exec_mf()
-	return ret
-
-def wrap_class(class_name):
-	"""
-	Manifest file processing and @response file workaround for command-line length limits on Windows systems
-	The indicated task class is replaced by a subclass to prevent conflicts in case the class is wrapped more than once
-	"""
-	cls = Task.classes.get(class_name, None)
-
-	if not cls:
-		return None
-
-	derived_class = type(class_name, (cls,), {})
-
-	def exec_command(self, *k, **kw):
-		if self.env['CC_NAME'] == 'msvc':
-			return self.exec_command_msvc(*k, **kw)
-		else:
-			return super(derived_class, self).exec_command(*k, **kw)
-
-	# Chain-up monkeypatch needed since exec_command() is in base class API
-	derived_class.exec_command = exec_command
-
-	# No chain-up behavior needed since the following methods aren't in
-	# base class API
-	derived_class.exec_response_command = exec_response_command
-	derived_class.quote_response_command = quote_response_command
-	derived_class.exec_command_msvc = exec_command_msvc
-	derived_class.exec_mf = exec_mf
-
-	if hasattr(cls, 'hcode'):
-		derived_class.hcode = cls.hcode
-
-	return derived_class
-
-for k in 'c cxx cprogram cxxprogram cshlib cxxshlib cstlib cxxstlib'.split():
-	wrap_class(k)
+		self.env.DO_MANIFEST = True
 
 def make_winapp(self, family):
 	append = self.env.append_unique
 	append('DEFINES', 'WINAPI_FAMILY=%s' % family)
-	append('CXXFLAGS', '/ZW')
-	append('CXXFLAGS', '/TP')
+	append('CXXFLAGS', ['/ZW', '/TP'])
 	for lib_path in self.env.LIBPATH:
 		append('CXXFLAGS','/AI%s'%lib_path)
 
@@ -1161,9 +1008,7 @@ def make_winphone_app(self):
 	Insert configuration flags for windows phone applications (adds /ZW, /TP...)
 	"""
 	make_winapp(self, 'WINAPI_FAMILY_PHONE_APP')
-	conf.env.append_unique('LINKFLAGS', '/NODEFAULTLIB:ole32.lib')
-	conf.env.append_unique('LINKFLAGS', 'PhoneAppModelHost.lib')
-
+	self.env.append_unique('LINKFLAGS', ['/NODEFAULTLIB:ole32.lib', 'PhoneAppModelHost.lib'])
 
 @feature('winapp')
 @after_method('process_use')
diff --git a/waflib/Tools/suncc.py b/waflib/Tools/suncc.py
index 4909854e..33d34fc9 100644
--- a/waflib/Tools/suncc.py
+++ b/waflib/Tools/suncc.py
@@ -1,27 +1,26 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 
+from waflib import Errors
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 
 @conf
 def find_scc(conf):
 	"""
-	Detect the Sun C compiler
+	Detects the Sun C compiler
 	"""
 	v = conf.env
 	cc = conf.find_program('cc', var='CC')
-
 	try:
 		conf.cmd_and_log(cc + ['-flags'])
-	except Exception:
+	except Errors.WafError:
 		conf.fatal('%r is not a Sun compiler' % cc)
 	v.CC_NAME = 'sun'
 	conf.get_suncc_version(cc)
 
-
 @conf
 def scc_common_flags(conf):
 	"""
@@ -29,36 +28,34 @@ def scc_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CC_SRC_F']            = []
-	v['CC_TGT_F']            = ['-c', '-o']
+	v.CC_SRC_F            = []
+	v.CC_TGT_F            = ['-c', '-o', '']
 
-	# linker
-	if not v['LINK_CC']: v['LINK_CC'] = v['CC']
-	v['CCLNK_SRC_F']         = ''
-	v['CCLNK_TGT_F']         = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CC:
+		v.LINK_CC = v.CC
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
+	v.CCLNK_SRC_F         = ''
+	v.CCLNK_TGT_F         = ['-o', '']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = '-Wl,-h,%s'
-	v['SHLIB_MARKER']        = '-Bdynamic'
-	v['STLIB_MARKER']        = '-Bstatic'
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
 
-	# program
-	v['cprogram_PATTERN']    = '%s'
+	v.SONAME_ST           = '-Wl,-h,%s'
+	v.SHLIB_MARKER        = '-Bdynamic'
+	v.STLIB_MARKER        = '-Bstatic'
 
-	# shared library
-	v['CFLAGS_cshlib']       = ['-xcode=pic32', '-DPIC']
-	v['LINKFLAGS_cshlib']    = ['-G']
-	v['cshlib_PATTERN']      = 'lib%s.so'
+	v.cprogram_PATTERN    = '%s'
 
-	# static lib
-	v['LINKFLAGS_cstlib']    = ['-Bstatic']
-	v['cstlib_PATTERN']      = 'lib%s.a'
+	v.CFLAGS_cshlib       = ['-xcode=pic32', '-DPIC']
+	v.LINKFLAGS_cshlib    = ['-G']
+	v.cshlib_PATTERN      = 'lib%s.so'
+
+	v.LINKFLAGS_cstlib    = ['-Bstatic']
+	v.cstlib_PATTERN      = 'lib%s.a'
 
 def configure(conf):
 	conf.find_scc()
@@ -67,3 +64,4 @@ def configure(conf):
 	conf.cc_load_tools()
 	conf.cc_add_flags()
 	conf.link_add_flags()
+
diff --git a/waflib/Tools/suncxx.py b/waflib/Tools/suncxx.py
index 5a04b483..3b384f6f 100644
--- a/waflib/Tools/suncxx.py
+++ b/waflib/Tools/suncxx.py
@@ -1,21 +1,22 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 
+from waflib import Errors
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 
 @conf
 def find_sxx(conf):
 	"""
-	Detect the sun C++ compiler
+	Detects the sun C++ compiler
 	"""
 	v = conf.env
 	cc = conf.find_program(['CC', 'c++'], var='CXX')
 	try:
 		conf.cmd_and_log(cc + ['-flags'])
-	except Exception:
+	except Errors.WafError:
 		conf.fatal('%r is not a Sun compiler' % cc)
 	v.CXX_NAME = 'sun'
 	conf.get_suncc_version(cc)
@@ -27,36 +28,34 @@ def sxx_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CXX_SRC_F']           = []
-	v['CXX_TGT_F']           = ['-c', '-o']
+	v.CXX_SRC_F           = []
+	v.CXX_TGT_F           = ['-c', '-o', '']
 
-	# linker
-	if not v['LINK_CXX']: v['LINK_CXX'] = v['CXX']
-	v['CXXLNK_SRC_F']        = []
-	v['CXXLNK_TGT_F']        = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CXX:
+		v.LINK_CXX = v.CXX
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
+	v.CXXLNK_SRC_F        = []
+	v.CXXLNK_TGT_F        = ['-o', '']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = '-Wl,-h,%s'
-	v['SHLIB_MARKER']        = '-Bdynamic'
-	v['STLIB_MARKER']        = '-Bstatic'
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
 
-	# program
-	v['cxxprogram_PATTERN']  = '%s'
+	v.SONAME_ST           = '-Wl,-h,%s'
+	v.SHLIB_MARKER        = '-Bdynamic'
+	v.STLIB_MARKER        = '-Bstatic'
 
-	# shared library
-	v['CXXFLAGS_cxxshlib']   = ['-xcode=pic32', '-DPIC']
-	v['LINKFLAGS_cxxshlib']  = ['-G']
-	v['cxxshlib_PATTERN']    = 'lib%s.so'
+	v.cxxprogram_PATTERN  = '%s'
 
-	# static lib
-	v['LINKFLAGS_cxxstlib']  = ['-Bstatic']
-	v['cxxstlib_PATTERN']    = 'lib%s.a'
+	v.CXXFLAGS_cxxshlib   = ['-xcode=pic32', '-DPIC']
+	v.LINKFLAGS_cxxshlib  = ['-G']
+	v.cxxshlib_PATTERN    = 'lib%s.so'
+
+	v.LINKFLAGS_cxxstlib  = ['-Bstatic']
+	v.cxxstlib_PATTERN    = 'lib%s.a'
 
 def configure(conf):
 	conf.find_sxx()
diff --git a/waflib/Tools/waf_unit_test.py b/waflib/Tools/waf_unit_test.py
index 42135e3f..a71ed1c0 100644
--- a/waflib/Tools/waf_unit_test.py
+++ b/waflib/Tools/waf_unit_test.py
@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # encoding: utf-8
 # Carlos Rafael Giani, 2006
-# Thomas Nagy, 2010
+# Thomas Nagy, 2010-2018 (ita)
 
 """
-Unit testing system for C/C++/D providing test execution:
+Unit testing system for C/C++/D and interpreted languages providing test execution:
 
 * in parallel, by using ``waf -j``
 * partial (only the tests that have changed) or full (by using ``waf --alltests``)
@@ -31,31 +31,128 @@ the predefined callback::
 		bld(features='cxx cxxprogram test', source='main.c', target='app')
 		from waflib.Tools import waf_unit_test
 		bld.add_post_fun(waf_unit_test.summary)
+
+By passing --dump-test-scripts the build outputs corresponding python files
+(with extension _run.py) that are useful for debugging purposes.
 """
 
-import os
+import os, shlex, sys
 from waflib.TaskGen import feature, after_method, taskgen_method
 from waflib import Utils, Task, Logs, Options
+from waflib.Tools import ccroot
 testlock = Utils.threading.Lock()
 
+SCRIPT_TEMPLATE = """#! %(python)s
+import subprocess, sys
+cmd = %(cmd)r
+# if you want to debug with gdb:
+#cmd = ['gdb', '-args'] + cmd
+env = %(env)r
+status = subprocess.call(cmd, env=env, cwd=%(cwd)r, shell=isinstance(cmd, str))
+sys.exit(status)
+"""
+
+@taskgen_method
+def handle_ut_cwd(self, key):
+	"""
+	Task generator method, used internally to limit code duplication.
+	This method may disappear anytime.
+	"""
+	cwd = getattr(self, key, None)
+	if cwd:
+		if isinstance(cwd, str):
+			# we want a Node instance
+			if os.path.isabs(cwd):
+				self.ut_cwd = self.bld.root.make_node(cwd)
+			else:
+				self.ut_cwd = self.path.make_node(cwd)
+
+@feature('test_scripts')
+def make_interpreted_test(self):
+	"""Create interpreted unit tests."""
+	for x in ['test_scripts_source', 'test_scripts_template']:
+		if not hasattr(self, x):
+			Logs.warn('a test_scripts taskgen i missing %s' % x)
+			return
+
+	self.ut_run, lst = Task.compile_fun(self.test_scripts_template, shell=getattr(self, 'test_scripts_shell', False))
+
+	script_nodes = self.to_nodes(self.test_scripts_source)
+	for script_node in script_nodes:
+		tsk = self.create_task('utest', [script_node])
+		tsk.vars = lst + tsk.vars
+		tsk.env['SCRIPT'] = script_node.path_from(tsk.get_cwd())
+
+	self.handle_ut_cwd('test_scripts_cwd')
+
+	env = getattr(self, 'test_scripts_env', None)
+	if env:
+		self.ut_env = env
+	else:
+		self.ut_env = dict(os.environ)
+
+	paths = getattr(self, 'test_scripts_paths', {})
+	for (k,v) in paths.items():
+		p = self.ut_env.get(k, '').split(os.pathsep)
+		if isinstance(v, str):
+			v = v.split(os.pathsep)
+		self.ut_env[k] = os.pathsep.join(p + v)
+
 @feature('test')
-@after_method('apply_link')
+@after_method('apply_link', 'process_use')
 def make_test(self):
 	"""Create the unit test task. There can be only one unit test task by task generator."""
-	if getattr(self, 'link_task', None):
-		self.create_task('utest', self.link_task.outputs)
-
+	if not getattr(self, 'link_task', None):
+		return
+
+	tsk = self.create_task('utest', self.link_task.outputs)
+	if getattr(self, 'ut_str', None):
+		self.ut_run, lst = Task.compile_fun(self.ut_str, shell=getattr(self, 'ut_shell', False))
+		tsk.vars = lst + tsk.vars
+
+	self.handle_ut_cwd('ut_cwd')
+
+	if not hasattr(self, 'ut_paths'):
+		paths = []
+		for x in self.tmp_use_sorted:
+			try:
+				y = self.bld.get_tgen_by_name(x).link_task
+			except AttributeError:
+				pass
+			else:
+				if not isinstance(y, ccroot.stlink_task):
+					paths.append(y.outputs[0].parent.abspath())
+		self.ut_paths = os.pathsep.join(paths) + os.pathsep
+
+	if not hasattr(self, 'ut_env'):
+		self.ut_env = dct = dict(os.environ)
+		def add_path(var):
+			dct[var] = self.ut_paths + dct.get(var,'')
+		if Utils.is_win32:
+			add_path('PATH')
+		elif Utils.unversioned_sys_platform() == 'darwin':
+			add_path('DYLD_LIBRARY_PATH')
+			add_path('LD_LIBRARY_PATH')
+		else:
+			add_path('LD_LIBRARY_PATH')
+
+	if not hasattr(self, 'ut_cmd'):
+		self.ut_cmd = getattr(Options.options, 'testcmd', False)
 
 @taskgen_method
 def add_test_results(self, tup):
 	"""Override and return tup[1] to interrupt the build immediately if a test does not run"""
 	Logs.debug("ut: %r", tup)
-	self.utest_result = tup
+	try:
+		self.utest_results.append(tup)
+	except AttributeError:
+		self.utest_results = [tup]
 	try:
 		self.bld.utest_results.append(tup)
 	except AttributeError:
 		self.bld.utest_results = [tup]
 
+@Task.deep_inputs
 class utest(Task.Task):
 	"""
 	Execute a unit test
@@ -63,6 +160,7 @@ class utest(Task.Task):
 	color = 'PINK'
 	after = ['vnum', 'inst']
 	vars = []
+
 	def runnable_status(self):
 		"""
 		Always execute the task if `waf --alltests` was used or no
@@ -77,37 +175,17 @@ class utest(Task.Task):
 				return Task.RUN_ME
 		return ret
 
-	def add_path(self, dct, path, var):
-		dct[var] = os.pathsep.join(Utils.to_list(path) + [os.environ.get(var, '')])
-
 	def get_test_env(self):
 		"""
 		In general, tests may require any library built anywhere in the project.
 		Override this method if fewer paths are needed
 		"""
-		try:
-			fu = getattr(self.generator.bld, 'all_test_paths')
-		except AttributeError:
-			# this operation may be performed by at most #maxjobs
-			fu = os.environ.copy()
-
-			lst = []
-			for g in self.generator.bld.groups:
-				for tg in g:
-					if getattr(tg, 'link_task', None):
-						s = tg.link_task.outputs[0].parent.abspath()
-						if s not in lst:
-							lst.append(s)
-
-			if Utils.is_win32:
-				self.add_path(fu, lst, 'PATH')
-			elif Utils.unversioned_sys_platform() == 'darwin':
-				self.add_path(fu, lst, 'DYLD_LIBRARY_PATH')
-				self.add_path(fu, lst, 'LD_LIBRARY_PATH')
-			else:
-				self.add_path(fu, lst, 'LD_LIBRARY_PATH')
-			self.generator.bld.all_test_paths = fu
-		return fu
+		return self.generator.ut_env
+
+	def post_run(self):
+		super(utest, self).post_run()
+		if getattr(Options.options, 'clear_failed_tests', False) and self.waf_unit_test_results[1]:
+			self.generator.bld.task_sigs[self.uid()] = None
 
 	def run(self):
 		"""
@@ -116,33 +194,43 @@ class utest(Task.Task):
 
 		Override ``add_test_results`` to interrupt the build
 		"""
-
-		filename = self.inputs[0].abspath()
-		self.ut_exec = getattr(self.generator, 'ut_exec', [filename])
-		if getattr(self.generator, 'ut_fun', None):
-			self.generator.ut_fun(self)
-
-
-		cwd = getattr(self.generator, 'ut_cwd', '') or self.inputs[0].parent.abspath()
-
-		testcmd = getattr(self.generator, 'ut_cmd', False) or getattr(Options.options, 'testcmd', False)
-		if testcmd:
-			self.ut_exec = (testcmd % " ".join(self.ut_exec)).split(' ')
-
-		proc = Utils.subprocess.Popen(self.ut_exec, cwd=cwd, env=self.get_test_env(), stderr=Utils.subprocess.PIPE, stdout=Utils.subprocess.PIPE)
+		if hasattr(self.generator, 'ut_run'):
+			return self.generator.ut_run(self)
+
+		self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()])
+		ut_cmd = getattr(self.generator, 'ut_cmd', False)
+		if ut_cmd:
+			self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec))
+
+		return self.exec_command(self.ut_exec)
+
+	def exec_command(self, cmd, **kw):
+		Logs.debug('runner: %r', cmd)
+		if getattr(Options.options, 'dump_test_scripts', False):
+			script_code = SCRIPT_TEMPLATE % {
+				'python': sys.executable,
+				'env': self.get_test_env(),
+				'cwd': self.get_cwd().abspath(),
+				'cmd': cmd
+			}
+			script_file = self.inputs[0].abspath() + '_run.py'
+			Utils.writef(script_file, script_code)
+			os.chmod(script_file, Utils.O755)
+			if Logs.verbose > 1:
+				Logs.info('Test debug file written as %r' % script_file)
+
+		proc = Utils.subprocess.Popen(cmd, cwd=self.get_cwd().abspath(), env=self.get_test_env(),
+			stderr=Utils.subprocess.PIPE, stdout=Utils.subprocess.PIPE, shell=isinstance(cmd,str))
 		(stdout, stderr) = proc.communicate()
-
-		self.waf_unit_test_results = tup = (filename, proc.returncode, stdout, stderr)
+		self.waf_unit_test_results = tup = (self.inputs[0].abspath(), proc.returncode, stdout, stderr)
 		testlock.acquire()
 		try:
 			return self.generator.add_test_results(tup)
 		finally:
 			testlock.release()
 
-	def post_run(self):
-		super(utest, self).post_run()
-		if getattr(Options.options, 'clear_failed_tests', False) and self.waf_unit_test_results[1]:
-			self.generator.bld.task_sigs[self.uid()] = None
+	def get_cwd(self):
+		return getattr(self.generator, 'ut_cwd', self.inputs[0].parent)
 
 def summary(bld):
 	"""
@@ -160,15 +248,15 @@ def summary(bld):
 		total = len(lst)
 		tfail = len([x for x in lst if x[1]])
 
-		Logs.pprint('CYAN', '  tests that pass %d/%d' % (total-tfail, total))
+		Logs.pprint('GREEN', '  tests that pass %d/%d' % (total-tfail, total))
 		for (f, code, out, err) in lst:
 			if not code:
-				Logs.pprint('CYAN', '    %s' % f)
+				Logs.pprint('GREEN', '    %s' % f)
 
-		Logs.pprint('CYAN', '  tests that fail %d/%d' % (tfail, total))
+		Logs.pprint('GREEN' if tfail == 0 else 'RED', '  tests that fail %d/%d' % (tfail, total))
 		for (f, code, out, err) in lst:
 			if code:
-				Logs.pprint('CYAN', '    %s' % f)
+				Logs.pprint('RED', '    %s' % f)
 
 def set_exit_code(bld):
 	"""
@@ -199,9 +287,10 @@ def options(opt):
 	"""
 	opt.add_option('--notests', action='store_true', default=False, help='Exec no unit tests', dest='no_tests')
 	opt.add_option('--alltests', action='store_true', default=False, help='Exec all unit tests', dest='all_tests')
-	opt.add_option('--clear-failed', action='store_true', default=False, help='Force failed unit tests to run again next time', dest='clear_failed_tests')
-	opt.add_option('--testcmd', action='store', default=False,
-	 help = 'Run the unit tests using the test-cmd string'
-	 ' example "--test-cmd="valgrind --error-exitcode=1'
-	 ' %s" to run under valgrind', dest='testcmd')
+	opt.add_option('--clear-failed', action='store_true', default=False,
+		help='Force failed unit tests to run again next time', dest='clear_failed_tests')
+	opt.add_option('--testcmd', action='store', default=False, dest='testcmd',
+		help='Run the unit tests using the test-cmd string example "--testcmd="valgrind --error-exitcode=1 %s" to run under valgrind')
+	opt.add_option('--dump-test-scripts', action='store_true', default=False,
+		help='Create python scripts to help debug tests', dest='dump_test_scripts')
 
diff --git a/waflib/Tools/xlc.py b/waflib/Tools/xlc.py
index 3bd8d026..134dd415 100644
--- a/waflib/Tools/xlc.py
+++ b/waflib/Tools/xlc.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 # Yinon Ehrlich, 2009
 # Michael Kuhn, 2009
@@ -11,7 +11,7 @@ from waflib.Configure import conf
 @conf
 def find_xlc(conf):
 	"""
-	Detect the Aix C compiler
+	Detects the Aix C compiler
 	"""
 	cc = conf.find_program(['xlc_r', 'xlc'], var='CC')
 	conf.get_xlc_version(cc)
@@ -24,38 +24,36 @@ def xlc_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CC_SRC_F']            = []
-	v['CC_TGT_F']            = ['-c', '-o']
+	v.CC_SRC_F            = []
+	v.CC_TGT_F            = ['-c', '-o']
 
-	# linker
-	if not v['LINK_CC']: v['LINK_CC'] = v['CC']
-	v['CCLNK_SRC_F']         = []
-	v['CCLNK_TGT_F']         = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CC:
+		v.LINK_CC = v.CC
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
-	v['RPATH_ST']            = '-Wl,-rpath,%s'
+	v.CCLNK_SRC_F         = []
+	v.CCLNK_TGT_F         = ['-o']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = []
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
+	v.RPATH_ST            = '-Wl,-rpath,%s'
 
-	# program
-	v['LINKFLAGS_cprogram']  = ['-Wl,-brtl']
-	v['cprogram_PATTERN']    = '%s'
+	v.SONAME_ST           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
 
-	# shared library
-	v['CFLAGS_cshlib']       = ['-fPIC']
-	v['LINKFLAGS_cshlib']    = ['-G', '-Wl,-brtl,-bexpfull']
-	v['cshlib_PATTERN']      = 'lib%s.so'
+	v.LINKFLAGS_cprogram  = ['-Wl,-brtl']
+	v.cprogram_PATTERN    = '%s'
 
-	# static lib
-	v['LINKFLAGS_cstlib']    = []
-	v['cstlib_PATTERN']      = 'lib%s.a'
+	v.CFLAGS_cshlib       = ['-fPIC']
+	v.LINKFLAGS_cshlib    = ['-G', '-Wl,-brtl,-bexpfull']
+	v.cshlib_PATTERN      = 'lib%s.so'
+
+	v.LINKFLAGS_cstlib    = []
+	v.cstlib_PATTERN      = 'lib%s.a'
 
 def configure(conf):
 	conf.find_xlc()
diff --git a/waflib/Tools/xlcxx.py b/waflib/Tools/xlcxx.py
index 150aeaa4..76aa59bc 100644
--- a/waflib/Tools/xlcxx.py
+++ b/waflib/Tools/xlcxx.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2006-2010 (ita)
+# Thomas Nagy, 2006-2018 (ita)
 # Ralf Habacker, 2006 (rh)
 # Yinon Ehrlich, 2009
 # Michael Kuhn, 2009
@@ -11,7 +11,7 @@ from waflib.Configure import conf
 @conf
 def find_xlcxx(conf):
 	"""
-	Detect the Aix C++ compiler
+	Detects the Aix C++ compiler
 	"""
 	cxx = conf.find_program(['xlc++_r', 'xlc++'], var='CXX')
 	conf.get_xlc_version(cxx)
@@ -24,38 +24,36 @@ def xlcxx_common_flags(conf):
 	"""
 	v = conf.env
 
-	v['CXX_SRC_F']           = []
-	v['CXX_TGT_F']           = ['-c', '-o']
+	v.CXX_SRC_F           = []
+	v.CXX_TGT_F           = ['-c', '-o']
 
-	# linker
-	if not v['LINK_CXX']: v['LINK_CXX'] = v['CXX']
-	v['CXXLNK_SRC_F']        = []
-	v['CXXLNK_TGT_F']        = ['-o']
-	v['CPPPATH_ST']          = '-I%s'
-	v['DEFINES_ST']          = '-D%s'
+	if not v.LINK_CXX:
+		v.LINK_CXX = v.CXX
 
-	v['LIB_ST']              = '-l%s' # template for adding libs
-	v['LIBPATH_ST']          = '-L%s' # template for adding libpaths
-	v['STLIB_ST']            = '-l%s'
-	v['STLIBPATH_ST']        = '-L%s'
-	v['RPATH_ST']            = '-Wl,-rpath,%s'
+	v.CXXLNK_SRC_F        = []
+	v.CXXLNK_TGT_F        = ['-o']
+	v.CPPPATH_ST          = '-I%s'
+	v.DEFINES_ST          = '-D%s'
 
-	v['SONAME_ST']           = []
-	v['SHLIB_MARKER']        = []
-	v['STLIB_MARKER']        = []
+	v.LIB_ST              = '-l%s' # template for adding libs
+	v.LIBPATH_ST          = '-L%s' # template for adding libpaths
+	v.STLIB_ST            = '-l%s'
+	v.STLIBPATH_ST        = '-L%s'
+	v.RPATH_ST            = '-Wl,-rpath,%s'
 
-	# program
-	v['LINKFLAGS_cxxprogram']= ['-Wl,-brtl']
-	v['cxxprogram_PATTERN']  = '%s'
+	v.SONAME_ST           = []
+	v.SHLIB_MARKER        = []
+	v.STLIB_MARKER        = []
 
-	# shared library
-	v['CXXFLAGS_cxxshlib']   = ['-fPIC']
-	v['LINKFLAGS_cxxshlib']  = ['-G', '-Wl,-brtl,-bexpfull']
-	v['cxxshlib_PATTERN']    = 'lib%s.so'
+	v.LINKFLAGS_cxxprogram= ['-Wl,-brtl']
+	v.cxxprogram_PATTERN  = '%s'
 
-	# static lib
-	v['LINKFLAGS_cxxstlib']  = []
-	v['cxxstlib_PATTERN']    = 'lib%s.a'
+	v.CXXFLAGS_cxxshlib   = ['-fPIC']
+	v.LINKFLAGS_cxxshlib  = ['-G', '-Wl,-brtl,-bexpfull']
+	v.cxxshlib_PATTERN    = 'lib%s.so'
+
+	v.LINKFLAGS_cxxstlib  = []
+	v.cxxstlib_PATTERN    = 'lib%s.a'
 
 def configure(conf):
 	conf.find_xlcxx()
diff --git a/waflib/Utils.py b/waflib/Utils.py
index 3d981ac7..b4665c4d 100644
--- a/waflib/Utils.py
+++ b/waflib/Utils.py
@@ -1,16 +1,37 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
 
 """
 Utilities and platform-specific fixes
 
 The portability fixes try to provide a consistent behavior of the Waf API
-through Python versions 2.3 to 3.X and across different platforms (win32, linux, etc)
+through Python versions 2.5 to 3.X and across different platforms (win32, linux, etc)
 """
 
-import os, sys, errno, traceback, inspect, re, shutil, datetime, gc, platform
-import subprocess # <- leave this!
+from __future__ import with_statement
+
+import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time
+
+try:
+	import cPickle
+except ImportError:
+	import pickle as cPickle
+
+# leave this
+if os.name == 'posix' and sys.version_info[0] < 3:
+	try:
+		import subprocess32 as subprocess
+	except ImportError:
+		import subprocess
+else:
+	import subprocess
+
+try:
+	TimeoutExpired = subprocess.TimeoutExpired
+except AttributeError:
+	class TimeoutExpired(Exception):
+		pass
 
 from collections import deque, defaultdict
 
@@ -25,11 +46,6 @@ except ImportError:
 from waflib import Errors
 
 try:
-	from collections import UserDict
-except ImportError:
-	from UserDict import UserDict
-
-try:
 	from hashlib import md5
 except ImportError:
 	try:
@@ -47,8 +63,8 @@ except ImportError:
 
 	class threading(object):
 		"""
-			A fake threading class for platforms lacking the threading module.
-			Use ``waf -j1`` on those platforms
+		A fake threading class for platforms lacking the threading module.
+		Use ``waf -j1`` on those platforms
 		"""
 		pass
 	class Lock(object):
@@ -58,19 +74,9 @@ except ImportError:
 		def release(self):
 			pass
 	threading.Lock = threading.Thread = Lock
-else:
-	run_old = threading.Thread.run
-	def run(*args, **kwargs):
-		try:
-			run_old(*args, **kwargs)
-		except (KeyboardInterrupt, SystemExit):
-			raise
-		except Exception:
-			sys.excepthook(*sys.exc_info())
-	threading.Thread.run = run
 
-SIG_NIL = 'iluvcuteoverload'.encode()
-"""Arbitrary null value for a md5 hash. This value must be changed when the hash value is replaced (size)"""
+SIG_NIL = 'SIG_NIL_SIG_NIL_'.encode()
+"""Arbitrary null value for hashes. Modify this value according to the hash function in use"""
 
 O644 = 420
 """Constant representing the permissions for regular files (0644 raises a syntax error on python 3)"""
@@ -84,42 +90,126 @@ rot_chr = ['\\', '|', '/', '-']
 rot_idx = 0
 "Index of the current throbber character (progress bar)"
 
-try:
-	from collections import OrderedDict as ordered_iter_dict
-except ImportError:
-	class ordered_iter_dict(dict):
-		def __init__(self, *k, **kw):
-			self.lst = []
-			dict.__init__(self, *k, **kw)
-		def clear(self):
-			dict.clear(self)
-			self.lst = []
-		def __setitem__(self, key, value):
-			dict.__setitem__(self, key, value)
-			try:
-				self.lst.remove(key)
-			except ValueError:
-				pass
-			self.lst.append(key)
-		def __delitem__(self, key):
-			dict.__delitem__(self, key)
-			try:
-				self.lst.remove(key)
-			except ValueError:
-				pass
-		def __iter__(self):
-			for x in self.lst:
-				yield x
-		def keys(self):
-			return self.lst
+class ordered_iter_dict(dict):
+	"""Ordered dictionary that provides iteration from the most recently inserted keys first"""
+	def __init__(self, *k, **kw):
+		self.lst = deque()
+		dict.__init__(self, *k, **kw)
+	def clear(self):
+		dict.clear(self)
+		self.lst = deque()
+	def __setitem__(self, key, value):
+		if key in dict.keys(self):
+			self.lst.remove(key)
+		dict.__setitem__(self, key, value)
+		self.lst.append(key)
+	def __delitem__(self, key):
+		dict.__delitem__(self, key)
+		try:
+			self.lst.remove(key)
+		except ValueError:
+			pass
+	def __iter__(self):
+		return reversed(self.lst)
+	def keys(self):
+		return reversed(self.lst)
 
-is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2
+class lru_node(object):
+	"""
+	Used by :py:class:`waflib.Utils.lru_cache`
+	"""
+	__slots__ = ('next', 'prev', 'key', 'val')
+	def __init__(self):
+		self.next = self
+		self.prev = self
+		self.key = None
+		self.val = None
 
-def readf(fname, m='r', encoding='ISO8859-1'):
+class lru_cache(object):
+	"""
+	A simple least-recently used cache with lazy allocation
 	"""
-	Read an entire file into a string, use this function instead of os.open() whenever possible.
+	__slots__ = ('maxlen', 'table', 'head')
+	def __init__(self, maxlen=100):
+		self.maxlen = maxlen
+		"""
+		Maximum amount of elements in the cache
+		"""
+		self.table = {}
+		"""
+		Mapping key-value
+		"""
+		self.head = lru_node()
+		self.head.next = self.head
+		self.head.prev = self.head
+
+	def __getitem__(self, key):
+		node = self.table[key]
+		# assert(key==node.key)
+		if node is self.head:
+			return node.val
+
+		# detach the node found
+		node.prev.next = node.next
+		node.next.prev = node.prev
+
+		# replace the head
+		node.next = self.head.next
+		node.prev = self.head
+		self.head = node.next.prev = node.prev.next = node
+
+		return node.val
+
+	def __setitem__(self, key, val):
+		if key in self.table:
+			# update the value for an existing key
+			node = self.table[key]
+			node.val = val
+			self.__getitem__(key)
+		else:
+			if len(self.table) < self.maxlen:
+				# the very first item is unused until the maximum is reached
+				node = lru_node()
+				node.prev = self.head
+				node.next = self.head.next
+				node.prev.next = node.next.prev = node
+			else:
+				node = self.head = self.head.next
+				try:
+					# that's another key
+					del self.table[node.key]
+				except KeyError:
+					pass
+
+			node.key = key
+			node.val = val
+			self.table[key] = node
+
+class lazy_generator(object):
+	def __init__(self, fun, params):
+		self.fun = fun
+		self.params = params
+
+	def __iter__(self):
+		return self
+
+	def __next__(self):
+		try:
+			it = self.it
+		except AttributeError:
+			it = self.it = self.fun(*self.params)
+		return next(it)
+
+	next = __next__
+
+is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2
+"""
+Whether this system is a Windows series
+"""
 
-	In practice the wrapper node.read(..) should be preferred to this function::
+def readf(fname, m='r', encoding='latin-1'):
+	"""
+	Reads an entire file into a string. See also :py:meth:`waflib.Node.Node.readf`::
 
 		def build(ctx):
 			from waflib import Utils
@@ -138,28 +228,21 @@ def readf(fname, m='r', encoding='ISO8859-1'):
 
 	if sys.hexversion > 0x3000000 and not 'b' in m:
 		m += 'b'
-		f = open(fname, m)
-		try:
+		with open(fname, m) as f:
 			txt = f.read()
-		finally:
-			f.close()
 		if encoding:
 			txt = txt.decode(encoding)
 		else:
 			txt = txt.decode()
 	else:
-		f = open(fname, m)
-		try:
+		with open(fname, m) as f:
 			txt = f.read()
-		finally:
-			f.close()
 	return txt
 
-def writef(fname, data, m='w', encoding='ISO8859-1'):
+def writef(fname, data, m='w', encoding='latin-1'):
 	"""
-	Write an entire file from a string, use this function instead of os.open() whenever possible.
-
-	In practice the wrapper node.write(..) should be preferred to this function::
+	Writes an entire file from a string.
+	See also :py:meth:`waflib.Node.Node.writef`::
 
 		def build(ctx):
 			from waflib import Utils
@@ -178,44 +261,27 @@ def writef(fname, data, m='w', encoding='ISO8859-1'):
 	if sys.hexversion > 0x3000000 and not 'b' in m:
 		data = data.encode(encoding)
 		m += 'b'
-	f = open(fname, m)
-	try:
+	with open(fname, m) as f:
 		f.write(data)
-	finally:
-		f.close()
 
 def h_file(fname):
 	"""
-	Compute a hash value for a file by using md5. This method may be replaced by
-	a faster version if necessary. The following uses the file size and the timestamp value::
-
-		import stat
-		from waflib import Utils
-		def h_file(fname):
-			st = os.stat(fname)
-			if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
-			m = Utils.md5()
-			m.update(str(st.st_mtime))
-			m.update(str(st.st_size))
-			m.update(fname)
-			return m.digest()
-		Utils.h_file = h_file
+	Computes a hash value for a file by using md5. Use the md5_tstamp
+	extension to get faster build hashes if necessary.
 
 	:type fname: string
 	:param fname: path to the file to hash
 	:return: hash of the file contents
+	:rtype: string or bytes
 	"""
-	f = open(fname, 'rb')
 	m = md5()
-	try:
+	with open(fname, 'rb') as f:
 		while fname:
 			fname = f.read(200000)
 			m.update(fname)
-	finally:
-		f.close()
 	return m.digest()
 
-def readf_win32(f, m='r', encoding='ISO8859-1'):
+def readf_win32(f, m='r', encoding='latin-1'):
 	flags = os.O_NOINHERIT | os.O_RDONLY
 	if 'b' in m:
 		flags |= os.O_BINARY
@@ -228,24 +294,18 @@ def readf_win32(f, m='r', encoding='ISO8859-1'):
 
 	if sys.hexversion > 0x3000000 and not 'b' in m:
 		m += 'b'
-		f = os.fdopen(fd, m)
-		try:
+		with os.fdopen(fd, m) as f:
 			txt = f.read()
-		finally:
-			f.close()
 		if encoding:
 			txt = txt.decode(encoding)
 		else:
 			txt = txt.decode()
 	else:
-		f = os.fdopen(fd, m)
-		try:
+		with os.fdopen(fd, m) as f:
 			txt = f.read()
-		finally:
-			f.close()
 	return txt
 
-def writef_win32(f, data, m='w', encoding='ISO8859-1'):
+def writef_win32(f, data, m='w', encoding='latin-1'):
 	if sys.hexversion > 0x3000000 and not 'b' in m:
 		data = data.encode(encoding)
 		m += 'b'
@@ -257,26 +317,20 @@ def writef_win32(f, data, m='w', encoding='ISO8859-1'):
 	try:
 		fd = os.open(f, flags)
 	except OSError:
-		raise IOError('Cannot write to %r' % f)
-	f = os.fdopen(fd, m)
-	try:
+		raise OSError('Cannot write to %r' % f)
+	with os.fdopen(fd, m) as f:
 		f.write(data)
-	finally:
-		f.close()
 
 def h_file_win32(fname):
 	try:
 		fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
 	except OSError:
-		raise IOError('Cannot read from %r' % fname)
-	f = os.fdopen(fd, 'rb')
+		raise OSError('Cannot read from %r' % fname)
 	m = md5()
-	try:
+	with os.fdopen(fd, 'rb') as f:
 		while fname:
 			fname = f.read(200000)
 			m.update(fname)
-	finally:
-		f.close()
 	return m.digest()
 
 # always save these
@@ -311,8 +365,8 @@ Return the hexadecimal representation of a string
 
 def listdir_win32(s):
 	"""
-	List the contents of a folder in a portable manner.
-	On Win32, return the list of drive letters: ['C:', 'X:', 'Z:']
+	Lists the contents of a folder in a portable manner.
+	On Win32, returns the list of drive letters: ['C:', 'X:', 'Z:'] when an empty string is given.
 
 	:type s: string
 	:param s: a string, which can be empty on Windows
@@ -322,7 +376,7 @@ def listdir_win32(s):
 			import ctypes
 		except ImportError:
 			# there is nothing much we can do
-			return [x + ':\\' for x in list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')]
+			return [x + ':\\' for x in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']
 		else:
 			dlen = 4 # length of "?:\\x00"
 			maxdrives = 26
@@ -345,7 +399,7 @@ if is_win32:
 
 def num2ver(ver):
 	"""
-	Convert a string, tuple or version number into an integer. The number is supposed to have at most 4 digits::
+	Converts a string, tuple or version number into an integer. The number is supposed to have at most 4 digits::
 
 		from waflib.Utils import num2ver
 		num2ver('1.3.2') == num2ver((1,3,2)) == num2ver((1,3,2,0))
@@ -363,33 +417,37 @@ def num2ver(ver):
 		return ret
 	return ver
 
-def ex_stack():
-	"""
-	Extract the stack to display exceptions
-
-	:return: a string represening the last exception
-	"""
-	exc_type, exc_value, tb = sys.exc_info()
-	exc_lines = traceback.format_exception(exc_type, exc_value, tb)
-	return ''.join(exc_lines)
-
-def to_list(sth):
+def to_list(val):
 	"""
-	Convert a string argument to a list by splitting on spaces, and pass
-	through a list argument unchanged::
+	Converts a string argument to a list by splitting it by spaces.
+	Returns the object if not a string::
 
 		from waflib.Utils import to_list
-		lst = to_list("a b c d")
+		lst = to_list('a b c d')
 
-	:param sth: List or a string of items separated by spaces
+	:param val: list of string or space-separated string
 	:rtype: list
 	:return: Argument converted to list
-
 	"""
-	if isinstance(sth, str):
-		return sth.split()
+	if isinstance(val, str):
+		return val.split()
 	else:
-		return sth
+		return val
+
+def console_encoding():
+	try:
+		import ctypes
+	except ImportError:
+		pass
+	else:
+		try:
+			codepage = ctypes.windll.kernel32.GetConsoleCP()
+		except AttributeError:
+			pass
+		else:
+			if codepage:
+				return 'cp%d' % codepage
+	return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1')
 
 def split_path_unix(path):
 	return path.split('/')
@@ -401,22 +459,24 @@ def split_path_cygwin(path):
 		return ret
 	return path.split('/')
 
-re_sp = re.compile('[/\\\\]')
+re_sp = re.compile('[/\\\\]+')
 def split_path_win32(path):
 	if path.startswith('\\\\'):
-		ret = re.split(re_sp, path)[2:]
-		ret[0] = '\\' + ret[0]
+		ret = re_sp.split(path)[1:]
+		ret[0] = '\\\\' + ret[0]
+		if ret[0] == '\\\\?':
+			return ret[1:]
 		return ret
-	return re.split(re_sp, path)
+	return re_sp.split(path)
 
 msysroot = None
 def split_path_msys(path):
-	if (path.startswith('/') or path.startswith('\\')) and not path.startswith('//') and not path.startswith('\\\\'):
+	if path.startswith(('/', '\\')) and not path.startswith(('//', '\\\\')):
 		# msys paths can be in the form /usr/bin
 		global msysroot
 		if not msysroot:
 			# msys has python 2.7 or 3, so we can use this
-			msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'iso8859-1')
+			msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'latin-1')
 			msysroot = msysroot.strip()
 		path = os.path.normpath(msysroot + os.sep + path)
 	return split_path_win32(path)
@@ -424,7 +484,7 @@ def split_path_msys(path):
 if sys.platform == 'cygwin':
 	split_path = split_path_cygwin
 elif is_win32:
-	if os.environ.get('MSYSTEM', None):
+	if os.environ.get('MSYSTEM'):
 		split_path = split_path_msys
 	else:
 		split_path = split_path_win32
@@ -432,19 +492,20 @@ else:
 	split_path = split_path_unix
 
 split_path.__doc__ = """
-Split a path by / or \\. This function is not like os.path.split
+Splits a path by / or \\; do not confuse this function with with ``os.path.split``
 
 :type  path: string
 :param path: path to split
-:return:     list of strings
+:return:     list of string
 """
 
 def check_dir(path):
 	"""
-	Ensure that a directory exists (similar to ``mkdir -p``).
+	Ensures that a directory exists (similar to ``mkdir -p``).
 
 	:type  path: string
 	:param path: Path to directory
+	:raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added.
 	"""
 	if not os.path.isdir(path):
 		try:
@@ -455,11 +516,14 @@ def check_dir(path):
 
 def check_exe(name, env=None):
 	"""
-	Ensure that a program exists
+	Ensures that a program exists
 
 	:type name: string
-	:param name: name or path to program
+	:param name: path to the program
+	:param env: configuration object
+	:type env: :py:class:`waflib.ConfigSet.ConfigSet`
 	:return: path of the program or None
+	:raises: :py:class:`waflib.Errors.WafError` if the folder cannot be added.
 	"""
 	if not name:
 		raise ValueError('Cannot execute an empty string!')
@@ -471,7 +535,7 @@ def check_exe(name, env=None):
 		return os.path.abspath(name)
 	else:
 		env = env or os.environ
-		for path in env["PATH"].split(os.pathsep):
+		for path in env['PATH'].split(os.pathsep):
 			path = path.strip('"')
 			exe_file = os.path.join(path, name)
 			if is_exe(exe_file):
@@ -480,7 +544,7 @@ def check_exe(name, env=None):
 
 def def_attrs(cls, **kw):
 	"""
-	Set default attributes on a class instance
+	Sets default attributes on a class instance
 
 	:type cls: class
 	:param cls: the class to update the given attributes in.
@@ -493,7 +557,7 @@ def def_attrs(cls, **kw):
 
 def quote_define_name(s):
 	"""
-	Convert a string to an identifier suitable for C defines.
+	Converts a string into an identifier suitable for C defines.
 
 	:type  s: string
 	:param s: String to convert
@@ -505,18 +569,32 @@ def quote_define_name(s):
 	fu = fu.upper()
 	return fu
 
+re_sh = re.compile('\\s|\'|"')
+"""
+Regexp used for shell_escape below
+"""
+
+def shell_escape(cmd):
+	"""
+	Escapes a command:
+	['ls', '-l', 'arg space'] -> ls -l 'arg space'
+	"""
+	if isinstance(cmd, str):
+		return cmd
+	return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd)
+
 def h_list(lst):
 	"""
-	Hash lists. For tuples, using hash(tup) is much more efficient,
-	except on python >= 3.3 where hash randomization assumes everybody is running a web application.
+	Hashes lists of ordered data.
+
+	Using hash(tup) for tuples would be much more efficient,
+	but Python now enforces hash randomization
 
 	:param lst: list to hash
 	:type lst: list of strings
 	:return: hash of the list
 	"""
-	m = md5()
-	m.update(str(lst).encode())
-	return m.digest()
+	return md5(repr(lst).encode()).digest()
 
 def h_fun(fun):
 	"""
@@ -525,14 +603,27 @@ def h_fun(fun):
 	:param fun: function to hash
 	:type  fun: function
 	:return: hash of the function
+	:rtype: string or bytes
 	"""
 	try:
 		return fun.code
 	except AttributeError:
+		if isinstance(fun, functools.partial):
+			code = list(fun.args)
+			# The method items() provides a sequence of tuples where the first element
+			# represents an optional argument of the partial function application
+			#
+			# The sorting result outcome will be consistent because:
+			# 1. tuples are compared in order of their elements
+			# 2. optional argument namess are unique
+			code.extend(sorted(fun.keywords.items()))
+			code.append(h_fun(fun.func))
+			fun.code = h_list(code)
+			return fun.code
 		try:
 			h = inspect.getsource(fun)
-		except IOError:
-			h = "nocode"
+		except EnvironmentError:
+			h = 'nocode'
 		try:
 			fun.code = h
 		except AttributeError:
@@ -541,8 +632,11 @@ def h_fun(fun):
 
 def h_cmd(ins):
 	"""
-	Task command hashes are calculated by calling this function. The inputs can be
-	strings, functions, tuples/lists containing strings/functions
+	Hashes objects recursively
+
+	:param ins: input object
+	:type ins: string or list or tuple or function
+	:rtype: string or bytes
 	"""
 	# this function is not meant to be particularly fast
 	if isinstance(ins, str):
@@ -555,13 +649,13 @@ def h_cmd(ins):
 		# or just a python function
 		ret = str(h_fun(ins))
 	if sys.hexversion > 0x3000000:
-		ret = ret.encode('iso8859-1', 'xmlcharrefreplace')
+		ret = ret.encode('latin-1', 'xmlcharrefreplace')
 	return ret
 
 reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}")
 def subst_vars(expr, params):
 	"""
-	Replace ${VAR} with the value of VAR taken from a dict or a config set::
+	Replaces ${VAR} with the value of VAR taken from a dict or a config set::
 
 		from waflib import Utils
 		s = Utils.subst_vars('${PREFIX}/bin', env)
@@ -586,7 +680,8 @@ def subst_vars(expr, params):
 
 def destos_to_binfmt(key):
 	"""
-	Return the binary format based on the unversioned platform name.
+	Returns the binary format based on the unversioned platform name,
+	and defaults to ``elf`` if nothing is found.
 
 	:param key: platform name
 	:type  key: string
@@ -600,7 +695,7 @@ def destos_to_binfmt(key):
 
 def unversioned_sys_platform():
 	"""
-	Return the unversioned platform name.
+	Returns the unversioned platform name.
 	Some Python platform names contain versions, that depend on
 	the build environment, e.g. linux2, freebsd6, etc.
 	This returns the name without the version number. Exceptions are
@@ -639,7 +734,7 @@ def unversioned_sys_platform():
 
 def nada(*k, **kw):
 	"""
-	A function that does nothing
+	Does nothing
 
 	:return: None
 	"""
@@ -648,7 +743,7 @@ def nada(*k, **kw):
 class Timer(object):
 	"""
 	Simple object for timing the execution of commands.
-	Its string representation is the current time::
+	Its string representation is the duration::
 
 		from waflib.Utils import Timer
 		timer = Timer()
@@ -656,10 +751,12 @@ class Timer(object):
 		s = str(timer)
 	"""
 	def __init__(self):
-		self.start_time = datetime.datetime.utcnow()
+		self.start_time = self.now()
 
 	def __str__(self):
-		delta = datetime.datetime.utcnow() - self.start_time
+		delta = self.now() - self.start_time
+		if not isinstance(delta, datetime.timedelta):
+			delta = datetime.timedelta(seconds=delta)
 		days = delta.days
 		hours, rem = divmod(delta.seconds, 3600)
 		minutes, seconds = divmod(rem, 60)
@@ -673,28 +770,16 @@ class Timer(object):
 			result += '%dm' % minutes
 		return '%s%.3fs' % (result, seconds)
 
-if is_win32:
-	old = shutil.copy2
-	def copy2(src, dst):
-		"""
-		shutil.copy2 does not copy the file attributes on windows, so we
-		hack into the shutil module to fix the problem
-		"""
-		old(src, dst)
-		shutil.copystat(src, dst)
-	setattr(shutil, 'copy2', copy2)
+	def now(self):
+		return datetime.datetime.utcnow()
 
-if os.name == 'java':
-	# Jython cannot disable the gc but they can enable it ... wtf?
-	try:
-		gc.disable()
-		gc.enable()
-	except NotImplementedError:
-		gc.disable = gc.enable
+	if hasattr(time, 'perf_counter'):
+		def now(self):
+			return time.perf_counter()
 
 def read_la_file(path):
 	"""
-	Read property files, used by msvc.py
+	Reads property files, used by msvc.py
 
 	:param path: file to read
 	:type path: string
@@ -709,25 +794,6 @@ def read_la_file(path):
 			pass
 	return dc
 
-def nogc(fun):
-	"""
-	Decorator: let a function disable the garbage collector during its execution.
-	It is used in the build context when storing/loading the build cache file (pickle)
-
-	:param fun: function to execute
-	:type fun: function
-	:return: the return value of the function executed
-	"""
-	def f(*k, **kw):
-		try:
-			gc.disable()
-			ret = fun(*k, **kw)
-		finally:
-			gc.enable()
-		return ret
-	f.__doc__ = fun.__doc__
-	return f
-
 def run_once(fun):
 	"""
 	Decorator: let a function cache its results, use like this::
@@ -736,16 +802,18 @@ def run_once(fun):
 		def foo(k):
 			return 345*2343
 
+	.. note:: in practice this can cause memory leaks, prefer a :py:class:`waflib.Utils.lru_cache`
+
 	:param fun: function to execute
 	:type fun: function
 	:return: the return value of the function executed
 	"""
 	cache = {}
-	def wrap(k):
+	def wrap(*k):
 		try:
 			return cache[k]
 		except KeyError:
-			ret = fun(k)
+			ret = fun(*k)
 			cache[k] = ret
 			return ret
 	wrap.__cache__ = cache
@@ -753,17 +821,29 @@ def run_once(fun):
 	return wrap
 
 def get_registry_app_path(key, filename):
+	"""
+	Returns the value of a registry key for an executable
+
+	:type key: string
+	:type filename: list of string
+	"""
 	if not winreg:
 		return None
 	try:
 		result = winreg.QueryValue(key, "Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\%s.exe" % filename[0])
-	except WindowsError:
+	except OSError:
 		pass
 	else:
 		if os.path.isfile(result):
 			return result
 
 def lib64():
+	"""
+	Guess the default ``/usr/lib`` extension for 64-bit applications
+
+	:return: '64' or ''
+	:rtype: string
+	"""
 	# default settings for /usr/lib
 	if os.sep == '/':
 		if platform.architecture()[0] == '64bit':
@@ -775,3 +855,167 @@ def sane_path(p):
 	# private function for the time being!
 	return os.path.abspath(os.path.expanduser(p))
 
+process_pool = []
+"""
+List of processes started to execute sub-process commands
+"""
+
+def get_process():
+	"""
+	Returns a process object that can execute commands as sub-processes
+
+	:rtype: subprocess.Popen
+	"""
+	try:
+		return process_pool.pop()
+	except IndexError:
+		filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py'
+		cmd = [sys.executable, '-c', readf(filepath)]
+		return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0)
+
+def run_prefork_process(cmd, kwargs, cargs):
+	"""
+	Delegates process execution to a pre-forked process instance.
+	"""
+	if not 'env' in kwargs:
+		kwargs['env'] = dict(os.environ)
+	try:
+		obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs]))
+	except (TypeError, AttributeError):
+		return run_regular_process(cmd, kwargs, cargs)
+
+	proc = get_process()
+	if not proc:
+		return run_regular_process(cmd, kwargs, cargs)
+
+	proc.stdin.write(obj)
+	proc.stdin.write('\n'.encode())
+	proc.stdin.flush()
+	obj = proc.stdout.readline()
+	if not obj:
+		raise OSError('Preforked sub-process %r died' % proc.pid)
+
+	process_pool.append(proc)
+	lst = cPickle.loads(base64.b64decode(obj))
+	# Jython wrapper failures (bash/execvp)
+	assert len(lst) == 5
+	ret, out, err, ex, trace = lst
+	if ex:
+		if ex == 'OSError':
+			raise OSError(trace)
+		elif ex == 'ValueError':
+			raise ValueError(trace)
+		elif ex == 'TimeoutExpired':
+			exc = TimeoutExpired(cmd, timeout=cargs['timeout'], output=out)
+			exc.stderr = err
+			raise exc
+		else:
+			raise Exception(trace)
+	return ret, out, err
+
+def lchown(path, user=-1, group=-1):
+	"""
+	Change the owner/group of a path, raises an OSError if the
+	ownership change fails.
+
+	:param user: user to change
+	:type user: int or str
+	:param group: group to change
+	:type group: int or str
+	"""
+	if isinstance(user, str):
+		import pwd
+		entry = pwd.getpwnam(user)
+		if not entry:
+			raise OSError('Unknown user %r' % user)
+		user = entry[2]
+	if isinstance(group, str):
+		import grp
+		entry = grp.getgrnam(group)
+		if not entry:
+			raise OSError('Unknown group %r' % group)
+		group = entry[2]
+	return os.lchown(path, user, group)
+
+def run_regular_process(cmd, kwargs, cargs={}):
+	"""
+	Executes a subprocess command by using subprocess.Popen
+	"""
+	proc = subprocess.Popen(cmd, **kwargs)
+	if kwargs.get('stdout') or kwargs.get('stderr'):
+		try:
+			out, err = proc.communicate(**cargs)
+		except TimeoutExpired:
+			if kwargs.get('start_new_session') and hasattr(os, 'killpg'):
+				os.killpg(proc.pid, signal.SIGKILL)
+			else:
+				proc.kill()
+			out, err = proc.communicate()
+			exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out)
+			exc.stderr = err
+			raise exc
+		status = proc.returncode
+	else:
+		out, err = (None, None)
+		try:
+			status = proc.wait(**cargs)
+		except TimeoutExpired as e:
+			if kwargs.get('start_new_session') and hasattr(os, 'killpg'):
+				os.killpg(proc.pid, signal.SIGKILL)
+			else:
+				proc.kill()
+			proc.wait()
+			raise e
+	return status, out, err
+
+def run_process(cmd, kwargs, cargs={}):
+	"""
+	Executes a subprocess by using a pre-forked process when possible
+	or falling back to subprocess.Popen. See :py:func:`waflib.Utils.run_prefork_process`
+	and :py:func:`waflib.Utils.run_regular_process`
+	"""
+	if kwargs.get('stdout') and kwargs.get('stderr'):
+		return run_prefork_process(cmd, kwargs, cargs)
+	else:
+		return run_regular_process(cmd, kwargs, cargs)
+
+def alloc_process_pool(n, force=False):
+	"""
+	Allocates an amount of processes to the default pool so its size is at least *n*.
+	It is useful to call this function early so that the pre-forked
+	processes use as little memory as possible.
+
+	:param n: pool size
+	:type n: integer
+	:param force: if True then *n* more processes are added to the existing pool
+	:type force: bool
+	"""
+	# mandatory on python2, unnecessary on python >= 3.2
+	global run_process, get_process, alloc_process_pool
+	if not force:
+		n = max(n - len(process_pool), 0)
+	try:
+		lst = [get_process() for x in range(n)]
+	except OSError:
+		run_process = run_regular_process
+		get_process = alloc_process_pool = nada
+	else:
+		for x in lst:
+			process_pool.append(x)
+
+def atexit_pool():
+	for k in process_pool:
+		try:
+			os.kill(k.pid, 9)
+		except OSError:
+			pass
+		else:
+			k.wait()
+# see #1889
+if (sys.hexversion<0x207000f and not is_win32) or sys.hexversion>=0x306000f:
+	atexit.register(atexit_pool)
+
+if os.environ.get('WAF_NO_PREFORK') or sys.platform == 'cli' or not sys.executable:
+	run_process = run_regular_process
+	get_process = alloc_process_pool = nada
+
diff --git a/waflib/__init__.py b/waflib/__init__.py
index c8a3c349..079df358 100644
--- a/waflib/__init__.py
+++ b/waflib/__init__.py
@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2005-2010 (ita)
+# Thomas Nagy, 2005-2018 (ita)
diff --git a/waflib/ansiterm.py b/waflib/ansiterm.py
index 64fa1407..0d20c637 100644
--- a/waflib/ansiterm.py
+++ b/waflib/ansiterm.py
@@ -120,7 +120,7 @@ else:
 		def clear_line(self, param):
 			mode = param and int(param) or 0
 			sbinfo = self.screen_buffer_info()
-			if mode == 1: # Clear from begining of line to cursor position
+			if mode == 1: # Clear from beginning of line to cursor position
 				line_start = COORD(0, sbinfo.CursorPosition.Y)
 				line_length = sbinfo.Size.X
 			elif mode == 2: # Clear entire line
@@ -136,7 +136,7 @@ else:
 		def clear_screen(self, param):
 			mode = to_int(param, 0)
 			sbinfo = self.screen_buffer_info()
-			if mode == 1: # Clear from begining of screen to cursor position
+			if mode == 1: # Clear from beginning of screen to cursor position
 				clear_start = COORD(0, 0)
 				clear_length = sbinfo.CursorPosition.X * sbinfo.CursorPosition.Y
 			elif mode == 2: # Clear entire screen and return cursor to home
@@ -320,7 +320,7 @@ else:
 		sbinfo = CONSOLE_SCREEN_BUFFER_INFO()
 		def get_term_cols():
 			windll.kernel32.GetConsoleScreenBufferInfo(console, byref(sbinfo))
-			# TODO Issue 1401
+			# Issue 1401 - the progress bar cannot reach the last character
 			return sbinfo.Size.X - 1
 
 # just try and see
diff --git a/waflib/extras/batched_cc.py b/waflib/extras/batched_cc.py
index 0b035e85..aad28722 100644
--- a/waflib/extras/batched_cc.py
+++ b/waflib/extras/batched_cc.py
@@ -3,21 +3,22 @@
 # Thomas Nagy, 2006-2015 (ita)
 
 """
-Build as batches.
-
 Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
 cc -c ../file1.c ../file2.c ../file3.c
 
 Files are output on the directory where the compiler is called, and dependencies are more difficult
 to track (do not run the command on all source files if only one file changes)
-
 As such, we do as if the files were compiled one by one, but no command is actually run:
 replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
 signatures from each slave and finds out the command-line to run.
 
-Just import this module in the configuration (no other change required).
-This is provided as an example, for performance unity builds are recommended (fewer tasks and
-fewer jobs to execute). See waflib/extras/unity.py.
+Just import this module to start using it:
+def build(bld):
+	bld.load('batched_cc')
+
+Note that this is provided as an example, unity builds are recommended
+for best performance results (fewer tasks and fewer jobs to execute).
+See waflib/extras/unity.py.
 """
 
 from waflib import Task, Utils
@@ -26,24 +27,21 @@ from waflib.Tools import c, cxx
 
 MAX_BATCH = 50
 
-c_str = '${CC} ${CFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED}'
+c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
 c_fun, _ = Task.compile_fun_noshell(c_str)
 
-cxx_str = '${CXX} ${CXXFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED}'
+cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
 cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
 
 count = 70000
-class batch_task(Task.Task):
+class batch(Task.Task):
 	color = 'PINK'
 
 	after = ['c', 'cxx']
 	before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
 
 	def uid(self):
-		m = Utils.md5()
-		m.update(Task.Task.uid(self))
-		m.update(str(self.generator.idx).encode())
-		return m.digest()
+		return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target])
 
 	def __str__(self):
 		return 'Batch compilation for %d slaves' % len(self.slaves)
@@ -74,6 +72,13 @@ class batch_task(Task.Task):
 
 		return Task.SKIP_ME
 
+	def get_cwd(self):
+		return self.slaves[0].outputs[0].parent
+
+	def batch_incpaths(self):
+		st = self.env.CPPPATH_ST
+		return [st % node.abspath() for node in self.generator.includes_nodes]
+
 	def run(self):
 		self.outputs = []
 
@@ -85,7 +90,6 @@ class batch_task(Task.Task):
 				srclst.append(t.inputs[0].abspath())
 
 		self.env.SRCLST = srclst
-		self.cwd = slaves[0].outputs[0].parent.abspath()
 
 		if self.slaves[0].__class__.__name__ == 'c':
 			ret = c_fun(self)
diff --git a/waflib/extras/build_file_tracker.py b/waflib/extras/build_file_tracker.py
index 4f9e7b6f..c4f26fd0 100644
--- a/waflib/extras/build_file_tracker.py
+++ b/waflib/extras/build_file_tracker.py
@@ -8,25 +8,21 @@ want to use this to force partial rebuilds, see playground/track_output_files/ f
 
 Note that there is a variety of ways to implement this, one may want use timestamps on source files too for example,
 or one may want to hash the files in the source directory only under certain conditions (md5_tstamp tool)
-or to hash the file in the build directory with its timestamp (similar to 'update_outputs')
+or to hash the file in the build directory with its timestamp
 """
 
 import os
 from waflib import Node, Utils
 
 def get_bld_sig(self):
+	if not self.is_bld() or self.ctx.bldnode is self.ctx.srcnode:
+		return Utils.h_file(self.abspath())
+
 	try:
-		return self.cache_sig
+		# add the creation time to the signature
+		return self.sig + str(os.stat(self.abspath()).st_mtime)
 	except AttributeError:
-		pass
-
-	if not self.is_bld() or self.ctx.bldnode is self.ctx.srcnode:
-		self.sig = Utils.h_file(self.abspath())
-		self.cache_sig = ret = self.sig
-	else:
-		# add the
-		self.cache_sig = ret = self.sig + str(os.stat(self.abspath()).st_mtime)
-	return ret
+		return None
 
 Node.Node.get_bld_sig = get_bld_sig
 
diff --git a/waflib/extras/build_logs.py b/waflib/extras/build_logs.py
index 9209ec33..cdf8ed09 100644
--- a/waflib/extras/build_logs.py
+++ b/waflib/extras/build_logs.py
@@ -17,7 +17,7 @@ try:
 	up = os.path.dirname(Context.g_module.__file__)
 except AttributeError:
 	up = '.'
-LOGFILE = os.path.join(up, 'logs', '%s.log' % time.strftime('%Y_%m_%d_%H_%M'))
+LOGFILE = os.path.join(up, 'logs', time.strftime('%Y_%m_%d_%H_%M.log'))
 
 wlock = threading.Lock()
 class log_to_file(object):
@@ -28,7 +28,7 @@ class log_to_file(object):
 		self.filename = filename
 		self.is_valid = True
 	def replace_colors(self, data):
-		for x in Logs.colors_lst.values(): 
+		for x in Logs.colors_lst.values():
 			if isinstance(x, str):
 				data = data.replace(x, '')
 		return data
@@ -96,7 +96,7 @@ def exit_cleanup():
 		fileobj.close()
 		filename = sys.stdout.filename
 
-		Logs.info('Output logged to %r' % filename)
+		Logs.info('Output logged to %r', filename)
 
 		# then copy the log file to "latest.log" if possible
 		up = os.path.dirname(os.path.abspath(filename))
@@ -104,7 +104,6 @@ def exit_cleanup():
 			shutil.copy(filename, os.path.join(up, 'latest.log'))
 		except OSError:
 			# this may fail on windows due to processes spawned
-			# 
 			pass
 
 atexit.register(exit_cleanup)
diff --git a/waflib/extras/c_nec.py b/waflib/extras/c_nec.py
index 87e0c055..96bfae4f 100644
--- a/waflib/extras/c_nec.py
+++ b/waflib/extras/c_nec.py
@@ -24,43 +24,46 @@ def find_sxc(conf):
 
 @conf
 def get_sxc_version(conf, fc):
-		version_re = re.compile(r"C\+\+/SX\s*Version\s*(?P<major>\d*)\.(?P<minor>\d*)", re.I).search
-		cmd = fc + ['-V']
-		p = Utils.subprocess.Popen(cmd, stdin=False, stdout=Utils.subprocess.PIPE, stderr=Utils.subprocess.PIPE, env=None)
-		out, err = p.communicate()
+	version_re = re.compile(r"C\+\+/SX\s*Version\s*(?P<major>\d*)\.(?P<minor>\d*)", re.I).search
+	cmd = fc + ['-V']
+	p = Utils.subprocess.Popen(cmd, stdin=False, stdout=Utils.subprocess.PIPE, stderr=Utils.subprocess.PIPE, env=None)
+	out, err = p.communicate()
 
-		if out: match = version_re(out)
-		else: match = version_re(err)
-		if not match:
-				conf.fatal('Could not determine the NEC C compiler version.')
-		k = match.groupdict()
-		conf.env['C_VERSION'] = (k['major'], k['minor'])
+	if out:
+		match = version_re(out)
+	else:
+		match = version_re(err)
+	if not match:
+		conf.fatal('Could not determine the NEC C compiler version.')
+	k = match.groupdict()
+	conf.env['C_VERSION'] = (k['major'], k['minor'])
 
 @conf
 def sxc_common_flags(conf):
-        v=conf.env
-        v['CC_SRC_F']=[]
-        v['CC_TGT_F']=['-c','-o']
-        if not v['LINK_CC']:v['LINK_CC']=v['CC']
-        v['CCLNK_SRC_F']=[]
-        v['CCLNK_TGT_F']=['-o']
-        v['CPPPATH_ST']='-I%s'
-        v['DEFINES_ST']='-D%s'
-        v['LIB_ST']='-l%s'
-        v['LIBPATH_ST']='-L%s'
-        v['STLIB_ST']='-l%s'
-        v['STLIBPATH_ST']='-L%s'
-        v['RPATH_ST']=''
-        v['SONAME_ST']=[]
-        v['SHLIB_MARKER']=[]
-        v['STLIB_MARKER']=[]
-        v['LINKFLAGS_cprogram']=['']
-        v['cprogram_PATTERN']='%s'
-        v['CFLAGS_cshlib']=['-fPIC']
-        v['LINKFLAGS_cshlib']=['']
-        v['cshlib_PATTERN']='lib%s.so'
-        v['LINKFLAGS_cstlib']=[]
-        v['cstlib_PATTERN']='lib%s.a'
+	v=conf.env
+	v['CC_SRC_F']=[]
+	v['CC_TGT_F']=['-c','-o']
+	if not v['LINK_CC']:
+		v['LINK_CC']=v['CC']
+	v['CCLNK_SRC_F']=[]
+	v['CCLNK_TGT_F']=['-o']
+	v['CPPPATH_ST']='-I%s'
+	v['DEFINES_ST']='-D%s'
+	v['LIB_ST']='-l%s'
+	v['LIBPATH_ST']='-L%s'
+	v['STLIB_ST']='-l%s'
+	v['STLIBPATH_ST']='-L%s'
+	v['RPATH_ST']=''
+	v['SONAME_ST']=[]
+	v['SHLIB_MARKER']=[]
+	v['STLIB_MARKER']=[]
+	v['LINKFLAGS_cprogram']=['']
+	v['cprogram_PATTERN']='%s'
+	v['CFLAGS_cshlib']=['-fPIC']
+	v['LINKFLAGS_cshlib']=['']
+	v['cshlib_PATTERN']='lib%s.so'
+	v['LINKFLAGS_cstlib']=[]
+	v['cstlib_PATTERN']='lib%s.a'
 
 def configure(conf):
 	conf.find_sxc()
diff --git a/waflib/extras/xcode.py b/waflib/extras/xcode.py
deleted file mode 100644
index e0b7b9de..00000000
--- a/waflib/extras/xcode.py
+++ /dev/null
@@ -1,312 +0,0 @@
-#! /usr/bin/env python
-# encoding: utf-8
-# XCode 3/XCode 4 generator for Waf
-# Nicolas Mercier 2011
-
-"""
-Usage:
-
-def options(opt):
-	opt.load('xcode')
-
-$ waf configure xcode
-"""
-
-# TODO: support iOS projects
-
-from waflib import Context, TaskGen, Build, Utils
-import os, sys
-
-HEADERS_GLOB = '**/(*.h|*.hpp|*.H|*.inl)'
-
-MAP_EXT = {
-	'.h' :  "sourcecode.c.h",
-
-	'.hh':  "sourcecode.cpp.h",
-	'.inl': "sourcecode.cpp.h",
-	'.hpp': "sourcecode.cpp.h",
-
-	'.c':   "sourcecode.c.c",
-
-	'.m':   "sourcecode.c.objc",
-
-	'.mm':  "sourcecode.cpp.objcpp",
-
-	'.cc':  "sourcecode.cpp.cpp",
-
-	'.cpp': "sourcecode.cpp.cpp",
-	'.C':   "sourcecode.cpp.cpp",
-	'.cxx': "sourcecode.cpp.cpp",
-	'.c++': "sourcecode.cpp.cpp",
-
-	'.l':   "sourcecode.lex", # luthor
-	'.ll':  "sourcecode.lex",
-
-	'.y':   "sourcecode.yacc",
-	'.yy':  "sourcecode.yacc",
-
-	'.plist': "text.plist.xml",
-	".nib":   "wrapper.nib",
-	".xib":   "text.xib",
-}
-
-
-part1 = 0
-part2 = 10000
-part3 = 0
-id = 562000999
-def newid():
-	global id
-	id = id + 1
-	return "%04X%04X%04X%012d" % (0, 10000, 0, id)
-
-class XCodeNode:
-	def __init__(self):
-		self._id = newid()
-
-	def tostring(self, value):
-		if isinstance(value, dict):
-			result = "{\n"
-			for k,v in value.items():
-				result = result + "\t\t\t%s = %s;\n" % (k, self.tostring(v))
-			result = result + "\t\t}"
-			return result
-		elif isinstance(value, str):
-			return "\"%s\"" % value
-		elif isinstance(value, list):
-			result = "(\n"
-			for i in value:
-				result = result + "\t\t\t%s,\n" % self.tostring(i)
-			result = result + "\t\t)"
-			return result
-		elif isinstance(value, XCodeNode):
-			return value._id
-		else:
-			return str(value)
-
-	def write_recursive(self, value, file):
-		if isinstance(value, dict):
-			for k,v in value.items():
-				self.write_recursive(v, file)
-		elif isinstance(value, list):
-			for i in value:
-				self.write_recursive(i, file)
-		elif isinstance(value, XCodeNode):
-			value.write(file)
-
-	def write(self, file):
-		for attribute,value in self.__dict__.items():
-			if attribute[0] != '_':
-				self.write_recursive(value, file)
-
-		w = file.write
-		w("\t%s = {\n" % self._id)
-		w("\t\tisa = %s;\n" % self.__class__.__name__)
-		for attribute,value in self.__dict__.items():
-			if attribute[0] != '_':
-				w("\t\t%s = %s;\n" % (attribute, self.tostring(value)))
-		w("\t};\n\n")
-
-
-
-# Configurations
-class XCBuildConfiguration(XCodeNode):
-	def __init__(self, name, settings = {}, env=None):
-		XCodeNode.__init__(self)
-		self.baseConfigurationReference = ""
-		self.buildSettings = settings
-		self.name = name
-		if env and env.ARCH:
-			settings['ARCHS'] = " ".join(env.ARCH)
-
-
-class XCConfigurationList(XCodeNode):
-	def __init__(self, settings):
-		XCodeNode.__init__(self)
-		self.buildConfigurations = settings
-		self.defaultConfigurationIsVisible = 0
-		self.defaultConfigurationName = settings and settings[0].name or ""
-
-# Group/Files
-class PBXFileReference(XCodeNode):
-	def __init__(self, name, path, filetype = '', sourcetree = "SOURCE_ROOT"):
-		XCodeNode.__init__(self)
-		self.fileEncoding = 4
-		if not filetype:
-			_, ext = os.path.splitext(name)
-			filetype = MAP_EXT.get(ext, 'text')
-		self.lastKnownFileType = filetype
-		self.name = name
-		self.path = path
-		self.sourceTree = sourcetree
-
-class PBXGroup(XCodeNode):
-	def __init__(self, name, sourcetree = "<group>"):
-		XCodeNode.__init__(self)
-		self.children = []
-		self.name = name
-		self.sourceTree = sourcetree
-
-	def add(self, root, sources):
-		folders = {}
-		def folder(n):
-			if not n.is_child_of(root):
-				return self
-			try:
-				return folders[n]
-			except KeyError:
-				f = PBXGroup(n.name)
-				p = folder(n.parent)
-				folders[n] = f
-				p.children.append(f)
-				return f
-		for s in sources:
-			f = folder(s.parent)
-			source = PBXFileReference(s.name, s.abspath())
-			f.children.append(source)
-
-
-# Targets
-class PBXLegacyTarget(XCodeNode):
-	def __init__(self, action, target=''):
-		XCodeNode.__init__(self)
-		self.buildConfigurationList = XCConfigurationList([XCBuildConfiguration('waf', {})])
-		if not target:
-			self.buildArgumentsString = "%s %s" % (sys.argv[0], action)
-		else:
-			self.buildArgumentsString = "%s %s --targets=%s" % (sys.argv[0], action, target)
-		self.buildPhases = []
-		self.buildToolPath = sys.executable
-		self.buildWorkingDirectory = ""
-		self.dependencies = []
-		self.name = target or action
-		self.productName = target or action
-		self.passBuildSettingsInEnvironment = 0
-
-class PBXShellScriptBuildPhase(XCodeNode):
-	def __init__(self, action, target):
-		XCodeNode.__init__(self)
-		self.buildActionMask = 2147483647
-		self.files = []
-		self.inputPaths = []
-		self.outputPaths = []
-		self.runOnlyForDeploymentPostProcessing = 0
-		self.shellPath = "/bin/sh"
-		self.shellScript = "%s %s %s --targets=%s" % (sys.executable, sys.argv[0], action, target)
-
-class PBXNativeTarget(XCodeNode):
-	def __init__(self, action, target, node, env):
-		XCodeNode.__init__(self)
-		conf = XCBuildConfiguration('waf', {'PRODUCT_NAME':target, 'CONFIGURATION_BUILD_DIR':node.parent.abspath()}, env)
-		self.buildConfigurationList = XCConfigurationList([conf])
-		self.buildPhases = [PBXShellScriptBuildPhase(action, target)]
-		self.buildRules = []
-		self.dependencies = []
-		self.name = target
-		self.productName = target
-		self.productType = "com.apple.product-type.application"
-		self.productReference = PBXFileReference(target, node.abspath(), 'wrapper.application', 'BUILT_PRODUCTS_DIR')
-
-# Root project object
-class PBXProject(XCodeNode):
-	def __init__(self, name, version):
-		XCodeNode.__init__(self)
-		self.buildConfigurationList = XCConfigurationList([XCBuildConfiguration('waf', {})])
-		self.compatibilityVersion = version[0]
-		self.hasScannedForEncodings = 1
-		self.mainGroup = PBXGroup(name)
-		self.projectRoot = ""
-		self.projectDirPath = ""
-		self.targets = []
-		self._objectVersion = version[1]
-		self._output = PBXGroup('out')
-		self.mainGroup.children.append(self._output)
-
-	def write(self, file):
-		w = file.write
-		w("// !$*UTF8*$!\n")
-		w("{\n")
-		w("\tarchiveVersion = 1;\n")
-		w("\tclasses = {\n")
-		w("\t};\n")
-		w("\tobjectVersion = %d;\n" % self._objectVersion)
-		w("\tobjects = {\n\n")
-
-		XCodeNode.write(self, file)
-
-		w("\t};\n")
-		w("\trootObject = %s;\n" % self._id)
-		w("}\n")
-
-	def add_task_gen(self, tg):
-		if not getattr(tg, 'mac_app', False):
-			self.targets.append(PBXLegacyTarget('build', tg.name))
-		else:
-			target = PBXNativeTarget('build', tg.name, tg.link_task.outputs[0].change_ext('.app'), tg.env)
-			self.targets.append(target)
-			self._output.children.append(target.productReference)
-
-class xcode(Build.BuildContext):
-	cmd = 'xcode'
-	fun = 'build'
-
-	def collect_source(self, tg):
-		source_files = tg.to_nodes(getattr(tg, 'source', []))
-		plist_files = tg.to_nodes(getattr(tg, 'mac_plist', []))
-		resource_files = [tg.path.find_node(i) for i in Utils.to_list(getattr(tg, 'mac_resources', []))]
-		include_dirs = Utils.to_list(getattr(tg, 'includes', [])) + Utils.to_list(getattr(tg, 'export_dirs', []))
-		include_files = []
-		for x in include_dirs:
-			if not isinstance(x, str):
-				include_files.append(x)
-				continue
-			d = tg.path.find_node(x)
-			if d:
-				lst = [y for y in d.ant_glob(HEADERS_GLOB, flat=False)]
-				include_files.extend(lst)
-
-		# remove duplicates
-		source = list(set(source_files + plist_files + resource_files + include_files))
-		source.sort(key=lambda x: x.abspath())
-		return source
-
-	def execute(self):
-		"""
-		Entry point
-		"""
-		self.restore()
-		if not self.all_envs:
-			self.load_envs()
-		self.recurse([self.run_dir])
-
-		appname = getattr(Context.g_module, Context.APPNAME, os.path.basename(self.srcnode.abspath()))
-		p = PBXProject(appname, ('Xcode 3.2', 46))
-
-		for g in self.groups:
-			for tg in g:
-				if not isinstance(tg, TaskGen.task_gen):
-					continue
-
-				tg.post()
-
-				features = Utils.to_list(getattr(tg, 'features', ''))
-
-				group = PBXGroup(tg.name)
-				group.add(tg.path, self.collect_source(tg))
-				p.mainGroup.children.append(group)
-
-				if 'cprogram' or 'cxxprogram' in features:
-					p.add_task_gen(tg)
-
-
-		# targets that don't produce the executable but that you might want to run
-		p.targets.append(PBXLegacyTarget('configure'))
-		p.targets.append(PBXLegacyTarget('dist'))
-		p.targets.append(PBXLegacyTarget('install'))
-		p.targets.append(PBXLegacyTarget('check'))
-		node = self.srcnode.make_node('%s.xcodeproj' % appname)
-		node.mkdir()
-		node = node.make_node('project.pbxproj')
-		p.write(open(node.abspath(), 'w'))
-
-
diff --git a/waflib/extras/xcode6.py b/waflib/extras/xcode6.py
index e5bf198c..91bbff18 100644
--- a/waflib/extras/xcode6.py
+++ b/waflib/extras/xcode6.py
@@ -1,61 +1,20 @@
 #! /usr/bin/env python
 # encoding: utf-8
-# XCode 3/XCode 4 generator for Waf
+# XCode 3/XCode 4/XCode 6/Xcode 7 generator for Waf
 # Based on work by Nicolas Mercier 2011
 # Extended by Simon Warg 2015, https://github.com/mimon
 # XCode project file format based on http://www.monobjc.net/xcode-project-file-format.html
 
 """
-Usage:
+See playground/xcode6/ for usage examples.
 
-See also demos/xcode6/ folder
-
-def options(opt):
-	opt.load('xcode6')
-
-def configure(cnf):
-	# <do your stuff>
-
-	# For example
-	cnf.env.SDKROOT = 'macosx10.9'
-
-	# Use cnf.PROJ_CONFIGURATION to completely set/override
-	# global project settings
-	# cnf.env.PROJ_CONFIGURATION = {
-	# 	'Debug': {
-	# 		'SDKROOT': 'macosx10.9'
-	# 	}
-	# 	'MyCustomReleaseConfig': {
-	# 		'SDKROOT': 'macosx10.10'
-	# 	}
-	# }
-
-	# In the end of configure() do
-	cnf.load('xcode6')
-
-def build(bld):
-
-	# Make a Framework target
-	bld.framework(
-		source_files={
-			'Include': bld.path.ant_glob('include/MyLib/*.h'),
-			'Source': bld.path.ant_glob('src/MyLib/*.cpp')
-		},
-		includes='include',
-		export_headers=bld.path.ant_glob('include/MyLib/*.h'),
-		target='MyLib',
-	)
-
-	# You can also make bld.dylib, bld.app, bld.stlib ...
-
-$ waf configure xcode6
 """
 
-# TODO: support iOS projects
+from waflib import Context, TaskGen, Build, Utils, Errors, Logs
+import os, sys
 
-from waflib import Context, TaskGen, Build, Utils, ConfigSet, Configure, Errors
-from waflib.Build import BuildContext
-import os, sys, random, time
+# FIXME too few extensions
+XCODE_EXTS = ['.c', '.cpp', '.m', '.mm']
 
 HEADERS_GLOB = '**/(*.h|*.hpp|*.H|*.inl)'
 
@@ -123,6 +82,13 @@ TARGET_TYPES = {
 	'exe' :TARGET_TYPE_EXECUTABLE,
 }
 
+def delete_invalid_values(dct):
+	""" Deletes entries that are dictionaries or sets """
+	for k, v in list(dct.items()):
+		if isinstance(v, dict) or isinstance(v, set):
+			del dct[k]
+	return dct
+
 """
 Configuration of the global project settings. Sets an environment variable 'PROJ_CONFIGURATION'
 which is a dictionary of configuration name and buildsettings pair.
@@ -150,15 +116,16 @@ def configure(self):
 
 	# Create default project configuration?
 	if 'PROJ_CONFIGURATION' not in self.env:
+		defaults = delete_invalid_values(self.env.get_merged_dict())
 		self.env.PROJ_CONFIGURATION = {
-			"Debug": self.env.get_merged_dict(),
-			"Release": self.env.get_merged_dict(),
+			"Debug": defaults,
+			"Release": defaults,
 		}
 
 	# Some build settings are required to be present by XCode. We will supply default values
 	# if user hasn't defined any.
 	defaults_required = [('PRODUCT_NAME', '$(TARGET_NAME)')]
-	for cfgname,settings in self.env.PROJ_CONFIGURATION.iteritems():
+	for cfgname,settings in self.env.PROJ_CONFIGURATION.items():
 		for default_var, default_val in defaults_required:
 			if default_var not in settings:
 				settings[default_var] = default_val
@@ -173,10 +140,17 @@ part3 = 0
 id = 562000999
 def newid():
 	global id
-	id = id + 1
+	id += 1
 	return "%04X%04X%04X%012d" % (0, 10000, 0, id)
 
-class XCodeNode:
+"""
+Represents a tree node in the XCode project plist file format.
+When written to a file, all attributes of XCodeNode are stringified together with
+its value. However, attributes starting with an underscore _ are ignored
+during that process and allows you to store arbitrary values that are not supposed
+to be written out.
+"""
+class XCodeNode(object):
 	def __init__(self):
 		self._id = newid()
 		self._been_written = False
@@ -247,12 +221,14 @@ class XCConfigurationList(XCodeNode):
 # Group/Files
 class PBXFileReference(XCodeNode):
 	def __init__(self, name, path, filetype = '', sourcetree = "SOURCE_ROOT"):
+
 		XCodeNode.__init__(self)
 		self.fileEncoding = 4
 		if not filetype:
 			_, ext = os.path.splitext(name)
 			filetype = MAP_EXT.get(ext, 'text')
 		self.lastKnownFileType = filetype
+		self.explicitFileType = filetype
 		self.name = name
 		self.path = path
 		self.sourceTree = sourcetree
@@ -267,11 +243,11 @@ class PBXBuildFile(XCodeNode):
 	""" This element indicate a file reference that is used in a PBXBuildPhase (either as an include or resource). """
 	def __init__(self, fileRef, settings={}):
 		XCodeNode.__init__(self)
-		
+
 		# fileRef is a reference to a PBXFileReference object
 		self.fileRef = fileRef
 
-		# A map of key/value pairs for additionnal settings.
+		# A map of key/value pairs for additional settings.
 		self.settings = settings
 
 	def __hash__(self):
@@ -281,16 +257,49 @@ class PBXBuildFile(XCodeNode):
 		return self.fileRef == other.fileRef
 
 class PBXGroup(XCodeNode):
-	def __init__(self, name, sourcetree = "<group>"):
+	def __init__(self, name, sourcetree = 'SOURCE_TREE'):
 		XCodeNode.__init__(self)
 		self.children = []
 		self.name = name
 		self.sourceTree = sourcetree
 
+		# Maintain a lookup table for all PBXFileReferences
+		# that are contained in this group.
+		self._filerefs = {}
+
 	def add(self, sources):
-		""" sources param should be a list of PBXFileReference objects """
+		"""
+		Add a list of PBXFileReferences to this group
+
+		:param sources: list of PBXFileReferences objects
+		"""
+		self._filerefs.update(dict(zip(sources, sources)))
 		self.children.extend(sources)
 
+	def get_sub_groups(self):
+		"""
+		Returns all child PBXGroup objects contained in this group
+		"""
+		return list(filter(lambda x: isinstance(x, PBXGroup), self.children))
+
+	def find_fileref(self, fileref):
+		"""
+		Recursively search this group for an existing PBXFileReference. Returns None
+		if none were found.
+
+		The reason you'd want to reuse existing PBXFileReferences from a PBXGroup is that XCode doesn't like PBXFileReferences that aren't part of a PBXGroup hierarchy.
+		If it isn't, the consequence is that certain UI features like 'Reveal in Finder'
+		stops working.
+		"""
+		if fileref in self._filerefs:
+			return self._filerefs[fileref]
+		elif self.children:
+			for childgroup in self.get_sub_groups():
+				f = childgroup.find_fileref(fileref)
+				if f:
+					return f
+		return None
+
 class PBXContainerItemProxy(XCodeNode):
 	""" This is the element for to decorate a target item. """
 	def __init__(self, containerPortal, remoteGlobalIDString, remoteInfo='', proxyType=1):
@@ -299,7 +308,6 @@ class PBXContainerItemProxy(XCodeNode):
 		self.remoteGlobalIDString = remoteGlobalIDString # PBXNativeTarget
 		self.remoteInfo = remoteInfo # Target name
 		self.proxyType = proxyType
-		
 
 class PBXTargetDependency(XCodeNode):
 	""" This is the element for referencing other target through content proxies. """
@@ -307,7 +315,7 @@ class PBXTargetDependency(XCodeNode):
 		XCodeNode.__init__(self)
 		self.target = native_target
 		self.targetProxy = proxy
-		
+
 class PBXFrameworksBuildPhase(XCodeNode):
 	""" This is the element for the framework link build phase, i.e. linking to frameworks """
 	def __init__(self, pbxbuildfiles):
@@ -409,7 +417,7 @@ class PBXProject(XCodeNode):
 		if not isinstance(env.PROJ_CONFIGURATION, dict):
 			raise Errors.WafError("Error: env.PROJ_CONFIGURATION must be a dictionary. This is done for you if you do not define one yourself. However, did you load the xcode module at the end of your wscript configure() ?")
 
-		# Retreive project configuration
+		# Retrieve project configuration
 		configurations = []
 		for config_name, settings in env.PROJ_CONFIGURATION.items():
 			cf = XCBuildConfiguration(config_name, settings)
@@ -427,8 +435,8 @@ class PBXProject(XCodeNode):
 	def create_target_dependency(self, target, name):
 		""" : param target : PXBNativeTarget """
 		proxy = PBXContainerItemProxy(self, target, name)
-		dependecy = PBXTargetDependency(target, proxy)
-		return dependecy
+		dependency = PBXTargetDependency(target, proxy)
+		return dependency
 
 	def write(self, file):
 
@@ -461,13 +469,145 @@ class PBXProject(XCodeNode):
 				return t
 		return None
 
+@TaskGen.feature('c', 'cxx')
+@TaskGen.after('propagate_uselib_vars', 'apply_incpaths')
+def process_xcode(self):
+	bld = self.bld
+	try:
+		p = bld.project
+	except AttributeError:
+		return
+
+	if not hasattr(self, 'target_type'):
+		return
+
+	products_group = bld.products_group
+
+	target_group = PBXGroup(self.name)
+	p.mainGroup.children.append(target_group)
+
+	# Determine what type to build - framework, app bundle etc.
+	target_type = getattr(self, 'target_type', 'app')
+	if target_type not in TARGET_TYPES:
+		raise Errors.WafError("Target type '%s' does not exists. Available options are '%s'. In target '%s'" % (target_type, "', '".join(TARGET_TYPES.keys()), self.name))
+	else:
+		target_type = TARGET_TYPES[target_type]
+	file_ext = target_type[2]
+
+	# Create the output node
+	target_node = self.path.find_or_declare(self.name+file_ext)
+	target = PBXNativeTarget(self.name, target_node, target_type, [], [])
+
+	products_group.children.append(target.productReference)
+
+	# Pull source files from the 'source' attribute and assign them to a UI group.
+	# Use a default UI group named 'Source' unless the user
+	# provides a 'group_files' dictionary to customize the UI grouping.
+	sources = getattr(self, 'source', [])
+	if hasattr(self, 'group_files'):
+		group_files = getattr(self, 'group_files', [])
+		for grpname,files in group_files.items():
+			group = bld.create_group(grpname, files)
+			target_group.children.append(group)
+	else:
+		group = bld.create_group('Source', sources)
+		target_group.children.append(group)
+
+	# Create a PBXFileReference for each source file.
+	# If the source file already exists as a PBXFileReference in any of the UI groups, then
+	# reuse that PBXFileReference object (XCode does not like it if we don't reuse)
+	for idx, path in enumerate(sources):
+		fileref = PBXFileReference(path.name, path.abspath())
+		existing_fileref = target_group.find_fileref(fileref)
+		if existing_fileref:
+			sources[idx] = existing_fileref
+		else:
+			sources[idx] = fileref
+
+	# If the 'source' attribute contains any file extension that XCode can't work with,
+	# then remove it. The allowed file extensions are defined in XCODE_EXTS.
+	is_valid_file_extension = lambda file: os.path.splitext(file.path)[1] in XCODE_EXTS
+	sources = list(filter(is_valid_file_extension, sources))
+
+	buildfiles = [bld.unique_buildfile(PBXBuildFile(x)) for x in sources]
+	target.add_build_phase(PBXSourcesBuildPhase(buildfiles))
+
+	# Check if any framework to link against is some other target we've made
+	libs = getattr(self, 'tmp_use_seen', [])
+	for lib in libs:
+		use_target = p.get_target(lib)
+		if use_target:
+			# Create an XCode dependency so that XCode knows to build the other target before this target
+			dependency = p.create_target_dependency(use_target, use_target.name)
+			target.add_dependency(dependency)
+
+			buildphase = PBXFrameworksBuildPhase([PBXBuildFile(use_target.productReference)])
+			target.add_build_phase(buildphase)
+			if lib in self.env.LIB:
+				self.env.LIB = list(filter(lambda x: x != lib, self.env.LIB))
+
+	# If 'export_headers' is present, add files to the Headers build phase in xcode.
+	# These are files that'll get packed into the Framework for instance.
+	exp_hdrs = getattr(self, 'export_headers', [])
+	hdrs = bld.as_nodes(Utils.to_list(exp_hdrs))
+	files = [p.mainGroup.find_fileref(PBXFileReference(n.name, n.abspath())) for n in hdrs]
+	files = [PBXBuildFile(f, {'ATTRIBUTES': ('Public',)}) for f in files]
+	buildphase = PBXHeadersBuildPhase(files)
+	target.add_build_phase(buildphase)
+
+	# Merge frameworks and libs into one list, and prefix the frameworks
+	frameworks = Utils.to_list(self.env.FRAMEWORK)
+	frameworks = ' '.join(['-framework %s' % (f.split('.framework')[0]) for f in frameworks])
+
+	libs = Utils.to_list(self.env.STLIB) + Utils.to_list(self.env.LIB)
+	libs = ' '.join(bld.env['STLIB_ST'] % t for t in libs)
+
+	# Override target specific build settings
+	bldsettings = {
+		'HEADER_SEARCH_PATHS': ['$(inherited)'] + self.env['INCPATHS'],
+		'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR) ,
+		'FRAMEWORK_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.FRAMEWORKPATH),
+		'OTHER_LDFLAGS': libs + ' ' + frameworks,
+		'OTHER_LIBTOOLFLAGS': bld.env['LINKFLAGS'],
+		'OTHER_CPLUSPLUSFLAGS': Utils.to_list(self.env['CXXFLAGS']),
+		'OTHER_CFLAGS': Utils.to_list(self.env['CFLAGS']),
+		'INSTALL_PATH': []
+	}
+
+	# Install path
+	installpaths = Utils.to_list(getattr(self, 'install', []))
+	prodbuildfile = PBXBuildFile(target.productReference)
+	for instpath in installpaths:
+		bldsettings['INSTALL_PATH'].append(instpath)
+		target.add_build_phase(PBXCopyFilesBuildPhase([prodbuildfile], instpath))
+
+	if not bldsettings['INSTALL_PATH']:
+		del bldsettings['INSTALL_PATH']
+
+	# Create build settings which can override the project settings. Defaults to none if user
+	# did not pass argument. This will be filled up with target specific
+	# search paths, libs to link etc.
+	settings = getattr(self, 'settings', {})
+
+	# The keys represents different build configuration, e.g. Debug, Release and so on..
+	# Insert our generated build settings to all configuration names
+	keys = set(settings.keys() + bld.env.PROJ_CONFIGURATION.keys())
+	for k in keys:
+		if k in settings:
+			settings[k].update(bldsettings)
+		else:
+			settings[k] = bldsettings
+
+	for k,v in settings.items():
+		target.add_configuration(XCBuildConfiguration(k, v))
+
+	p.add_target(target)
+
+
 class xcode(Build.BuildContext):
 	cmd = 'xcode6'
 	fun = 'build'
 
-	file_refs = dict()
-	build_files = dict()
-
 	def as_nodes(self, files):
 		""" Returns a list of waflib.Nodes from a list of string of file paths """
 		nodes = []
@@ -476,42 +616,42 @@ class xcode(Build.BuildContext):
 				d = x
 			else:
 				d = self.srcnode.find_node(x)
+				if not d:
+					raise Errors.WafError('File \'%s\' was not found' % x)
 			nodes.append(d)
 		return nodes
 
 	def create_group(self, name, files):
 		"""
-			Returns a new PBXGroup containing the files (paths) passed in the files arg 
-			:type files: string
+		Returns a new PBXGroup containing the files (paths) passed in the files arg
+		:type files: string
 		"""
 		group = PBXGroup(name)
 		"""
 		Do not use unique file reference here, since XCode seem to allow only one file reference
 		to be referenced by a group.
 		"""
-		files = [(PBXFileReference(d.name, d.abspath())) for d in self.as_nodes(files)]
-		group.add(files)
+		files_ = []
+		for d in self.as_nodes(Utils.to_list(files)):
+			fileref = PBXFileReference(d.name, d.abspath())
+			files_.append(fileref)
+		group.add(files_)
 		return group
 
-	def unique_filereference(self, fileref):
-		"""
-		Returns a unique fileref, possibly an existing one if the paths are the same.
-		Use this after you've constructed a PBXFileReference to make sure there is
-		only one PBXFileReference for the same file in the same project.
-		"""
-		if fileref not in self.file_refs:
-			self.file_refs[fileref] = fileref
-		return self.file_refs[fileref]
-
 	def unique_buildfile(self, buildfile):
 		"""
 		Returns a unique buildfile, possibly an existing one.
 		Use this after you've constructed a PBXBuildFile to make sure there is
 		only one PBXBuildFile for the same file in the same project.
 		"""
-		if buildfile not in self.build_files:
-			self.build_files[buildfile] = buildfile
-		return self.build_files[buildfile]
+		try:
+			build_files = self.build_files
+		except AttributeError:
+			build_files = self.build_files = {}
+
+		if buildfile not in build_files:
+			build_files[buildfile] = buildfile
+		return build_files[buildfile]
 
 	def execute(self):
 		"""
@@ -525,7 +665,7 @@ class xcode(Build.BuildContext):
 		appname = getattr(Context.g_module, Context.APPNAME, os.path.basename(self.srcnode.abspath()))
 
 		p = PBXProject(appname, ('Xcode 3.2', 46), self.env)
-		
+
 		# If we don't create a Products group, then
 		# XCode will create one, which entails that
 		# we'll start to see duplicate files in the UI
@@ -533,124 +673,55 @@ class xcode(Build.BuildContext):
 		products_group = PBXGroup('Products')
 		p.mainGroup.children.append(products_group)
 
-		for g in self.groups:
-			for tg in g:
-				if not isinstance(tg, TaskGen.task_gen):
-					continue
-
-				tg.post()
-
-				target_group = PBXGroup(tg.name)
-				p.mainGroup.children.append(target_group)
-
-				# Determine what type to build - framework, app bundle etc.
-				target_type = getattr(tg, 'target_type', 'app')
-				if target_type not in TARGET_TYPES:
-					raise Errors.WafError("Target type '%s' does not exists. Available options are '%s'. In target '%s'" % (target_type, "', '".join(TARGET_TYPES.keys()), tg.name))
-				else:
-					target_type = TARGET_TYPES[target_type]
-				file_ext = target_type[2]
-
-				# Create the output node
-				target_node = tg.path.find_or_declare(tg.name+file_ext)
-				
-				target = PBXNativeTarget(tg.name, target_node, target_type, [], [])
-
-				products_group.children.append(target.productReference)
-
-				if hasattr(tg, 'source_files'):
-					# Create list of PBXFileReferences
-					sources = []
-					if isinstance(tg.source_files, dict):
-						for grpname,files in tg.source_files.items():
-							group = self.create_group(grpname, files)
-							target_group.children.append(group)
-							sources.extend(group.children)
-					elif isinstance(tg.source_files, list):
-						group = self.create_group("Source", tg.source_files)
-						target_group.children.append(group)
-						sources.extend(group.children)
-					else:
-						self.to_log("Argument 'source_files' passed to target '%s' was not a dictionary. Hence, some source files may not be included. Please provide a dictionary of source files, with group name as key and list of source files as value.\n" % tg.name)
-
-					supported_extensions = ['.c', '.cpp', '.m', '.mm']
-					sources = filter(lambda fileref: os.path.splitext(fileref.path)[1] in supported_extensions, sources)
-					buildfiles = [self.unique_buildfile(PBXBuildFile(fileref)) for fileref in sources]
-					target.add_build_phase(PBXSourcesBuildPhase(buildfiles))
-
-				# Create build settings which can override the project settings. Defaults to none if user
-				# did not pass argument. However, this will be filled up further below with target specfic
-				# search paths, libs to link etc.
-				settings = getattr(tg, 'settings', {})
-
-				# Check if any framework to link against is some other target we've made
-				libs = getattr(tg, 'tmp_use_seen', [])
-				for lib in libs:
-					use_target = p.get_target(lib)
-					if use_target:
-						# Create an XCode dependency so that XCode knows to build the other target before this target
-						target.add_dependency(p.create_target_dependency(use_target, use_target.name))
-						target.add_build_phase(PBXFrameworksBuildPhase([PBXBuildFile(use_target.productReference)]))
-						if lib in tg.env.LIB:
-							tg.env.LIB = list(filter(lambda x: x != lib, tg.env.LIB))
-
-				# If 'export_headers' is present, add files to the Headers build phase in xcode.
-				# These are files that'll get packed into the Framework for instance.
-				exp_hdrs = getattr(tg, 'export_headers', [])
-				hdrs = self.as_nodes(Utils.to_list(exp_hdrs))
-				files = [self.unique_filereference(PBXFileReference(n.name, n.abspath())) for n in hdrs]
-				target.add_build_phase(PBXHeadersBuildPhase([PBXBuildFile(f, {'ATTRIBUTES': ('Public',)}) for f in files]))
-
-				# Install path
-				installpaths = Utils.to_list(getattr(tg, 'install', []))
-				prodbuildfile = PBXBuildFile(target.productReference)
-				for instpath in installpaths:
-					target.add_build_phase(PBXCopyFilesBuildPhase([prodbuildfile], instpath))
-
-				# Merge frameworks and libs into one list, and prefix the frameworks
-				ld_flags = ['-framework %s' % lib.split('.framework')[0] for lib in Utils.to_list(tg.env.FRAMEWORK)]
-				ld_flags.extend(Utils.to_list(tg.env.STLIB) + Utils.to_list(tg.env.LIB))
-
-				# Override target specfic build settings
-				bldsettings = {
-					'HEADER_SEARCH_PATHS': ['$(inherited)'] + tg.env['INCPATHS'],
-					'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(tg.env.LIBPATH) + Utils.to_list(tg.env.STLIBPATH),
-					'FRAMEWORK_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(tg.env.FRAMEWORKPATH),
-					'OTHER_LDFLAGS': r'\n'.join(ld_flags)
-				}
-
-				# The keys represents different build configuration, e.g. Debug, Release and so on..
-				# Insert our generated build settings to all configuration names
-				keys = set(settings.keys() + self.env.PROJ_CONFIGURATION.keys())
-				for k in keys:
-					if k in settings:
-						settings[k].update(bldsettings)
-					else:
-						settings[k] = bldsettings
-
-				for k,v in settings.items():
-					target.add_configuration(XCBuildConfiguration(k, v))
-
-				p.add_target(target)
-				
+		self.project = p
+		self.products_group = products_group
+
+		# post all task generators
+		# the process_xcode method above will be called for each target
+		if self.targets and self.targets != '*':
+			(self._min_grp, self._exact_tg) = self.get_targets()
+
+		self.current_group = 0
+		while self.current_group < len(self.groups):
+			self.post_group()
+			self.current_group += 1
+
 		node = self.bldnode.make_node('%s.xcodeproj' % appname)
 		node.mkdir()
 		node = node.make_node('project.pbxproj')
-		p.write(open(node.abspath(), 'w'))
-	
-	def build_target(self, tgtype, *k, **kw):
-		"""
-		Provide user-friendly methods to build different target types
-		E.g. bld.framework(source='..', ...) to build a Framework target.
-		E.g. bld.dylib(source='..', ...) to build a Dynamic library target. etc...
-		"""
-		self.load('ccroot')
-		kw['features'] = 'cxx cxxprogram'
+		with open(node.abspath(), 'w') as f:
+			p.write(f)
+		Logs.pprint('GREEN', 'Wrote %r' % node.abspath())
+
+def bind_fun(tgtype):
+	def fun(self, *k, **kw):
+		tgtype = fun.__name__
+		if tgtype == 'shlib' or tgtype == 'dylib':
+			features = 'cxx cxxshlib'
+			tgtype = 'dylib'
+		elif tgtype == 'framework':
+			features = 'cxx cxxshlib'
+			tgtype = 'framework'
+		elif tgtype == 'program':
+			features = 'cxx cxxprogram'
+			tgtype = 'exe'
+		elif tgtype == 'app':
+			features = 'cxx cxxprogram'
+			tgtype = 'app'
+		elif tgtype == 'stlib':
+			features = 'cxx cxxstlib'
+			tgtype = 'stlib'
+		lst = kw['features'] = Utils.to_list(kw.get('features', []))
+		for x in features.split():
+			if not x in kw['features']:
+				lst.append(x)
+
 		kw['target_type'] = tgtype
 		return self(*k, **kw)
+	fun.__name__ = tgtype
+	setattr(Build.BuildContext, tgtype, fun)
+	return fun
+
+for xx in 'app framework dylib shlib stlib program'.split():
+	bind_fun(xx)
 
-	def app(self, *k, **kw): return self.build_target('app', *k, **kw)
-	def framework(self, *k, **kw): return self.build_target('framework', *k, **kw)
-	def dylib(self, *k, **kw): return self.build_target('dylib', *k, **kw)
-	def stlib(self, *k, **kw): return self.build_target('stlib', *k, **kw)
-	def exe(self, *k, **kw): return self.build_target('exe', *k, **kw)
diff --git a/waflib/fixpy2.py b/waflib/fixpy2.py
index 1adc9e0c..24176e06 100644
--- a/waflib/fixpy2.py
+++ b/waflib/fixpy2.py
@@ -1,24 +1,21 @@
 #!/usr/bin/env python
 # encoding: utf-8
-# Thomas Nagy, 2010-2015 (ita)
+# Thomas Nagy, 2010-2018 (ita)
 
-"""
-burn a book, save a tree
-"""
+from __future__ import with_statement
 
 import os
 
 all_modifs = {}
 
 def fixdir(dir):
-	"""call all the substitution functions on the waf folders"""
-	global all_modifs
+	"""Call all substitution functions on Waf folders"""
 	for k in all_modifs:
 		for v in all_modifs[k]:
 			modif(os.path.join(dir, 'waflib'), k, v)
 
 def modif(dir, name, fun):
-	"""execute a substitution function"""
+	"""Call a substitution function"""
 	if name == '*':
 		lst = []
 		for y in '. Tools extras'.split():
@@ -30,24 +27,17 @@ def modif(dir, name, fun):
 		return
 
 	filename = os.path.join(dir, name)
-	f = open(filename, 'r')
-	try:
+	with open(filename, 'r') as f:
 		txt = f.read()
-	finally:
-		f.close()
 
 	txt = fun(txt)
 
-	f = open(filename, 'w')
-	try:
+	with open(filename, 'w') as f:
 		f.write(txt)
-	finally:
-		f.close()
 
 def subst(*k):
 	"""register a substitution function"""
 	def do_subst(fun):
-		global all_modifs
 		for x in k:
 			try:
 				all_modifs[x].append(fun)
@@ -60,13 +50,15 @@ def subst(*k):
 def r1(code):
 	"utf-8 fixes for python < 2.6"
 	code = code.replace('as e:', ',e:')
-	code = code.replace(".decode(sys.stdout.encoding or 'iso8859-1')", '')
-	code = code.replace('.encode()', '')
-	return code
+	code = code.replace(".decode(sys.stdout.encoding or'latin-1',errors='replace')", '')
+	return code.replace('.encode()', '')
 
 @subst('Runner.py')
 def r4(code):
 	"generator syntax"
-	code = code.replace('next(self.biter)', 'self.biter.next()')
-	return code
+	return code.replace('next(self.biter)', 'self.biter.next()')
+
+@subst('Context.py')
+def r5(code):
+	return code.replace("('Execution failure: %s'%str(e),ex=e)", "('Execution failure: %s'%str(e),ex=e),None,sys.exc_info()[2]")
 
diff --git a/waflib/processor.py b/waflib/processor.py
new file mode 100755
index 00000000..2eecf3bd
--- /dev/null
+++ b/waflib/processor.py
@@ -0,0 +1,64 @@
+#! /usr/bin/env python
+# encoding: utf-8
+# Thomas Nagy, 2016-2018 (ita)
+
+import os, sys, traceback, base64, signal
+try:
+	import cPickle
+except ImportError:
+	import pickle as cPickle
+
+try:
+	import subprocess32 as subprocess
+except ImportError:
+	import subprocess
+
+try:
+	TimeoutExpired = subprocess.TimeoutExpired
+except AttributeError:
+	class TimeoutExpired(Exception):
+		pass
+
+def run():
+	txt = sys.stdin.readline().strip()
+	if not txt:
+		# parent process probably ended
+		sys.exit(1)
+	[cmd, kwargs, cargs] = cPickle.loads(base64.b64decode(txt))
+	cargs = cargs or {}
+
+	ret = 1
+	out, err, ex, trace = (None, None, None, None)
+	try:
+		proc = subprocess.Popen(cmd, **kwargs)
+		try:
+			out, err = proc.communicate(**cargs)
+		except TimeoutExpired:
+			if kwargs.get('start_new_session') and hasattr(os, 'killpg'):
+				os.killpg(proc.pid, signal.SIGKILL)
+			else:
+				proc.kill()
+			out, err = proc.communicate()
+			exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out)
+			exc.stderr = err
+			raise exc
+		ret = proc.returncode
+	except Exception as e:
+		exc_type, exc_value, tb = sys.exc_info()
+		exc_lines = traceback.format_exception(exc_type, exc_value, tb)
+		trace = str(cmd) + '\n' + ''.join(exc_lines)
+		ex = e.__class__.__name__
+
+	# it is just text so maybe we do not need to pickle()
+	tmp = [ret, out, err, ex, trace]
+	obj = base64.b64encode(cPickle.dumps(tmp))
+	sys.stdout.write(obj.decode())
+	sys.stdout.write('\n')
+	sys.stdout.flush()
+
+while 1:
+	try:
+		run()
+	except KeyboardInterrupt:
+		break
+