Merged default into default

22 files changed, 181 insertions, 8 deletions

diff --git a/doc/man/scons.xml b/doc/man/scons.xml
index b68f27ae..3268860b 100644
--- a/doc/man/scons.xml
+++ b/doc/man/scons.xml
@@ -2187,6 +2187,22 @@ platform name when the Environment is constructed. Changing the PATH
 variable after the Environment is constructed will not cause the tools to
 be redetected.</para>
 
+<para> One feature now present within Scons is the ability to have nested tools.
+Tools which can be located within a subdirectory in the toolpath.
+With a nested tool name the dot represents a directory seperator</para>
+
+<programlisting>
+# namespaced builder
+env = Environment(ENV = os.environ, tools = ['SubDir1.SubDir2.SomeTool'])
+env.SomeTool(targets, sources)
+
+# Search Paths
+# SCons\Tool\SubDir1\SubDir2\SomeTool.py
+# SCons\Tool\SubDir1\SubDir2\SomeTool\__init__.py
+# .\site_scons\site_tools\SubDir1\SubDir2\SomeTool.py
+# .\site_scons\site_tools\SubDir1\SubDir2\SomeTool\__init__.py
+</programlisting>
+
 <para>SCons supports the following tool specifications out of the box:</para>
 
 <!-- '\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -->
diff --git a/doc/user/builders-writing.xml b/doc/user/builders-writing.xml
index 07f2dec6..35dd989d 100644
--- a/doc/user/builders-writing.xml
+++ b/doc/user/builders-writing.xml
@@ -880,6 +880,53 @@ env2.Foo('file2')
 
   </section>
 
+  <section>
+  <title>Nested and namespace builders;</title>
+
+    <para>
+    &SCons; now supports the ability for a Builder to be located within a sub-directory of the toolpath.
+	This is similar to namespacing within python.
+
+    Normally when loading a tool into the environment, scons will search for the tool within two locations
+    </para>
+
+    <sconstruct>
+# Regular non namespace target
+env = Environment(ENV = os.environ, tools = ['SomeTool'])
+env.SomeTool(targets, sources)
+    </sconstruct>
+
+    <para>
+    The locations would include
+	<filename>SCons\Tool\SomeTool.py</filename>
+    <filename>SCons\Tool\SomeTool\__init__.py</filename>
+    <filename>.\site_scons\site_tools\SomeTool.py</filename>
+	<filename>.\site_scons\site_tools\SomeTool\__init__.py</filename>
+
+    If a toolpath is specified this is also searched as well.
+	With nested or namespaced tools we can use the dot notation to specify a sub-directoty that the tool is located under
+    </para>
+
+    <sconstruct>
+# namespaced target
+env = Environment(ENV = os.environ, tools = ['SubDir1.SubDir2.SomeTool'])
+env.SomeTool(targets, sources)
+    </sconstruct>
+
+    <para>
+    With this example the search locations would include
+	<filename>SCons\Tool\SubDir1\SubDir2\SomeTool.py</filename>
+    <filename>SCons\Tool\SubDir1\SubDir2\SomeTool\__init__.py</filename>
+    <filename>.\site_scons\site_tools\SubDir1\SubDir2\SomeTool.py</filename>
+	<filename>.\site_scons\site_tools\SubDir1\SubDir2\SomeTool\__init__.py</filename>
+
+    It's important to note when creating tools within sub-directories, there needs to be a __init__.py file within each directory.
+	This file can just be empty however.
+	This is the same constraint used by python when loading modules from within sub-directories (packages).
+
+    </para>
+  </section>
+
   <!--
 
   <section>
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 537aca39..745c69dc 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -7,6 +7,10 @@
 
 RELEASE 3.0.0.alpha.20170614 - Mon, 14 Jun 2017 12:23:56 -0400
 
+  From Richard West:
+    - Added nested / namespace tool support
+    - Added a small fix to the python3 tool loader when loading a tool as a package
+
   From William Blevins:
     - Updated D language scanner support to latest: 2.071.1. (PR #1924)
       https://dlang.org/spec/module.html accessed 11 August 2016
diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py
index 61b7788d..e5b4b05a 100644
--- a/src/engine/SCons/Tool/__init__.py
+++ b/src/engine/SCons/Tool/__init__.py
@@ -118,6 +118,16 @@ class Tool(object):
         if hasattr(module, 'options'):
             self.options = module.options
 
+    def _load_dotted_module_py2(self, short_name, full_name, searchpaths=None):
+        splitname = short_name.split('.')
+        index = 0
+        srchpths = searchpaths
+        for item in splitname:
+            file, path, desc = imp.find_module(item, srchpths)
+            mod = imp.load_module(full_name, file, path, desc)
+            srchpths = [path]
+        return mod, file
+
     def _tool_module(self):
         oldpythonpath = sys.path
         sys.path = self.toolpath + sys.path
@@ -127,15 +137,16 @@ class Tool(object):
             # Py 2 code
             try:
                 try:
-                    file, path, desc = imp.find_module(self.name, self.toolpath)
+                    file = None
                     try:
-                        return imp.load_module(self.name, file, path, desc)
-
+                        mod, file = self._load_dotted_module_py2(self.name, self.name, self.toolpath)
+                        return mod
                     finally:
                         if file:
                             file.close()
                 except ImportError as e:
-                    if str(e)!="No module named %s"%self.name:
+                    splitname = self.name.split('.')
+                    if str(e)!="No module named %s"%splitname[0]:
                         raise SCons.Errors.EnvironmentError(e)
                     try:
                         import zipimport
@@ -169,8 +180,9 @@ class Tool(object):
             found_name = self.name
             add_to_scons_tools_namespace = False
             for path in self.toolpath:
-                file_path = os.path.join(path, "%s.py"%self.name)
-                file_package = os.path.join(path, self.name)
+                sepname = self.name.replace('.', os.path.sep)
+                file_path = os.path.join(path, "%s.py"%sepname)
+                file_package = os.path.join(path, sepname)
 
                 if debug: sys.stderr.write("Trying:%s %s\n"%(file_path, file_package))
 
@@ -179,6 +191,7 @@ class Tool(object):
                     if debug: print("file_Path:%s FOUND"%file_path)
                     break
                 elif os.path.isdir(file_package):
+                    file_package = os.path.join(file_package, '__init__.py')
                     spec = importlib.util.spec_from_file_location(self.name, file_package)
                     if debug: print("PACKAGE:%s Found"%file_package)
                     break
@@ -231,8 +244,7 @@ class Tool(object):
             try:
                 smpath = sys.modules['SCons.Tool'].__path__
                 try:
-                    file, path, desc = imp.find_module(self.name, smpath)
-                    module = imp.load_module(full_name, file, path, desc)
+                    module, file = self._load_dotted_module_py2(self.name, full_name, smpath)
                     setattr(SCons.Tool, self.name, module)
                     if file:
                         file.close()
diff --git a/test/toolpath/nested/image/SConstruct b/test/toolpath/nested/image/SConstruct
new file mode 100644
index 00000000..284f21c5
--- /dev/null
+++ b/test/toolpath/nested/image/SConstruct
@@ -0,0 +1,15 @@
+# Test where tools are located under site_scons/site_tools

+env1 = Environment(tools=['subdir1.Site_TestTool1', 'subdir1.subdir2.Site_TestTool2', 'subdir1.Site_TestTool3'])

+print("env1['Site_TestTool1'] =", env1.get('Site_TestTool1'))

+print("env1['Site_TestTool2'] =", env1.get('Site_TestTool2'))

+print("env1['Site_TestTool3'] =", env1.get('Site_TestTool3'))

+

+# Test where toolpath is set in the env constructor

+env2 = Environment(tools=['subdir1.Toolpath_TestTool1', 'subdir1.subdir2.Toolpath_TestTool2', 'subdir1.Toolpath_TestTool3'], toolpath=['tools'])

+print("env2['Toolpath_TestTool1'] =", env2.get('Toolpath_TestTool1'))

+print("env2['Toolpath_TestTool2'] =", env2.get('Toolpath_TestTool2'))

+print("env2['Toolpath_TestTool3'] =", env2.get('Toolpath_TestTool3'))

+

+base = Environment(tools=[], toolpath=['tools'])

+derived = base.Clone(tools=['subdir1.Toolpath_TestTool1'])

+print("derived['Toolpath_TestTool1'] =", derived.get('Toolpath_TestTool1'))

diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool1.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool1.py
new file mode 100644
index 00000000..d4a19a82
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool1.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Site_TestTool1'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/__init__.py
new file mode 100644
index 00000000..60bbd02f
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/__init__.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Site_TestTool3'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/sconstest.skip b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Site_TestTool3/sconstest.skip
diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/__init__.py
diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/sconstest.skip b/test/toolpath/nested/image/site_scons/site_tools/subdir1/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/sconstest.skip
diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Site_TestTool2.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Site_TestTool2.py
new file mode 100644
index 00000000..adae55bb
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Site_TestTool2.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Site_TestTool2'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/__init__.py
diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/sconstest.skip b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/sconstest.skip
diff --git a/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool1.py b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool1.py
new file mode 100644
index 00000000..072daf0f
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool1.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Toolpath_TestTool1'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/__init__.py b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/__init__.py
new file mode 100644
index 00000000..26bc7482
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/__init__.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Toolpath_TestTool3'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/sconstest.skip b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/Toolpath_TestTool3/sconstest.skip
diff --git a/test/toolpath/nested/image/tools/subdir1/__init__.py b/test/toolpath/nested/image/tools/subdir1/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/__init__.py
diff --git a/test/toolpath/nested/image/tools/subdir1/sconstest.skip b/test/toolpath/nested/image/tools/subdir1/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/sconstest.skip
diff --git a/test/toolpath/nested/image/tools/subdir1/subdir2/Toolpath_TestTool2.py b/test/toolpath/nested/image/tools/subdir1/subdir2/Toolpath_TestTool2.py
new file mode 100644
index 00000000..f4ccefee
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/subdir2/Toolpath_TestTool2.py
@@ -0,0 +1,4 @@
+def generate(env):

+    env['Toolpath_TestTool2'] = 1

+def exists(env):

+    return 1

diff --git a/test/toolpath/nested/image/tools/subdir1/subdir2/__init__.py b/test/toolpath/nested/image/tools/subdir1/subdir2/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/subdir2/__init__.py
diff --git a/test/toolpath/nested/image/tools/subdir1/subdir2/sconstest.skip b/test/toolpath/nested/image/tools/subdir1/subdir2/sconstest.skip
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/toolpath/nested/image/tools/subdir1/subdir2/sconstest.skip
diff --git a/test/toolpath/nested/nested.py b/test/toolpath/nested/nested.py
new file mode 100644
index 00000000..7304dd57
--- /dev/null
+++ b/test/toolpath/nested/nested.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import os.path
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+test.dir_fixture('image')
+
+test.run(arguments = '.', stdout = """\
+scons: Reading SConscript files ...
+env1['Site_TestTool1'] = 1
+env1['Site_TestTool2'] = 1
+env1['Site_TestTool3'] = 1
+env2['Toolpath_TestTool1'] = 1
+env2['Toolpath_TestTool2'] = 1
+env2['Toolpath_TestTool3'] = 1
+derived['Toolpath_TestTool1'] = 1
+scons: done reading SConscript files.
+scons: Building targets ...
+scons: `.' is up to date.
+scons: done building targets.
+""")
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4: