add win_flex as option for windows, add choco default path, and add flag for nounistd on windows. also more testing

11 files changed, 231 insertions, 42 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 1849707ec..cbf723376 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -18,12 +18,12 @@ install:
   - cmd: "C:\\%WINPYTHON%\\python.exe --version"
   - cmd: for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -m site --user-site') do (set PYSITEDIR=%%g)
   # use mingw 32 bit until #3291 is resolved
-  - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\ProgramData\\chocolatey\\bin;%PATH%"
+  - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;%PATH%"
   - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel "
   - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pypiwin32 coverage codecov"
   - cmd: set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml
   # install 3rd party tools to test with
-  - cmd: choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc
+  - cmd: choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc winflexbison
   - cmd: set
 
   ### LINUX ###
diff --git a/src/engine/SCons/Platform/win32.py b/src/engine/SCons/Platform/win32.py
index 2d40fb8f3..be3054613 100644
--- a/src/engine/SCons/Platform/win32.py
+++ b/src/engine/SCons/Platform/win32.py
@@ -43,6 +43,10 @@ from SCons.Platform.virtualenv import ImportVirtualenv
 from SCons.Platform.virtualenv import ignore_virtualenv, enable_virtualenv
 import SCons.Util
 
+CHOCO_DEFAULT_PATH = [
+    r'C:\ProgramData\chocolatey\bin'
+]
+
 try:
     import msvcrt
     import win32api
diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py
index 70c0c5f3f..3506f7c12 100644
--- a/src/engine/SCons/Tool/lex.py
+++ b/src/engine/SCons/Tool/lex.py
@@ -41,6 +41,7 @@ import SCons.Tool
 import SCons.Util
 from SCons.Platform.mingw import MINGW_DEFAULT_PATHS
 from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS
+from SCons.Platform.win32 import CHOCO_DEFAULT_PATH
 
 LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR")
 
@@ -74,17 +75,19 @@ def get_lex_path(env, append_paths=False):
     """
     # save existing path to reset if we don't want to append any paths
     envPath = env['ENV']['PATH']
-    bins = ['lex', 'flex']
+    bins = ['win_flex', 'lex', 'flex']
 
     for prog in bins:
-        bin_path = SCons.Tool.find_program_path(env, prog, default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
+        bin_path = SCons.Tool.find_program_path(
+            env, 
+            prog, 
+            default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
         if bin_path:
             if not append_paths:
                 env['ENV']['PATH'] = envPath
             else:
                 env.AppendENVPath('PATH', os.path.dirname(bin_path))
             return bin_path
-
     SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH')
 
 
@@ -92,9 +95,6 @@ def generate(env):
     """Add Builders and construction variables for lex to an Environment."""
     c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
 
-    if sys.platform == 'win32':
-        get_lex_path(env, append_paths=True)
-
     # C
     c_file.add_action(".l", LexAction)
     c_file.add_emitter(".l", lexEmitter)
@@ -109,10 +109,16 @@ def generate(env):
     # C++
     cxx_file.add_action(".ll", LexAction)
     cxx_file.add_emitter(".ll", lexEmitter)
-    
-    env["LEX"] = env.Detect("flex") or "lex"
-    env["LEXFLAGS"] = SCons.Util.CLVar("")
-    env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
+
+    if sys.platform == 'win32':
+        get_lex_path(env, append_paths=True)
+        env["LEX"] = env.Detect(['win_flex', 'lex', 'flex'])
+        env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd")
+        env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET"
+    else:
+        env["LEX"] = env.Detect(["flex", "lex"])
+        env["LEXFLAGS"] = SCons.Util.CLVar("")
+        env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
 
 def exists(env):
     if sys.platform == 'win32':
diff --git a/src/engine/SCons/Tool/lex.xml b/src/engine/SCons/Tool/lex.xml
index 0388ee355..f933451e5 100644
--- a/src/engine/SCons/Tool/lex.xml
+++ b/src/engine/SCons/Tool/lex.xml
@@ -33,6 +33,7 @@ Sets construction variables for the &lex; lexical analyser.
 <item>LEX</item>
 <item>LEXFLAGS</item>
 <item>LEXCOM</item>
+<item>LEXUNISTD</item>
 </sets>
 <uses>
 <item>LEXCOMSTR</item>
@@ -78,4 +79,12 @@ General options passed to the lexical analyzer generator.
 </summary>
 </cvar>
 
+<cvar name="LEXUNISTD">
+<summary>
+<para>
+Used only on windows environments to set a lex flag to prevent 'unistd.h' from being included. The default value is '--nounistd'.
+</para>
+</summary>
+</cvar>
+
 </sconsdoc>
diff --git a/test/CFILESUFFIX.py b/test/CFILESUFFIX.py
index 0a3a81a7a..410ece5fe 100644
--- a/test/CFILESUFFIX.py
+++ b/test/CFILESUFFIX.py
@@ -39,7 +39,11 @@ test = TestSCons.TestSCons()
 test.write('mylex.py', """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 't', [])
+if sys.platform == 'win32':
+    longopts = ['nounistd']
+else:
+    longopts = []
+cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts)
 for a in args:
     contents = open(a, 'rb').read()
     sys.stdout.write((contents.replace(b'LEX', b'mylex.py')).decode())
diff --git a/test/CXX/CXXFILESUFFIX.py b/test/CXX/CXXFILESUFFIX.py
index 9442408fe..c8dbf0aa7 100644
--- a/test/CXX/CXXFILESUFFIX.py
+++ b/test/CXX/CXXFILESUFFIX.py
@@ -35,7 +35,11 @@ test = TestSCons.TestSCons()
 test.write('mylex.py', """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 't', [])
+if sys.platform == 'win32':
+    longopts = ['nounistd']
+else:
+    longopts = []
+cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts)
 for a in args:
     contents = open(a, 'r').read()
     sys.stdout.write(contents.replace('LEX', 'mylex.py'))
diff --git a/test/LEX/LEX.py b/test/LEX/LEX.py
index 1239c6b49..65e449725 100644
--- a/test/LEX/LEX.py
+++ b/test/LEX/LEX.py
@@ -38,7 +38,11 @@ test = TestSCons.TestSCons()
 test.write('mylex.py', """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 't', [])
+if sys.platform == 'win32':
+    longopts = ['nounistd']
+else:
+    longopts = []
+cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts)
 for a in args:
     contents = open(a, 'rb').read()
     sys.stdout.write(contents.replace(b'LEX', b'mylex.py').decode())
diff --git a/test/LEX/LEXFLAGS.py b/test/LEX/LEXFLAGS.py
index 54df16177..51b6614f2 100644
--- a/test/LEX/LEXFLAGS.py
+++ b/test/LEX/LEXFLAGS.py
@@ -25,6 +25,7 @@
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os
+import sys
 
 import TestSCons
 
@@ -35,22 +36,25 @@ test = TestSCons.TestSCons()
 
 test.subdir('in')
 
-
-
 test.write('mylex.py', """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 'I:tx', [])
+import os
+if sys.platform == 'win32':
+    longopts = ['nounistd']
+else:
+    longopts = []
+cmd_opts, args = getopt.getopt(sys.argv[1:], 'I:tx', longopts)
 opt_string = ''
 i_arguments = ''
 for opt, arg in cmd_opts:
     if opt == '-I': i_arguments = i_arguments + ' ' + arg
     else: opt_string = opt_string + ' ' + opt
 for a in args:
-    contents = open(a, 'rb').read()
-    contents = contents.replace(b'LEXFLAGS', opt_string.encode())
-    contents = contents.replace(b'I_ARGS', i_arguments.encode())
-    sys.stdout.write(contents.decode())
+    contents = open(a, 'r').read()
+    contents = contents.replace('LEXFLAGS', opt_string)
+    contents = contents.replace('I_ARGS', i_arguments)
+    sys.stdout.write(contents)
 sys.exit(0)
 """)
 
@@ -61,13 +65,16 @@ env = Environment(LEX = r'%(_python_)s mylex.py',
 env.CFile(target = 'out/aaa', source = 'in/aaa.l')
 """ % locals())
 
-test.write(['in', 'aaa.l'],		"aaa.l\nLEXFLAGS\nI_ARGS\n")
+test.write(['in', 'aaa.l'], "aaa.l\nLEXFLAGS\nI_ARGS\n")
 
 test.run('.', stderr = None)
 
+lexflags = ' -x -t'
+if sys.platform == 'win32':
+    lexflags = ' --nounistd' + lexflags
 # Read in with mode='r' because mylex.py implicitley wrote to stdout
 # with mode='w'.
-test.must_match(['out', 'aaa.c'],	"aaa.l\n -x -t\n out in\n", mode='r')
+test.must_match(['out', 'aaa.c'],	"aaa.l\n%s\n out in\n" % lexflags, mode='r')
 
 
 
diff --git a/test/LEX/live.py b/test/LEX/live.py
index 853e97ca3..91a2d4989 100644
--- a/test/LEX/live.py
+++ b/test/LEX/live.py
@@ -28,8 +28,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 Test LEX and LEXFLAGS with a live lex.
 """
 
-import sys
-
 import TestSCons
 
 _exe = TestSCons._exe
@@ -37,29 +35,18 @@ _python_ = TestSCons._python_
 
 test = TestSCons.TestSCons()
 
-lex = test.where_is('lex') or test.where_is('flex')
+lex = test.where_is('win_flex') or test.where_is('lex') or test.where_is('flex')
 
 if not lex:
     test.skip_test('No lex or flex found; skipping test.\n')
 
-tools = "'default'"
-if sys.platform == 'win32':
-    # make sure mingw is installed on win32
-    if not test.where_is('gcc'):
-        test.skip_test('No mingw on windows; skipping test.\n')
-    # lex on win32 has a dependencies on mingw for unix headers
-    # so add it as a tool to the environment.
-    tools += ", 'mingw'"
-
-
 test.file_fixture('wrapper.py')
 
 test.write('SConstruct', """
-foo = Environment(tools=[%(tools)s])
+foo = Environment()
 lex = foo.Dictionary('LEX')
 bar = Environment(LEX = r'%(_python_)s wrapper.py ' + lex,
-                  LEXFLAGS = '-b',
-                  tools=[%(tools)s])
+                  LEXFLAGS = '-b')
 foo.Program(target = 'foo', source = 'foo.l')
 bar.Program(target = 'bar', source = 'bar.l')
 """ % locals())
@@ -100,8 +87,6 @@ test.must_exist(test.workpath('lex.backup'))
 
 test.run(program = test.workpath('bar'), stdin = "b\n", stdout = "Bbar.lB\n")
 
-
-
 test.pass_test()
 
 # Local Variables:
diff --git a/test/LEX/live_mingw.py b/test/LEX/live_mingw.py
new file mode 100644
index 000000000..13e23429b
--- /dev/null
+++ b/test/LEX/live_mingw.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""
+Test LEX and LEXFLAGS and unistd.h with a live lex in mingw environment.
+"""
+
+import sys
+
+import TestSCons
+
+_exe = TestSCons._exe
+_python_ = TestSCons._python_
+
+test = TestSCons.TestSCons()
+
+if sys.platform != 'win32':
+    test.skip_test('Not windows environment; skipping test.\n')
+
+if not test.where_is('gcc'):
+        test.skip_test('No mingw or cygwin on windows; skipping test.\n')
+
+lex = test.where_is('lex') or test.where_is('flex')
+
+if not lex:
+    test.skip_test('No lex or flex found; skipping test.\n')
+
+test.file_fixture('wrapper.py')
+
+test.write('SConstruct', """
+foo = Environment(tools=['default', 'mingw', 'lex'], LEXUNISTD="")
+lex = foo.Dictionary('LEX')
+bar = Environment(LEX = r'%(_python_)s wrapper.py ' + lex,
+                  LEXFLAGS = '-b',
+                  LEXUNISTD="",
+                  tools=['default', 'mingw', 'lex'])
+foo.Program(target = 'foo', source = 'foo.l')
+bar.Program(target = 'bar', source = 'bar.l')
+""" % locals())
+
+lex = r"""
+%%%%
+a       printf("A%sA");
+b       printf("B%sB");
+%%%%
+int
+yywrap()
+{
+    return 1;
+}
+
+int
+main()
+{
+    yylex();
+}
+"""
+
+test.write('foo.l', lex % ('foo.l', 'foo.l'))
+
+test.write('bar.l', lex % ('bar.l', 'bar.l'))
+
+test.run(arguments = 'foo' + _exe, stderr = None)
+
+test.must_not_exist(test.workpath('wrapper.out'))
+test.must_not_exist(test.workpath('lex.backup'))
+
+test.run(program = test.workpath('foo'), stdin = "a\n", stdout = "Afoo.lA\n")
+
+test.run(arguments = 'bar' + _exe)
+
+test.must_match(test.workpath('wrapper.out'), "wrapper.py\n")
+test.must_exist(test.workpath('lex.backup'))
+
+test.run(program = test.workpath('bar'), stdin = "b\n", stdout = "Bbar.lB\n")
+test.must_contain(test.workpath('bar.c'), "unistd.h")
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/LEX/no_lex.py b/test/LEX/no_lex.py
new file mode 100644
index 000000000..89ffdc70b
--- /dev/null
+++ b/test/LEX/no_lex.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""
+Test Environments are functional and return None when no lex tool is found.
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+test.write('SConstruct', """
+import SCons
+
+def no_lex(env, key_program, default_paths=[]):
+    return None
+
+class TestEnvironment(SCons.Environment.Environment):
+    def Detect(self, progs):
+        return None
+
+SCons.Tool.find_program_path = no_lex
+
+foo = TestEnvironment(tools=['default', 'lex'])
+print(foo.Dictionary('LEX'))
+""" % locals())
+
+test.run(arguments = '-Q -s', stdout = 'None\n' )
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4: