Merge branch 'master' into patch-1

20 files changed, 248 insertions, 152 deletions

diff --git a/.travis.yml b/.travis.yml
index 8fdcc6b..b33e49b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,5 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-install:
-  - "pip install . "
+install: true
 script: "cd test && python testlex.py && python testyacc.py"
diff --git a/ANNOUNCE b/ANNOUNCE
deleted file mode 100644
index 3e58250..0000000
--- a/ANNOUNCE
+++ /dev/null
@@ -1,40 +0,0 @@
-February 15, 2018
-
-                  Announcing :  PLY-3.11 (Python Lex-Yacc)
-
-                        http://www.dabeaz.com/ply
-
-I'm pleased to announce PLY-3.11--a pure Python implementation of the
-common parsing tools lex and yacc.  PLY-3.11 is a minor bug fix
-release.  It supports both Python 2 and Python 3.
-
-If you are new to PLY, here are a few highlights:
-
--  PLY is closely modeled after traditional lex/yacc.  If you know how 
-   to use these or similar tools in other languages, you will find
-   PLY to be comparable.
-
--  PLY provides very extensive error reporting and diagnostic
-   information to assist in parser construction.  The original
-   implementation was developed for instructional purposes.  As
-   a result, the system tries to identify the most common types
-   of errors made by novice users.
-
--  PLY provides full support for empty productions, error recovery,
-   precedence rules, and ambiguous grammars.
-
--  Parsing is based on LR-parsing which is fast, memory efficient,
-   better suited to large grammars, and which has a number of nice
-   properties when dealing with syntax errors and other parsing 
-   problems. Currently, PLY can build its parsing tables using 
-   either SLR or LALR(1) algorithms. 
-
-More information about PLY can be obtained on the PLY webpage at:
-
-                   http://www.dabeaz.com/ply
-
-PLY is freely available.
-
-Cheers,
-
-David Beazley (http://www.dabeaz.com)
\ No newline at end of file
diff --git a/CHANGES b/CHANGES
index 4405007..9a90a55 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,22 @@
+Current Version
+---------------
+IMPORTANT NOTE (2018-12-22): PLY is no longer be released in any
+package-installable format. If you want to use the latest version, you
+need to COPY the contents of the ply/ directory into your own project
+and use it. Although PLY is no longer distributed as a package, it is
+maintained as a mature library.  No new features are planned, but 
+issues and pull requests for bugs are still welcome. Any changes to the 
+software will be noted here.
+
+01/19/19  Some improvements to the preprocessor module contributed by
+          Rob Reilink.  Issue #195 fixes the evaluation of expressions
+          such as #if a != b.   Issue #196 fixes some some issues
+          related to the lack of a specified Unicode encoding on reading
+          text files.
+
+12/22/18  Incorporated some minor changes to eliminate warning messages
+          related to open() calls.
+
 Version 3.11
 ---------------------
 02/15/18  beazley
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..6da57c4
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,18 @@
+Contributing to PLY
+===================
+
+PLY is a mature project that no longer makes releases. New features
+are no longer being added to it.  However, if you feel that you have
+found a bug in PLY or its documentation, please submit an issue or a
+pull request.
+
+Important note: The Github repo for PLY always contains the most
+up-to-date version of the software.   If you want to use the current
+version, you should COPY the contents of the `ply/` directory into
+your own project and use it.  There will be no future package-installable
+releases of PLY.
+
+
+
+
+
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index 0d37431..0000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,8 +0,0 @@
-recursive-include example *
-recursive-include doc *
-recursive-include test *
-include ANNOUNCE
-include README.md
-include CHANGES
-include TODO
-global-exclude *.pyc
diff --git a/README.md b/README.md
index bef5b3b..52fa31d 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-# PLY (Python Lex-Yacc)                   Version 3.11
+# PLY (Python Lex-Yacc)
 
 [![Build Status](https://travis-ci.org/dabeaz/ply.svg?branch=master)](https://travis-ci.org/dabeaz/ply)
 
-Copyright (C) 2001-2018
+Copyright (C) 2001-2019
 David M. Beazley (Dabeaz LLC)
 All rights reserved.
 
@@ -78,19 +78,19 @@ To use PLY, simply copy the 'ply' directory to your project and import
 lex and yacc from the associated 'ply' package.  For example:
 
 ```python
-import ply.lex as lex
-import ply.yacc as yacc
+from .ply import lex
+from .ply import yacc
 ```
 
 Alternatively, you can copy just the files lex.py and yacc.py
-individually and use them as modules.  For example:
+individually and use them as modules however you see fit.  For example:
 
 ```python
 import lex
 import yacc
 ```
 
-The file setup.py can be used to install ply using distutils.
+PLY has no third-party dependencies. 
 
 The file doc/ply.html contains complete documentation on how to use
 the system.
@@ -122,7 +122,7 @@ The GitHub page for PLY can be found at:
 
 * https://github.com/dabeaz/ply
 
-An old and relatively inactive discussion group for PLY is found at:
+An old and inactive discussion group for PLY is found at:
 
 * http://groups.google.com/group/ply-hack
 
@@ -262,11 +262,10 @@ Bug Reports and Patches
 My goal with PLY is to simply have a decent lex/yacc implementation
 for Python.  As a general rule, I don't spend huge amounts of time
 working on it unless I receive very specific bug reports and/or
-patches to fix problems. I also try to incorporate submitted feature
-requests and enhancements into each new version.  Please visit the PLY
-github page at https://github.com/dabeaz/ply to submit issues and pull
-requests.  To contact me about bugs and/or new features, please send
-email to dave@dabeaz.com.
+patches to fix problems. At this time, PLY is mature software and new
+features are no longer being added.  If you think you have found a
+bug, please visit the PLY Github page at https://github.com/dabeaz/ply
+to report an issue.
  
 -- Dave
 
diff --git a/TODO b/TODO
deleted file mode 100644
index f4800aa..0000000
--- a/TODO
+++ /dev/null
@@ -1,16 +0,0 @@
-The PLY to-do list:
-
-1. Finish writing the C Preprocessor module.  Started in the
-   file ply/cpp.py
-
-2. Create and document libraries of useful tokens.
-
-3. Expand the examples/yply tool that parses bison/yacc
-   files.
-
-4. Think of various diabolical things to do with the
-   new yacc internals.  For example, it is now possible
-   to specify grammrs using completely different schemes
-   than the reflection approach used by PLY.
-
-
diff --git a/doc/makedoc.py b/doc/makedoc.py
index 415a53a..e5cbdb0 100644
--- a/doc/makedoc.py
+++ b/doc/makedoc.py
@@ -25,7 +25,7 @@ alink = re.compile(r"<a *name *= *\"(.*)\"></a>", re.IGNORECASE)
 heading = re.compile(r"(_nn\d)", re.IGNORECASE)
 
 def getheadingname(m):
-    autogeneratedheading = True;
+    autogeneratedheading = True
     if m.group(1) != None:
         amatch = alink.match(m.group(1))
         if amatch:
@@ -65,8 +65,9 @@ h3 = re.compile(r".*?<H3>(<a.*a>)*[\d\.\s]*(.*?)</H3>", re.IGNORECASE)
 h4 = re.compile(r".*?<H4>(<a.*a>)*[\d\.\s]*(.*?)</H4>", re.IGNORECASE)
 h5 = re.compile(r".*?<H5>(<a.*a>)*[\d\.\s]*(.*?)</H5>", re.IGNORECASE)
 
-data = open(filename).read()            # Read data
-open(filename+".bak","w").write(data)   # Make backup
+# Make backup
+with open(filename) as src, open(filename+".bak","w") as dst:
+    dst.write(src.read())
 
 lines = data.splitlines()
 result = [ ] # This is the result of postprocessing the file
@@ -82,7 +83,7 @@ for s in lines:
             skip = 1
         else:
             skip = 0
-        continue;
+        continue
     if skip:
         continue
 
@@ -186,9 +187,8 @@ index += "</div>\n<!-- INDEX -->\n"
 
 data = "\n".join(result)
 
-data = data.replace("@INDEX@",index) + "\n";
+data = data.replace("@INDEX@",index) + "\n"
 
 # Write the file back out
-open(filename,"w").write(data)
-
-
+with open(filename,"w") as f:
+    f.write(data)
diff --git a/doc/ply.html b/doc/ply.html
index b35ba44..6b8aca9 100644
--- a/doc/ply.html
+++ b/doc/ply.html
@@ -353,7 +353,7 @@ while True:
 
 The <tt>tok.type</tt> and <tt>tok.value</tt> attributes contain the
 type and value of the token itself. 
-<tt>tok.line</tt> and <tt>tok.lexpos</tt> contain information about
+<tt>tok.lineno</tt> and <tt>tok.lexpos</tt> contain information about
 the location of the token.  <tt>tok.lexpos</tt> is the index of the
 token relative to the start of the input text.
 
@@ -3463,7 +3463,7 @@ large grammar, you should probably reconsider--the parsing tables are meant to d
 </p>
 
 <p>
-During operation, is is normal for PLY to produce diagnostic error
+During operation, it is normal for PLY to produce diagnostic error
 messages (usually printed to standard error).  These are generated
 entirely using the <tt>logging</tt> module.  If you want to redirect
 these messages or silence them, you can provide your own logging
diff --git a/example/BASIC/basic.py b/example/BASIC/basic.py
index 70ac9e7..17687b1 100644
--- a/example/BASIC/basic.py
+++ b/example/BASIC/basic.py
@@ -15,7 +15,8 @@ import basinterp
 # If a runtime error occurs, we bail out and enter
 # interactive mode below
 if len(sys.argv) == 2:
-    data = open(sys.argv[1]).read()
+    with open(sys.argv[1]) as f:
+        data = f.read()
     prog = basparse.parse(data)
     if not prog:
         raise SystemExit
diff --git a/example/BASIC/basiclog.py b/example/BASIC/basiclog.py
index 9dcc7fe..9258e29 100644
--- a/example/BASIC/basiclog.py
+++ b/example/BASIC/basiclog.py
@@ -23,7 +23,8 @@ import basinterp
 # If a runtime error occurs, we bail out and enter
 # interactive mode below
 if len(sys.argv) == 2:
-    data = open(sys.argv[1]).read()
+    with open(sys.argv[1]) as f:
+        data = f.read()
     prog = basparse.parse(data, debug=log)
     if not prog:
         raise SystemExit
diff --git a/ply/cpp.py b/ply/cpp.py
index 2422916..50a44a1 100644
--- a/ply/cpp.py
+++ b/ply/cpp.py
@@ -1,10 +1,38 @@
 # -----------------------------------------------------------------------------
-# cpp.py
+# ply: cpp.py
 #
-# Author:  David Beazley (http://www.dabeaz.com)
-# Copyright (C) 2007
-# All rights reserved
+# Copyright (C) 2001-2019
+# David M. Beazley (Dabeaz LLC)
+# All rights reserved.
 #
+# Latest version: https://github.com/dabeaz/ply
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of David Beazley or Dabeaz LLC may be used to
+#   endorse or promote products derived from this software without
+#   specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+
 # This module implements an ANSI-C style lexical preprocessor for PLY.
 # -----------------------------------------------------------------------------
 from __future__ import generators
@@ -436,7 +464,7 @@ class Preprocessor(object):
     # representing the replacement macro tokens
     # ----------------------------------------------------------------------
 
-    def macro_expand_args(self,macro,args):
+    def macro_expand_args(self,macro,args,expanded):
         # Make a copy of the macro token sequence
         rep = [copy.copy(_x) for _x in macro.value]
 
@@ -460,16 +488,16 @@ class Preprocessor(object):
         # has been sorted in reverse order of patch location since replacements will cause the
         # size of the replacement sequence to expand from the patch point.
 
-        expanded = { }
+        expanded_args = { }
         for ptype, argnum, i in macro.patch:
             # Concatenation.   Argument is left unexpanded
             if ptype == 'c':
                 rep[i:i+1] = args[argnum]
             # Normal expansion.  Argument is macro expanded first
             elif ptype == 'e':
-                if argnum not in expanded:
-                    expanded[argnum] = self.expand_macros(args[argnum])
-                rep[i:i+1] = expanded[argnum]
+                if argnum not in expanded_args:
+                    expanded_args[argnum] = self.expand_macros(args[argnum],expanded)
+                rep[i:i+1] = expanded_args[argnum]
 
         # Get rid of removed comma if necessary
         if comma_patch:
@@ -530,7 +558,7 @@ class Preprocessor(object):
                                         del args[len(m.arglist):]
 
                                 # Get macro replacement text
-                                rep = self.macro_expand_args(m,args)
+                                rep = self.macro_expand_args(m,args,expanded)
                                 rep = self.expand_macros(rep,expanded)
                                 for r in rep:
                                     r.lineno = t.lineno
@@ -589,11 +617,21 @@ class Preprocessor(object):
                 del tokens[i+1:j+1]
             i += 1
         tokens = self.expand_macros(tokens)
+        return self.evalexpr_expanded(tokens)
+
+    # ----------------------------------------------------------------------
+    # evalexpr_expanded()
+    #
+    # Helper for evalexpr that evaluates the expression that had its macros
+    # and defined(...) expressions expanded by evalexpr
+    # ----------------------------------------------------------------------
+
+    def evalexpr_expanded(self, tokens):
         for i,t in enumerate(tokens):
             if t.type == self.t_ID:
                 tokens[i] = copy.copy(t)
                 tokens[i].type = self.t_INTEGER
-                tokens[i].value = self.t_INTEGER_TYPE("0L")
+                tokens[i].value = self.t_INTEGER_TYPE("0")
             elif t.type == self.t_INTEGER:
                 tokens[i] = copy.copy(t)
                 # Strip off any trailing suffixes
@@ -601,10 +639,19 @@ class Preprocessor(object):
                 while tokens[i].value[-1] not in "0123456789abcdefABCDEF":
                     tokens[i].value = tokens[i].value[:-1]
 
-        expr = "".join([str(x.value) for x in tokens])
+        return self.evalexpr_string("".join([str(x.value) for x in tokens]))
+
+    # ----------------------------------------------------------------------
+    # evalexpr_string()
+    #
+    # Helper for evalexpr that evaluates a string expression
+    # This implementation does basic C->python conversion and then uses eval()
+    # ----------------------------------------------------------------------
+    def evalexpr_string(self, expr):
         expr = expr.replace("&&"," and ")
         expr = expr.replace("||"," or ")
         expr = expr.replace("!"," not ")
+        expr = expr.replace(" not ="," !=")
         try:
             result = eval(expr)
         except Exception:
@@ -777,7 +824,7 @@ class Preprocessor(object):
         for p in path:
             iname = os.path.join(p,filename)
             try:
-                data = open(iname,"r").read()
+                data = self.read_include_file(iname)
                 dname = os.path.dirname(iname)
                 if dname:
                     self.temp_path.insert(0,dname)
@@ -792,6 +839,19 @@ class Preprocessor(object):
             print("Couldn't find '%s'" % filename)
 
     # ----------------------------------------------------------------------
+    # read_include_file()
+    #
+    # Reads a source file for inclusion using #include
+    # Could be overridden to e.g. customize encoding, limit access to
+    # certain paths on the filesystem, or provide the contents of system
+    # include files
+    # ----------------------------------------------------------------------
+
+    def read_include_file(self, filepath):
+        with open(filepath, 'r', encoding='utf-8', errors='surrogateescape') as file:
+            return file.read()
+
+    # ----------------------------------------------------------------------
     # define()
     #
     # Define a new macro
@@ -903,8 +963,8 @@ if __name__ == '__main__':
 
     # Run a preprocessor
     import sys
-    f = open(sys.argv[1])
-    input = f.read()
+    with open(sys.argv[1]) as f:
+        input = f.read()
 
     p = Preprocessor(lexer)
     p.parse(input,sys.argv[1])
diff --git a/ply/lex.py b/ply/lex.py
index f95bcdb..bc9ed34 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -1,10 +1,12 @@
 # -----------------------------------------------------------------------------
 # ply: lex.py
 #
-# Copyright (C) 2001-2018
+# Copyright (C) 2001-2019
 # David M. Beazley (Dabeaz LLC)
 # All rights reserved.
 #
+# Latest version: https://github.com/dabeaz/ply
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
@@ -14,9 +16,9 @@
 # * Redistributions in binary form must reproduce the above copyright notice,
 #   this list of conditions and the following disclaimer in the documentation
 #   and/or other materials provided with the distribution.
-# * Neither the name of the David Beazley or Dabeaz LLC may be used to
+# * Neither the name of David Beazley or Dabeaz LLC may be used to
 #   endorse or promote products derived from this software without
-#  specific prior written permission.
+#   specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
@@ -1055,9 +1057,8 @@ def runmain(lexer=None, data=None):
     if not data:
         try:
             filename = sys.argv[1]
-            f = open(filename)
-            data = f.read()
-            f.close()
+            with open(filename) as f:
+                data = f.read()
         except IndexError:
             sys.stdout.write('Reading from standard input (type EOF to end):\n')
             data = sys.stdin.read()
diff --git a/ply/yacc.py b/ply/yacc.py
index 88188a1..108c43d 100644
--- a/ply/yacc.py
+++ b/ply/yacc.py
@@ -1,10 +1,12 @@
 # -----------------------------------------------------------------------------
 # ply: yacc.py
 #
-# Copyright (C) 2001-2018
+# Copyright (C) 2001-2019
 # David M. Beazley (Dabeaz LLC)
 # All rights reserved.
 #
+# Latest version: https://github.com/dabeaz/ply
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
@@ -14,9 +16,9 @@
 # * Redistributions in binary form must reproduce the above copyright notice,
 #   this list of conditions and the following disclaimer in the documentation
 #   and/or other materials provided with the distribution.
-# * Neither the name of the David Beazley or Dabeaz LLC may be used to
+# * Neither the name of David Beazley or Dabeaz LLC may be used to
 #   endorse or promote products derived from this software without
-#  specific prior written permission.
+#   specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 1eee7db..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-[bdist_wheel]
-universal = 1
-
-[metadata]
-description-file = README.md
diff --git a/setup.md b/setup.md
new file mode 100644
index 0000000..967874a
--- /dev/null
+++ b/setup.md
@@ -0,0 +1,40 @@
+# Maintained, No Package Releases
+
+PLY is maintained software, but no longer produces package releases.
+There is no `setup.py` file.  It is not something that you install
+with `pip` or a similar tool. You must COPY the necessary code from
+PLY into your project and take ownership of it.
+
+Why this policy? PLY is a highly specialized tool for expert-level
+programmers who are writing parsers and compilers.  If you are writing
+a compiler, there's a good chance that it's part of a substantially
+larger project.  Managing external dependencies (such as PLY) in such
+projects is an ongoing challenge.  However, the truth of the matter is
+that PLY just isn't that big.  All of the core functionality is
+contained in just two files.  PLY has no external dependencies of its
+own.  It changes very rarely.  Plus, there are various customizations
+that you might want to apply to how it works.  So, all things equal,
+it's probably better for you to copy it.  
+
+But what about getting all of the latest improvements and bug fixes?
+What improvements? PLY is implementing a 1970s-era parsing algorithm.
+It's not cutting edge.  As for bug fixes, you'll know pretty rapidly
+if PLY works for your project or not.  If it's working, there's
+literally no reason to ever upgrade it. Keep using the version of code
+that you copied.  If you think you've found a bug, check back with the
+repository to see if it's been fixed. Or submit it as an issue so that
+it can be looked at.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 46bc6b3..0000000
--- a/setup.py
+++ /dev/null
@@ -1,31 +0,0 @@
-try:
-    from setuptools import setup
-except ImportError:
-    from distutils.core import setup
-
-setup(name = "ply",
-            description="Python Lex & Yacc",
-            long_description = """
-PLY is yet another implementation of lex and yacc for Python. Some notable
-features include the fact that its implemented entirely in Python and it
-uses LALR(1) parsing which is efficient and well suited for larger grammars.
-
-PLY provides most of the standard lex/yacc features including support for empty 
-productions, precedence rules, error recovery, and support for ambiguous grammars. 
-
-PLY is extremely easy to use and provides very extensive error checking. 
-It is compatible with both Python 2 and Python 3.
-""",
-            license="""BSD""",
-            version = "3.11",
-            author = "David Beazley",
-            author_email = "dave@dabeaz.com",
-            maintainer = "David Beazley",
-            maintainer_email = "dave@dabeaz.com",
-            url = "http://www.dabeaz.com/ply/",
-            packages = ['ply'],
-            classifiers = [
-              'Programming Language :: Python :: 3',
-              'Programming Language :: Python :: 2',
-              ]
-            )
diff --git a/test/test_cpp_nonascii.c b/test/test_cpp_nonascii.c
new file mode 100644
index 0000000..3e97d81
--- /dev/null
+++ b/test/test_cpp_nonascii.c
@@ -0,0 +1,2 @@
+/* ë */
+#define x 1
\ No newline at end of file
diff --git a/test/testcpp.py b/test/testcpp.py
index 2e98edd..dbfb3e4 100644
--- a/test/testcpp.py
+++ b/test/testcpp.py
@@ -4,6 +4,7 @@ from multiprocessing import Process, Queue
 from six.moves.queue import Empty
 
 import sys
+import locale
 
 if ".." not in sys.path:
     sys.path.insert(0, "..")
@@ -45,6 +46,22 @@ class CPPTests(TestCase):
         else:
             self.assertMultiLineEqual(out, expected)
 
+    def test_infinite_argument_expansion(self):
+        # CPP does not drags set of currently expanded macros through macro
+        # arguments expansion. If there is a match between an argument value
+        # and name of an already expanded macro then CPP falls into infinite
+        # recursion.
+        self.__test_preprocessing("""\
+#define a(x) x
+#define b a(b)
+b
+"""         , """\
+
+
+b"""
+        )
+
+
     def test_concatenation(self):
         self.__test_preprocessing("""\
 #define a(x) x##_
@@ -98,4 +115,39 @@ a"""
 a"""
         )
 
+    def test_evalexpr(self):
+        # #if 1 != 2 is not processed correctly; undefined values are converted
+        # to 0L instead of 0 (issue #195)
+        #
+        self.__test_preprocessing("""\
+#if (1!=0) && (!x || (!(1==2)))
+a;
+#else
+b;
+#endif
+"""
+            , """\
+
+a;
+
+"""
+        )
+
+    def test_include_nonascii(self):
+        # Issue #196: #included files are read using the current locale's
+        # getdefaultencoding. if a #included file contains non-ascii characters,
+        # while default encoding is e.g. US_ASCII, this causes an error
+        locale.setlocale(locale.LC_ALL, 'C')
+        self.__test_preprocessing("""\
+#include "test_cpp_nonascii.c"
+x;
+
+"""
+            , """\
+
+ 
+1;
+"""
+        )
+
 main()
diff --git a/test/testlex.py b/test/testlex.py
index 83070a7..a94ed64 100755
--- a/test/testlex.py
+++ b/test/testlex.py
@@ -479,8 +479,10 @@ class LexBuildOptionTests(unittest.TestCase):
         
         os.mkdir("lexdir")
         os.mkdir("lexdir/sub")
-        open("lexdir/__init__.py","w").write("")
-        open("lexdir/sub/__init__.py","w").write("")
+        with open("lexdir/__init__.py","w") as f:
+            f.write("")
+        with open("lexdir/sub/__init__.py","w") as f:
+            f.write("")
         run_import("lex_optimize3")
         result = sys.stdout.getvalue()
         self.assert_(check_expected(result,