Merge pull request #135 from laerreal/bugfixes

Bugfixes for C preprocessor

3 files changed, 111 insertions, 2 deletions

diff --git a/ply/cpp.py b/ply/cpp.py
index b6bfc69..f527655 100644
--- a/ply/cpp.py
+++ b/ply/cpp.py
@@ -411,10 +411,11 @@ class Preprocessor(object):
                 elif (i > 0 and macro.value[i-1].value == '##'):
                     macro.patch.append(('c',argnum,i-1))
                     del macro.value[i-1]
+                    i -= 1
                     continue
                 elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'):
                     macro.patch.append(('c',argnum,i))
-                    i += 1
+                    del macro.value[i + 1]
                     continue
                 # Standard expansion
                 else:
@@ -509,7 +510,7 @@ class Preprocessor(object):
                         j = i + 1
                         while j < len(tokens) and tokens[j].type in self.t_WS:
                             j += 1
-                        if tokens[j].value == '(':
+                        if j < len(tokens) and tokens[j].value == '(':
                             tokcount,args,positions = self.collect_args(tokens[j:])
                             if not m.variadic and len(args) !=  len(m.arglist):
                                 self.error(self.source,t.lineno,"Macro %s requires %d arguments" % (t.value,len(m.arglist)))
@@ -535,6 +536,12 @@ class Preprocessor(object):
                                     r.lineno = t.lineno
                                 tokens[i:j+tokcount] = rep
                                 i += len(rep)
+                        else:
+                            # This is not a macro. It is just a word which
+                            # equals to name of the macro. Hence, go to the
+                            # next token.
+                            i += 1
+
                     del expanded[t.value]
                     continue
                 elif t.value == '__LINE__':
diff --git a/test/README b/test/README
index 52f032a..03b167c 100644
--- a/test/README
+++ b/test/README
@@ -3,5 +3,6 @@ conditions.  To run:
 
   $ python testlex.py 
   $ python testyacc.py 
+  $ python testcpp.py
 
 The script 'cleanup.sh' cleans up this directory to its original state.
diff --git a/test/testcpp.py b/test/testcpp.py
new file mode 100644
index 0000000..2e98edd
--- /dev/null
+++ b/test/testcpp.py
@@ -0,0 +1,101 @@
+from unittest import TestCase, main
+
+from multiprocessing import Process, Queue
+from six.moves.queue import Empty
+
+import sys
+
+if ".." not in sys.path:
+    sys.path.insert(0, "..")
+
+from ply.lex import lex
+from ply.cpp import *
+
+
+def preprocessing(in_, out_queue):
+    out = None
+
+    try:
+        p = Preprocessor(lex())
+        p.parse(in_)
+        tokens = [t.value for t in p.parser]
+        out = "".join(tokens)
+    finally:
+        out_queue.put(out)
+
+class CPPTests(TestCase):
+    "Tests related to ANSI-C style lexical preprocessor."
+
+    def __test_preprocessing(self, in_, expected, time_limit = 1.0):
+        out_queue = Queue()
+
+        preprocessor = Process(
+            name = "PLY`s C preprocessor",
+            target = preprocessing,
+            args = (in_, out_queue)
+        )
+
+        preprocessor.start()
+
+        try:
+            out = out_queue.get(timeout = time_limit)
+        except Empty:
+            preprocessor.terminate()
+            raise RuntimeError("Time limit exceeded!")
+        else:
+            self.assertMultiLineEqual(out, expected)
+
+    def test_concatenation(self):
+        self.__test_preprocessing("""\
+#define a(x) x##_
+#define b(x) _##x
+#define c(x) _##x##_
+#define d(x,y) _##x##y##_
+
+a(i)
+b(j)
+c(k)
+d(q,s)"""
+            , """\
+
+
+
+
+
+i_
+_j
+_k_
+_qs_"""
+        )
+
+    def test_deadloop_macro(self):
+        # If there is a word which equals to name of a parametrized macro, then
+        # attempt to expand such word as a macro manages the parser to fall
+        # into an infinite loop.
+
+        self.__test_preprocessing("""\
+#define a(x) x
+
+a;"""
+            , """\
+
+
+a;"""
+        )
+
+    def test_index_error(self):
+        # If there are no tokens after a word ("a") which equals to name of
+        # a parameterized macro, then attempt to expand this word leads to
+        # IndexError.
+
+        self.__test_preprocessing("""\
+#define a(x) x
+
+a"""
+            , """\
+
+
+a"""
+        )
+
+main()