1 files changed, 136 insertions, 0 deletions
diff --git a/Cython/Compiler/Tests/TestScanning.py b/Cython/Compiler/Tests/TestScanning.py
new file mode 100644
index 000000000..e9cac1b47
--- /dev/null
+++ b/Cython/Compiler/Tests/TestScanning.py
@@ -0,0 +1,136 @@
+from __future__ import unicode_literals
+
+import unittest
+from io import StringIO
+import string
+
+from .. import Scanning
+from ..Symtab import ModuleScope
+from ..TreeFragment import StringParseContext
+from ..Errors import init_thread
+
+# generate some fake code - just a bunch of lines of the form "a0 a1 ..."
+code = []
+for ch in string.ascii_lowercase:
+    line = " ".join(["%s%s" % (ch, n) for n in range(10)])
+    code.append(line)
+code = "\n".join(code)
+
+init_thread()
+
+
+class TestScanning(unittest.TestCase):
+    def make_scanner(self):
+        source = Scanning.StringSourceDescriptor("fake code", code)
+        buf = StringIO(code)
+        context = StringParseContext("fake context")
+        scope = ModuleScope("fake_module", None, None)
+
+        return Scanning.PyrexScanner(buf, source, scope=scope, context=context)
+
+    def test_put_back_positions(self):
+        scanner = self.make_scanner()
+
+        self.assertEqual(scanner.sy, "IDENT")
+        self.assertEqual(scanner.systring, "a0")
+        scanner.next()
+        self.assertEqual(scanner.sy, "IDENT")
+        self.assertEqual(scanner.systring, "a1")
+        a1pos = scanner.position()
+        self.assertEqual(a1pos[1:], (1, 3))
+        a2peek = scanner.peek()  # shouldn't mess up the position
+        self.assertEqual(a1pos, scanner.position())
+        scanner.next()
+        self.assertEqual(a2peek, (scanner.sy, scanner.systring))
+
+        # find next line
+        while scanner.sy != "NEWLINE":
+            scanner.next()
+
+        line_sy = []
+        line_systring = []
+        line_pos = []
+
+        scanner.next()
+        while scanner.sy != "NEWLINE":
+            line_sy.append(scanner.sy)
+            line_systring.append(scanner.systring)
+            line_pos.append(scanner.position())
+            scanner.next()
+
+        for sy, systring, pos in zip(
+            line_sy[::-1], line_systring[::-1], line_pos[::-1]
+        ):
+            scanner.put_back(sy, systring, pos)
+
+        n = 0
+        while scanner.sy != "NEWLINE":
+            self.assertEqual(scanner.sy, line_sy[n])
+            self.assertEqual(scanner.systring, line_systring[n])
+            self.assertEqual(scanner.position(), line_pos[n])
+            scanner.next()
+            n += 1
+
+        self.assertEqual(n, len(line_pos))
+
+    def test_tentatively_scan(self):
+        scanner = self.make_scanner()
+        with Scanning.tentatively_scan(scanner) as errors:
+            while scanner.sy != "NEWLINE":
+                scanner.next()
+        self.assertFalse(errors)
+
+        scanner.next()
+        self.assertEqual(scanner.systring, "b0")
+        pos = scanner.position()
+        with Scanning.tentatively_scan(scanner) as errors:
+            while scanner.sy != "NEWLINE":
+                scanner.next()
+                if scanner.systring == "b7":
+                    scanner.error("Oh no not b7!")
+                    break
+        self.assertTrue(errors)
+        self.assertEqual(scanner.systring, "b0")  # state has been restored
+        self.assertEqual(scanner.position(), pos)
+        scanner.next()
+        self.assertEqual(scanner.systring, "b1")  # and we can keep going again
+        scanner.next()
+        self.assertEqual(scanner.systring, "b2")  # and we can keep going again
+
+        with Scanning.tentatively_scan(scanner) as error:
+            scanner.error("Something has gone wrong with the current symbol")
+        self.assertEqual(scanner.systring, "b2")
+        scanner.next()
+        self.assertEqual(scanner.systring, "b3")
+
+        # test a few combinations of nested scanning
+        sy1, systring1 = scanner.sy, scanner.systring
+        pos1 = scanner.position()
+        with Scanning.tentatively_scan(scanner):
+            scanner.next()
+            sy2, systring2 = scanner.sy, scanner.systring
+            pos2 = scanner.position()
+            with Scanning.tentatively_scan(scanner):
+                with Scanning.tentatively_scan(scanner):
+                    scanner.next()
+                    scanner.next()
+                    scanner.error("Ooops")
+                self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
+            self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
+            scanner.error("eee")
+        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
+        with Scanning.tentatively_scan(scanner):
+            scanner.next()
+            scanner.next()
+            with Scanning.tentatively_scan(scanner):
+                scanner.next()
+                # no error - but this block should be unwound by the outer block too
+            scanner.next()
+            scanner.error("Oooops")
+        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
+
+
+
+
+if __name__ == "__main__":
+    unittest.main()