Merged in dscorbett/pygments-tads3-2 (pull request #407)

Add a lexer for TADS 3

8 files changed, 138 insertions, 2 deletions

diff --git a/tests/examplefiles/demo.css.in b/tests/examplefiles/demo.css.in
new file mode 100644
index 00000000..36330a9d
--- /dev/null
+++ b/tests/examplefiles/demo.css.in
@@ -0,0 +1,6 @@
+%if defined(__foo__)
+.cls {
+  color: #fff;
+}
+%endif
+%literal %foo
diff --git a/tests/examplefiles/demo.js.in b/tests/examplefiles/demo.js.in
new file mode 100644
index 00000000..f44fc53d
--- /dev/null
+++ b/tests/examplefiles/demo.js.in
@@ -0,0 +1,6 @@
+window.foo = {
+#if defined(__foo__)
+  'key': 'value'
+#endif
+}
+#literal #foo
diff --git a/tests/examplefiles/demo.xul.in b/tests/examplefiles/demo.xul.in
new file mode 100644
index 00000000..9e1f4938
--- /dev/null
+++ b/tests/examplefiles/demo.xul.in
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<window
+   xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+#if defined(__foo__)
+<description value="Text" />
+#endif
+</window>
diff --git a/tests/examplefiles/resourcebundle_demo b/tests/examplefiles/resourcebundle_demo
new file mode 100644
index 00000000..e1daa56a
--- /dev/null
+++ b/tests/examplefiles/resourcebundle_demo
@@ -0,0 +1,9 @@
+root:table {
+  usage:string { "Usage: genrb [Options] files" }
+  version:int { 122 }
+  errorcodes:array {
+    :string { "Invalid argument" }
+    :string { "File not found" }
+    :string { "\x00 \r \t \n \u1234" }
+  }
+}
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 9e26ce17..2e02ae52 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -28,8 +28,8 @@ def run_cmdline(*args):
     if sys.version_info > (3,):
         stdout_buffer = BytesIO()
         stderr_buffer = BytesIO()
-        new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer)
-        new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer)
+        new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
+        new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
     else:
         stdout_buffer = new_stdout = sys.stdout = StringIO()
         stderr_buffer = new_stderr = sys.stderr = StringIO()
diff --git a/tests/test_regexopt.py b/tests/test_regexopt.py
index f62a57ef..66f7c609 100644
--- a/tests/test_regexopt.py
+++ b/tests/test_regexopt.py
@@ -50,3 +50,19 @@ class RegexOptTestCase(unittest.TestCase):
                 self.assertTrue(rex.match(w))
             for w in no_match:
                 self.assertFalse(rex.match(w))
+
+    def test_prefix(self):
+        opt = regex_opt(('a', 'b'), prefix=r':{1,2}')
+        print(opt)
+        rex = re.compile(opt)
+        self.assertFalse(rex.match('a'))
+        self.assertTrue(rex.match('::a'))
+        self.assertFalse(rex.match(':::')) # fullmatch
+
+    def test_suffix(self):
+        opt = regex_opt(('a', 'b'), suffix=r':{1,2}')
+        print(opt)
+        rex = re.compile(opt)
+        self.assertFalse(rex.match('a'))
+        self.assertTrue(rex.match('a::'))
+        self.assertFalse(rex.match(':::')) # fullmatch
diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py
new file mode 100644
index 00000000..de94545a
--- /dev/null
+++ b/tests/test_textfmts.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+    Basic Tests for textfmts
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Operator, Number, Text, Token
+from pygments.lexers.textfmts import HttpLexer
+
+
+class RubyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = HttpLexer()
+        self.maxDiff = None
+
+    def testApplicationXml(self):
+        fragment = u'GET / HTTP/1.0\nContent-Type: application/xml\n\n<foo>\n'
+        tokens = [
+            (Token.Name.Tag, u'<foo'),
+            (Token.Name.Tag, u'>'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(
+            tokens, list(self.lexer.get_tokens(fragment))[-len(tokens):])
+
+    def testApplicationCalendarXml(self):
+        fragment = u'GET / HTTP/1.0\nContent-Type: application/calendar+xml\n\n<foo>\n'
+        tokens = [
+            (Token.Name.Tag, u'<foo'),
+            (Token.Name.Tag, u'>'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(
+            tokens, list(self.lexer.get_tokens(fragment))[-len(tokens):])
+
diff --git a/tests/test_unistring.py b/tests/test_unistring.py
new file mode 100644
index 00000000..a8791e0c
--- /dev/null
+++ b/tests/test_unistring.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the unistring module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import unittest
+import random
+
+from pygments import unistring as uni
+from pygments.util import unichr
+
+class UnistringTest(unittest.TestCase):
+    def test_cats_exist_and_compilable(self):
+        for cat in uni.cats:
+            s = getattr(uni, cat)
+            if s == '': # Probably Cs on Jython
+                continue
+            print("%s %r" % (cat, s))
+            re.compile('[%s]' % s)
+
+    def _cats_that_match(self, c):
+        matching_cats = []
+        for cat in uni.cats:
+            s = getattr(uni, cat)
+            if s == '': # Probably Cs on Jython
+                continue
+            if re.compile('[%s]' % s).match(c):
+                matching_cats.append(cat)
+        return matching_cats
+
+    def test_spot_check_types(self):
+        # Each char should match one, and precisely one, category
+        random.seed(0)
+        for i in range(1000):
+            o = random.randint(0, 65535)
+            c = unichr(o)
+            if o > 0xd800 and o <= 0xdfff and not uni.Cs:
+                continue # Bah, Jython.
+            print(hex(o))
+            cats = self._cats_that_match(c)
+            self.assertEqual(len(cats), 1,
+                             "%d (%s): %s" % (o, c, cats))
+
+
+
+