TNT: fix reliance on "assert" stmt, handle recursion

3 files changed, 37 insertions, 18 deletions

diff --git a/CHANGES b/CHANGES
index b5f2ec8f..52660d3b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -11,6 +11,14 @@ Version 2.10.0
 --------------
 (not released yet)
 
+
+Version 2.9.1
+-------------
+(released May 31st, 2021)
+
+- Fix assert statements in TNT lexer.
+
+
 Version 2.9.0
 -------------
 (released May 3rd, 2021)
diff --git a/pygments/lexers/tnt.py b/pygments/lexers/tnt.py
index 50fa3335..e6e71961 100644
--- a/pygments/lexers/tnt.py
+++ b/pygments/lexers/tnt.py
@@ -12,7 +12,7 @@ import re
 
 from pygments.lexer import Lexer
 from pygments.token import Text, Comment, Operator, Keyword, Name, Number, \
-     Punctuation, Error
+    Punctuation, Error
 
 __all__ = ['TNTLexer']
 
@@ -66,15 +66,16 @@ class TNTLexer(Lexer):
                 end += 1
         except IndexError:
             end = len(text)
-        if required:
-            assert end != start
+        if required and end == start:
+            raise AssertionError
         if end != start:
             self.cur.append((start, Text, text[start:end]))
         return end
 
     def variable(self, start, text):
         """Tokenize a variable."""
-        assert text[start] in self.VARIABLES
+        if text[start] not in self.VARIABLES:
+            raise AssertionError
         end = start+1
         while text[end] in self.PRIMES:
             end += 1
@@ -97,10 +98,12 @@ class TNTLexer(Lexer):
         if text[start] == '(':  # (...+...)
             self.cur.append((start, Punctuation, text[start]))
             start = self.term(start+1, text)
-            assert text[start] in self.OPERATORS
+            if text[start] not in self.OPERATORS:
+                raise AssertionError
             self.cur.append((start, Operator, text[start]))
             start = self.term(start+1, text)
-            assert text[start] == ')'
+            if text[start] != ')':
+                raise AssertionError
             self.cur.append((start, Punctuation, text[start]))
             return start+1
         raise AssertionError  # no matches
@@ -116,21 +119,25 @@ class TNTLexer(Lexer):
         if text[start] in self.QUANTIFIERS:  # Aa:<...>
             self.cur.append((start, Keyword.Declaration, text[start]))
             start = self.variable(start+1, text)
-            assert text[start] == ':'
+            if text[start] != ':':
+                raise AssertionError
             self.cur.append((start, Punctuation, text[start]))
             return self.formula(start+1, text)
         if text[start] == '<':  # <...&...>
             self.cur.append((start, Punctuation, text[start]))
             start = self.formula(start+1, text)
-            assert text[start] in self.LOGIC
+            if text[start] not in self.LOGIC:
+                raise AssertionError
             self.cur.append((start, Operator, text[start]))
             start = self.formula(start+1, text)
-            assert text[start] == '>'
+            if text[start] != '>':
+                raise AssertionError
             self.cur.append((start, Punctuation, text[start]))
             return start+1
         # ...=...
         start = self.term(start, text)
-        assert text[start] == '='
+        if text[start] != '=':
+            raise AssertionError
         self.cur.append((start, Operator, text[start]))
         start = self.term(start+1, text)
         return start
@@ -138,7 +145,8 @@ class TNTLexer(Lexer):
     def rule(self, start, text):
         """Tokenize a rule."""
         match = self.RULES.match(text, start)
-        assert match is not None
+        if match is None:
+            raise AssertionError
         groups = sorted(match.regs[1:])  # exclude whole match
         for group in groups:
             if group[0] >= 0:  # this group matched
@@ -162,8 +170,10 @@ class TNTLexer(Lexer):
         self.cur.append((start+1, Text, text[start+1:end]))
         start = end
         match = self.LINENOS.match(text, start)
-        assert match is not None
-        assert text[match.end()] == ')'
+        if match is None:
+            raise AssertionError
+        if text[match.end()] != ')':
+            raise AssertionError
         self.cur.append((match.start(), Number.Integer, match.group(0)))
         self.cur.append((match.end(), Punctuation, text[match.end()]))
         return match.end() + 1
@@ -219,7 +229,7 @@ class TNTLexer(Lexer):
                     orig = len(self.cur)
                     try:
                         start = end = self.formula(start, text)
-                    except AssertionError:  # not well-formed
+                    except (AssertionError, RecursionError):  # not well-formed
                         del self.cur[orig:]
                         while text[end] not in self.WHITESPACE:
                             end += 1
@@ -257,6 +267,6 @@ class TNTLexer(Lexer):
                 try:
                     del self.cur[orig:]
                 except NameError:
-                    pass # if orig was never defined, fine
+                    pass  # if orig was never defined, fine
                 self.error_till_line_end(start, text)
         return self.cur
diff --git a/scripts/debug_lexer.py b/scripts/debug_lexer.py
index a76265f5..5cc0ef6f 100755
--- a/scripts/debug_lexer.py
+++ b/scripts/debug_lexer.py
@@ -191,7 +191,8 @@ def main(fn, lexer=None, options={}):
         reprs = list(map(repr, tok))
         print('   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ')
         if debug_lexer:
-            print(' ' + ' ' * (29-len(reprs[0])) + ' : '.join(state) if state else '', end=' ')
+            print(' ' + ' ' * (29-len(reprs[0])) + ' : '.join(state)
+                  if state else '', end=' ')
         print()
 
     for type, val in lx.get_tokens(text):
@@ -206,10 +207,10 @@ def main(fn, lexer=None, options={}):
                     else:
                         show_token(tokens[i], None)
             print('Error token:')
-            l = len(repr(val))
+            vlen = len(repr(val))
             print('   ' + repr(val), end=' ')
             if debug_lexer and hasattr(lx, 'ctx'):
-                print(' ' * (60-l) + ' : '.join(lx.ctx.stack), end=' ')
+                print(' ' * (60-vlen) + ' : '.join(lx.ctx.stack), end=' ')
             print()
             print()
             return 1