diff options
| -rw-r--r-- | CHANGES | 8 | ||||
| -rw-r--r-- | ply/lex.py | 4 | ||||
| -rw-r--r-- | ply/yacc.py | 3 | ||||
| -rw-r--r-- | test/lex_optimize4.py | 26 | ||||
| -rwxr-xr-x | test/testlex.py | 20 |
5 files changed, 59 insertions, 2 deletions
@@ -1,5 +1,13 @@ Version 3.11 --------------------- +02/15/18 beazley + Fixed some minor bugs related to re flags and token order. + Github pull requests #151 and #153. + +02/15/18 beazley + Added a set_lexpos() method to grammar symbols. Github issue #148. + + 04/13/17 beazley Mostly minor bug fixes and small code cleanups. @@ -179,8 +179,8 @@ class Lexer: with open(filename, 'w') as tf: tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__)) tf.write('_tabversion = %s\n' % repr(__tabversion__)) - tf.write('_lextokens = set(%s)\n' % repr(tuple(self.lextokens))) - tf.write('_lexreflags = %s\n' % repr(self.lexreflags)) + tf.write('_lextokens = set(%s)\n' % repr(tuple(sorted(self.lextokens)))) + tf.write('_lexreflags = %s\n' % repr(int(self.lexreflags))) tf.write('_lexliterals = %s\n' % repr(self.lexliterals)) tf.write('_lexstateinfo = %s\n' % repr(self.lexstateinfo)) diff --git a/ply/yacc.py b/ply/yacc.py index ceaaefd..d9692a9 100644 --- a/ply/yacc.py +++ b/ply/yacc.py @@ -267,6 +267,9 @@ class YaccProduction: def lexpos(self, n): return getattr(self.slice[n], 'lexpos', 0) + def set_lexpos(self, n, lexpos): + self.slice[n].lexpos = lexpos + def lexspan(self, n): startpos = getattr(self.slice[n], 'lexpos', 0) endpos = getattr(self.slice[n], 'endlexpos', startpos) diff --git a/test/lex_optimize4.py b/test/lex_optimize4.py new file mode 100644 index 0000000..cc6e2a9 --- /dev/null +++ b/test/lex_optimize4.py @@ -0,0 +1,26 @@ +# ----------------------------------------------------------------------------- +# lex_optimize4.py +# ----------------------------------------------------------------------------- +import re +import sys + +if ".." not in sys.path: sys.path.insert(0,"..") +import ply.lex as lex + +tokens = [ + "PLUS", + "MINUS", + "NUMBER", + ] + +t_PLUS = r'\+?' +t_MINUS = r'-' +t_NUMBER = r'(\d+)' + +def t_error(t): + pass + + +# Build the lexer +lex.lex(optimize=True, lextab="opt4tab", reflags=re.UNICODE) +lex.runmain(data="3+4") diff --git a/test/testlex.py b/test/testlex.py index 3880f6f..83070a7 100755 --- a/test/testlex.py +++ b/test/testlex.py @@ -514,6 +514,26 @@ class LexBuildOptionTests(unittest.TestCase): except OSError: pass + def test_lex_optimize4(self): + + # Regression test to make sure that reflags works correctly + # on Python 3. + + for extension in ['py', 'pyc']: + try: + os.remove("opt4tab.{0}".format(extension)) + except OSError: + pass + + run_import("lex_optimize4") + run_import("lex_optimize4") + + for extension in ['py', 'pyc']: + try: + os.remove("opt4tab.{0}".format(extension)) + except OSError: + pass + def test_lex_opt_alias(self): try: os.remove("aliastab.py") |