Improve lexer with better string handling and grouping parens/brackets

This allows the lexer to correctly handle strings like:
    ${'backslash quote right-curly is \\\'}'}

And also allows users to use the bitwise-or operator to mean bitwise or
simply by enclosing the expression inside parens:
    ${(0x5432 | 0x8000)}

or by using it in the middle of a dictionary literal:
    ${ {'foo-val': 0x43 | 0x100, 'bar-val': 0x22 | 0x100}[thing+'-val']}

or inside brackets:
    ${ big_lookup_dict[index_low | (indexhigh << 3)] }

Basically, only "top level" uses of the vertical bar mean pipe.

(Note that currently, any non-top-level use of the vertical bar in
an expression just results in a syntax error in the generated python,
so no working code is affected by this change)

1 files changed, 15 insertions, 2 deletions

diff --git a/test/test_lexer.py b/test/test_lexer.py
index 3e97bef..06ebb05 100644
--- a/test/test_lexer.py
+++ b/test/test_lexer.py
@@ -477,9 +477,10 @@ more text
             ])
         )
 
-        template = """
+        template = r"""
 
             ${hello + '''heres '{|}' text | | }''' | escape1}
+            ${'Tricky string: ' + '\\\"\\\'|\\'}
         """
         nodes = Lexer(template).parse()
         self._compare(
@@ -488,7 +489,10 @@ more text
                 Text('\n\n            ', (1, 1)),
                 Expression("hello + '''heres '{|}' text | | }''' ",
                                 ['escape1'], (3, 13)),
-                Text('\n        ', (3, 62))
+                Text('\n            ', (3, 62)),
+                Expression(r"""'Tricky string: ' + '\\\"\\\'|\\'""",
+                                [], (4, 13)),
+                Text('\n        ', (4, 49))
             ])
         )
 
@@ -587,6 +591,15 @@ print('''
                       Expression(" {'key': 'value'} ", [], (1, 8)),
                       Text(' after', (1, 29))]))
 
+    def test_tricky_code_6(self):
+        template = \
+            """before ${ (0x5302 | 0x0400) } after"""
+        nodes = Lexer(template).parse()
+        self._compare(nodes, TemplateNode({},
+                      [Text('before ', (1, 1)),
+                      Expression(" (0x5302 | 0x0400) ", [], (1, 8)),
+                      Text(' after', (1, 30))]))
+
     def test_control_lines(self):
         template = \
             """