diff options
author | Adrian Thurston <thurston@complang.org> | 2012-12-18 20:56:10 -0500 |
---|---|---|
committer | Adrian Thurston <thurston@complang.org> | 2012-12-18 20:56:10 -0500 |
commit | 82466231c6401bf490726f8a469b7139c6a37242 (patch) | |
tree | 0f5dc8478a0fd0ab7c651c12c5f8d02a4302821d | |
parent | 5bb34e270b42ab19d5839a551fae0f42ea688f6b (diff) | |
download | colm-82466231c6401bf490726f8a469b7139c6a37242.tar.gz |
updated test for lex scope syntax
71 files changed, 124 insertions, 250 deletions
diff --git a/test/accum1.lm b/test/accum1.lm index 4b3edb88..b3beb821 100644 --- a/test/accum1.lm +++ b/test/accum1.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ -} +end def item [id] diff --git a/test/accum2.lm b/test/accum2.lm index ae387b27..7f6bb2cb 100644 --- a/test/accum2.lm +++ b/test/accum2.lm @@ -5,11 +5,10 @@ context ctx k: int lex - { ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ - } + end def foo [id] diff --git a/test/accum3.lm b/test/accum3.lm index 9d19e218..54c1c3b4 100644 --- a/test/accum3.lm +++ b/test/accum3.lm @@ -1,17 +1,15 @@ lex -{ literal '-' token file /^('-'|0)(^0)*/ -} +end token word /(^0)+/ token zero /0/ lex -{ token single /[qvh]/ token with_opt /[oi]/ -} +end def item [file zero] diff --git a/test/accumbt1.lm b/test/accumbt1.lm index fb2e9c6d..be8c3464 100644 --- a/test/accumbt1.lm +++ b/test/accumbt1.lm @@ -1,12 +1,11 @@ # Token names. lex -{ token number /[0-9]+/ token id /[a-z]+/ token string /'"' [^"]* '"'/ ignore ws / [ \t\n]+ / -} +end def prefix [id] diff --git a/test/accumbt2.lm b/test/accumbt2.lm index 57407d65..40bd2bbd 100644 --- a/test/accumbt2.lm +++ b/test/accumbt2.lm @@ -1,17 +1,15 @@ context accum_bt { lex - { ignore /[ \t\n]+/ token id1 /[a-zA-Z_][a-zA-Z_0-9]*/ def one [ id1* ] - } + end OneParser: accum<one> lex - { ignore /[ \t]+/ token id2 /[a-zA-Z_][a-zA-Z_0-9]*/ literal '!', ';', '\n' @@ -32,7 +30,7 @@ context accum_bt def two [A1 item2* '!' '\n'] | [A2 item2* ';' '\n'] - } + end } AccumBt: accum_bt = cons accum_bt[] diff --git a/test/accumbt3.lm b/test/accumbt3.lm index 38b6d3d2..7046cbf5 100644 --- a/test/accumbt3.lm +++ b/test/accumbt3.lm @@ -5,7 +5,6 @@ # Any single character can be a literal lex -{ # Ignore whitespace. ignore /[ \t\n\r\v]+/ @@ -17,17 +16,16 @@ lex parse_stop NC: nested_comment[ input ] print( 'discarding: ' NC.tree '\n' ) } -} +end # # Token translation # lex -{ literal '(', ')' token nc_data /[^()]+/ -} +end def nc_item [nc_data] @@ -43,7 +41,6 @@ context accum_bt NestedParser: accum<nested> lex - { ignore /[ \t]+/ token word /[a-zA-Z0-9/*+_\-]+/ token stuff /[a-zA-Z0-9()/*+_\- ]+/ @@ -74,7 +71,7 @@ context accum_bt [A1 item* '!' '\n'] | [A2 item* ';' '\n'] - } + end } cons AccumBt: accum_bt[] diff --git a/test/ambig1.lm b/test/ambig1.lm index 5d7ba962..ad7dd6e9 100644 --- a/test/ambig1.lm +++ b/test/ambig1.lm @@ -4,7 +4,6 @@ # The items in this scanner may have newline in front of them. lex -{ # Reserved Words. literal '__LINE__', '__FILE__', '__ENCODING__', 'BEGIN', 'END', 'alias', 'and', 'begin', 'break', 'case', 'class', 'def', 'defined?', 'do', @@ -48,11 +47,10 @@ lex ignore /[ \t\n]+/ ignore comment /'#' [^\n]* '\n'/ -} +end # These items cannot appear at the beginning of a line (except maybe the first). lex -{ ignore /[\t ]+/ literal '+', '-', '*', '**', '/', '%', '^' @@ -68,34 +66,30 @@ lex literal '<', '>', '>=', '<=' literal '!=', '==', '===' literal '..', '...' -} +end lex -{ ignore /[\t ]+/ ignore /'#' [^\n]*/ literal ';' literal '\n' -} +end lex -{ token dstring_contents /[^"]+/ token tDSTRING_END /'"'/ -} +end lex -{ token sstring_contents /[^']+/ token tSSTRING_END /'\''/ -} +end lex -{ token xstring_contents /[^`]+/ token tXSTRING_END /'`'/ -} +end def ruby [compstmt] diff --git a/test/argv2.lm b/test/argv2.lm index f1c045f7..9d7792ab 100644 --- a/test/argv2.lm +++ b/test/argv2.lm @@ -1,16 +1,14 @@ lex -{ literal '-' token file /^('-'|0)(^0)*/ -} +end lex -{ token single /[qvh]/ token with_opt /[oi]/ token dash /'-'/ literal 'help', 'verbose', 'input', '=' -} +end def long ['help'] diff --git a/test/backtrack1.lm b/test/backtrack1.lm index 0db2f607..13370444 100644 --- a/test/backtrack1.lm +++ b/test/backtrack1.lm @@ -1,11 +1,10 @@ # Token names. lex -{ literal '+', '*' token number /[0-9]+/ ignore ws / [ \t\n]+ / -} +end def F [number '+'] diff --git a/test/backtrack2.lm b/test/backtrack2.lm index 821483ba..6bd67a41 100644 --- a/test/backtrack2.lm +++ b/test/backtrack2.lm @@ -1,10 +1,9 @@ # Token names. lex -{ token id /[a-z]+/ ignore ws /[ \t\n]+/ -} +end token bang1 /'!'/ token bang2 /'!'/ diff --git a/test/backtrack3.lm b/test/backtrack3.lm index 773edbe9..6aca30dd 100644 --- a/test/backtrack3.lm +++ b/test/backtrack3.lm @@ -1,12 +1,11 @@ # Token names. lex -{ token number /[0-9]+/ token id /[a-z]+/ token string /'"' [^"]* '"'/ ignore ws / [ \t\n]+ / -} +end def prefix [id] diff --git a/test/btscan1.lm b/test/btscan1.lm index c9ebd6f4..041462eb 100644 --- a/test/btscan1.lm +++ b/test/btscan1.lm @@ -1,10 +1,9 @@ namespace r1 { lex - { literal '!', 'a', 'b' ignore /[ \n\t]+/ - } + end def line [ '!' 'a' 'b' 'b' 'a'] } @@ -12,11 +11,10 @@ namespace r1 namespace r2 { lex - { literal '!' token id /[a-zA-Z_]+/ ignore /[ \n\t]+/ - } + end def line [ '!' id ] } diff --git a/test/btscan2.lm b/test/btscan2.lm index 93fb5522..1dfc8ec1 100644 --- a/test/btscan2.lm +++ b/test/btscan2.lm @@ -1,10 +1,9 @@ namespace r1 { lex - { literal '!', 'a', 'b' ignore /[ \n\t]+/ - } + end def line [ '!' 'a' 'b' 'b' 'a'] } @@ -12,11 +11,10 @@ namespace r1 namespace r2 { lex - { literal '!' token id /[a-zA-Z_]+/ ignore /[ \n\t]+/ - } + end def line [ '!' id ] } diff --git a/test/commitbt.lm b/test/commitbt.lm index c819f36c..e409fe1c 100644 --- a/test/commitbt.lm +++ b/test/commitbt.lm @@ -18,7 +18,6 @@ # 'parsed' items because it must entertain the possibility of backtracking. lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -27,7 +26,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def expression [term expression_op*] diff --git a/test/concat1.lm b/test/concat1.lm index 49281663..e8453939 100644 --- a/test/concat1.lm +++ b/test/concat1.lm @@ -1,18 +1,15 @@ lex -{ - literal 'type', 'include' token id /[A-Za-z_][A-Za-z_0-9]*/ ignore /'#' [^\n]* '\n'/ ignore /[ \t\r\n]+/ -} +end lex -{ token ifn_part /[a-zA-Z0-9_.\-]+/ token ifn_slash /'/'/ -} +end def ifn_path_part [ifn_part] @@ -26,9 +23,8 @@ def ifn_path literal '%%' lex -{ token em_ws /( any - 33..126 )+/ -} +end def em_item [em_ws] diff --git a/test/concat2.lm b/test/concat2.lm index 49281663..e8edfd82 100644 --- a/test/concat2.lm +++ b/test/concat2.lm @@ -1,18 +1,16 @@ lex -{ literal 'type', 'include' token id /[A-Za-z_][A-Za-z_0-9]*/ ignore /'#' [^\n]* '\n'/ ignore /[ \t\r\n]+/ -} +end lex -{ token ifn_part /[a-zA-Z0-9_.\-]+/ token ifn_slash /'/'/ -} +end def ifn_path_part [ifn_part] @@ -26,9 +24,8 @@ def ifn_path literal '%%' lex -{ token em_ws /( any - 33..126 )+/ -} +end def em_item [em_ws] diff --git a/test/construct1.lm b/test/construct1.lm index 6bbe5a98..9813bc33 100644 --- a/test/construct1.lm +++ b/test/construct1.lm @@ -2,11 +2,10 @@ rl ident_pattern /[a-zA-Z_][a-zA-Z_0-9]*/ rl number_pattern /[0-9]+/ lex -{ ignore /[ \t\n]+/ token id /ident_pattern/ token number /number_pattern/ -} +end def four_ids [id id id id] diff --git a/test/construct2.lm b/test/construct2.lm index c3b50367..2f9af821 100644 --- a/test/construct2.lm +++ b/test/construct2.lm @@ -1,10 +1,9 @@ lex -{ ignore /[ \t\n]+/ token id /[a-z]+/ literal ',', '.', '*', '(', ')' -} +end def lang [id*] diff --git a/test/construct3.lm b/test/construct3.lm index b2c73b37..a6977d96 100644 --- a/test/construct3.lm +++ b/test/construct3.lm @@ -1,10 +1,9 @@ lex -{ ignore /[ \t\n]+/ token id /[a-z0-9]+/ literal ',', '.', '*', '(', ')' -} +end def bigger ['(' item* ')'] diff --git a/test/constructex.lm b/test/constructex.lm index 8a5d284e..f41cb5c1 100644 --- a/test/constructex.lm +++ b/test/constructex.lm @@ -1,9 +1,8 @@ lex -{ token id /[a-zA-Z_][a-zA-Z0-9_]*/ literal '=', '<', '>', '/' ignore /[ \t\n\r\v]+/ -} +end def attr [id '=' id] diff --git a/test/context1.lm b/test/context1.lm index 706e288e..919c6157 100644 --- a/test/context1.lm +++ b/test/context1.lm @@ -5,11 +5,10 @@ context ctx k: int lex - { ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ - } + end def foo [id] diff --git a/test/context2.lm b/test/context2.lm index c5ffc963..48371f3b 100644 --- a/test/context2.lm +++ b/test/context2.lm @@ -4,19 +4,17 @@ context ruby_here rl number_pattern /[0-9]+/ lex - { ignore /[ \t\n]+/ token id /ident_pattern/ token number /number_pattern/ literal '<<', '*', ',', '(', ')', '!' - } + end HereId: str token rest_of_line /[^\n]*'\n'/ lex - { ignore /[ \t\n]+/ token here_id HereData: here_data @@ -37,10 +35,9 @@ context ruby_here # Send the here_id token. Attach the heredoc data as an attribute. input.push( make_token( typeid<here_id> HereId HereData.tree ) ) } - } + end lex - { token here_close_id / ident_pattern '\n' / { @@ -55,7 +52,7 @@ context ruby_here token here_line / [^\n]* '\n' / - } + end def here_data [here_line* here_close_id] diff --git a/test/context3.lm b/test/context3.lm index 18ec41f2..41f8de0b 100644 --- a/test/context3.lm +++ b/test/context3.lm @@ -5,11 +5,10 @@ context ctx k: int lex - { ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ - } + end def foo [id] diff --git a/test/counting1.lm b/test/counting1.lm index 8b961422..9d73a4fd 100644 --- a/test/counting1.lm +++ b/test/counting1.lm @@ -13,14 +13,13 @@ context counting # lex - { # Ignore whitespace. ignore /rl_ws/ # Tokens. token id /rl_id/ token number /rl_num/ - } + end # # Global Data diff --git a/test/counting2.lm b/test/counting2.lm index ca43493c..9dacf083 100644 --- a/test/counting2.lm +++ b/test/counting2.lm @@ -12,14 +12,13 @@ rl rl_num /[0-9]+/ # lex -{ # Ignore whitespace. ignore /rl_ws/ # Tokens. token id /rl_id/ token number /rl_num/ -} +end # # Productions diff --git a/test/counting3.lm b/test/counting3.lm index 0914ceed..ada6c1d9 100644 --- a/test/counting3.lm +++ b/test/counting3.lm @@ -12,7 +12,6 @@ context counting # lex - { # Ignore whitespace. ignore /rl_ws/ @@ -21,7 +20,7 @@ context counting # Tokens. token id /rl_id/ token number /rl_num/ - } + end # # Global Data diff --git a/test/counting4.lm b/test/counting4.lm index 3cd77dac..ea0f1ac9 100644 --- a/test/counting4.lm +++ b/test/counting4.lm @@ -13,7 +13,6 @@ context counting # lex - { # Ignore whitespace. ignore /rl_ws/ @@ -22,7 +21,7 @@ context counting # Tokens. token id /rl_id/ token number /rl_num/ - } + end # # Global Data diff --git a/test/export1.lm b/test/export1.lm index f6544f87..ae1e6c4b 100644 --- a/test/export1.lm +++ b/test/export1.lm @@ -1,8 +1,7 @@ lex -{ token id /[a-z]+/ ignore /[ \t]+/ -} +end def start [id*] diff --git a/test/func.lm b/test/func.lm index f45970a2..72a1e49c 100644 --- a/test/func.lm +++ b/test/func.lm @@ -1,13 +1,12 @@ lex -{ literal '{', '}' literal 'struct', 'type' token id /[A-Za-z_][A-Za-z_0-9]*/ ignore /'/*' any* :>> '*/'/ ignore /[ \t\r\n]+/ -} +end def attribute ['type' id] diff --git a/test/generate1.lm b/test/generate1.lm index 0eb4279b..20869b14 100644 --- a/test/generate1.lm +++ b/test/generate1.lm @@ -11,7 +11,6 @@ context generate # Tokens. lex - { # Python keywords. literal 'and', 'del', 'from', 'not', 'while', 'as', 'elif', 'global', 'or', 'with', 'assert', 'else', 'if', 'pass', 'yield', 'break', 'except', @@ -130,7 +129,7 @@ context generate # First the newline. input.push( make_token( typeid<NEWLINE> '' ) ) } - } + end # Blank lines or comment lines at the beginning of the file. token LEADER / ( [ \t]* ('#' [^\n]*)? '\n' )* / diff --git a/test/generate2.lm b/test/generate2.lm index 2a8e5ca9..3d89a6e5 100644 --- a/test/generate2.lm +++ b/test/generate2.lm @@ -8,7 +8,6 @@ context generate OpenStack: list<open_item> lex - { token stray_close // token ocurly /'{'+/ @@ -122,7 +121,7 @@ context generate } } } - } + end # # Internal Links @@ -130,11 +129,10 @@ context generate lex - { literal 'http:' literal 'ftp:' literal 'mailto:' - } + end def el_prefix ['http:'] diff --git a/test/heredoc.lm b/test/heredoc.lm index 6355b2b0..23a8b114 100644 --- a/test/heredoc.lm +++ b/test/heredoc.lm @@ -3,7 +3,6 @@ context heredoc rl ident_char /[a-zA-Z_]/ lex - { # Tokens token other /(^(ident_char|0|'\n'))+/ @@ -22,7 +21,7 @@ context heredoc } token nl /'\n'/ - } + end def here_name [id] diff --git a/test/ignore1.lm b/test/ignore1.lm index b31ff46b..e74bda93 100644 --- a/test/ignore1.lm +++ b/test/ignore1.lm @@ -10,7 +10,6 @@ rl rl_id /[a-zA-Z_][a-zA-Z0-9_]*/ # lex -{ literal '=', '<', '>', '/' # Ignore whitespace. @@ -18,7 +17,7 @@ lex # Open and close id token id /rl_id/ -} +end # # Productions diff --git a/test/ignore2.lm b/test/ignore2.lm index 25053045..5b3497d2 100644 --- a/test/ignore2.lm +++ b/test/ignore2.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ -} +end def item [id] diff --git a/test/ignore3.lm b/test/ignore3.lm index 4b3ce01a..e1003251 100644 --- a/test/ignore3.lm +++ b/test/ignore3.lm @@ -1,17 +1,15 @@ lex -{ ignore /space+/ literal '*', '(' ni, ni ')', '!', ';' token id /[a-zA-Z_0-9]+/ -} +end lex -{ ignore /space+/ token inner_t /[a-zA-Z_0-9]+/ literal '' -} +end def inner ['' inner_t*] diff --git a/test/island.lm b/test/island.lm index 74e7f1af..a42e87aa 100644 --- a/test/island.lm +++ b/test/island.lm @@ -1,10 +1,9 @@ lex -{ token func_chr /[^{}]+/ token func_open /'{'/ token func_close /'}'/ -} +end def func_item [func_chr] @@ -17,7 +16,6 @@ def func [ident '(' ')' '{' func_body func_close ] lex -{ token ident /[a-zA-Z_]+/ token number /[0-9]+/ @@ -28,7 +26,7 @@ lex literal '+', '*', ';', '(', ')', '{', '}' ignore wp / [ \t\n]+ / -} +end def class_item [func] diff --git a/test/lhs1.lm b/test/lhs1.lm index fea02be0..adcfcf99 100644 --- a/test/lhs1.lm +++ b/test/lhs1.lm @@ -1,10 +1,9 @@ lex -{ ignore /space+/ literal '*', '(', ')', '!', ';\n' token id /[a-zA-Z_0-9]+/ -} +end def item [id] diff --git a/test/liftattrs.lm b/test/liftattrs.lm index 6a3884f6..9478acae 100644 --- a/test/liftattrs.lm +++ b/test/liftattrs.lm @@ -10,7 +10,6 @@ rl rl_id /[a-zA-Z_][a-zA-Z0-9_]*/ # lex -{ literal '=', '<', '>', '/' # Ignore whitespace. @@ -18,7 +17,7 @@ lex # Open and close id token id /rl_id/ -} +end # # Productions diff --git a/test/lookup1.lm b/test/lookup1.lm index 97aa3b46..a6fcd4dd 100644 --- a/test/lookup1.lm +++ b/test/lookup1.lm @@ -139,7 +139,6 @@ context lookup # The C++ scanner. lex - { rl fract_const / digit* '.' digit+ | digit+ '.' / rl exponent / [eE] [+\-]? digit+ / rl float_suffix / [flFL] / @@ -236,7 +235,7 @@ context lookup ignore /( '/*' (any | '\n')* :>> '*/' )/ ignore /( '//' any* :> '\n' )/ ignore /( any - 33..126 )+/ - } + end # # Support functions @@ -1635,7 +1634,6 @@ context lookup [function_def_declaration ctor_initializer_opt class_function_body function_def_end] lex - { token cfb_open /'{'/ token cfb_close /'}'/ token cfb_string / @@ -1645,7 +1643,7 @@ context lookup ( '/*' (any | '\n')* :>> '*/' ) | ( '//' any* :> '\n' )/ token cfb_data /[^{}'"/]+ | '/'/ - } + end def cfb_item [cfb_data] diff --git a/test/mailbox.lm b/test/mailbox.lm index e0a891a9..df45a65a 100644 --- a/test/mailbox.lm +++ b/test/mailbox.lm @@ -1,7 +1,6 @@ # lines, and fromlines lex -{ rl day /[A-Z][a-z][a-z]/ rl month /[A-Z][a-z][a-z]/ rl year /[0-9][0-9][0-9][0-9]/ @@ -22,7 +21,7 @@ lex # entirely correct from line is matched. token from_line / 'From ' (any-'\n')* ' ' date '\n' / token simple_line / [^\n]* '\n' / -} +end rl hchar /print - [ :]/ token header_name /hchar+/ diff --git a/test/matchex.lm b/test/matchex.lm index e6be64e0..c8104299 100644 --- a/test/matchex.lm +++ b/test/matchex.lm @@ -1,9 +1,8 @@ lex -{ token id /[a-zA-Z_][a-zA-Z0-9_]*/ literal '=', '<', '>', '/' ignore /[ \t\n\r\v]+/ -} +end def attr [id '=' id] diff --git a/test/maxlen.lm b/test/maxlen.lm index 68a7c902..4291bc17 100644 --- a/test/maxlen.lm +++ b/test/maxlen.lm @@ -13,10 +13,9 @@ context maxlen # lex - { ignore /rl_ws/ token id /rl_id/ - } + end num: int allow: int diff --git a/test/multiregion1.lm b/test/multiregion1.lm index 95181c91..3eb05714 100644 --- a/test/multiregion1.lm +++ b/test/multiregion1.lm @@ -11,14 +11,13 @@ token separator_line / '='+ '\n' / # Whitespace separated word list lex -{ token word /[^\t \n]+/ ignore /[\t ]+/ def word_list [word word_list] | [] -} +end token old_file_start / '---' [\t ]+ / token new_file_start / '+++' [\t ]+ / diff --git a/test/multiregion2.lm b/test/multiregion2.lm index 1b61235e..0b19b15b 100644 --- a/test/multiregion2.lm +++ b/test/multiregion2.lm @@ -46,11 +46,10 @@ def request_line token field_name /token_char+/ lex -{ token fv_plain /(^(CR|LF))*/ token fv_ext /CR LF (SP|HT)/ token fv_term /CR LF/ -} +end def fv [fv_plain] diff --git a/test/nestedcomm.lm b/test/nestedcomm.lm index 153fb1ab..5adc8902 100644 --- a/test/nestedcomm.lm +++ b/test/nestedcomm.lm @@ -4,7 +4,6 @@ # Any single character can be a literal lex -{ # Ignore whitespace. ignore /[ \t\n\r\v]+/ @@ -17,17 +16,16 @@ lex print( %NC.tree '\n' ) input.push_ignore( NC.tree ) } -} +end # # Token translation # lex -{ literal '(', ')' token nc_data /[^()]+/ -} +end def nc_item [nc_data] diff --git a/test/order1.lm b/test/order1.lm index e965bdf2..e7260118 100644 --- a/test/order1.lm +++ b/test/order1.lm @@ -1,6 +1,5 @@ lex -{ token c_single_lit /( 'L'? "'" ( [^'\\\n] | '\\' any )* "'" )/ token c_double_lit /( 'L'? '"' ( [^"\\\n] | '\\' any )* '"' )/ @@ -15,7 +14,7 @@ lex token comm_c /( '/*' (any | '\n')* :>> '*/' )/ token comm_cxx /( '//' any* :> '\n' )/ token ws /( any - 33..126 )+/ -} +end def c_token [c_single_lit] @@ -37,7 +36,6 @@ def c literal '%%' lex -{ literal '{', '}' literal 'protocol', 'client', 'server', 'port', 'by', 'tcp', 'udp' token id /[A-Za-z_][A-Za-z_0-9]*/ @@ -45,7 +43,7 @@ lex ignore /'/*' any* :>> '*/'/ ignore /[ \t\r\n]+/ -} +end def tcp_by_port ['tcp' 'by' 'port'] diff --git a/test/order2.lm b/test/order2.lm index e7142abb..f19168a8 100644 --- a/test/order2.lm +++ b/test/order2.lm @@ -1,6 +1,5 @@ lex -{ token c_single_lit /( 'L'? "'" ( [^'\\\n] | '\\' any )* "'" )/ token c_double_lit /( 'L'? '"' ( [^"\\\n] | '\\' any )* '"' )/ @@ -15,7 +14,7 @@ lex token comm_c /( '/*' (any | '\n')* :>> '*/' )/ token comm_cxx /( '//' any* :> '\n' )/ token ws /( any - 33..126 )+/ -} +end def c_token [c_single_lit] @@ -38,7 +37,6 @@ def c [c_token_list] lex -{ literal '%%' literal '{', '}' literal 'protocol', 'client', 'server', 'port', 'by', 'tcp', 'udp' @@ -47,7 +45,7 @@ lex ignore /'/*' any* :>> '*/'/ ignore /[ \t\r\n]+/ -} +end def tcp_by_port ['tcp' 'by' 'port'] diff --git a/test/ragelambig1.lm b/test/ragelambig1.lm index da0a6a28..c4921192 100644 --- a/test/ragelambig1.lm +++ b/test/ragelambig1.lm @@ -1,5 +1,4 @@ lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -8,7 +7,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def start diff --git a/test/ragelambig2.lm b/test/ragelambig2.lm index 709271e1..67e84503 100644 --- a/test/ragelambig2.lm +++ b/test/ragelambig2.lm @@ -1,5 +1,4 @@ lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -8,7 +7,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def start diff --git a/test/ragelambig3.lm b/test/ragelambig3.lm index 364ffb1e..10061f6a 100644 --- a/test/ragelambig3.lm +++ b/test/ragelambig3.lm @@ -1,5 +1,4 @@ lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -8,7 +7,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def start diff --git a/test/ragelambig4.lm b/test/ragelambig4.lm index 75e2298e..9ae6458d 100644 --- a/test/ragelambig4.lm +++ b/test/ragelambig4.lm @@ -1,5 +1,4 @@ lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -8,7 +7,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def start diff --git a/test/rediv.lm b/test/rediv.lm index f270f6a4..d22d66cf 100644 --- a/test/rediv.lm +++ b/test/rediv.lm @@ -1,12 +1,11 @@ # Or-literal scanner lex -{ token orlit_dash /'-' / token orlit_close /']'/ rl orlit_specials /[\-\]]/ token orlit_chr /^orlit_specials | '\\' any/ -} +end def orlit_item [orlit_chr] @@ -17,7 +16,6 @@ def orlit # Regex scanner lex -{ token orlit_open /'['/ token orlit_neg_open /'[^'/ token regex_dot /'.'/ @@ -26,7 +24,7 @@ lex rl regex_specials /[\[\.\*\/\\]/ token regex_chr /(^regex_specials)+ | '\\' any/ -} +end def regex_rep [regex_star] @@ -49,7 +47,6 @@ rl d_string /'"' ([^"\\\n] | '\\' any )* '"'/ # Root scanner lex -{ token ident /[a-zA-Z_]+/ token number /[0-9]+/ token string /s_string | d_string/ @@ -59,7 +56,7 @@ lex token semi /';'/ ignore wp /[ \t\n]+/ -} +end def factor [ident] diff --git a/test/reparse.lm b/test/reparse.lm index f972a497..317d5c01 100644 --- a/test/reparse.lm +++ b/test/reparse.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ -} +end def item [id] diff --git a/test/repeat1.lm b/test/repeat1.lm index eed176f2..ddefaa1e 100644 --- a/test/repeat1.lm +++ b/test/repeat1.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ -} +end def item [id] diff --git a/test/repeat2.lm b/test/repeat2.lm index 0a3a92da..d9104492 100644 --- a/test/repeat2.lm +++ b/test/repeat2.lm @@ -20,7 +20,6 @@ lex -{ token word /( [^. \t\n]+ | '.' )/ token lws /[ \t]+/ token nl / '\n'/ @@ -70,25 +69,22 @@ lex token cmd_li /'.li' lws/ token cmd_license /'.license' lws? '\n'/ -} +end lex -{ token bar_data /[^|]*/ token end_bar /'|'/ -} +end lex -{ token slash_data /[^/]*/ token end_slash /'/'/ -} +end lex -{ token curly_data /[^}]*/ token end_curly /'}'/ -} +end def cmd_il [cmd_verb1 bar_data end_bar] @@ -104,28 +100,25 @@ def text | [cmd_il] lex -{ token end_verbatim /lws? '.' lws? 'end' lws 'verbatim' lws? '\n'/ token verbatim_line /[^\n]* '\n'/ -} +end def verbatim [cmd_verbatim verbatim_line* end_verbatim] lex -{ token end_code /lws? '.' lws? 'end' lws 'code' lws? '\n'/ token code_line /[^\n]* '\n'/ -} +end def code [cmd_code code_line* end_code] lex -{ token end_comment /lws? '.' lws? 'end' lws 'comment' lws? '\n'/ token comment_line /[^\n]* '\n'/ -} +end def comment [cmd_comment comment_line* end_comment] diff --git a/test/rhsref1.lm b/test/rhsref1.lm index bc5cbf40..ff7d0e4d 100644 --- a/test/rhsref1.lm +++ b/test/rhsref1.lm @@ -1,5 +1,4 @@ lex -{ literal 'var', 'if', 'then', 'else', 'while', 'do', 'for', 'read', 'write', 'end', 'to', 'goto' literal ':=', '!=', ';', '+', '-', '*', '/', '=', '(', ')', ':' @@ -9,7 +8,7 @@ lex token id /[a-zA-Z_]+/ token integernumber /[0-9]+/ token stringlit /'"' [^"]* '"'/ -} +end def program [statement*] diff --git a/test/rubyhere.lm b/test/rubyhere.lm index 91fcc628..39fc9848 100644 --- a/test/rubyhere.lm +++ b/test/rubyhere.lm @@ -4,19 +4,17 @@ context rubyhere rl number_pattern /[0-9]+/ lex - { ignore /[ \t\n]+/ token id /ident_pattern/ token number /number_pattern/ literal '<<', '*', ',', '(', ')', '!' - } + end HereId: str token rest_of_line /[^\n]*'\n'/ lex - { ignore /[ \t\n]+/ token here_id HereData: here_data @@ -37,10 +35,9 @@ context rubyhere # Send the here_id token. Attach the heredoc data as an attribute. input.push( make_token( typeid<here_id> HereId HereData.tree ) ) } - } + end lex - { token here_close_id / ident_pattern '\n' / { @@ -55,7 +52,7 @@ context rubyhere token here_line / [^\n]* '\n' / - } + end def here_data [here_line* here_close_id] diff --git a/test/string.lm b/test/string.lm index 1d24525e..c16561fe 100644 --- a/test/string.lm +++ b/test/string.lm @@ -1,8 +1,7 @@ lex -{ token str_escape /'\\' any/ token str_chr /[^\\"]+/ -} +end def str_item [str_escape] @@ -12,7 +11,6 @@ def string ['"' str_item* '"'] lex -{ token ident /[a-zA-Z_]+/ token number /[0-9]+/ @@ -20,7 +18,7 @@ lex literal '+=', '-=', '*=' ignore wp /[ \t\n]+/ -} +end def expr [expr '+' term] diff --git a/test/superid.lm b/test/superid.lm index 9d8c3ffb..75bebe69 100644 --- a/test/superid.lm +++ b/test/superid.lm @@ -1,7 +1,6 @@ context si { lex - { literal '!', 'a', ';\n' token id /'a'|'b'/ @@ -13,7 +12,7 @@ context si token foo // ignore ws / [ \n\t]+ / - } + end trans_id_to: int diff --git a/test/tags1.lm b/test/tags1.lm index 0730664e..06edfe33 100644 --- a/test/tags1.lm +++ b/test/tags1.lm @@ -16,7 +16,6 @@ context tags # Any single character can be a literal lex - { literal '!\n', ';\n' # Ignore whitespace. @@ -24,7 +23,7 @@ context tags # Open and close id token id /rl_id/ - } + end # # Global Data diff --git a/test/tags2.lm b/test/tags2.lm index 58dcc789..c3acd437 100644 --- a/test/tags2.lm +++ b/test/tags2.lm @@ -27,7 +27,6 @@ rl char_ref_pat / '&#' [0-9]+ ';' | '&0x' [0-9a-fA-F]+ ';' / # Single quotes. # lex -{ token sq_close /'\''/ # References in single quotes @@ -44,13 +43,12 @@ lex # The opening quote belongs to the tag region. def sq_string [ '\'' sq_item* sq_close ] -} +end # # Double quotes. # lex -{ token dq_close /'"'/ # References in double quotes @@ -67,13 +65,12 @@ lex # The opening quote belongs to the tag region. def dq_string [ '"' dq_item* dq_close ] -} +end # # Tag elements. # lex -{ literal '\'', '\"', '=', '\/' # Within this region whitespace is not significant. @@ -83,7 +80,7 @@ lex # Attributes # token attr_name / name / -} +end literal '>' @@ -91,7 +88,6 @@ literal '>' # Top Level # lex -{ # # Comments # @@ -112,7 +108,7 @@ lex token char_data / [^<&]+ / token entity_ref /entity_ref_pat/ token char_ref /char_ref_pat/ -} +end def attribute_value diff --git a/test/tags3.lm b/test/tags3.lm index 7cd7f03f..75ac2ab3 100644 --- a/test/tags3.lm +++ b/test/tags3.lm @@ -11,29 +11,26 @@ context tags # Scanner for tag names. # lex - { ignore /space+/ token tag_id /def_name/ - } + end # # Scanner for attributes names # lex - { ignore /space+/ token attr_name /def_name_char+/ literal '=' - } + end # Scanner for attribute values. lex - { ignore /space+/ token dquote_val /'"' ([^"] | '\\' any)* '"'/ token squote_val /"'" ([^'] | '\\' any)* "'"/ token unq_val /[^ \t\r\n<>"'] [^ \t\r\n<>]*/ - } + end literal '>', '/>' @@ -42,12 +39,11 @@ context tags # lex - { ignore /space+/ literal '<', '</', '<!DOCTYPE' token doc_data /[^<]+/ token comment /"<!--" any* :>> "-->"/ - } + end # # Tags @@ -56,7 +52,6 @@ context tags # This scanner is just for the id in close tags. The id needs to be looked up # in the tag stack so we can determine if it is a stray. lex - { # Ignore whitespace. ignore /space+/ @@ -78,7 +73,7 @@ context tags input.push( make_token( send_id input.pull(match_length) ) ) } - } + end # # Tag Stack @@ -95,22 +90,20 @@ context tags # # This scanner handles inside DOCTYPE tags (except keywords). lex - { ignore /space+/ token dt_name /def_name/ token dt_literal /def_system_literal/ token dt_bl /"[" [^\]]* "]"/ - } + end token dt_close /'>'/ # Using a separate scanner for the keywords in DOCTYPE prevents them from # covering dt_name lex - { ignore /space+/ literal 'SYSTEM', 'PUBLIC' - } + end def DOCTYPE ['<!DOCTYPE' dt_name external_id dt_bl? dt_close] diff --git a/test/tags4.lm b/test/tags4.lm index 0e01ee8d..6983a574 100644 --- a/test/tags4.lm +++ b/test/tags4.lm @@ -19,38 +19,34 @@ context tags # Scanner for tag names. # lex - { ignore /space+/ token tag_id /def_name/ - } + end # # Scanner for attributes names # lex - { ignore /space+/ token attr_name /def_name_char+/ literal '=' - } + end literal '>', '/>' # Scanner for attribute values. lex - { ignore /space+/ token dquote_val /'"' ([^"] | '\\' any)* '"'/ token squote_val /"'" ([^'] | '\\' any)* "'"/ token unq_val /[^ \t\r\n<>"'] [^ \t\r\n<>]*/ - } + end # # Tokens # lex - { ignore /space+/ literal '<', '</', '<!DOCTYPE' @@ -59,7 +55,7 @@ context tags token doc_data /[^<]+/ token comment /'<!--' any* :>> '-->'/ - } + end # # Tags @@ -78,7 +74,6 @@ context tags # This scanner is just for the id in close tags. The id needs to be looked up # in the tag stack so we can determine if it is a stray. lex - { # Ignore whitespace. ignore /space+/ @@ -114,7 +109,7 @@ context tags input.push( make_token( typeid<stray_close> input.pull( match_length ) ) ) } } - } + end # # Tag Stack @@ -131,21 +126,19 @@ context tags # # This scanner handles inside DOCTYPE tags (except keywords). lex - { ignore /space+/ token dt_name /def_name/ token dt_literal /def_system_literal/ token dt_bl /"[" [^\]]* "]"/ token dt_close /'>'/ - } + end # Using a separate scanner for the keywords in DOCTYPE prevents them from # covering dt_name lex - { ignore /space+/ literal 'SYSTEM', 'PUBLIC' - } + end def DOCTYPE ['<!DOCTYPE' dt_name external_id dt_bl? dt_close] diff --git a/test/til.lm b/test/til.lm index 04563ae7..615f203c 100644 --- a/test/til.lm +++ b/test/til.lm @@ -1,5 +1,4 @@ lex -{ literal 'var', 'if', 'then', 'else', 'while', 'do', 'for', 'read', 'write', 'end', 'to', 'goto' literal ':=', '!=', ';', '+', '-', '*', '/', '=', '(', ')', ':' @@ -9,7 +8,7 @@ lex token id /[a-zA-Z_]+/ token integernumber /[0-9]+/ token stringlit /'"' [^"]* '"'/ -} +end def program [statement*] diff --git a/test/translate1.lm b/test/translate1.lm index 6de3af85..01cc9ba8 100644 --- a/test/translate1.lm +++ b/test/translate1.lm @@ -1,5 +1,4 @@ lex -{ ignore /space+/ literal '*', '(', ')' token id /[a-zA-Z_]+/ @@ -7,7 +6,7 @@ lex t: str = input.pull( match_length ) input.push( make_token( typeid<id> t ) ) } -} +end def foo [id] diff --git a/test/translate2.lm b/test/translate2.lm index bb0d88e8..7d746e28 100644 --- a/test/translate2.lm +++ b/test/translate2.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '#', '{', '}' token id2 /[a-zA-Z_]+/ -} +end def item2 [id2] @@ -14,9 +13,7 @@ def start2 context ctx { - lex - { ignore /space+/ literal '*', '(', ')', '!', ';\n' token id /[a-zA-Z_0-9]+/ @@ -28,7 +25,7 @@ context ctx input.push( make_token( typeid<id> "dot" ) ) input.push( make_token( typeid<id> "dot" ) ) } - } + end def item [id] diff --git a/test/travs1.lm b/test/travs1.lm index 20145ac9..b5f756c3 100644 --- a/test/travs1.lm +++ b/test/travs1.lm @@ -1,5 +1,4 @@ lex -{ ignore /[\t\n ]+/ literal '^', '|', '-', ',', ':', '!', '?', '.' literal '(', ')', '{', '}', '*', '&', '+' @@ -8,7 +7,7 @@ lex token word /[a-zA-Z_][a-zA-Z0-9_]*/ token uint /[0-9]+/ -} +end def start [expression] diff --git a/test/treecmp1.lm b/test/treecmp1.lm index c7fcc028..c8f351e3 100644 --- a/test/treecmp1.lm +++ b/test/treecmp1.lm @@ -2,11 +2,10 @@ rl ident_pattern /[a-zA-Z_][a-zA-Z_0-9]*/ rl number_pattern /[0-9]+/ lex -{ ignore /[ \t\n]+/ token id /ident_pattern/ token number /number_pattern/ -} +end def four_ids [id id id id] diff --git a/test/undofrag1.lm b/test/undofrag1.lm index 117b635f..de0a7cd2 100644 --- a/test/undofrag1.lm +++ b/test/undofrag1.lm @@ -1,9 +1,8 @@ lex -{ ignore /space+/ literal '#', '{', '}' token id2 /[a-zA-Z_]+/ -} +end def item2 [id2] @@ -18,11 +17,10 @@ context ctx SP: parser<start2> lex - { ignore /space+/ literal '*', '(', ')', '!', ';\n' token id /[a-zA-Z_]+/ - } + end def item [id] diff --git a/test/undofrag2.lm b/test/undofrag2.lm index b65b17ce..f8659bae 100644 --- a/test/undofrag2.lm +++ b/test/undofrag2.lm @@ -1,11 +1,10 @@ context undo { lex - { ignore /[ \t]+/ literal '*', '(', ')', '^', ';', '\n' token id /[a-zA-Z_]+/ - } + end Out: parser<out> diff --git a/test/undofrag3.lm b/test/undofrag3.lm index c7066fa7..9a39e98d 100644 --- a/test/undofrag3.lm +++ b/test/undofrag3.lm @@ -1,11 +1,10 @@ context undo { lex - { ignore /[ \t]+/ literal '*', '(', ')', '^', ';', '\n', '.' token id /[a-zA-Z_]+/ - } + end Out: parser<out> |