From 4714be8ce5f5f1ffb0bc7481504b60bad7399908 Mon Sep 17 00:00:00 2001 From: Adrian Thurston Date: Sun, 21 Jun 2020 22:05:23 +0300 Subject: testing a token redef --- src/colm.lm | 8 ++++++++ src/consinit.cc | 10 +++++----- src/loadfinal.cc | 30 ++++++++++++++++++++++++++++-- src/loadinit.cc | 4 ++-- src/parser.cc | 6 ++++-- src/parser.h | 4 ++-- src/parsetree.h | 2 ++ 7 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/colm.lm b/src/colm.lm index 16a9eeb5..bab4f77a 100644 --- a/src/colm.lm +++ b/src/colm.lm @@ -214,6 +214,7 @@ def root_item [rl_def] :Rl commit | [literal_def] :Literal commit | [token_def] :Token commit +| [token_redef] :TokenRedef commit | [ic_def] :IgnoreCollector commit | [ignore_def] :Ignore commit | [cfl_def] :CflDef commit @@ -467,6 +468,13 @@ def token_def no_ignore_right opt_translate] +def token_redef + [REDEF TOKEN id VarDefList: var_def<* + no_ignore_left + LEX_FSLASH opt_lex_expr LEX_FSLASH + no_ignore_right + opt_translate] + def ic_def [TOKEN id MINUS] diff --git a/src/consinit.cc b/src/consinit.cc index 50775084..bed6a636 100644 --- a/src/consinit.cc +++ b/src/consinit.cc @@ -229,7 +229,7 @@ void ConsInit::wsIgnore() LexExpression *expr = LexExpression::cons( term ); LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, String(), join, objectDef, 0, true, false, false ); + defineToken( internal, false, String(), join, objectDef, 0, true, false, false ); } void ConsInit::commentIgnore() @@ -249,7 +249,7 @@ void ConsInit::commentIgnore() LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, String(), join, objectDef, 0, true, false, false ); + defineToken( internal, false, String(), join, objectDef, 0, true, false, false ); } void ConsInit::idToken() @@ -275,7 +275,7 @@ void ConsInit::idToken() LexExpression *expr = LexExpression::cons( concat ); LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, hello, join, objectDef, 0, false, false, false ); + defineToken( internal, false, hello, join, objectDef, 0, false, false, false ); } void ConsInit::literalToken() @@ -308,7 +308,7 @@ void ConsInit::literalToken() LexExpression *expr = LexExpression::cons( concat ); LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, hello, join, objectDef, 0, false, false, false ); + defineToken( internal, false, hello, join, objectDef, 0, false, false, false ); } void ConsInit::keyword( const String &name, const String &lit ) @@ -317,7 +317,7 @@ void ConsInit::keyword( const String &name, const String &lit ) LexTerm *term = litTerm( lit ); LexExpression *expr = LexExpression::cons( term ); LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, name, join, objectDef, 0, false, false, false ); + defineToken( internal, false, name, join, objectDef, 0, false, false, false ); } void ConsInit::keyword( const String &kw ) diff --git a/src/loadfinal.cc b/src/loadfinal.cc index d478a446..84f85368 100644 --- a/src/loadfinal.cc +++ b/src/loadfinal.cc @@ -397,10 +397,33 @@ struct LoadColm CodeBlock *translate = walkOptTranslate( TokenDef.opt_translate() ); - defineToken( TokenDef.id().loc(), name, join, objectDef, + defineToken( TokenDef.id().loc(), false, name, join, objectDef, translate, false, niLeft, niRight ); } + void walkTokenRedef( token_redef TokenRedef ) + { + String name = TokenRedef.id().data(); + + bool niLeft = walkNoIgnoreLeft( TokenRedef.no_ignore_left() ); + bool niRight = walkNoIgnoreRight( TokenRedef.no_ignore_right() ); + + ObjectDef *objectDef = walkVarDefList( TokenRedef.VarDefList() ); + objectDef->name = name; + + LexJoin *join = 0; + if ( TokenRedef.opt_lex_expr().lex_expr() != 0 ) { + LexExpression *expr = walkLexExpr( TokenRedef.opt_lex_expr().lex_expr() ); + join = LexJoin::cons( expr ); + } + + CodeBlock *translate = walkOptTranslate( TokenRedef.opt_translate() ); + + defineToken( TokenRedef.id().loc(), true, name, join, objectDef, + translate, false, niLeft, niRight ); + } + + void walkIgnoreCollector( ic_def IgnoreCollector ) { String id = IgnoreCollector.id().data(); @@ -452,7 +475,7 @@ struct LoadColm join = LexJoin::cons( expr ); } - defineToken( IgnoreDef.IGNORE().loc(), name, join, objectDef, + defineToken( IgnoreDef.IGNORE().loc(), false, name, join, objectDef, 0, true, false, false ); } @@ -2771,6 +2794,9 @@ struct LoadColm case root_item::Token: walkTokenDef( rootItem.token_def() ); break; + case root_item::TokenRedef: + walkTokenRedef( rootItem.token_redef() ); + break; case root_item::IgnoreCollector: walkIgnoreCollector( rootItem.ic_def() ); break; diff --git a/src/loadinit.cc b/src/loadinit.cc index d9a047d3..045345f4 100644 --- a/src/loadinit.cc +++ b/src/loadinit.cc @@ -237,7 +237,7 @@ void LoadInit::walkTokenList( token_list &tokenList ) bool leftNi = walkNoIgnore( tokenDef.LeftNi() ); bool rightNi = walkNoIgnore( tokenDef.RightNi() ); - defineToken( internal, name, join, objectDef, 0, false, leftNi, rightNi ); + defineToken( internal, false, name, join, objectDef, 0, false, leftNi, rightNi ); } if ( tokenList.IgnoreDef() != 0 ) { @@ -249,7 +249,7 @@ void LoadInit::walkTokenList( token_list &tokenList ) LexExpression *expr = walkLexExpr( LexExpr ); LexJoin *join = LexJoin::cons( expr ); - defineToken( internal, String(), join, objectDef, 0, true, false, false ); + defineToken( internal, false, String(), join, objectDef, 0, true, false, false ); } } diff --git a/src/parser.cc b/src/parser.cc index ce598e38..9009c0e9 100644 --- a/src/parser.cc +++ b/src/parser.cc @@ -297,8 +297,8 @@ LexJoin *BaseParser::literalJoin( const InputLoc &loc, const String &data ) return join; } -void BaseParser::defineToken( const InputLoc &loc, String name, LexJoin *join, - ObjectDef *objectDef, CodeBlock *transBlock, bool ignore, +void BaseParser::defineToken( const InputLoc &loc, bool isRedef, String name, + LexJoin *join, ObjectDef *objectDef, CodeBlock *transBlock, bool ignore, bool noPreIgnore, bool noPostIgnore ) { bool pushedRegion = false; @@ -323,6 +323,8 @@ void BaseParser::defineToken( const InputLoc &loc, String name, LexJoin *join, TokenDef *tokenDef = TokenDef::cons( name, String(), false, ignore, join, transBlock, loc, 0, nspace, regionSet, objectDef, curStruct() ); + + tokenDef->isRedef = isRedef; regionSet->tokenDefList.append( tokenDef ); nspace->tokenDefList.append( tokenDef ); diff --git a/src/parser.h b/src/parser.h index 98956c0a..b55cc7bf 100644 --- a/src/parser.h +++ b/src/parser.h @@ -92,8 +92,8 @@ struct BaseParser Reduction *createReduction( const InputLoc loc, const String &name ); - void defineToken( const InputLoc &loc, String name, LexJoin *join, - ObjectDef *objectDef, CodeBlock *transBlock, + void defineToken( const InputLoc &loc, bool isRedef, String name, + LexJoin *join, ObjectDef *objectDef, CodeBlock *transBlock, bool ignore, bool noPreIgnore, bool noPostIgnore ); void zeroDef( const InputLoc &loc, const String &name ); diff --git a/src/parsetree.h b/src/parsetree.h index 1890bbae..bb63b7a2 100644 --- a/src/parsetree.h +++ b/src/parsetree.h @@ -404,6 +404,7 @@ struct TokenDef t->noPostIgnore = false; t->noPreIgnore = false; t->isZero = false; + t->isRedef = false; return t; } @@ -436,6 +437,7 @@ struct TokenDef bool noPostIgnore; bool noPreIgnore; bool isZero; + bool isRedef; }; struct TokenInstancePtr -- cgit v1.2.1