diff options
| -rw-r--r-- | colm.vim | 2 | ||||
| -rw-r--r-- | src/colm.lm | 8 | ||||
| -rw-r--r-- | src/declare.cc | 26 | ||||
| -rw-r--r-- | src/loadcolm.cc | 20 | ||||
| -rw-r--r-- | src/parser.cc | 28 | ||||
| -rw-r--r-- | src/parser.h | 3 | ||||
| -rw-r--r-- | src/pdabuild.cc | 4 | ||||
| -rw-r--r-- | test/ignore3.lm | 6 | ||||
| -rw-r--r-- | test/ignore5.lm | 4 | ||||
| -rw-r--r-- | test/superid.lm | 4 |
10 files changed, 58 insertions, 47 deletions
@@ -40,7 +40,7 @@ syntax match char "\\." contained syntax match otLit "\~.*$" syntax match otLit "'\(\\.\|[^'\\]\)*\('[i]*\)\?" -syntax match otLit "`[^ \t\r\]]*" +syntax match otLit "`[^ \t\r\]]\+" syntax match otLit "`\]" " diff --git a/src/colm.lm b/src/colm.lm index 35a155c2..feeec83c 100644 --- a/src/colm.lm +++ b/src/colm.lm @@ -72,7 +72,7 @@ lex / ( '0' .. '9' ) + / token backtick_lit - / '`' . ^( ' ' | '\n' | '\t' | ']' )* | '`]' / + / '`' . ^( ' ' | '\n' | '\t' | ']' )+ | '`]' / token DQ / '\"' / ni token SQ / '\'' / ni @@ -196,6 +196,7 @@ def root_item [rl_def] :Rl commit | [literal_def] :Literal commit | [token_def] :Token commit +| [ic_def] :IgnoreCollector commit | [ignore_def] :Ignore commit | [cfl_def] :Cfl commit | [region_def] :Region commit @@ -241,6 +242,7 @@ def context_item | [literal_def] :Literal commit | [rl_def] :Rl commit | [token_def] :Token commit +| [ic_def] :IgnoreCollector commit | [ignore_def] :Ignore commit | [cfl_def] :Cfl commit | [region_def] :Region commit @@ -309,6 +311,7 @@ def namespace_item [rl_def] :Rl commit | [literal_def] :Literal commit | [token_def] :Token commit +| [ic_def] :IgnoreCollector commit | [ignore_def] :Ignore commit | [cfl_def] :Cfl commit | [region_def] :Region commit @@ -351,6 +354,9 @@ def token_def NiRight: opt_no_ignore opt_translate] +def ic_def + [TOKEN id MINUS] + def opt_translate [COPEN lang_stmt_list CCLOSE] :Translate | [] diff --git a/src/declare.cc b/src/declare.cc index 5ee113fc..4388c452 100644 --- a/src/declare.cc +++ b/src/declare.cc @@ -468,6 +468,12 @@ void Namespace::declare( Compiler *pd ) tokEl->noPostIgnore = true; tokenDef->tdLangEl = tokEl; + + if ( tokenDef->isZero ) { + /* Setting zero lel to newly created tokEl. */ + tokenDef->regionSet->collectIgnore->zeroLel = tokEl; + tokEl->isZero = true; + } } } @@ -505,13 +511,15 @@ void Namespace::declare( Compiler *pd ) void Compiler::makeIgnoreCollectors() { for ( RegionSetList::Iter regionSet = regionSetList; regionSet.lte(); regionSet++ ) { - String name( 128, "_ign_%p", regionSet->tokenIgnore ); - LangEl *zeroLel = new LangEl( rootNamespace, name, LangEl::Term ); - langEls.append( zeroLel ); - zeroLel->isZero = true; - zeroLel->regionSet = regionSet; - - regionSet->collectIgnore->zeroLel = zeroLel; + if ( regionSet->collectIgnore->zeroLel == 0 ) { + String name( 128, "_ign_%p", regionSet->tokenIgnore ); + LangEl *zeroLel = new LangEl( rootNamespace, name, LangEl::Term ); + langEls.append( zeroLel ); + zeroLel->isZero = true; + zeroLel->regionSet = regionSet; + + regionSet->collectIgnore->zeroLel = zeroLel; + } } } @@ -1222,13 +1230,15 @@ void Compiler::declarePass() declareReVars(); makeDefaultIterators(); - makeIgnoreCollectors(); for ( FunctionList::Iter f = functionList; f.lte(); f++ ) makeFuncVisible( f, f->isUserIter ); rootNamespace->declare( this ); + /* Will fill in zero lels that were not declared. */ + makeIgnoreCollectors(); + declareByteCode(); initIntObject(); diff --git a/src/loadcolm.cc b/src/loadcolm.cc index 3ed822fc..40576040 100644 --- a/src/loadcolm.cc +++ b/src/loadcolm.cc @@ -336,6 +336,12 @@ struct LoadColm translate, false, niLeft, niRight ); } + void walkIgnoreCollector( ic_def IgnoreCollector ) + { + String id = IgnoreCollector.id().data(); + zeroDef( IgnoreCollector.id().loc(), id ); + } + String walkOptId( opt_id optId ) { String name; @@ -2054,6 +2060,9 @@ struct LoadColm case context_item::_Token: walkTokenDef( contextItem.token_def() ); break; + case context_item::_IgnoreCollector: + walkIgnoreCollector( contextItem.ic_def() ); + break; case context_item::_Ignore: walkIgnoreDef( contextItem.ignore_def() ); break; @@ -2119,6 +2128,9 @@ struct LoadColm case root_item::_Token: walkTokenDef( rootItem.token_def() ); break; + case root_item::_IgnoreCollector: + walkIgnoreCollector( rootItem.ic_def() ); + break; case root_item::_Ignore: walkIgnoreDef( rootItem.ignore_def() ); break; @@ -2186,6 +2198,9 @@ struct LoadColm case namespace_item::_Token: walkTokenDef( item.token_def() ); break; + case root_item::_IgnoreCollector: + walkIgnoreCollector( item.ic_def() ); + break; case namespace_item::_Ignore: walkIgnoreDef( item.ignore_def() ); break; @@ -2243,10 +2258,7 @@ struct LoadColm bool niRight = walkOptNoIgnore( literalItem.NiRight() ); String lit = literalItem.backtick_lit().data(); - if ( strcmp( lit, "`" ) == 0 ) - zeroDef( literalItem.backtick_lit().loc(), lit, niLeft, niRight ); - else - literalDef( literalItem.backtick_lit().loc(), lit, niLeft, niRight ); + literalDef( literalItem.backtick_lit().loc(), lit, niLeft, niRight ); } void walkLiteralList( literal_list literalList ) diff --git a/src/parser.cc b/src/parser.cc index 462255d5..f1a0dfc6 100644 --- a/src/parser.cc +++ b/src/parser.cc @@ -235,41 +235,25 @@ void BaseParser::defineToken( const InputLoc &loc, String name, LexJoin *join, O } } -void BaseParser::zeroDef( const InputLoc &loc, const String &data, - bool noPreIgnore, bool noPostIgnore ) +void BaseParser::zeroDef( const InputLoc &loc, const String &name ) { - /* Create a name for the literal. */ - String name( 32, "_literal_%.4x", pd->nextTokenId ); - if ( !insideRegion() ) error(loc) << "zero token should be inside token" << endp; - String interp("");; - - /* Look for the production's associated region. */ - Namespace *nspace = curNspace(); RegionSet *regionSet = regionStack.top(); + Namespace *nspace = curNspace(); - LiteralDictEl *ldel = nspace->literalDict.find( interp ); - if ( ldel != 0 ) - error( loc ) << "literal already defined in this namespace" << endp; - - LexJoin *join = literalJoin( loc, data ); + LexJoin *join = literalJoin( loc, String("`") ); - TokenDef *tokenDef = TokenDef::cons( name, data, true, false, join, - 0, loc, 0, nspace, regionSet, 0, 0 ); + TokenDef *tokenDef = TokenDef::cons( name, String(), false, false, join, + 0, loc, 0, nspace, regionSet, 0, curContext() ); tokenDef->isZero = true; regionSet->tokenDefList.append( tokenDef ); nspace->tokenDefList.append( tokenDef ); - TokenInstance *tokenInstance = TokenInstance::cons( tokenDef, join, - loc, pd->nextTokenId++, nspace, regionSet->tokenIgnore ); - - /* Doesn't go into instance list. */ - - ldel = nspace->literalDict.insert( interp, tokenInstance ); + /* No token instance created. */ } void BaseParser::literalDef( const InputLoc &loc, const String &data, diff --git a/src/parser.h b/src/parser.h index 37af7e7e..c4d02fe4 100644 --- a/src/parser.h +++ b/src/parser.h @@ -74,8 +74,7 @@ struct BaseParser void defineToken( const InputLoc &loc, String name, LexJoin *join, ObjectDef *objectDef, CodeBlock *transBlock, bool ignore, bool noPreIgnore, bool noPostIgnore ); - void zeroDef( const InputLoc &loc, const String &data, - bool noPreIgnore, bool noPostIgnore ); + void zeroDef( const InputLoc &loc, const String &name ); void literalDef( const InputLoc &loc, const String &data, bool noPreIgnore, bool noPostIgnore ); diff --git a/src/pdabuild.cc b/src/pdabuild.cc index fcbbcfb4..7978edff 100644 --- a/src/pdabuild.cc +++ b/src/pdabuild.cc @@ -488,8 +488,8 @@ void Compiler::addRegion( PdaState *tabState, PdaTrans *tabTrans, /* If it is not the eof, then use the region associated * with the token definition. */ if ( langEl->isZero ) { - region = langEl->regionSet->collectIgnore; - regionSet = langEl->regionSet; + region = langEl->tokenDef->regionSet->collectIgnore; + regionSet = langEl->tokenDef->regionSet; } else if ( !langEl->isEOF && langEl->tokenDef != 0 ) { region = langEl->tokenDef->regionSet->tokenIgnore; diff --git a/test/ignore3.lm b/test/ignore3.lm index 71aeb133..681e0ed0 100644 --- a/test/ignore3.lm +++ b/test/ignore3.lm @@ -11,12 +11,12 @@ end lex ignore /space+/ token inner_t /[a-zA-Z_0-9]+/ - - literal ` + token empty - end def inner - [` inner_t*] + [inner_t*] +| [empty] def item [id] diff --git a/test/ignore5.lm b/test/ignore5.lm index 23604c56..f626c572 100644 --- a/test/ignore5.lm +++ b/test/ignore5.lm @@ -11,11 +11,11 @@ lex ignore /space+/ token inner_t /[a-zA-Z_0-9]+/ - literal ` + token empty - end def inner - [` inner_t*] + [empty inner_t*] def item [id] diff --git a/test/superid.lm b/test/superid.lm index 0d4060f0..eb190201 100644 --- a/test/superid.lm +++ b/test/superid.lm @@ -68,9 +68,9 @@ print( '\n' ) !a b b a; ##### EXP ##### old_id = NIL -new_id = 14 -old_id = NIL new_id = 13 +old_id = NIL +new_id = 12 this is item2 <si::start><si::item2><si::e2></si::e2><si::_literal_0001>!</si::_literal_0001><si::_literal_0003>a</si::_literal_0003><si::super_id>b</si::super_id><si::super_id>b</si::super_id><si::_literal_0003>a</si::_literal_0003></si::item2><si::SEMI_NL>; </si::SEMI_NL></si::start> |