rl ident_pattern /[a-zA-Z_][a-zA-Z_0-9]*/
rl number_pattern /[0-9]+/

lex start
{
	ignore /[ \t\n]+/
	token id /ident_pattern/
	token number /number_pattern/
	literal '<<', '*', ',', '(', ')', '!'
}

global HereId: str

token rest_of_line /[^\n]*'\n'/

lex here_start
{
	ignore /[ \t\n]+/
	token here_id
		HereData: here_data
		/ident_pattern/
		{
			# Take the text of the here_id from the input stream.
			HereId = input.pull( match_length )

			# Get the data up to the rest of the line.
			ROL: rest_of_line = parse_stop rest_of_line( stdin )

			# Parse the heredoc data.
			HereData: here_data HereData = parse_stop here_data( stdin )

			# Push the rest-of-line data back to the input stream.
			input.push( $ROL )

			# Send the here_id token. Attach the heredoc data as an attribute.
			input.push( make_token( typeid here_id HereId HereData ) )
		}
}

lex here_data
{
	token here_close_id 
		/ ident_pattern '\n' /
		{
			if match_text == HereId + '\n' {
				input.push( make_token( 
					typeid here_close_id
					input.pull( match_length ) ) )
			}
			else 
				input.push( make_token( typeid here_line input.pull(match_length) ) )
		}

	token here_line
		/ [^\n]* '\n' /
}

def here_data
	[here_line* here_close_id]

def heredoc
	['<<' here_id]

def primary
	[id]
|	[number]
|	[heredoc]

def arglist
	[primary arglist_more*]

def arglist_more
	[',' primary]

def call
	[id '(' arglist? ')']

def statement
	[primary]
|	[call]

token foobar /any*/

def item
	[statement '!']
|	[foobar]

def start 
	[item*]

S: start = parse start( stdin )
print_xml(S)