diff options
Diffstat (limited to 'cheetah/Parser.py')
| -rw-r--r-- | cheetah/Parser.py | 2662 |
1 files changed, 2662 insertions, 0 deletions
diff --git a/cheetah/Parser.py b/cheetah/Parser.py new file mode 100644 index 0000000..7436e9c --- /dev/null +++ b/cheetah/Parser.py @@ -0,0 +1,2662 @@ +# $Id: Parser.py,v 1.137 2008/03/10 05:25:13 tavis_rudd Exp $ +"""Parser classes for Cheetah's Compiler + +Classes: + ParseError( Exception ) + _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer + _HighLevelParser( _LowLevelParser ) + Parser === _HighLevelParser (an alias) + +Meta-Data +================================================================================ +Author: Tavis Rudd <tavis@damnsimple.com> +Version: $Revision: 1.137 $ +Start Date: 2001/08/01 +Last Revision Date: $Date: 2008/03/10 05:25:13 $ +""" +__author__ = "Tavis Rudd <tavis@damnsimple.com>" +__revision__ = "$Revision: 1.137 $"[11:-2] + +import os +import sys +import re +from re import DOTALL, MULTILINE +from types import StringType, ListType, TupleType, ClassType, TypeType +import time +from tokenize import pseudoprog +import inspect +import new +import traceback + +from Cheetah.SourceReader import SourceReader +from Cheetah import Filters +from Cheetah import ErrorCatchers +from Cheetah.Unspecified import Unspecified +from Cheetah.Macros.I18n import I18n + +# re tools +_regexCache = {} +def cachedRegex(pattern): + if pattern not in _regexCache: + _regexCache[pattern] = re.compile(pattern) + return _regexCache[pattern] + +def escapeRegexChars(txt, + escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')): + + """Return a txt with all special regular expressions chars escaped.""" + + return escapeRE.sub(r'\\\1' , txt) + +def group(*choices): return '(' + '|'.join(choices) + ')' +def nongroup(*choices): return '(?:' + '|'.join(choices) + ')' +def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')' +def any(*choices): return apply(group, choices) + '*' +def maybe(*choices): return apply(group, choices) + '?' + +################################################## +## CONSTANTS & GLOBALS ## + +NO_CACHE = 0 +STATIC_CACHE = 1 +REFRESH_CACHE = 2 + +SET_LOCAL = 0 +SET_GLOBAL = 1 +SET_MODULE = 2 + +################################################## +## Tokens for the parser ## + +#generic +identchars = "abcdefghijklmnopqrstuvwxyz" \ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ_" +namechars = identchars + "0123456789" + +#operators +powerOp = '**' +unaryArithOps = ('+', '-', '~') +binaryArithOps = ('+', '-', '/', '//','%') +shiftOps = ('>>','<<') +bitwiseOps = ('&','|','^') +assignOp = '=' +augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=', + '>>=','<<=','&=','|=', ) +assignmentOps = (assignOp,) + augAssignOps + +compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',) +booleanOps = ('and','or','not') +operators = (powerOp,) + unaryArithOps + binaryArithOps \ + + shiftOps + bitwiseOps + assignmentOps \ + + compOps + booleanOps + +delimeters = ('(',')','{','}','[',']', + ',','.',':',';','=','`') + augAssignOps + + +keywords = ('and', 'del', 'for', 'is', 'raise', + 'assert', 'elif', 'from', 'lambda', 'return', + 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', + 'continue', 'exec', 'import', 'pass', + 'def', 'finally', 'in', 'print', + ) + +single3 = "'''" +double3 = '"""' + +tripleQuotedStringStarts = ("'''", '"""', + "r'''", 'r"""', "R'''", 'R"""', + "u'''", 'u"""', "U'''", 'U"""', + "ur'''", 'ur"""', "Ur'''", 'Ur"""', + "uR'''", 'uR"""', "UR'''", 'UR"""') + +tripleQuotedStringPairs = {"'''": single3, '"""': double3, + "r'''": single3, 'r"""': double3, + "u'''": single3, 'u"""': double3, + "ur'''": single3, 'ur"""': double3, + "R'''": single3, 'R"""': double3, + "U'''": single3, 'U"""': double3, + "uR'''": single3, 'uR"""': double3, + "Ur'''": single3, 'Ur"""': double3, + "UR'''": single3, 'UR"""': double3, + } + +closurePairs= {')':'(',']':'[','}':'{'} +closurePairsRev= {'(':')','[':']','{':'}'} + +################################################## +## Regex chunks for the parser ## + +tripleQuotedStringREs = {} +def makeTripleQuoteRe(start, end): + start = escapeRegexChars(start) + end = escapeRegexChars(end) + return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL) + +for start, end in tripleQuotedStringPairs.items(): + tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end) + +WS = r'[ \f\t]*' +EOL = r'\r\n|\n|\r' +EOLZ = EOL + r'|\Z' +escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)') +nameCharLookAhead = r'(?=[A-Za-z_])' +identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*') +EOLre=re.compile(r'(?:\r\n|\r|\n)') + +specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments +# e.g. ##author@ Tavis Rudd + +unicodeDirectiveRE = re.compile( + r'(?:^|\r\n|\r|\n)\s*#\s{0,5}unicode[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE) +encodingDirectiveRE = re.compile( + r'(?:^|\r\n|\r|\n)\s*#\s{0,5}encoding[:\s]*([-\w.]*)\s*(?:\r\n|\r|\n)', re.MULTILINE) + +escapedNewlineRE = re.compile(r'(?<!\\)\\n') + +directiveNamesAndParsers = { + # importing and inheritance + 'import':None, + 'from':None, + 'extends': 'eatExtends', + 'implements': 'eatImplements', + 'super': 'eatSuper', + + # output, filtering, and caching + 'slurp': 'eatSlurp', + 'raw': 'eatRaw', + 'include': 'eatInclude', + 'cache': 'eatCache', + 'filter': 'eatFilter', + 'echo': None, + 'silent': None, + 'transform' : 'eatTransform', + + 'call': 'eatCall', + 'arg': 'eatCallArg', + + 'capture': 'eatCapture', + + # declaration, assignment, and deletion + 'attr': 'eatAttr', + 'def': 'eatDef', + 'block': 'eatBlock', + '@': 'eatDecorator', + 'defmacro': 'eatDefMacro', + + 'closure': 'eatClosure', + + 'set': 'eatSet', + 'del': None, + + # flow control + 'if': 'eatIf', + 'while': None, + 'for': None, + 'else': None, + 'elif': None, + 'pass': None, + 'break': None, + 'continue': None, + 'stop': None, + 'return': None, + 'yield': None, + + # little wrappers + 'repeat': None, + 'unless': None, + + # error handling + 'assert': None, + 'raise': None, + 'try': None, + 'except': None, + 'finally': None, + 'errorCatcher': 'eatErrorCatcher', + + # intructions to the parser and compiler + 'breakpoint': 'eatBreakPoint', + 'compiler': 'eatCompiler', + 'compiler-settings': 'eatCompilerSettings', + + # misc + 'shBang': 'eatShbang', + 'encoding': 'eatEncoding', + + 'end': 'eatEndDirective', + } + +endDirectiveNamesAndHandlers = { + 'def': 'handleEndDef', # has short-form + 'block': None, # has short-form + 'closure': None, # has short-form + 'cache': None, # has short-form + 'call': None, # has short-form + 'capture': None, # has short-form + 'filter': None, + 'errorCatcher':None, + 'while': None, # has short-form + 'for': None, # has short-form + 'if': None, # has short-form + 'try': None, # has short-form + 'repeat': None, # has short-form + 'unless': None, # has short-form + } + +################################################## +## CLASSES ## + +# @@TR: SyntaxError doesn't call exception.__str__ for some reason! +#class ParseError(SyntaxError): +class ParseError(ValueError): + def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None): + self.stream = stream + if stream.pos() >= len(stream): + stream.setPos(len(stream) -1) + self.msg = msg + self.extMsg = extMsg + self.lineno = lineno + self.col = col + + def __str__(self): + return self.report() + + def report(self): + stream = self.stream + if stream.filename(): + f = " in file %s" % stream.filename() + else: + f = '' + report = '' + if self.lineno: + lineno = self.lineno + row, col, line = (lineno, (self.col or 0), + self.stream.splitlines()[lineno-1]) + else: + row, col, line = self.stream.getRowColLine() + + ## get the surrounding lines + lines = stream.splitlines() + prevLines = [] # (rowNum, content) + for i in range(1,4): + if row-1-i <=0: + break + prevLines.append( (row-i,lines[row-1-i]) ) + + nextLines = [] # (rowNum, content) + for i in range(1,4): + if not row-1+i < len(lines): + break + nextLines.append( (row+i,lines[row-1+i]) ) + nextLines.reverse() + + ## print the main message + report += "\n\n%s\n" %self.msg + report += "Line %i, column %i%s\n\n" % (row, col, f) + report += 'Line|Cheetah Code\n' + report += '----|-------------------------------------------------------------\n' + while prevLines: + lineInfo = prevLines.pop() + report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]} + report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line} + report += ' '*5 +' '*(col-1) + "^\n" + + while nextLines: + lineInfo = nextLines.pop() + report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]} + ## add the extra msg + if self.extMsg: + report += self.extMsg + '\n' + + return report + +class ForbiddenSyntax(ParseError): pass +class ForbiddenExpression(ForbiddenSyntax): pass +class ForbiddenDirective(ForbiddenSyntax): pass + +class CheetahVariable: + def __init__(self, nameChunks, useNameMapper=True, cacheToken=None, + rawSource=None): + self.nameChunks = nameChunks + self.useNameMapper = useNameMapper + self.cacheToken = cacheToken + self.rawSource = rawSource + +class Placeholder(CheetahVariable): pass + +class ArgList: + """Used by _LowLevelParser.getArgList()""" + + def __init__(self): + self.argNames = [] + self.defVals = [] + self.i = 0 + + def addArgName(self, name): + self.argNames.append( name ) + self.defVals.append( None ) + + def next(self): + self.i += 1 + + def addToDefVal(self, token): + i = self.i + if self.defVals[i] == None: + self.defVals[i] = '' + self.defVals[i] += token + + def merge(self): + defVals = self.defVals + for i in range(len(defVals)): + if type(defVals[i]) == StringType: + defVals[i] = defVals[i].strip() + + return map(None, [i.strip() for i in self.argNames], defVals) + + def __str__(self): + return str(self.merge()) + +class _LowLevelParser(SourceReader): + """This class implements the methods to match or extract ('get*') the basic + elements of Cheetah's grammar. It does NOT handle any code generation or + state management. + """ + + _settingsManager = None + + def setSettingsManager(self, settingsManager): + self._settingsManager = settingsManager + + def setting(self, key, default=Unspecified): + if default is Unspecified: + return self._settingsManager.setting(key) + else: + return self._settingsManager.setting(key, default=default) + + def setSetting(self, key, val): + self._settingsManager.setSetting(key, val) + + def settings(self): + return self._settingsManager.settings() + + def updateSettings(self, settings): + self._settingsManager.updateSettings(settings) + + def _initializeSettings(self): + self._settingsManager._initializeSettings() + + def configureParser(self): + """Is called by the Compiler instance after the parser has had a + settingsManager assigned with self.setSettingsManager() + """ + self._makeCheetahVarREs() + self._makeCommentREs() + self._makeDirectiveREs() + self._makePspREs() + self._possibleNonStrConstantChars = ( + self.setting('commentStartToken')[0] + + self.setting('multiLineCommentStartToken')[0] + + self.setting('cheetahVarStartToken')[0] + + self.setting('directiveStartToken')[0] + + self.setting('PSPStartToken')[0]) + self._nonStrConstMatchers = [ + self.matchCommentStartToken, + self.matchMultiLineCommentStartToken, + self.matchVariablePlaceholderStart, + self.matchExpressionPlaceholderStart, + self.matchDirective, + self.matchPSPStartToken, + self.matchEOLSlurpToken, + ] + + ## regex setup ## + + def _makeCheetahVarREs(self): + + """Setup the regexs for Cheetah $var parsing.""" + + num = r'[0-9\.]+' + interval = (r'(?P<interval>' + + num + r's|' + + num + r'm|' + + num + r'h|' + + num + r'd|' + + num + r'w|' + + num + ')' + ) + + cacheToken = (r'(?:' + + r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+ + '|' + + r'(?P<STATIC_CACHE>\*)' + + '|' + + r'(?P<NO_CACHE>)' + + ')') + self.cacheTokenRE = cachedRegex(cacheToken) + + silentPlaceholderToken = (r'(?:' + + r'(?P<SILENT>' +escapeRegexChars('!')+')'+ + '|' + + r'(?P<NOT_SILENT>)' + + ')') + self.silentPlaceholderTokenRE = cachedRegex(silentPlaceholderToken) + + self.cheetahVarStartRE = cachedRegex( + escCharLookBehind + + r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+ + r'(?P<silenceToken>'+silentPlaceholderToken+')'+ + r'(?P<cacheToken>'+cacheToken+')'+ + r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure + r'(?=[A-Za-z_])') + validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])' + self.cheetahVarStartToken = self.setting('cheetahVarStartToken') + self.cheetahVarStartTokenRE = cachedRegex( + escCharLookBehind + + escapeRegexChars(self.setting('cheetahVarStartToken')) + +validCharsLookAhead + ) + + self.cheetahVarInExpressionStartTokenRE = cachedRegex( + escapeRegexChars(self.setting('cheetahVarStartToken')) + +r'(?=[A-Za-z_])' + ) + + self.expressionPlaceholderStartRE = cachedRegex( + escCharLookBehind + + r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' + + r'(?P<cacheToken>' + cacheToken + ')' + + #r'\[[ \t\f]*' + r'(?:\{|\(|\[)[ \t\f]*' + + r'(?=[^\)\}\]])' + ) + + if self.setting('EOLSlurpToken'): + self.EOLSlurpRE = cachedRegex( + escapeRegexChars(self.setting('EOLSlurpToken')) + + r'[ \t\f]*' + + r'(?:'+EOL+')' + ) + else: + self.EOLSlurpRE = None + + + def _makeCommentREs(self): + """Construct the regex bits that are used in comment parsing.""" + startTokenEsc = escapeRegexChars(self.setting('commentStartToken')) + self.commentStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc) + del startTokenEsc + + startTokenEsc = escapeRegexChars( + self.setting('multiLineCommentStartToken')) + endTokenEsc = escapeRegexChars( + self.setting('multiLineCommentEndToken')) + self.multiLineCommentTokenStartRE = cachedRegex(escCharLookBehind + + startTokenEsc) + self.multiLineCommentEndTokenRE = cachedRegex(escCharLookBehind + + endTokenEsc) + + def _makeDirectiveREs(self): + """Construct the regexs that are used in directive parsing.""" + startToken = self.setting('directiveStartToken') + endToken = self.setting('directiveEndToken') + startTokenEsc = escapeRegexChars(startToken) + endTokenEsc = escapeRegexChars(endToken) + validSecondCharsLookAhead = r'(?=[A-Za-z_@])' + reParts = [escCharLookBehind, startTokenEsc] + if self.setting('allowWhitespaceAfterDirectiveStartToken'): + reParts.append('[ \t]*') + reParts.append(validSecondCharsLookAhead) + self.directiveStartTokenRE = cachedRegex(''.join(reParts)) + self.directiveEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc) + + def _makePspREs(self): + """Setup the regexs for PSP parsing.""" + startToken = self.setting('PSPStartToken') + startTokenEsc = escapeRegexChars(startToken) + self.PSPStartTokenRE = cachedRegex(escCharLookBehind + startTokenEsc) + endToken = self.setting('PSPEndToken') + endTokenEsc = escapeRegexChars(endToken) + self.PSPEndTokenRE = cachedRegex(escCharLookBehind + endTokenEsc) + + + def isLineClearToStartToken(self, pos=None): + return self.isLineClearToPos(pos) + + def matchTopLevelToken(self): + """Returns the first match found from the following methods: + self.matchCommentStartToken + self.matchMultiLineCommentStartToken + self.matchVariablePlaceholderStart + self.matchExpressionPlaceholderStart + self.matchDirective + self.matchPSPStartToken + self.matchEOLSlurpToken + + Returns None if no match. + """ + match = None + if self.peek() in self._possibleNonStrConstantChars: + for matcher in self._nonStrConstMatchers: + match = matcher() + if match: + break + return match + + def matchPyToken(self): + match = pseudoprog.match(self.src(), self.pos()) + + if match and match.group() in tripleQuotedStringStarts: + TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos()) + if TQSmatch: + return TQSmatch + return match + + def getPyToken(self): + match = self.matchPyToken() + if match is None: + raise ParseError(self) + elif match.group() in tripleQuotedStringStarts: + raise ParseError(self, msg='Malformed triple-quoted string') + return self.readTo(match.end()) + + def matchEOLSlurpToken(self): + if self.EOLSlurpRE: + return self.EOLSlurpRE.match(self.src(), self.pos()) + + def getEOLSlurpToken(self): + match = self.matchEOLSlurpToken() + if not match: + raise ParseError(self, msg='Invalid EOL slurp token') + return self.readTo(match.end()) + + def matchCommentStartToken(self): + return self.commentStartTokenRE.match(self.src(), self.pos()) + + def getCommentStartToken(self): + match = self.matchCommentStartToken() + if not match: + raise ParseError(self, msg='Invalid single-line comment start token') + return self.readTo(match.end()) + + def matchMultiLineCommentStartToken(self): + return self.multiLineCommentTokenStartRE.match(self.src(), self.pos()) + + def getMultiLineCommentStartToken(self): + match = self.matchMultiLineCommentStartToken() + if not match: + raise ParseError(self, msg='Invalid multi-line comment start token') + return self.readTo(match.end()) + + def matchMultiLineCommentEndToken(self): + return self.multiLineCommentEndTokenRE.match(self.src(), self.pos()) + + def getMultiLineCommentEndToken(self): + match = self.matchMultiLineCommentEndToken() + if not match: + raise ParseError(self, msg='Invalid multi-line comment end token') + return self.readTo(match.end()) + + def getCommaSeparatedSymbols(self): + """ + Loosely based on getDottedName to pull out comma separated + named chunks + """ + srcLen = len(self) + pieces = [] + nameChunks = [] + + if not self.peek() in identchars: + raise ParseError(self) + + while self.pos() < srcLen: + c = self.peek() + if c in namechars: + nameChunk = self.getIdentifier() + nameChunks.append(nameChunk) + elif c == '.': + if self.pos()+1 <srcLen and self.peek(1) in identchars: + nameChunks.append(self.getc()) + else: + break + elif c == ',': + self.getc() + pieces.append(''.join(nameChunks)) + nameChunks = [] + elif c in (' ', '\t'): + self.getc() + else: + break + + if nameChunks: + pieces.append(''.join(nameChunks)) + + return pieces + + def getDottedName(self): + srcLen = len(self) + nameChunks = [] + + if not self.peek() in identchars: + raise ParseError(self) + + while self.pos() < srcLen: + c = self.peek() + if c in namechars: + nameChunk = self.getIdentifier() + nameChunks.append(nameChunk) + elif c == '.': + if self.pos()+1 <srcLen and self.peek(1) in identchars: + nameChunks.append(self.getc()) + else: + break + else: + break + + return ''.join(nameChunks) + + def matchIdentifier(self): + return identRE.match(self.src(), self.pos()) + + def getIdentifier(self): + match = self.matchIdentifier() + if not match: + raise ParseError(self, msg='Invalid identifier') + return self.readTo(match.end()) + + def matchOperator(self): + match = self.matchPyToken() + if match and match.group() not in operators: + match = None + return match + + def getOperator(self): + match = self.matchOperator() + if not match: + raise ParseError(self, msg='Expected operator') + return self.readTo( match.end() ) + + def matchAssignmentOperator(self): + match = self.matchPyToken() + if match and match.group() not in assignmentOps: + match = None + return match + + def getAssignmentOperator(self): + match = self.matchAssignmentOperator() + if not match: + raise ParseError(self, msg='Expected assignment operator') + return self.readTo( match.end() ) + + def matchDirective(self): + """Returns False or the name of the directive matched. + """ + startPos = self.pos() + if not self.matchDirectiveStartToken(): + return False + self.getDirectiveStartToken() + directiveName = self.matchDirectiveName() + self.setPos(startPos) + return directiveName + + def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'): + startPos = self.pos() + possibleMatches = self._directiveNamesAndParsers.keys() + name = '' + match = None + + while not self.atEnd(): + c = self.getc() + if not c in directiveNameChars: + break + name += c + if name == '@': + if not self.atEnd() and self.peek() in identchars: + match = '@' + break + possibleMatches = [dn for dn in possibleMatches if dn.startswith(name)] + if not possibleMatches: + break + elif (name in possibleMatches and (self.atEnd() or self.peek() not in directiveNameChars)): + match = name + break + + self.setPos(startPos) + return match + + def matchDirectiveStartToken(self): + return self.directiveStartTokenRE.match(self.src(), self.pos()) + + def getDirectiveStartToken(self): + match = self.matchDirectiveStartToken() + if not match: + raise ParseError(self, msg='Invalid directive start token') + return self.readTo(match.end()) + + def matchDirectiveEndToken(self): + return self.directiveEndTokenRE.match(self.src(), self.pos()) + + def getDirectiveEndToken(self): + match = self.matchDirectiveEndToken() + if not match: + raise ParseError(self, msg='Invalid directive end token') + return self.readTo(match.end()) + + + def matchColonForSingleLineShortFormDirective(self): + if not self.atEnd() and self.peek()==':': + restOfLine = self[self.pos()+1:self.findEOL()] + restOfLine = restOfLine.strip() + if not restOfLine: + return False + elif self.commentStartTokenRE.match(restOfLine): + return False + else: # non-whitespace, non-commment chars found + return True + return False + + def matchPSPStartToken(self): + return self.PSPStartTokenRE.match(self.src(), self.pos()) + + def matchPSPEndToken(self): + return self.PSPEndTokenRE.match(self.src(), self.pos()) + + def getPSPStartToken(self): + match = self.matchPSPStartToken() + if not match: + raise ParseError(self, msg='Invalid psp start token') + return self.readTo(match.end()) + + def getPSPEndToken(self): + match = self.matchPSPEndToken() + if not match: + raise ParseError(self, msg='Invalid psp end token') + return self.readTo(match.end()) + + def matchCheetahVarStart(self): + """includes the enclosure and cache token""" + return self.cheetahVarStartRE.match(self.src(), self.pos()) + + def matchCheetahVarStartToken(self): + """includes the enclosure and cache token""" + return self.cheetahVarStartTokenRE.match(self.src(), self.pos()) + + def matchCheetahVarInExpressionStartToken(self): + """no enclosures or cache tokens allowed""" + return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos()) + + def matchVariablePlaceholderStart(self): + """includes the enclosure and cache token""" + return self.cheetahVarStartRE.match(self.src(), self.pos()) + + def matchExpressionPlaceholderStart(self): + """includes the enclosure and cache token""" + return self.expressionPlaceholderStartRE.match(self.src(), self.pos()) + + def getCheetahVarStartToken(self): + """just the start token, not the enclosure or cache token""" + match = self.matchCheetahVarStartToken() + if not match: + raise ParseError(self, msg='Expected Cheetah $var start token') + return self.readTo( match.end() ) + + + def getCacheToken(self): + try: + token = self.cacheTokenRE.match(self.src(), self.pos()) + self.setPos( token.end() ) + return token.group() + except: + raise ParseError(self, msg='Expected cache token') + + def getSilentPlaceholderToken(self): + try: + token = self.silentPlaceholderTokenRE.match(self.src(), self.pos()) + self.setPos( token.end() ) + return token.group() + except: + raise ParseError(self, msg='Expected silent placeholder token') + + + + def getTargetVarsList(self): + varnames = [] + while not self.atEnd(): + if self.peek() in ' \t\f': + self.getWhiteSpace() + elif self.peek() in '\r\n': + break + elif self.startswith(','): + self.advance() + elif self.startswith('in ') or self.startswith('in\t'): + break + #elif self.matchCheetahVarStart(): + elif self.matchCheetahVarInExpressionStartToken(): + self.getCheetahVarStartToken() + self.getSilentPlaceholderToken() + self.getCacheToken() + varnames.append( self.getDottedName() ) + elif self.matchIdentifier(): + varnames.append( self.getDottedName() ) + else: + break + return varnames + + def getCheetahVar(self, plain=False, skipStartToken=False): + """This is called when parsing inside expressions. Cache tokens are only + valid in placeholders so this method discards any cache tokens found. + """ + if not skipStartToken: + self.getCheetahVarStartToken() + self.getSilentPlaceholderToken() + self.getCacheToken() + return self.getCheetahVarBody(plain=plain) + + def getCheetahVarBody(self, plain=False): + # @@TR: this should be in the compiler + return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(), plain=plain) + + def getCheetahVarNameChunks(self): + + """ + nameChunks = list of Cheetah $var subcomponents represented as tuples + [ (namemapperPart,autoCall,restOfName), + ] + where: + namemapperPart = the dottedName base + autocall = where NameMapper should use autocalling on namemapperPart + restOfName = any arglist, index, or slice + + If restOfName contains a call arglist (e.g. '(1234)') then autocall is + False, otherwise it defaults to True. + + EXAMPLE + ------------------------------------------------------------------------ + + if the raw CheetahVar is + $a.b.c[1].d().x.y.z + + nameChunks is the list + [ ('a.b.c',True,'[1]'), + ('d',False,'()'), + ('x.y.z',True,''), + ] + + """ + + chunks = [] + while self.pos() < len(self): + rest = '' + autoCall = True + if not self.peek() in identchars + '.': + break + elif self.peek() == '.': + + if self.pos()+1 < len(self) and self.peek(1) in identchars: + self.advance() # discard the period as it isn't needed with NameMapper + else: + break + + dottedName = self.getDottedName() + if not self.atEnd() and self.peek() in '([': + if self.peek() == '(': + rest = self.getCallArgString() + else: + rest = self.getExpression(enclosed=True) + + period = max(dottedName.rfind('.'), 0) + if period: + chunks.append( (dottedName[:period], autoCall, '') ) + dottedName = dottedName[period+1:] + if rest and rest[0]=='(': + autoCall = False + chunks.append( (dottedName, autoCall, rest) ) + + return chunks + + + def getCallArgString(self, + enclosures=[], # list of tuples (char, pos), where char is ({ or [ + useNameMapper=Unspecified): + + """ Get a method/function call argument string. + + This method understands *arg, and **kw + """ + + # @@TR: this settings mangling should be removed + if useNameMapper is not Unspecified: + useNameMapper_orig = self.setting('useNameMapper') + self.setSetting('useNameMapper', useNameMapper) + + if enclosures: + pass + else: + if not self.peek() == '(': + raise ParseError(self, msg="Expected '('") + startPos = self.pos() + self.getc() + enclosures = [('(', startPos), + ] + + argStringBits = ['('] + addBit = argStringBits.append + + while 1: + if self.atEnd(): + open = enclosures[-1][0] + close = closurePairsRev[open] + self.setPos(enclosures[-1][1]) + raise ParseError( + self, msg="EOF was reached before a matching '" + close + + "' was found for the '" + open + "'") + + c = self.peek() + if c in ")}]": # get the ending enclosure and break + if not enclosures: + raise ParseError(self) + c = self.getc() + open = closurePairs[c] + if enclosures[-1][0] == open: + enclosures.pop() + addBit(')') + break + else: + raise ParseError(self) + elif c in " \t\f\r\n": + addBit(self.getc()) + elif self.matchCheetahVarInExpressionStartToken(): + startPos = self.pos() + codeFor1stToken = self.getCheetahVar() + WS = self.getWhiteSpace() + if not self.atEnd() and self.peek() == '=': + nextToken = self.getPyToken() + if nextToken == '=': + endPos = self.pos() + self.setPos(startPos) + codeFor1stToken = self.getCheetahVar(plain=True) + self.setPos(endPos) + + ## finally + addBit( codeFor1stToken + WS + nextToken ) + else: + addBit( codeFor1stToken + WS) + elif self.matchCheetahVarStart(): + # it has syntax that is only valid at the top level + self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() + else: + beforeTokenPos = self.pos() + token = self.getPyToken() + if token in ('{','(','['): + self.rev() + token = self.getExpression(enclosed=True) + token = self.transformToken(token, beforeTokenPos) + addBit(token) + + if useNameMapper is not Unspecified: + self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above + + return ''.join(argStringBits) + + def getDefArgList(self, exitPos=None, useNameMapper=False): + + """ Get an argument list. Can be used for method/function definition + argument lists or for #directive argument lists. Returns a list of + tuples in the form (argName, defVal=None) with one tuple for each arg + name. + + These defVals are always strings, so (argName, defVal=None) is safe even + with a case like (arg1, arg2=None, arg3=1234*2), which would be returned as + [('arg1', None), + ('arg2', 'None'), + ('arg3', '1234*2'), + ] + + This method understands *arg, and **kw + + """ + + if self.peek() == '(': + self.advance() + else: + exitPos = self.findEOL() # it's a directive so break at the EOL + argList = ArgList() + onDefVal = False + + # @@TR: this settings mangling should be removed + useNameMapper_orig = self.setting('useNameMapper') + self.setSetting('useNameMapper', useNameMapper) + + while 1: + if self.atEnd(): + raise ParseError( + self, msg="EOF was reached before a matching ')'"+ + " was found for the '('") + + if self.pos() == exitPos: + break + + c = self.peek() + if c == ")" or self.matchDirectiveEndToken(): + break + elif c == ":": + break + elif c in " \t\f\r\n": + if onDefVal: + argList.addToDefVal(c) + self.advance() + elif c == '=': + onDefVal = True + self.advance() + elif c == ",": + argList.next() + onDefVal = False + self.advance() + elif self.startswith(self.cheetahVarStartToken) and not onDefVal: + self.advance(len(self.cheetahVarStartToken)) + elif self.matchIdentifier() and not onDefVal: + argList.addArgName( self.getIdentifier() ) + elif onDefVal: + if self.matchCheetahVarInExpressionStartToken(): + token = self.getCheetahVar() + elif self.matchCheetahVarStart(): + # it has syntax that is only valid at the top level + self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() + else: + beforeTokenPos = self.pos() + token = self.getPyToken() + if token in ('{','(','['): + self.rev() + token = self.getExpression(enclosed=True) + token = self.transformToken(token, beforeTokenPos) + argList.addToDefVal(token) + elif c == '*' and not onDefVal: + varName = self.getc() + if self.peek() == '*': + varName += self.getc() + if not self.matchIdentifier(): + raise ParseError(self) + varName += self.getIdentifier() + argList.addArgName(varName) + else: + raise ParseError(self) + + + self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above + return argList.merge() + + def getExpressionParts(self, + enclosed=False, + enclosures=None, # list of tuples (char, pos), where char is ({ or [ + pyTokensToBreakAt=None, # only works if not enclosed + useNameMapper=Unspecified, + ): + + """ Get a Cheetah expression that includes $CheetahVars and break at + directive end tokens, the end of an enclosure, or at a specified + pyToken. + """ + + if useNameMapper is not Unspecified: + useNameMapper_orig = self.setting('useNameMapper') + self.setSetting('useNameMapper', useNameMapper) + + if enclosures is None: + enclosures = [] + + srcLen = len(self) + exprBits = [] + while 1: + if self.atEnd(): + if enclosures: + open = enclosures[-1][0] + close = closurePairsRev[open] + self.setPos(enclosures[-1][1]) + raise ParseError( + self, msg="EOF was reached before a matching '" + close + + "' was found for the '" + open + "'") + else: + break + + c = self.peek() + if c in "{([": + exprBits.append(c) + enclosures.append( (c, self.pos()) ) + self.advance() + elif enclosed and not enclosures: + break + elif c in "])}": + if not enclosures: + raise ParseError(self) + open = closurePairs[c] + if enclosures[-1][0] == open: + enclosures.pop() + exprBits.append(c) + else: + open = enclosures[-1][0] + close = closurePairsRev[open] + row, col = self.getRowCol() + self.setPos(enclosures[-1][1]) + raise ParseError( + self, msg= "A '" + c + "' was found at line " + str(row) + + ", col " + str(col) + + " before a matching '" + close + + "' was found\nfor the '" + open + "'") + self.advance() + + elif c in " \f\t": + exprBits.append(self.getWhiteSpace()) + elif self.matchDirectiveEndToken() and not enclosures: + break + elif c == "\\" and self.pos()+1 < srcLen: + eolMatch = EOLre.match(self.src(), self.pos()+1) + if not eolMatch: + self.advance() + raise ParseError(self, msg='Line ending expected') + self.setPos( eolMatch.end() ) + elif c in '\r\n': + if enclosures: + self.advance() + else: + break + elif self.matchCheetahVarInExpressionStartToken(): + expr = self.getCheetahVar() + exprBits.append(expr) + elif self.matchCheetahVarStart(): + # it has syntax that is only valid at the top level + self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr() + else: + beforeTokenPos = self.pos() + token = self.getPyToken() + if (not enclosures + and pyTokensToBreakAt + and token in pyTokensToBreakAt): + + self.setPos(beforeTokenPos) + break + + token = self.transformToken(token, beforeTokenPos) + + exprBits.append(token) + if identRE.match(token): + if token == 'for': + expr = self.getExpression(useNameMapper=False, pyTokensToBreakAt=['in']) + exprBits.append(expr) + else: + exprBits.append(self.getWhiteSpace()) + if not self.atEnd() and self.peek() == '(': + exprBits.append(self.getCallArgString()) + ## + if useNameMapper is not Unspecified: + self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above + return exprBits + + def getExpression(self, + enclosed=False, + enclosures=None, # list of tuples (char, pos), where # char is ({ or [ + pyTokensToBreakAt=None, + useNameMapper=Unspecified, + ): + """Returns the output of self.getExpressionParts() as a concatenated + string rather than as a list. + """ + return ''.join(self.getExpressionParts( + enclosed=enclosed, enclosures=enclosures, + pyTokensToBreakAt=pyTokensToBreakAt, + useNameMapper=useNameMapper)) + + + def transformToken(self, token, beforeTokenPos): + """Takes a token from the expression being parsed and performs and + special transformations required by Cheetah. + + At the moment only Cheetah's c'$placeholder strings' are transformed. + """ + if token=='c' and not self.atEnd() and self.peek() in '\'"': + nextToken = self.getPyToken() + token = nextToken.upper() + theStr = eval(token) + endPos = self.pos() + if not theStr: + return + + if token.startswith(single3) or token.startswith(double3): + startPosIdx = 3 + else: + startPosIdx = 1 + #print 'CHEETAH STRING', nextToken, theStr, startPosIdx + self.setPos(beforeTokenPos+startPosIdx+1) + outputExprs = [] + strConst = '' + while self.pos() < (endPos-startPosIdx): + if self.matchCheetahVarStart() or self.matchExpressionPlaceholderStart(): + if strConst: + outputExprs.append(repr(strConst)) + strConst = '' + placeholderExpr = self.getPlaceholder() + outputExprs.append('str('+placeholderExpr+')') + else: + strConst += self.getc() + self.setPos(endPos) + if strConst: + outputExprs.append(repr(strConst)) + #if not self.atEnd() and self.matches('.join('): + # print 'DEBUG***' + token = "''.join(["+','.join(outputExprs)+"])" + return token + + def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self): + match = self.matchCheetahVarStart() + groupdict = match.groupdict() + if groupdict.get('cacheToken'): + raise ParseError( + self, + msg='Cache tokens are not valid inside expressions. ' + 'Use them in top-level $placeholders only.') + elif groupdict.get('enclosure'): + raise ParseError( + self, + msg='Long-form placeholders - ${}, $(), $[], etc. are not valid inside expressions. ' + 'Use them in top-level $placeholders only.') + else: + raise ParseError( + self, + msg='This form of $placeholder syntax is not valid here.') + + + def getPlaceholder(self, allowCacheTokens=False, plain=False, returnEverything=False): + # filtered + for callback in self.setting('preparsePlaceholderHooks'): + callback(parser=self) + + startPos = self.pos() + lineCol = self.getRowCol(startPos) + startToken = self.getCheetahVarStartToken() + silentPlaceholderToken = self.getSilentPlaceholderToken() + if silentPlaceholderToken: + isSilentPlaceholder = True + else: + isSilentPlaceholder = False + + + if allowCacheTokens: + cacheToken = self.getCacheToken() + cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict() + else: + cacheTokenParts = {} + + if self.peek() in '({[': + pos = self.pos() + enclosureOpenChar = self.getc() + enclosures = [ (enclosureOpenChar, pos) ] + self.getWhiteSpace() + else: + enclosures = [] + + filterArgs = None + if self.matchIdentifier(): + nameChunks = self.getCheetahVarNameChunks() + expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain) + restOfExpr = None + if enclosures: + WS = self.getWhiteSpace() + expr += WS + if self.setting('allowPlaceholderFilterArgs') and self.peek()==',': + filterArgs = self.getCallArgString(enclosures=enclosures)[1:-1] + else: + if self.peek()==closurePairsRev[enclosureOpenChar]: + self.getc() + else: + restOfExpr = self.getExpression(enclosed=True, enclosures=enclosures) + if restOfExpr[-1] == closurePairsRev[enclosureOpenChar]: + restOfExpr = restOfExpr[:-1] + expr += restOfExpr + rawPlaceholder = self[startPos: self.pos()] + else: + expr = self.getExpression(enclosed=True, enclosures=enclosures) + if expr[-1] == closurePairsRev[enclosureOpenChar]: + expr = expr[:-1] + rawPlaceholder=self[startPos: self.pos()] + + expr = self._applyExpressionFilters(expr,'placeholder', + rawExpr=rawPlaceholder,startPos=startPos) + for callback in self.setting('postparsePlaceholderHooks'): + callback(parser=self) + + if returnEverything: + return (expr, rawPlaceholder, lineCol, cacheTokenParts, + filterArgs, isSilentPlaceholder) + else: + return expr + + +class _HighLevelParser(_LowLevelParser): + """This class is a StateMachine for parsing Cheetah source and + sending state dependent code generation commands to + Cheetah.Compiler.Compiler. + """ + def __init__(self, src, filename=None, breakPoint=None, compiler=None): + _LowLevelParser.__init__(self, src, filename=filename, breakPoint=breakPoint) + self.setSettingsManager(compiler) + self._compiler = compiler + self.setupState() + self.configureParser() + + def setupState(self): + self._macros = {} + self._macroDetails = {} + self._openDirectivesStack = [] + + def cleanup(self): + """Cleanup to remove any possible reference cycles + """ + self._macros.clear() + for macroname, macroDetails in self._macroDetails.items(): + macroDetails.template.shutdown() + del macroDetails.template + self._macroDetails.clear() + + def configureParser(self): + _LowLevelParser.configureParser(self) + self._initDirectives() + + def _initDirectives(self): + def normalizeParserVal(val): + if isinstance(val, (str,unicode)): + handler = getattr(self, val) + elif type(val) in (ClassType, TypeType): + handler = val(self) + elif callable(val): + handler = val + elif val is None: + handler = val + else: + raise Exception('Invalid parser/handler value %r for %s'%(val, name)) + return handler + + normalizeHandlerVal = normalizeParserVal + + _directiveNamesAndParsers = directiveNamesAndParsers.copy() + customNamesAndParsers = self.setting('directiveNamesAndParsers',{}) + _directiveNamesAndParsers.update(customNamesAndParsers) + + _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy() + customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',{}) + _endDirectiveNamesAndHandlers.update(customNamesAndHandlers) + + self._directiveNamesAndParsers = {} + for name, val in _directiveNamesAndParsers.items(): + if val in (False, 0): + continue + self._directiveNamesAndParsers[name] = normalizeParserVal(val) + + self._endDirectiveNamesAndHandlers = {} + for name, val in _endDirectiveNamesAndHandlers.items(): + if val in (False, 0): + continue + self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val) + + self._closeableDirectives = ['def','block','closure','defmacro', + 'call', + 'capture', + 'cache', + 'filter', + 'if','unless', + 'for','while','repeat', + 'try', + ] + for directiveName in self.setting('closeableDirectives',[]): + self._closeableDirectives.append(directiveName) + + + + macroDirectives = self.setting('macroDirectives',{}) + macroDirectives['i18n'] = I18n + + + for macroName, callback in macroDirectives.items(): + if type(callback) in (ClassType, TypeType): + callback = callback(parser=self) + assert callback + self._macros[macroName] = callback + self._directiveNamesAndParsers[macroName] = self.eatMacroCall + + def _applyExpressionFilters(self, expr, exprType, rawExpr=None, startPos=None): + """Pipes cheetah expressions through a set of optional filter hooks. + + The filters are functions which may modify the expressions or raise + a ForbiddenExpression exception if the expression is not allowed. They + are defined in the compiler setting 'expressionFilterHooks'. + + Some intended use cases: + + - to implement 'restricted execution' safeguards in cases where you + can't trust the author of the template. + + - to enforce style guidelines + + filter call signature: (parser, expr, exprType, rawExpr=None, startPos=None) + - parser is the Cheetah parser + - expr is the expression to filter. In some cases the parser will have + already modified it from the original source code form. For example, + placeholders will have been translated into namemapper calls. If you + need to work with the original source, see rawExpr. + - exprType is the name of the directive, 'psp', or 'placeholder'. All + lowercase. @@TR: These will eventually be replaced with a set of + constants. + - rawExpr is the original source string that Cheetah parsed. This + might be None in some cases. + - startPos is the character position in the source string/file + where the parser started parsing the current expression. + + @@TR: I realize this use of the term 'expression' is a bit wonky as many + of the 'expressions' are actually statements, but I haven't thought of + a better name yet. Suggestions? + """ + for callback in self.setting('expressionFilterHooks'): + expr = callback(parser=self, expr=expr, exprType=exprType, + rawExpr=rawExpr, startPos=startPos) + return expr + + def _filterDisabledDirectives(self, directiveName): + directiveName = directiveName.lower() + if (directiveName in self.setting('disabledDirectives') + or (self.setting('enabledDirectives') + and directiveName not in self.setting('enabledDirectives'))): + for callback in self.setting('disabledDirectiveHooks'): + callback(parser=self, directiveName=directiveName) + raise ForbiddenDirective(self, msg='This %r directive is disabled'%directiveName) + + ## main parse loop + + def parse(self, breakPoint=None, assertEmptyStack=True): + if breakPoint: + origBP = self.breakPoint() + self.setBreakPoint(breakPoint) + assertEmptyStack = False + + while not self.atEnd(): + if self.matchCommentStartToken(): + self.eatComment() + elif self.matchMultiLineCommentStartToken(): + self.eatMultiLineComment() + elif self.matchVariablePlaceholderStart(): + self.eatPlaceholder() + elif self.matchExpressionPlaceholderStart(): + self.eatPlaceholder() + elif self.matchDirective(): + self.eatDirective() + elif self.matchPSPStartToken(): + self.eatPSP() + elif self.matchEOLSlurpToken(): + self.eatEOLSlurpToken() + else: + self.eatPlainText() + if assertEmptyStack: + self.assertEmptyOpenDirectivesStack() + if breakPoint: + self.setBreakPoint(origBP) + + ## non-directive eat methods + + def eatPlainText(self): + startPos = self.pos() + match = None + while not self.atEnd(): + match = self.matchTopLevelToken() + if match: + break + else: + self.advance() + strConst = self.readTo(self.pos(), start=startPos) + self._compiler.addStrConst(strConst) + return match + + def eatComment(self): + isLineClearToStartToken = self.isLineClearToStartToken() + if isLineClearToStartToken: + self._compiler.handleWSBeforeDirective() + self.getCommentStartToken() + comm = self.readToEOL(gobble=isLineClearToStartToken) + self._compiler.addComment(comm) + + def eatMultiLineComment(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + + self.getMultiLineCommentStartToken() + endPos = startPos = self.pos() + level = 1 + while 1: + endPos = self.pos() + if self.atEnd(): + break + if self.matchMultiLineCommentStartToken(): + self.getMultiLineCommentStartToken() + level += 1 + elif self.matchMultiLineCommentEndToken(): + self.getMultiLineCommentEndToken() + level -= 1 + if not level: + break + self.advance() + comm = self.readTo(endPos, start=startPos) + + if not self.atEnd(): + self.getMultiLineCommentEndToken() + + if (not self.atEnd()) and self.setting('gobbleWhitespaceAroundMultiLineComments'): + restOfLine = self[self.pos():self.findEOL()] + if not restOfLine.strip(): # WS only to EOL + self.readToEOL(gobble=isLineClearToStartToken) + + if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLine): + self._compiler.handleWSBeforeDirective() + + self._compiler.addComment(comm) + + def eatPlaceholder(self): + (expr, rawPlaceholder, + lineCol, cacheTokenParts, + filterArgs, isSilentPlaceholder) = self.getPlaceholder( + allowCacheTokens=True, returnEverything=True) + + self._compiler.addPlaceholder( + expr, + filterArgs=filterArgs, + rawPlaceholder=rawPlaceholder, + cacheTokenParts=cacheTokenParts, + lineCol=lineCol, + silentMode=isSilentPlaceholder) + return + + def eatPSP(self): + # filtered + self._filterDisabledDirectives(directiveName='psp') + self.getPSPStartToken() + endToken = self.setting('PSPEndToken') + startPos = self.pos() + while not self.atEnd(): + if self.peek() == endToken[0]: + if self.matchPSPEndToken(): + break + self.advance() + pspString = self.readTo(self.pos(), start=startPos).strip() + pspString = self._applyExpressionFilters(pspString, 'psp', startPos=startPos) + self._compiler.addPSP(pspString) + self.getPSPEndToken() + + ## generic directive eat methods + _simpleIndentingDirectives = ''' + else elif for while repeat unless try except finally'''.split() + _simpleExprDirectives = ''' + pass continue stop return yield break + del assert raise + silent echo + import from'''.split() + _directiveHandlerNames = {'import':'addImportStatement', + 'from':'addImportStatement', } + def eatDirective(self): + directiveName = self.matchDirective() + self._filterDisabledDirectives(directiveName) + + for callback in self.setting('preparseDirectiveHooks'): + callback(parser=self, directiveName=directiveName) + + # subclasses can override the default behaviours here by providing an + # eater method in self._directiveNamesAndParsers[directiveName] + directiveParser = self._directiveNamesAndParsers.get(directiveName) + if directiveParser: + directiveParser() + elif directiveName in self._simpleIndentingDirectives: + handlerName = self._directiveHandlerNames.get(directiveName) + if not handlerName: + handlerName = 'add'+directiveName.capitalize() + handler = getattr(self._compiler, handlerName) + self.eatSimpleIndentingDirective(directiveName, callback=handler) + elif directiveName in self._simpleExprDirectives: + handlerName = self._directiveHandlerNames.get(directiveName) + if not handlerName: + handlerName = 'add'+directiveName.capitalize() + handler = getattr(self._compiler, handlerName) + if directiveName in ('silent', 'echo'): + includeDirectiveNameInExpr = False + else: + includeDirectiveNameInExpr = True + expr = self.eatSimpleExprDirective( + directiveName, + includeDirectiveNameInExpr=includeDirectiveNameInExpr) + handler(expr) + ## + for callback in self.setting('postparseDirectiveHooks'): + callback(parser=self, directiveName=directiveName) + + def _eatRestOfDirectiveTag(self, isLineClearToStartToken, endOfFirstLinePos): + foundComment = False + if self.matchCommentStartToken(): + pos = self.pos() + self.advance() + if not self.matchDirective(): + self.setPos(pos) + foundComment = True + self.eatComment() # this won't gobble the EOL + else: + self.setPos(pos) + + if not foundComment and self.matchDirectiveEndToken(): + self.getDirectiveEndToken() + elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n': + # still gobble the EOL if a comment was found. + self.readToEOL(gobble=True) + + if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLinePos): + self._compiler.handleWSBeforeDirective() + + def _eatToThisEndDirective(self, directiveName): + finalPos = endRawPos = startPos = self.pos() + directiveChar = self.setting('directiveStartToken')[0] + isLineClearToStartToken = False + while not self.atEnd(): + if self.peek() == directiveChar: + if self.matchDirective() == 'end': + endRawPos = self.pos() + self.getDirectiveStartToken() + self.advance(len('end')) + self.getWhiteSpace() + if self.startswith(directiveName): + if self.isLineClearToStartToken(endRawPos): + isLineClearToStartToken = True + endRawPos = self.findBOL(endRawPos) + self.advance(len(directiveName)) # to end of directiveName + self.getWhiteSpace() + finalPos = self.pos() + break + self.advance() + finalPos = endRawPos = self.pos() + + textEaten = self.readTo(endRawPos, start=startPos) + self.setPos(finalPos) + + endOfFirstLinePos = self.findEOL() + + if self.matchDirectiveEndToken(): + self.getDirectiveEndToken() + elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n': + self.readToEOL(gobble=True) + + if isLineClearToStartToken and self.pos() > endOfFirstLinePos: + self._compiler.handleWSBeforeDirective() + return textEaten + + + def eatSimpleExprDirective(self, directiveName, includeDirectiveNameInExpr=True): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + if not includeDirectiveNameInExpr: + self.advance(len(directiveName)) + startPos = self.pos() + expr = self.getExpression().strip() + directiveName = expr.split()[0] + expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos) + if directiveName in self._closeableDirectives: + self.pushToOpenDirectivesStack(directiveName) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + return expr + + def eatSimpleIndentingDirective(self, directiveName, callback, + includeDirectiveNameInExpr=False): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + lineCol = self.getRowCol() + self.getDirectiveStartToken() + if directiveName not in 'else elif for while try except finally'.split(): + self.advance(len(directiveName)) + startPos = self.pos() + + self.getWhiteSpace() + + expr = self.getExpression(pyTokensToBreakAt=[':']) + expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos) + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + if directiveName in 'else elif except finally'.split(): + callback(expr, dedent=False, lineCol=lineCol) + else: + callback(expr, lineCol=lineCol) + + self.getWhiteSpace(max=1) + self.parse(breakPoint=self.findEOL(gobble=True)) + self._compiler.commitStrConst() + self._compiler.dedent() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + if directiveName in self._closeableDirectives: + self.pushToOpenDirectivesStack(directiveName) + callback(expr, lineCol=lineCol) + + def eatEndDirective(self): + isLineClearToStartToken = self.isLineClearToStartToken() + self.getDirectiveStartToken() + self.advance(3) # to end of 'end' + self.getWhiteSpace() + pos = self.pos() + directiveName = False + for key in self._endDirectiveNamesAndHandlers.keys(): + if self.find(key, pos) == pos: + directiveName = key + break + if not directiveName: + raise ParseError(self, msg='Invalid end directive') + + endOfFirstLinePos = self.findEOL() + self.getExpression() # eat in any extra comment-like crap + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + if directiveName in self._closeableDirectives: + self.popFromOpenDirectivesStack(directiveName) + + # subclasses can override the default behaviours here by providing an + # end-directive handler in self._endDirectiveNamesAndHandlers[directiveName] + if self._endDirectiveNamesAndHandlers.get(directiveName): + handler = self._endDirectiveNamesAndHandlers[directiveName] + handler() + elif directiveName in 'block capture cache call filter errorCatcher'.split(): + if key == 'block': + self._compiler.closeBlock() + elif key == 'capture': + self._compiler.endCaptureRegion() + elif key == 'cache': + self._compiler.endCacheRegion() + elif key == 'call': + self._compiler.endCallRegion() + elif key == 'filter': + self._compiler.closeFilterBlock() + elif key == 'errorCatcher': + self._compiler.turnErrorCatcherOff() + elif directiveName in 'while for if try repeat unless'.split(): + self._compiler.commitStrConst() + self._compiler.dedent() + elif directiveName=='closure': + self._compiler.commitStrConst() + self._compiler.dedent() + # @@TR: temporary hack of useSearchList + self.setSetting('useSearchList', self._useSearchList_orig) + + ## specific directive eat methods + + def eatBreakPoint(self): + """Tells the parser to stop parsing at this point and completely ignore + everything else. + + This is a debugging tool. + """ + self.setBreakPoint(self.pos()) + + def eatShbang(self): + # filtered + self.getDirectiveStartToken() + self.advance(len('shBang')) + self.getWhiteSpace() + startPos = self.pos() + shBang = self.readToEOL() + shBang = self._applyExpressionFilters(shBang, 'shbang', startPos=startPos) + self._compiler.setShBang(shBang.strip()) + + def eatEncoding(self): + # filtered + self.getDirectiveStartToken() + self.advance(len('encoding')) + self.getWhiteSpace() + startPos = self.pos() + encoding = self.readToEOL() + encoding = self._applyExpressionFilters(encoding, 'encoding', startPos=startPos) + self._compiler.setModuleEncoding(encoding.strip()) + + def eatCompiler(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + startPos = self.pos() + self.getDirectiveStartToken() + self.advance(len('compiler')) # to end of 'compiler' + self.getWhiteSpace() + + startPos = self.pos() + settingName = self.getIdentifier() + + if settingName.lower() == 'reset': + self.getExpression() # gobble whitespace & junk + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + self._initializeSettings() + self.configureParser() + return + + self.getWhiteSpace() + if self.peek() == '=': + self.advance() + else: + raise ParseError(self) + valueExpr = self.getExpression() + endPos = self.pos() + + # @@TR: it's unlikely that anyone apply filters would have left this + # directive enabled: + # @@TR: fix up filtering, regardless + self._applyExpressionFilters('%s=%r'%(settingName, valueExpr), + 'compiler', startPos=startPos) + + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + try: + self._compiler.setCompilerSetting(settingName, valueExpr) + except: + out = sys.stderr + print >> out, 'An error occurred while processing the following #compiler directive.' + print >> out, '-'*80 + print >> out, self[startPos:endPos] + print >> out, '-'*80 + print >> out, 'Please check the syntax of these settings.' + print >> out, 'A full Python exception traceback follows.' + raise + + + def eatCompilerSettings(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('compiler-settings')) # to end of 'settings' + + keywords = self.getTargetVarsList() + self.getExpression() # gobble any garbage + + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + + if 'reset' in keywords: + self._compiler._initializeSettings() + self.configureParser() + # @@TR: this implies a single-line #compiler-settings directive, and + # thus we should parse forward for an end directive. + # Subject to change in the future + return + startPos = self.pos() + settingsStr = self._eatToThisEndDirective('compiler-settings') + settingsStr = self._applyExpressionFilters(settingsStr, 'compilerSettings', + startPos=startPos) + try: + self._compiler.setCompilerSettings(keywords=keywords, settingsStr=settingsStr) + except: + out = sys.stderr + print >> out, 'An error occurred while processing the following compiler settings.' + print >> out, '-'*80 + print >> out, settingsStr.strip() + print >> out, '-'*80 + print >> out, 'Please check the syntax of these settings.' + print >> out, 'A full Python exception traceback follows.' + raise + + def eatAttr(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + startPos = self.pos() + self.getDirectiveStartToken() + self.advance(len('attr')) + self.getWhiteSpace() + startPos = self.pos() + if self.matchCheetahVarStart(): + self.getCheetahVarStartToken() + attribName = self.getIdentifier() + self.getWhiteSpace() + self.getAssignmentOperator() + expr = self.getExpression() + expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos) + self._compiler.addAttribute(attribName, expr) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + + def eatDecorator(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + startPos = self.pos() + self.getDirectiveStartToken() + #self.advance() # eat @ + startPos = self.pos() + decoratorExpr = self.getExpression() + decoratorExpr = self._applyExpressionFilters(decoratorExpr, 'decorator', startPos=startPos) + self._compiler.addDecorator(decoratorExpr) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self.getWhiteSpace() + + directiveName = self.matchDirective() + if not directiveName or directiveName not in ('def', 'block', 'closure', '@'): + raise ParseError( + self, msg='Expected #def, #block, #closure or another @decorator') + self.eatDirective() + + def eatDef(self): + # filtered + self._eatDefOrBlock('def') + + def eatBlock(self): + # filtered + startPos = self.pos() + methodName, rawSignature = self._eatDefOrBlock('block') + self._compiler._blockMetaData[methodName] = { + 'raw':rawSignature, + 'lineCol':self.getRowCol(startPos), + } + + def eatClosure(self): + # filtered + self._eatDefOrBlock('closure') + + def _eatDefOrBlock(self, directiveName): + # filtered + assert directiveName in ('def','block','closure') + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + startPos = self.pos() + self.getDirectiveStartToken() + self.advance(len(directiveName)) + self.getWhiteSpace() + if self.matchCheetahVarStart(): + self.getCheetahVarStartToken() + methodName = self.getIdentifier() + self.getWhiteSpace() + if self.peek() == '(': + argsList = self.getDefArgList() + self.advance() # past the closing ')' + if argsList and argsList[0][0] == 'self': + del argsList[0] + else: + argsList=[] + + def includeBlockMarkers(): + if self.setting('includeBlockMarkers'): + startMarker = self.setting('blockMarkerStart') + self._compiler.addStrConst(startMarker[0] + methodName + startMarker[1]) + + # @@TR: fix up filtering + self._applyExpressionFilters(self[startPos:self.pos()], 'def', startPos=startPos) + + if self.matchColonForSingleLineShortFormDirective(): + isNestedDef = (self.setting('allowNestedDefScopes') + and [name for name in self._openDirectivesStack if name=='def']) + self.getc() + rawSignature = self[startPos:endOfFirstLinePos] + self._eatSingleLineDef(directiveName=directiveName, + methodName=methodName, + argsList=argsList, + startPos=startPos, + endPos=endOfFirstLinePos) + if directiveName == 'def' and not isNestedDef: + #@@TR: must come before _eatRestOfDirectiveTag ... for some reason + self._compiler.closeDef() + elif directiveName == 'block': + includeBlockMarkers() + self._compiler.closeBlock() + elif directiveName == 'closure' or isNestedDef: + self._compiler.dedent() + + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + else: + if self.peek()==':': + self.getc() + self.pushToOpenDirectivesStack(directiveName) + rawSignature = self[startPos:self.pos()] + self._eatMultiLineDef(directiveName=directiveName, + methodName=methodName, + argsList=argsList, + startPos=startPos, + isLineClearToStartToken=isLineClearToStartToken) + if directiveName == 'block': + includeBlockMarkers() + + return methodName, rawSignature + + def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos, + isLineClearToStartToken=False): + # filtered in calling method + self.getExpression() # slurp up any garbage left at the end + signature = self[startPos:self.pos()] + endOfFirstLinePos = self.findEOL() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + signature = ' '.join([line.strip() for line in signature.splitlines()]) + parserComment = ('## CHEETAH: generated from ' + signature + + ' at line %s, col %s' % self.getRowCol(startPos) + + '.') + + isNestedDef = (self.setting('allowNestedDefScopes') + and len([name for name in self._openDirectivesStack if name=='def'])>1) + if directiveName=='block' or (directiveName=='def' and not isNestedDef): + self._compiler.startMethodDef(methodName, argsList, parserComment) + else: #closure + self._useSearchList_orig = self.setting('useSearchList') + self.setSetting('useSearchList', False) + self._compiler.addClosure(methodName, argsList, parserComment) + + return methodName + + def _eatSingleLineDef(self, directiveName, methodName, argsList, startPos, endPos): + # filtered in calling method + fullSignature = self[startPos:endPos] + parserComment = ('## Generated from ' + fullSignature + + ' at line %s, col %s' % self.getRowCol(startPos) + + '.') + isNestedDef = (self.setting('allowNestedDefScopes') + and [name for name in self._openDirectivesStack if name=='def']) + if directiveName=='block' or (directiveName=='def' and not isNestedDef): + self._compiler.startMethodDef(methodName, argsList, parserComment) + else: #closure + # @@TR: temporary hack of useSearchList + useSearchList_orig = self.setting('useSearchList') + self.setSetting('useSearchList', False) + self._compiler.addClosure(methodName, argsList, parserComment) + + self.getWhiteSpace(max=1) + self.parse(breakPoint=endPos) + if directiveName=='closure' or isNestedDef: # @@TR: temporary hack of useSearchList + self.setSetting('useSearchList', useSearchList_orig) + + def eatExtends(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('extends')) + self.getWhiteSpace() + startPos = self.pos() + if self.setting('allowExpressionsInExtendsDirective'): + baseName = self.getExpression() + else: + baseName = self.getCommaSeparatedSymbols() + baseName = ', '.join(baseName) + + baseName = self._applyExpressionFilters(baseName, 'extends', startPos=startPos) + self._compiler.setBaseClass(baseName) # in compiler + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + + def eatImplements(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('implements')) + self.getWhiteSpace() + startPos = self.pos() + methodName = self.getIdentifier() + if not self.atEnd() and self.peek() == '(': + argsList = self.getDefArgList() + self.advance() # past the closing ')' + if argsList and argsList[0][0] == 'self': + del argsList[0] + else: + argsList=[] + + # @@TR: need to split up filtering of the methodname and the args + #methodName = self._applyExpressionFilters(methodName, 'implements', startPos=startPos) + self._applyExpressionFilters(self[startPos:self.pos()], 'implements', startPos=startPos) + + self._compiler.setMainMethodName(methodName) + self._compiler.setMainMethodArgs(argsList) + + self.getExpression() # throw away and unwanted crap that got added in + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + + def eatSuper(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('super')) + self.getWhiteSpace() + startPos = self.pos() + if not self.atEnd() and self.peek() == '(': + argsList = self.getDefArgList() + self.advance() # past the closing ')' + if argsList and argsList[0][0] == 'self': + del argsList[0] + else: + argsList=[] + + self._applyExpressionFilters(self[startPos:self.pos()], 'super', startPos=startPos) + + #parserComment = ('## CHEETAH: generated from ' + signature + + # ' at line %s, col %s' % self.getRowCol(startPos) + # + '.') + + self.getExpression() # throw away and unwanted crap that got added in + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + self._compiler.addSuper(argsList) + + def eatSet(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + self.getDirectiveStartToken() + self.advance(3) + self.getWhiteSpace() + style = SET_LOCAL + if self.startswith('local'): + self.getIdentifier() + self.getWhiteSpace() + elif self.startswith('global'): + self.getIdentifier() + self.getWhiteSpace() + style = SET_GLOBAL + elif self.startswith('module'): + self.getIdentifier() + self.getWhiteSpace() + style = SET_MODULE + + startsWithDollar = self.matchCheetahVarStart() + startPos = self.pos() + LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps, useNameMapper=False).strip() + OP = self.getAssignmentOperator() + RVALUE = self.getExpression() + expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip() + + expr = self._applyExpressionFilters(expr, 'set', startPos=startPos) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + + class Components: pass # used for 'set global' + exprComponents = Components() + exprComponents.LVALUE = LVALUE + exprComponents.OP = OP + exprComponents.RVALUE = RVALUE + self._compiler.addSet(expr, exprComponents, style) + + def eatSlurp(self): + if self.isLineClearToStartToken(): + self._compiler.handleWSBeforeDirective() + self._compiler.commitStrConst() + self.readToEOL(gobble=True) + + def eatEOLSlurpToken(self): + if self.isLineClearToStartToken(): + self._compiler.handleWSBeforeDirective() + self._compiler.commitStrConst() + self.readToEOL(gobble=True) + + def eatRaw(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('raw')) + self.getWhiteSpace() + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self.getWhiteSpace(max=1) + rawBlock = self.readToEOL(gobble=False) + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + rawBlock = self._eatToThisEndDirective('raw') + self._compiler.addRawText(rawBlock) + + def eatInclude(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('include')) + + self.getWhiteSpace() + includeFrom = 'file' + isRaw = False + if self.startswith('raw'): + self.advance(3) + isRaw=True + + self.getWhiteSpace() + if self.startswith('source'): + self.advance(len('source')) + includeFrom = 'str' + self.getWhiteSpace() + if not self.peek() == '=': + raise ParseError(self) + self.advance() + startPos = self.pos() + sourceExpr = self.getExpression() + sourceExpr = self._applyExpressionFilters(sourceExpr, 'include', startPos=startPos) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.addInclude(sourceExpr, includeFrom, isRaw) + + + def eatDefMacro(self): + # @@TR: not filtered yet + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('defmacro')) + + self.getWhiteSpace() + if self.matchCheetahVarStart(): + self.getCheetahVarStartToken() + macroName = self.getIdentifier() + self.getWhiteSpace() + if self.peek() == '(': + argsList = self.getDefArgList(useNameMapper=False) + self.advance() # past the closing ')' + if argsList and argsList[0][0] == 'self': + del argsList[0] + else: + argsList=[] + + assert not self._directiveNamesAndParsers.has_key(macroName) + argsList.insert(0, ('src',None)) + argsList.append(('parser','None')) + argsList.append(('macros','None')) + argsList.append(('compilerSettings','None')) + argsList.append(('isShortForm','None')) + argsList.append(('EOLCharsInShortForm','None')) + argsList.append(('startPos','None')) + argsList.append(('endPos','None')) + + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self.getWhiteSpace(max=1) + macroSrc = self.readToEOL(gobble=False) + self.readToEOL(gobble=True) + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + macroSrc = self._eatToThisEndDirective('defmacro') + + #print argsList + normalizedMacroSrc = ''.join( + ['%def callMacro('+','.join([defv and '%s=%s'%(n,defv) or n + for n,defv in argsList]) + +')\n', + macroSrc, + '%end def']) + + + from Cheetah.Template import Template + templateAPIClass = self.setting('templateAPIClassForDefMacro', default=Template) + compilerSettings = self.setting('compilerSettingsForDefMacro', default={}) + searchListForMacros = self.setting('searchListForDefMacro', default=[]) + searchListForMacros = list(searchListForMacros) # copy to avoid mutation bugs + searchListForMacros.append({'macros':self._macros, + 'parser':self, + 'compilerSettings':self.settings(), + }) + + templateAPIClass._updateSettingsWithPreprocessTokens( + compilerSettings, placeholderToken='@', directiveToken='%') + macroTemplateClass = templateAPIClass.compile(source=normalizedMacroSrc, + compilerSettings=compilerSettings) + #print normalizedMacroSrc + #t = macroTemplateClass() + #print t.callMacro('src') + #print t.generatedClassCode() + + class MacroDetails: pass + macroDetails = MacroDetails() + macroDetails.macroSrc = macroSrc + macroDetails.argsList = argsList + macroDetails.template = macroTemplateClass(searchList=searchListForMacros) + + self._macroDetails[macroName] = macroDetails + self._macros[macroName] = macroDetails.template.callMacro + self._directiveNamesAndParsers[macroName] = self.eatMacroCall + + def eatMacroCall(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + startPos = self.pos() + self.getDirectiveStartToken() + macroName = self.getIdentifier() + macro = self._macros[macroName] + if hasattr(macro, 'parse'): + return macro.parse(parser=self, startPos=startPos) + + if hasattr(macro, 'parseArgs'): + args = macro.parseArgs(parser=self, startPos=startPos) + else: + self.getWhiteSpace() + args = self.getExpression(useNameMapper=False, + pyTokensToBreakAt=[':']).strip() + + if self.matchColonForSingleLineShortFormDirective(): + isShortForm = True + self.advance() # skip over : + self.getWhiteSpace(max=1) + srcBlock = self.readToEOL(gobble=False) + EOLCharsInShortForm = self.readToEOL(gobble=True) + #self.readToEOL(gobble=False) + else: + isShortForm = False + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + srcBlock = self._eatToThisEndDirective(macroName) + + + if hasattr(macro, 'convertArgStrToDict'): + kwArgs = macro.convertArgStrToDict(args, parser=self, startPos=startPos) + else: + def getArgs(*pargs, **kws): + return pargs, kws + exec 'positionalArgs, kwArgs = getArgs(%(args)s)'%locals() + + assert not kwArgs.has_key('src') + kwArgs['src'] = srcBlock + + if type(macro)==new.instancemethod: + co = macro.im_func.func_code + elif (hasattr(macro, '__call__') + and hasattr(macro.__call__, 'im_func')): + co = macro.__call__.im_func.func_code + else: + co = macro.func_code + availableKwArgs = inspect.getargs(co)[0] + + if 'parser' in availableKwArgs: + kwArgs['parser'] = self + if 'macros' in availableKwArgs: + kwArgs['macros'] = self._macros + if 'compilerSettings' in availableKwArgs: + kwArgs['compilerSettings'] = self.settings() + if 'isShortForm' in availableKwArgs: + kwArgs['isShortForm'] = isShortForm + if isShortForm and 'EOLCharsInShortForm' in availableKwArgs: + kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm + + if 'startPos' in availableKwArgs: + kwArgs['startPos'] = startPos + if 'endPos' in availableKwArgs: + kwArgs['endPos'] = self.pos() + + srcFromMacroOutput = macro(**kwArgs) + + origParseSrc = self._src + origBreakPoint = self.breakPoint() + origPos = self.pos() + # add a comment to the output about the macro src that is being parsed + # or add a comment prefix to all the comments added by the compiler + self._src = srcFromMacroOutput + self.setPos(0) + self.setBreakPoint(len(srcFromMacroOutput)) + + self.parse(assertEmptyStack=False) + + self._src = origParseSrc + self.setBreakPoint(origBreakPoint) + self.setPos(origPos) + + + #self._compiler.addRawText('end') + + def eatCache(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + lineCol = self.getRowCol() + self.getDirectiveStartToken() + self.advance(len('cache')) + + startPos = self.pos() + argList = self.getDefArgList(useNameMapper=True) + argList = self._applyExpressionFilters(argList, 'cache', startPos=startPos) + + def startCache(): + cacheInfo = self._compiler.genCacheInfoFromArgList(argList) + self._compiler.startCacheRegion(cacheInfo, lineCol) + + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self.getWhiteSpace(max=1) + startCache() + self.parse(breakPoint=self.findEOL(gobble=True)) + self._compiler.endCacheRegion() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self.pushToOpenDirectivesStack('cache') + startCache() + + def eatCall(self): + # @@TR: need to enable single line version of this + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + lineCol = self.getRowCol() + self.getDirectiveStartToken() + self.advance(len('call')) + startPos = self.pos() + + useAutocallingOrig = self.setting('useAutocalling') + self.setSetting('useAutocalling', False) + self.getWhiteSpace() + if self.matchCheetahVarStart(): + functionName = self.getCheetahVar() + else: + functionName = self.getCheetahVar(plain=True, skipStartToken=True) + self.setSetting('useAutocalling', useAutocallingOrig) + # @@TR: fix up filtering + self._applyExpressionFilters(self[startPos:self.pos()], 'call', startPos=startPos) + + self.getWhiteSpace() + args = self.getExpression(pyTokensToBreakAt=[':']).strip() + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self._compiler.startCallRegion(functionName, args, lineCol) + self.getWhiteSpace(max=1) + self.parse(breakPoint=self.findEOL(gobble=False)) + self._compiler.endCallRegion() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self.pushToOpenDirectivesStack("call") + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.startCallRegion(functionName, args, lineCol) + + def eatCallArg(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + lineCol = self.getRowCol() + self.getDirectiveStartToken() + + self.advance(len('arg')) + startPos = self.pos() + self.getWhiteSpace() + argName = self.getIdentifier() + self.getWhiteSpace() + argName = self._applyExpressionFilters(argName, 'arg', startPos=startPos) + self._compiler.setCallArg(argName, lineCol) + if self.peek() == ':': + self.getc() + else: + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + + def eatFilter(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + + self.getDirectiveStartToken() + self.advance(len('filter')) + self.getWhiteSpace() + startPos = self.pos() + if self.matchCheetahVarStart(): + isKlass = True + theFilter = self.getExpression(pyTokensToBreakAt=[':']) + else: + isKlass = False + theFilter = self.getIdentifier() + self.getWhiteSpace() + theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos) + + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self.getWhiteSpace(max=1) + self._compiler.setFilter(theFilter, isKlass) + self.parse(breakPoint=self.findEOL(gobble=False)) + self._compiler.closeFilterBlock() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self.pushToOpenDirectivesStack("filter") + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.setFilter(theFilter, isKlass) + + def eatTransform(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + + self.getDirectiveStartToken() + self.advance(len('transform')) + self.getWhiteSpace() + startPos = self.pos() + if self.matchCheetahVarStart(): + isKlass = True + transformer = self.getExpression(pyTokensToBreakAt=[':']) + else: + isKlass = False + transformer = self.getIdentifier() + self.getWhiteSpace() + transformer = self._applyExpressionFilters(transformer, 'transform', startPos=startPos) + + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.setTransform(transformer, isKlass) + + + def eatErrorCatcher(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + self.getDirectiveStartToken() + self.advance(len('errorCatcher')) + self.getWhiteSpace() + startPos = self.pos() + errorCatcherName = self.getIdentifier() + errorCatcherName = self._applyExpressionFilters( + errorCatcherName, 'errorcatcher', startPos=startPos) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.setErrorCatcher(errorCatcherName) + + def eatCapture(self): + # @@TR: this could be refactored to use the code in eatSimpleIndentingDirective + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + lineCol = self.getRowCol() + + self.getDirectiveStartToken() + self.advance(len('capture')) + startPos = self.pos() + self.getWhiteSpace() + + expr = self.getExpression(pyTokensToBreakAt=[':']) + expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos) + if self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol) + self.getWhiteSpace(max=1) + self.parse(breakPoint=self.findEOL(gobble=False)) + self._compiler.endCaptureRegion() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self.pushToOpenDirectivesStack("capture") + self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol) + + + def eatIf(self): + # filtered + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLine = self.findEOL() + lineCol = self.getRowCol() + self.getDirectiveStartToken() + startPos = self.pos() + + expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':']) + expr = ''.join(expressionParts).strip() + expr = self._applyExpressionFilters(expr, 'if', startPos=startPos) + + isTernaryExpr = ('then' in expressionParts and 'else' in expressionParts) + if isTernaryExpr: + conditionExpr = [] + trueExpr = [] + falseExpr = [] + currentExpr = conditionExpr + for part in expressionParts: + if part.strip()=='then': + currentExpr = trueExpr + elif part.strip()=='else': + currentExpr = falseExpr + else: + currentExpr.append(part) + + conditionExpr = ''.join(conditionExpr) + trueExpr = ''.join(trueExpr) + falseExpr = ''.join(falseExpr) + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + self._compiler.addTernaryExpr(conditionExpr, trueExpr, falseExpr, lineCol=lineCol) + elif self.matchColonForSingleLineShortFormDirective(): + self.advance() # skip over : + self._compiler.addIf(expr, lineCol=lineCol) + self.getWhiteSpace(max=1) + self.parse(breakPoint=self.findEOL(gobble=True)) + self._compiler.commitStrConst() + self._compiler.dedent() + else: + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine) + self.pushToOpenDirectivesStack('if') + self._compiler.addIf(expr, lineCol=lineCol) + + ## end directive handlers + def handleEndDef(self): + isNestedDef = (self.setting('allowNestedDefScopes') + and [name for name in self._openDirectivesStack if name=='def']) + if not isNestedDef: + self._compiler.closeDef() + else: + # @@TR: temporary hack of useSearchList + self.setSetting('useSearchList', self._useSearchList_orig) + self._compiler.commitStrConst() + self._compiler.dedent() + ### + + def pushToOpenDirectivesStack(self, directiveName): + assert directiveName in self._closeableDirectives + self._openDirectivesStack.append(directiveName) + + def popFromOpenDirectivesStack(self, directiveName): + if not self._openDirectivesStack: + raise ParseError(self, msg="#end found, but nothing to end") + + if self._openDirectivesStack[-1] == directiveName: + del self._openDirectivesStack[-1] + else: + raise ParseError(self, msg="#end %s found, expected #end %s" %( + directiveName, self._openDirectivesStack[-1])) + + def assertEmptyOpenDirectivesStack(self): + if self._openDirectivesStack: + errorMsg = ( + "Some #directives are missing their corresponding #end ___ tag: %s" %( + ', '.join(self._openDirectivesStack))) + raise ParseError(self, msg=errorMsg) + +################################################## +## Make an alias to export +Parser = _HighLevelParser |