diff options
-rw-r--r-- | scss/_native.py | 265 | ||||
-rw-r--r-- | scss/compiler.py | 7 | ||||
-rw-r--r-- | scss/expression.py | 5 | ||||
-rw-r--r-- | scss/grammar/__init__.py | 9 | ||||
-rw-r--r-- | scss/grammar/expression.g (renamed from scss/src/grammar/grammar.g) | 12 | ||||
-rw-r--r-- | scss/grammar/expression.py (renamed from scss/_grammar.py) | 12 | ||||
-rw-r--r-- | scss/grammar/scanner.py | 274 | ||||
-rw-r--r-- | scss/src/_speedups.c | 8 | ||||
-rw-r--r-- | scss/src/grammar/LICENSE | 18 | ||||
-rw-r--r-- | scss/src/grammar/README | 6 | ||||
-rw-r--r-- | scss/src/grammar/yappsrt.py | 275 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rwxr-xr-x | yapps2.py (renamed from scss/src/grammar/yapps2.py) | 288 |
13 files changed, 585 insertions, 596 deletions
diff --git a/scss/_native.py b/scss/_native.py deleted file mode 100644 index 2be3d75..0000000 --- a/scss/_native.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Pure-Python scanner and parser, used if _speedups is not available.""" -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import deque - -import re - -DEBUG = False - -# TODO copied from __init__ -_blocks_re = re.compile(r'[{},;()\'"\n]') - - -def locate_blocks(codestr): - """ - For processing CSS like strings. - - Either returns all selectors (that can be "smart" multi-lined, as - long as it's joined by `,`, or enclosed in `(` and `)`) with its code block - (the one between `{` and `}`, which can be nested), or the "lose" code - (properties) that doesn't have any blocks. - """ - lineno = 1 - - par = 0 - instr = None - depth = 0 - skip = False - i = init = lose = 0 - start = end = None - lineno_stack = deque() - - for m in _blocks_re.finditer(codestr): - i = m.start(0) - c = codestr[i] - if c == '\n': - lineno += 1 - - if instr is not None: - if c == instr: - instr = None # A string ends (FIXME: needs to accept escaped characters) - elif c in ('"', "'"): - instr = c # A string starts - elif c == '(': # parenthesis begins: - par += 1 - elif c == ')': # parenthesis ends: - par -= 1 - elif not par and not instr: - if c == '{': # block begins: - if depth == 0: - if i > 0 and codestr[i - 1] == '#': # Do not process #{...} as blocks! - skip = True - else: - lineno_stack.append(lineno) - start = i - if lose < init: - _property = codestr[lose:init].strip() - if _property: - yield lineno, _property, None - lose = init - depth += 1 - elif c == '}': # block ends: - if depth <= 0: - raise SyntaxError("Unexpected closing brace on line {0}".format(lineno)) - else: - depth -= 1 - if depth == 0: - if not skip: - end = i - _selectors = codestr[init:start].strip() - _codestr = codestr[start + 1:end].strip() - if _selectors: - yield lineno_stack.pop(), _selectors, _codestr - init = lose = end + 1 - skip = False - elif depth == 0: - if c == ';': # End of property (or block): - init = i - if lose < init: - _property = codestr[lose:init].strip() - if _property: - yield lineno, _property, None - init = lose = i + 1 - if depth > 0: - if not skip: - _selectors = codestr[init:start].strip() - _codestr = codestr[start + 1:].strip() - if _selectors: - yield lineno, _selectors, _codestr - if par: - raise Exception("Missing closing parenthesis somewhere in block: '%s'" % _selectors) - elif instr: - raise Exception("Missing closing string somewhere in block: '%s'" % _selectors) - else: - raise Exception("Block never closed: '%s'" % _selectors) - losestr = codestr[lose:] - for _property in losestr.split(';'): - _property = _property.strip() - lineno += _property.count('\n') - if _property: - yield lineno, _property, None - - -################################################################################ -# Parser - -class Parser(object): - # NOTE: This class has no C equivalent - def __init__(self, scanner): - self._scanner = scanner - self._pos = 0 - self._char_pos = 0 - - def reset(self, input): - self._scanner.reset(input) - self._pos = 0 - self._char_pos = 0 - - def _peek(self, types): - """ - Returns the token type for lookahead; if there are any args - then the list of args is the set of token types to allow - """ - try: - tok = self._scanner.token(self._pos, types) - return tok[2] - except SyntaxError: - return None - - def _scan(self, type): - """ - Returns the matched text, and moves to the next token - """ - tok = self._scanner.token(self._pos, set([type])) - self._char_pos = tok[0] - if tok[2] != type: - raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type)) - self._pos += 1 - return tok[3] - - -class NoMoreTokens(Exception): - """ - Another exception object, for when we run out of tokens - """ - pass - - -class Scanner(object): - def __init__(self, patterns, ignore, input=None): - """ - Patterns is [(terminal,regex)...] - Ignore is [terminal,...]; - Input is a string - """ - self.reset(input) - self.ignore = ignore - # The stored patterns are a pair (compiled regex,source - # regex). If the patterns variable passed in to the - # constructor is None, we assume that the class already has a - # proper .patterns list constructed - if patterns is not None: - self.patterns = [] - for k, r in patterns: - self.patterns.append((k, re.compile(r))) - - def reset(self, input): - self.tokens = [] - self.restrictions = [] - self.input = input - self.pos = 0 - - def __repr__(self): - """ - Print the last 10 tokens that have been scanned in - """ - output = '' - for t in self.tokens[-10:]: - output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) - return output - - def _scan(self, restrict): - """ - Should scan another token and add it to the list, self.tokens, - and add the restriction to self.restrictions - """ - # Keep looking for a token, ignoring any in self.ignore - token = None - while True: - best_pat = None - # Search the patterns for a match, with earlier - # tokens in the list having preference - best_pat_len = 0 - for tok, regex in self.patterns: - if DEBUG: - print("\tTrying %s: %s at pos %d -> %s" % (repr(tok), repr(regex.pattern), self.pos, repr(self.input))) - # First check to see if we're restricting to this token - if restrict and tok not in restrict and tok not in self.ignore: - if DEBUG: - print("\tSkipping %r!" % (tok,)) - continue - m = regex.match(self.input, self.pos) - if m: - # We got a match - best_pat = tok - best_pat_len = len(m.group(0)) - if DEBUG: - print("Match OK! %s: %s at pos %d" % (repr(tok), repr(regex.pattern), self.pos)) - break - - # If we didn't find anything, raise an error - if best_pat is None: - msg = "Bad token found" - if restrict: - msg = "Bad token found while trying to find one of the restricted tokens: %s" % (", ".join(repr(r) for r in restrict)) - raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(self.pos), msg)) - - # If we found something that isn't to be ignored, return it - if best_pat in self.ignore: - # This token should be ignored... - self.pos += best_pat_len - else: - end_pos = self.pos + best_pat_len - # Create a token with this data - token = ( - self.pos, - end_pos, - best_pat, - self.input[self.pos:end_pos] - ) - break - if token is not None: - self.pos = token[1] - # Only add this token if it's not in the list - # (to prevent looping) - if not self.tokens or token != self.tokens[-1]: - self.tokens.append(token) - self.restrictions.append(restrict) - return 1 - return 0 - - def token(self, i, restrict=None): - """ - Get the i'th token, and if i is one past the end, then scan - for another token; restrict is a list of tokens that - are allowed, or 0 for any token. - """ - tokens_len = len(self.tokens) - if i == tokens_len: # We are at the end, get the next... - tokens_len += self._scan(restrict) - if i < tokens_len: - if restrict and self.restrictions[i] and restrict > self.restrictions[i]: - raise NotImplementedError("Unimplemented: restriction set changed") - return self.tokens[i] - raise NoMoreTokens - - def rewind(self, i): - tokens_len = len(self.tokens) - if i <= tokens_len: - token = self.tokens[i] - self.tokens = self.tokens[:i] - self.restrictions = self.restrictions[:i] - self.pos = token[0] diff --git a/scss/compiler.py b/scss/compiler.py index dceacef..fd40b74 100644 --- a/scss/compiler.py +++ b/scss/compiler.py @@ -27,6 +27,7 @@ from scss.extension import Extension from scss.extension.core import CoreExtension from scss.extension import NamespaceAdapterExtension from scss.extension.compass.sprites import sprite_map +from scss.grammar import locate_blocks from scss.rule import BlockAtRuleHeader from scss.rule import Namespace from scss.rule import RuleAncestry @@ -45,12 +46,6 @@ from scss.types import Url from scss.util import dequote from scss.util import normalize_var # TODO put in... namespace maybe? -try: - # Use C version if available - from scss._speedups import locate_blocks -except ImportError: - from scss._native import locate_blocks - # TODO should mention logging for the programmatic interface in the # documentation diff --git a/scss/expression.py b/scss/expression.py index a249f50..54001f6 100644 --- a/scss/expression.py +++ b/scss/expression.py @@ -4,20 +4,17 @@ from __future__ import unicode_literals import sys import logging -import operator -import re from warnings import warn import six from scss.cssdefs import _expr_glob_re, _interpolate_re from scss.errors import SassError, SassEvaluationError, SassParseError +from scss.grammar.expression import SassExpression, SassExpressionScanner from scss.rule import Namespace from scss.types import String from scss.util import dequote -from scss._grammar import SassExpression, SassExpressionScanner - log = logging.getLogger(__name__) diff --git a/scss/grammar/__init__.py b/scss/grammar/__init__.py new file mode 100644 index 0000000..ffd5ca8 --- /dev/null +++ b/scss/grammar/__init__.py @@ -0,0 +1,9 @@ +"""Grammar and parser plumbing for Sass. Much of this is generated or compiled +in some fashion. +""" +from .scanner import NoMoreTokens +from .scanner import Parser +from .scanner import Scanner +from .scanner import locate_blocks + +__all__ = ('NoMoreTokens', 'Parser', 'Scanner', 'locate_blocks') diff --git a/scss/src/grammar/grammar.g b/scss/grammar/expression.g index 5580cfb..9cc833d 100644 --- a/scss/src/grammar/grammar.g +++ b/scss/grammar/expression.g @@ -1,7 +1,8 @@ """Grammar for parsing Sass expressions.""" # This is a GENERATED FILE -- DO NOT EDIT DIRECTLY! -# Edit scss/src/grammar/grammar.g, then run: -# python2 scss/src/grammar/yapps2.py scss/src/grammar/grammar.g scss/_grammar.py +# Edit scss/grammar/expression.g, then run: +# +# python2 yapps2.py scss/grammar/expression.g import operator import re @@ -24,11 +25,8 @@ from scss.types import String from scss.types import Url from scss.util import dequote -from scss._native import Parser -try: - from scss._speedups import Scanner -except ImportError: - from scss._native import Scanner +from scss.grammar import Parser +from scss.grammar import Scanner %% diff --git a/scss/_grammar.py b/scss/grammar/expression.py index 6296e0b..aed08b9 100644 --- a/scss/_grammar.py +++ b/scss/grammar/expression.py @@ -1,7 +1,8 @@ """Grammar for parsing Sass expressions.""" # This is a GENERATED FILE -- DO NOT EDIT DIRECTLY! -# Edit scss/src/grammar/grammar.g, then run: -# python2 scss/src/grammar/yapps2.py scss/src/grammar/grammar.g scss/_grammar.py +# Edit scss/grammar/expression.g, then run: +# +# python2 yapps2.py scss/grammar/expression.g import operator import re @@ -24,11 +25,8 @@ from scss.types import String from scss.types import Url from scss.util import dequote -from scss._native import Parser -try: - from scss._speedups import Scanner -except ImportError: - from scss._native import Scanner +from scss.grammar import Parser +from scss.grammar import Scanner diff --git a/scss/grammar/scanner.py b/scss/grammar/scanner.py new file mode 100644 index 0000000..d466254 --- /dev/null +++ b/scss/grammar/scanner.py @@ -0,0 +1,274 @@ +"""Pure-Python scanner and parser, used if the C module is not available.""" +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import deque + +import re + +DEBUG = False + +# TODO copied from __init__ +_blocks_re = re.compile(r'[{},;()\'"\n]') + + +try: + from ._scanner import locate_blocks +except ImportError: + def locate_blocks(codestr): + """ + For processing CSS like strings. + + Either returns all selectors (that can be "smart" multi-lined, as + long as it's joined by `,`, or enclosed in `(` and `)`) with its code block + (the one between `{` and `}`, which can be nested), or the "lose" code + (properties) that doesn't have any blocks. + """ + lineno = 1 + + par = 0 + instr = None + depth = 0 + skip = False + i = init = lose = 0 + start = end = None + lineno_stack = deque() + + for m in _blocks_re.finditer(codestr): + i = m.start(0) + c = codestr[i] + if c == '\n': + lineno += 1 + + if instr is not None: + if c == instr: + instr = None # A string ends (FIXME: needs to accept escaped characters) + elif c in ('"', "'"): + instr = c # A string starts + elif c == '(': # parenthesis begins: + par += 1 + elif c == ')': # parenthesis ends: + par -= 1 + elif not par and not instr: + if c == '{': # block begins: + if depth == 0: + if i > 0 and codestr[i - 1] == '#': # Do not process #{...} as blocks! + skip = True + else: + lineno_stack.append(lineno) + start = i + if lose < init: + _property = codestr[lose:init].strip() + if _property: + yield lineno, _property, None + lose = init + depth += 1 + elif c == '}': # block ends: + if depth <= 0: + raise SyntaxError("Unexpected closing brace on line {0}".format(lineno)) + else: + depth -= 1 + if depth == 0: + if not skip: + end = i + _selectors = codestr[init:start].strip() + _codestr = codestr[start + 1:end].strip() + if _selectors: + yield lineno_stack.pop(), _selectors, _codestr + init = lose = end + 1 + skip = False + elif depth == 0: + if c == ';': # End of property (or block): + init = i + if lose < init: + _property = codestr[lose:init].strip() + if _property: + yield lineno, _property, None + init = lose = i + 1 + if depth > 0: + if not skip: + _selectors = codestr[init:start].strip() + _codestr = codestr[start + 1:].strip() + if _selectors: + yield lineno, _selectors, _codestr + if par: + raise Exception("Missing closing parenthesis somewhere in block: '%s'" % _selectors) + elif instr: + raise Exception("Missing closing string somewhere in block: '%s'" % _selectors) + else: + raise Exception("Block never closed: '%s'" % _selectors) + losestr = codestr[lose:] + for _property in losestr.split(';'): + _property = _property.strip() + lineno += _property.count('\n') + if _property: + yield lineno, _property, None + + +################################################################################ +# Parser + +# NOTE: This class has no C equivalent +class Parser(object): + def __init__(self, scanner): + self._scanner = scanner + self._pos = 0 + self._char_pos = 0 + + def reset(self, input): + self._scanner.reset(input) + self._pos = 0 + self._char_pos = 0 + + def _peek(self, types): + """ + Returns the token type for lookahead; if there are any args + then the list of args is the set of token types to allow + """ + try: + tok = self._scanner.token(self._pos, types) + return tok[2] + except SyntaxError: + return None + + def _scan(self, type): + """ + Returns the matched text, and moves to the next token + """ + tok = self._scanner.token(self._pos, set([type])) + self._char_pos = tok[0] + if tok[2] != type: + raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type)) + self._pos += 1 + return tok[3] + + +try: + from ._scanner import NoMoreTokens +except ImportError: + class NoMoreTokens(Exception): + """ + Another exception object, for when we run out of tokens + """ + pass + + +try: + from ._scanner import Scanner +except ImportError: + class Scanner(object): + def __init__(self, patterns, ignore, input=None): + """ + Patterns is [(terminal,regex)...] + Ignore is [terminal,...]; + Input is a string + """ + self.reset(input) + self.ignore = ignore + # The stored patterns are a pair (compiled regex,source + # regex). If the patterns variable passed in to the + # constructor is None, we assume that the class already has a + # proper .patterns list constructed + if patterns is not None: + self.patterns = [] + for k, r in patterns: + self.patterns.append((k, re.compile(r))) + + def reset(self, input): + self.tokens = [] + self.restrictions = [] + self.input = input + self.pos = 0 + + def __repr__(self): + """ + Print the last 10 tokens that have been scanned in + """ + output = '' + for t in self.tokens[-10:]: + output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) + return output + + def _scan(self, restrict): + """ + Should scan another token and add it to the list, self.tokens, + and add the restriction to self.restrictions + """ + # Keep looking for a token, ignoring any in self.ignore + token = None + while True: + best_pat = None + # Search the patterns for a match, with earlier + # tokens in the list having preference + best_pat_len = 0 + for tok, regex in self.patterns: + if DEBUG: + print("\tTrying %s: %s at pos %d -> %s" % (repr(tok), repr(regex.pattern), self.pos, repr(self.input))) + # First check to see if we're restricting to this token + if restrict and tok not in restrict and tok not in self.ignore: + if DEBUG: + print("\tSkipping %r!" % (tok,)) + continue + m = regex.match(self.input, self.pos) + if m: + # We got a match + best_pat = tok + best_pat_len = len(m.group(0)) + if DEBUG: + print("Match OK! %s: %s at pos %d" % (repr(tok), repr(regex.pattern), self.pos)) + break + + # If we didn't find anything, raise an error + if best_pat is None: + msg = "Bad token found" + if restrict: + msg = "Bad token found while trying to find one of the restricted tokens: %s" % (", ".join(repr(r) for r in restrict)) + raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(self.pos), msg)) + + # If we found something that isn't to be ignored, return it + if best_pat in self.ignore: + # This token should be ignored... + self.pos += best_pat_len + else: + end_pos = self.pos + best_pat_len + # Create a token with this data + token = ( + self.pos, + end_pos, + best_pat, + self.input[self.pos:end_pos] + ) + break + if token is not None: + self.pos = token[1] + # Only add this token if it's not in the list + # (to prevent looping) + if not self.tokens or token != self.tokens[-1]: + self.tokens.append(token) + self.restrictions.append(restrict) + return 1 + return 0 + + def token(self, i, restrict=None): + """ + Get the i'th token, and if i is one past the end, then scan + for another token; restrict is a list of tokens that + are allowed, or 0 for any token. + """ + tokens_len = len(self.tokens) + if i == tokens_len: # We are at the end, get the next... + tokens_len += self._scan(restrict) + if i < tokens_len: + if restrict and self.restrictions[i] and restrict > self.restrictions[i]: + raise NotImplementedError("Unimplemented: restriction set changed") + return self.tokens[i] + raise NoMoreTokens + + def rewind(self, i): + tokens_len = len(self.tokens) + if i <= tokens_len: + token = self.tokens[i] + self.tokens = self.tokens[:i] + self.restrictions = self.restrictions[:i] + self.pos = token[0] diff --git a/scss/src/_speedups.c b/scss/src/_speedups.c index 460cac3..66c8d36 100644 --- a/scss/src/_speedups.c +++ b/scss/src/_speedups.c @@ -566,7 +566,7 @@ static PyMethodDef scss_methods[] = { static struct PyModuleDef speedups_module_def = { PyModuleDef_HEAD_INIT, - "_speedups", /* m_name */ + "_scanner", /* m_name */ NULL, /* m_doc */ (Py_ssize_t) -1, /* m_size */ scss_methods, /* m_methods */ @@ -586,12 +586,12 @@ static struct PyModuleDef speedups_module_def = { #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) #endif -MOD_INIT(_speedups) +MOD_INIT(_scanner) { #if PY_MAJOR_VERSION >= 3 PyObject* m = PyModule_Create(&speedups_module_def); #else - PyObject* m = Py_InitModule("_speedups", scss_methods); + PyObject* m = Py_InitModule("_scanner", scss_methods); #endif scss_BlockLocatorType.tp_new = PyType_GenericNew; @@ -613,7 +613,7 @@ MOD_INIT(_speedups) Py_INCREF(&scss_ScannerType); PyModule_AddObject(m, "Scanner", (PyObject *)&scss_ScannerType); - PyExc_scss_NoMoreTokens = PyErr_NewException("_speedups.NoMoreTokens", NULL, NULL); + PyExc_scss_NoMoreTokens = PyErr_NewException("_scanner.NoMoreTokens", NULL, NULL); Py_INCREF(PyExc_scss_NoMoreTokens); PyModule_AddObject(m, "NoMoreTokens", (PyObject *)PyExc_scss_NoMoreTokens); #if PY_MAJOR_VERSION >= 3 diff --git a/scss/src/grammar/LICENSE b/scss/src/grammar/LICENSE deleted file mode 100644 index 64f38b8..0000000 --- a/scss/src/grammar/LICENSE +++ /dev/null @@ -1,18 +0,0 @@ -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scss/src/grammar/README b/scss/src/grammar/README deleted file mode 100644 index 5a13dde..0000000 --- a/scss/src/grammar/README +++ /dev/null @@ -1,6 +0,0 @@ -To build the parser from the grammar do: - `python ./yapps2.py grammar.g` - -This will produce `grammar.py`; to that file modify the UNITS to be: '(?<!\\s)(?:' + '|'.join(_units) + ')(?![-\\w])' - -Paste the parser into pyScss's relevant block. diff --git a/scss/src/grammar/yappsrt.py b/scss/src/grammar/yappsrt.py deleted file mode 100644 index e7c18cd..0000000 --- a/scss/src/grammar/yappsrt.py +++ /dev/null @@ -1,275 +0,0 @@ -# Yapps 3.0 Runtime (by Kronuz) -# -# This module is needed to run generated parsers. - -import re - -try: - from _scss import Scanner, NoMoreTokens -except ImportError: - Scanner = None - -################################################################################ -# Parser - -if not Scanner: - class NoMoreTokens(Exception): - """ - Another exception object, for when we run out of tokens - """ - pass - - class Scanner(object): - def __init__(self, patterns, ignore, input=None): - """ - Patterns is [(terminal,regex)...] - Ignore is [terminal,...]; - Input is a string - """ - self.reset(input) - self.ignore = ignore - # The stored patterns are a pair (compiled regex,source - # regex). If the patterns variable passed in to the - # constructor is None, we assume that the class already has a - # proper .patterns list constructed - if patterns is not None: - self.patterns = [] - for k, r in patterns: - self.patterns.append((k, re.compile(r))) - - def reset(self, input): - self.tokens = [] - self.restrictions = [] - self.input = input - self.pos = 0 - - def __repr__(self): - """ - Print the last 10 tokens that have been scanned in - """ - output = '' - for t in self.tokens[-10:]: - output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) - return output - - def _scan(self, restrict): - """ - Should scan another token and add it to the list, self.tokens, - and add the restriction to self.restrictions - """ - # Keep looking for a token, ignoring any in self.ignore - token = None - while True: - best_pat = None - # Search the patterns for a match, with earlier - # tokens in the list having preference - best_pat_len = 0 - for p, regexp in self.patterns: - # First check to see if we're restricting to this token - if restrict and p not in restrict and p not in self.ignore: - continue - m = regexp.match(self.input, self.pos) - if m: - # We got a match - best_pat = p - best_pat_len = len(m.group(0)) - break - - # If we didn't find anything, raise an error - if best_pat is None: - msg = "Bad Token" - if restrict: - msg = "Trying to find one of " + ", ".join(restrict) - raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(self.pos), msg)) - - # If we found something that isn't to be ignored, return it - if best_pat in self.ignore: - # This token should be ignored... - self.pos += best_pat_len - else: - end_pos = self.pos + best_pat_len - # Create a token with this data - token = ( - self.pos, - end_pos, - best_pat, - self.input[self.pos:end_pos] - ) - break - if token is not None: - self.pos = token[1] - # Only add this token if it's not in the list - # (to prevent looping) - if not self.tokens or token != self.tokens[-1]: - self.tokens.append(token) - self.restrictions.append(restrict) - return 1 - return 0 - - def token(self, i, restrict=None): - """ - Get the i'th token, and if i is one past the end, then scan - for another token; restrict is a list of tokens that - are allowed, or 0 for any token. - """ - tokens_len = len(self.tokens) - if i == tokens_len: # We are at the end, get the next... - tokens_len += self._scan(restrict) - if i < tokens_len: - if restrict and self.restrictions[i] and restrict > self.restrictions[i]: - raise NotImplementedError("Unimplemented: restriction set changed") - return self.tokens[i] - raise NoMoreTokens - - def rewind(self, i): - tokens_len = len(self.tokens) - if i <= tokens_len: - token = self.tokens[i] - self.tokens = self.tokens[:i] - self.restrictions = self.restrictions[:i] - self.pos = token[0] - - -class CachedScanner(Scanner): - """ - Same as Scanner, but keeps cached tokens for any given input - """ - _cache_ = {} - _goals_ = ['END'] - - @classmethod - def cleanup(cls): - cls._cache_ = {} - - def __init__(self, patterns, ignore, input=None): - try: - self._tokens = self._cache_[input] - except KeyError: - self._tokens = None - self.__tokens = {} - self.__input = input - super(CachedScanner, self).__init__(patterns, ignore, input) - - def reset(self, input): - try: - self._tokens = self._cache_[input] - except KeyError: - self._tokens = None - self.__tokens = {} - self.__input = input - super(CachedScanner, self).reset(input) - - def __repr__(self): - if self._tokens is None: - return super(CachedScanner, self).__repr__() - output = '' - for t in self._tokens[-10:]: - output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) - return output - - def token(self, i, restrict=None): - if self._tokens is None: - token = super(CachedScanner, self).token(i, restrict) - self.__tokens[i] = token - if token[2] in self._goals_: # goal tokens - self._cache_[self.__input] = self._tokens = self.__tokens - return token - else: - token = self._tokens.get(i) - if token is None: - raise NoMoreTokens - return token - - def rewind(self, i): - if self._tokens is None: - super(CachedScanner, self).rewind(i) - - -class Parser(object): - def __init__(self, scanner): - self._scanner = scanner - self._pos = 0 - - def reset(self, input): - self._scanner.reset(input) - self._pos = 0 - - def _peek(self, types): - """ - Returns the token type for lookahead; if there are any args - then the list of args is the set of token types to allow - """ - tok = self._scanner.token(self._pos, types) - return tok[2] - - def _scan(self, type): - """ - Returns the matched text, and moves to the next token - """ - tok = self._scanner.token(self._pos, set([type])) - if tok[2] != type: - raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type)) - self._pos += 1 - return tok[3] - - def _rewind(self, n=1): - self._pos -= min(n, self._pos) - self._scanner.rewind(self._pos) - - -################################################################################ - - -def print_error(input, err, scanner): - """This is a really dumb long function to print error messages nicely.""" - p = err.pos - # Figure out the line number - line = input[:p].count('\n') - print err.msg + " on line " + repr(line + 1) + ":" - # Now try printing part of the line - text = input[max(p - 80, 0): - p + 80] - p = p - max(p - 80, 0) - - # Strip to the left - i = text[:p].rfind('\n') - j = text[:p].rfind('\r') - if i < 0 or (0 <= j < i): - i = j - if 0 <= i < p: - p = p - i - 1 - text = text[i + 1:] - - # Strip to the right - i = text.find('\n', p) - j = text.find('\r', p) - if i < 0 or (0 <= j < i): - i = j - if i >= 0: - text = text[:i] - - # Now shorten the text - while len(text) > 70 and p > 60: - # Cut off 10 chars - text = "..." + text[10:] - p = p - 7 - - # Now print the string, along with an indicator - print '> ', text - print '> ', ' ' * p + '^' - print 'List of nearby tokens:', scanner - - -def wrap_error_reporter(parser, rule, *args): - try: - return getattr(parser, rule)(*args) - except SyntaxError, s: - input = parser._scanner.input - try: - print_error(input, s, parser._scanner) - raise - except ImportError: - print "Syntax Error %s on line %d" % (s.msg, input[:s.pos].count('\n') + 1) - except NoMoreTokens: - print "Could not complete parsing; stopped around here:" - print parser._scanner @@ -22,7 +22,7 @@ speedups = Feature( # NOTE: header files are included by MANIFEST.in; Extension does not # include headers in an sdist (since they're typically in /usr/lib) Extension( - 'scss._speedups', + 'scss.grammar._scanner', sources=['scss/src/_speedups.c', 'scss/src/block_locator.c', 'scss/src/scanner.c'], libraries=['pcre'] ), diff --git a/scss/src/grammar/yapps2.py b/yapps2.py index 1cea3d3..e94a7b2 100755 --- a/scss/src/grammar/yapps2.py +++ b/yapps2.py @@ -1,4 +1,22 @@ #!/usr/bin/env python +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # Yapps 3.0 - yet another python parser system # Amit J Patel, January 1999 @@ -30,7 +48,8 @@ # * Style change: replaced raise "string exception" with raise # ClassException(...) (thanks Alex Verstak) -from yappsrt import * +from string import find +from string import join import sys import re @@ -656,8 +675,271 @@ def resolve_name(tokens, id, args): return NonTerminal(id, args) -from string import * -from yappsrt import * +################################################################################ +# Contents of yappsrt follow. + +# Parser + +class NoMoreTokens(Exception): + """ + Another exception object, for when we run out of tokens + """ + pass + +class Scanner(object): + def __init__(self, patterns, ignore, input=None): + """ + Patterns is [(terminal,regex)...] + Ignore is [terminal,...]; + Input is a string + """ + self.reset(input) + self.ignore = ignore + # The stored patterns are a pair (compiled regex,source + # regex). If the patterns variable passed in to the + # constructor is None, we assume that the class already has a + # proper .patterns list constructed + if patterns is not None: + self.patterns = [] + for k, r in patterns: + self.patterns.append((k, re.compile(r))) + + def reset(self, input): + self.tokens = [] + self.restrictions = [] + self.input = input + self.pos = 0 + + def __repr__(self): + """ + Print the last 10 tokens that have been scanned in + """ + output = '' + for t in self.tokens[-10:]: + output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) + return output + + def _scan(self, restrict): + """ + Should scan another token and add it to the list, self.tokens, + and add the restriction to self.restrictions + """ + # Keep looking for a token, ignoring any in self.ignore + token = None + while True: + best_pat = None + # Search the patterns for a match, with earlier + # tokens in the list having preference + best_pat_len = 0 + for p, regexp in self.patterns: + # First check to see if we're restricting to this token + if restrict and p not in restrict and p not in self.ignore: + continue + m = regexp.match(self.input, self.pos) + if m: + # We got a match + best_pat = p + best_pat_len = len(m.group(0)) + break + + # If we didn't find anything, raise an error + if best_pat is None: + msg = "Bad Token" + if restrict: + msg = "Trying to find one of " + ", ".join(restrict) + raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(self.pos), msg)) + + # If we found something that isn't to be ignored, return it + if best_pat in self.ignore: + # This token should be ignored... + self.pos += best_pat_len + else: + end_pos = self.pos + best_pat_len + # Create a token with this data + token = ( + self.pos, + end_pos, + best_pat, + self.input[self.pos:end_pos] + ) + break + if token is not None: + self.pos = token[1] + # Only add this token if it's not in the list + # (to prevent looping) + if not self.tokens or token != self.tokens[-1]: + self.tokens.append(token) + self.restrictions.append(restrict) + return 1 + return 0 + + def token(self, i, restrict=None): + """ + Get the i'th token, and if i is one past the end, then scan + for another token; restrict is a list of tokens that + are allowed, or 0 for any token. + """ + tokens_len = len(self.tokens) + if i == tokens_len: # We are at the end, get the next... + tokens_len += self._scan(restrict) + if i < tokens_len: + if restrict and self.restrictions[i] and restrict > self.restrictions[i]: + raise NotImplementedError("Unimplemented: restriction set changed") + return self.tokens[i] + raise NoMoreTokens + + def rewind(self, i): + tokens_len = len(self.tokens) + if i <= tokens_len: + token = self.tokens[i] + self.tokens = self.tokens[:i] + self.restrictions = self.restrictions[:i] + self.pos = token[0] + + +class CachedScanner(Scanner): + """ + Same as Scanner, but keeps cached tokens for any given input + """ + _cache_ = {} + _goals_ = ['END'] + + @classmethod + def cleanup(cls): + cls._cache_ = {} + + def __init__(self, patterns, ignore, input=None): + try: + self._tokens = self._cache_[input] + except KeyError: + self._tokens = None + self.__tokens = {} + self.__input = input + super(CachedScanner, self).__init__(patterns, ignore, input) + + def reset(self, input): + try: + self._tokens = self._cache_[input] + except KeyError: + self._tokens = None + self.__tokens = {} + self.__input = input + super(CachedScanner, self).reset(input) + + def __repr__(self): + if self._tokens is None: + return super(CachedScanner, self).__repr__() + output = '' + for t in self._tokens[-10:]: + output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3])) + return output + + def token(self, i, restrict=None): + if self._tokens is None: + token = super(CachedScanner, self).token(i, restrict) + self.__tokens[i] = token + if token[2] in self._goals_: # goal tokens + self._cache_[self.__input] = self._tokens = self.__tokens + return token + else: + token = self._tokens.get(i) + if token is None: + raise NoMoreTokens + return token + + def rewind(self, i): + if self._tokens is None: + super(CachedScanner, self).rewind(i) + + +class Parser(object): + def __init__(self, scanner): + self._scanner = scanner + self._pos = 0 + + def reset(self, input): + self._scanner.reset(input) + self._pos = 0 + + def _peek(self, types): + """ + Returns the token type for lookahead; if there are any args + then the list of args is the set of token types to allow + """ + tok = self._scanner.token(self._pos, types) + return tok[2] + + def _scan(self, type): + """ + Returns the matched text, and moves to the next token + """ + tok = self._scanner.token(self._pos, set([type])) + if tok[2] != type: + raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type)) + self._pos += 1 + return tok[3] + + def _rewind(self, n=1): + self._pos -= min(n, self._pos) + self._scanner.rewind(self._pos) + + +def print_error(input, err, scanner): + """This is a really dumb long function to print error messages nicely.""" + p = err.pos + # Figure out the line number + line = input[:p].count('\n') + print err.msg + " on line " + repr(line + 1) + ":" + # Now try printing part of the line + text = input[max(p - 80, 0): + p + 80] + p = p - max(p - 80, 0) + + # Strip to the left + i = text[:p].rfind('\n') + j = text[:p].rfind('\r') + if i < 0 or (0 <= j < i): + i = j + if 0 <= i < p: + p = p - i - 1 + text = text[i + 1:] + + # Strip to the right + i = text.find('\n', p) + j = text.find('\r', p) + if i < 0 or (0 <= j < i): + i = j + if i >= 0: + text = text[:i] + + # Now shorten the text + while len(text) > 70 and p > 60: + # Cut off 10 chars + text = "..." + text[10:] + p = p - 7 + + # Now print the string, along with an indicator + print '> ', text + print '> ', ' ' * p + '^' + print 'List of nearby tokens:', scanner + + +def wrap_error_reporter(parser, rule, *args): + try: + return getattr(parser, rule)(*args) + except SyntaxError, s: + input = parser._scanner.input + try: + print_error(input, s, parser._scanner) + raise + except ImportError: + print "Syntax Error %s on line %d" % (s.msg, input[:s.pos].count('\n') + 1) + except NoMoreTokens: + print "Could not complete parsing; stopped around here:" + print parser._scanner + +# End yappsrt +################################################################################ class ParserDescriptionScanner(Scanner): |