summaryrefslogtreecommitdiff
path: root/scss/src/scanner.py
diff options
context:
space:
mode:
Diffstat (limited to 'scss/src/scanner.py')
-rwxr-xr-xscss/src/scanner.py127
1 files changed, 1 insertions, 126 deletions
diff --git a/scss/src/scanner.py b/scss/src/scanner.py
index 97c0b34..d148b30 100755
--- a/scss/src/scanner.py
+++ b/scss/src/scanner.py
@@ -84,132 +84,7 @@ PATTERNS = [
# Parser
DEBUG = False
-
-class NoMoreTokens(Exception):
- """
- Another exception object, for when we run out of tokens
- """
- pass
-
-
-class Scanner(object):
- def __init__(self, patterns, ignore, input=None):
- """
- Patterns is [(terminal,regex)...]
- Ignore is [terminal,...];
- Input is a string
- """
- self.reset(input)
- self.ignore = ignore
- # The stored patterns are a pair (compiled regex,source
- # regex). If the patterns variable passed in to the
- # constructor is None, we assume that the class already has a
- # proper .patterns list constructed
- if patterns is not None:
- self.patterns = []
- for k, r in patterns:
- self.patterns.append((k, re.compile(r)))
-
- def reset(self, input):
- self.tokens = []
- self.restrictions = []
- self.input = input
- self.pos = 0
-
- def __repr__(self):
- """
- Print the last 10 tokens that have been scanned in
- """
- output = ''
- for t in self.tokens[-10:]:
- output = "%s\n (@%s) %s = %s" % (output, t[0], t[2], repr(t[3]))
- return output
-
- def _scan(self, restrict):
- """
- Should scan another token and add it to the list, self.tokens,
- and add the restriction to self.restrictions
- """
- # Keep looking for a token, ignoring any in self.ignore
- token = None
- while True:
- tok = None
- best_pat = None
- # Search the patterns for a match, with earlier
- # tokens in the list having preference
- best_pat_len = 0
- for tok, regex in self.patterns:
- if DEBUG:
- print("\tTrying %s: %s at pos %d -> %s" % (repr(tok), repr(regex.pattern), self.pos, repr(self.input)))
- # First check to see if we're restricting to this token
- if restrict and tok not in restrict and tok not in self.ignore:
- if DEBUG:
- print "\tSkipping %s!" % repr(tok)
- continue
- m = regex.match(self.input, self.pos)
- if m:
- # We got a match
- best_pat = tok
- best_pat_len = len(m.group(0))
- if DEBUG:
- print("Match OK! %s: %s at pos %d" % (repr(tok), repr(regex.pattern), self.pos))
- break
-
- # If we didn't find anything, raise an error
- if best_pat is None:
- msg = "Bad token: %s" % ("???" if tok is None else repr(tok),)
- if restrict:
- msg = "%s found while trying to find one of the restricted tokens: %s" % ("???" if tok is None else repr(tok), ", ".join(repr(r) for r in restrict))
- raise SyntaxError("SyntaxError[@ char %s: %s]" % (repr(self.pos), msg))
-
- # If we found something that isn't to be ignored, return it
- if best_pat in self.ignore:
- # This token should be ignored...
- self.pos += best_pat_len
- else:
- end_pos = self.pos + best_pat_len
- # Create a token with this data
- token = (
- self.pos,
- end_pos,
- best_pat,
- self.input[self.pos:end_pos]
- )
- break
- if token is not None:
- self.pos = token[1]
- # Only add this token if it's not in the list
- # (to prevent looping)
- if not self.tokens or token != self.tokens[-1]:
- self.tokens.append(token)
- self.restrictions.append(restrict)
- return 1
- return 0
-
- def token(self, i, restrict=None):
- """
- Get the i'th token, and if i is one past the end, then scan
- for another token; restrict is a list of tokens that
- are allowed, or 0 for any token.
- """
- tokens_len = len(self.tokens)
- if i == tokens_len: # We are at the end, get the next...
- tokens_len += self._scan(restrict)
- elif i >= 0 and i < tokens_len:
- if restrict and self.restrictions[i] and restrict > self.restrictions[i]:
- raise NotImplementedError("Unimplemented: restriction set changed")
- if i >= 0 and i < tokens_len:
- return self.tokens[i]
- raise NoMoreTokens()
-
- def rewind(self, i):
- tokens_len = len(self.tokens)
- if i <= tokens_len:
- token = self.tokens[i]
- self.tokens = self.tokens[:i]
- self.restrictions = self.restrictions[:i]
- self.pos = token[0]
-
+from yapps.runtime import Scanner
class _Scanner_a(Scanner):
patterns = None