even more decoder optimizations

git-svn-id: http://simplejson.googlecode.com/svn/trunk@103 a4795897-2c25-0410-b006-0d3caba88fa1

1 files changed, 39 insertions, 75 deletions

diff --git a/simplejson/decoder.py b/simplejson/decoder.py
index 4d0146c..a4be8ab 100644
--- a/simplejson/decoder.py
+++ b/simplejson/decoder.py
@@ -4,7 +4,7 @@ Implementation of JSONDecoder
 import re
 import sys
 
-from simplejson.scanner import make_scanner, pattern
+from simplejson.scanner import make_scanner
 try:
     from simplejson._speedups import scanstring as c_scanstring
 except ImportError:
@@ -46,36 +46,8 @@ _CONSTANTS = {
     '-Infinity': NegInf,
     'Infinity': PosInf,
     'NaN': NaN,
-    'true': True,
-    'false': False,
-    'null': None,
 }
 
-def JSONConstant(match, context, c=_CONSTANTS):
-    s = match.group(0)
-    fn = getattr(context, 'parse_constant', None)
-    if fn is None:
-        rval = c[s]
-    else:
-        rval = fn(s)
-    return rval, match.end()
-pattern(r'(-?Infinity|NaN|true|false|null)')(JSONConstant)
-
-
-def JSONNumber(match, context):
-    # m1 = JSONNumber.regex.match(match.string, *match.span())
-    # assert m1.groups()[:3] == match.groups()[:3]
-    integer, frac, exp = match.groups()[:3]
-    if frac or exp:
-        fn = getattr(context, 'parse_float', None) or float
-        res = fn(integer + (frac or '') + (exp or ''))
-    else:
-        fn = getattr(context, 'parse_int', None) or int
-        res = fn(integer)
-    return res, match.end()
-pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
-
-
 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
 BACKSLASH = {
     '"': u'"', '\\': u'\\', '/': u'/',
@@ -150,13 +122,6 @@ def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHU
 # Use speedup if available
 scanstring = c_scanstring or py_scanstring
 
-def JSONString((string, end), context):
-    encoding = getattr(context, 'encoding', None)
-    strict = getattr(context, 'strict', True)
-    return scanstring(string, end, encoding, strict)
-pattern(r'"')(JSONString)
-
-
 WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
 WHITESPACE_STR = ' \t\n\r'
 
@@ -174,9 +139,9 @@ def JSONObject((s, end), context, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
         elif nextchar != '"':
             raise ValueError(errmsg("Expecting property name", s, end))
     end += 1
-    encoding = getattr(context, 'encoding', None)
-    strict = getattr(context, 'strict', True)
-    scan_once = JSONScanner
+    encoding = context.encoding
+    strict = context.strict
+    scan_once = context.scan_once
     while True:
         key, end = scanstring(s, end, encoding, strict)
 
@@ -193,7 +158,7 @@ def JSONObject((s, end), context, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
             if s[end] in _ws:
                 end += 1
                 if s[end] in _ws:
-                    end = _w(s, end).end()
+                    end = _w(s, end + 1).end()
         except IndexError:
             pass
 
@@ -202,10 +167,14 @@ def JSONObject((s, end), context, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
         pairs[key] = value
-        nextchar = s[end:end + 1]
-        if nextchar in _ws:
-            end = _w(s, end).end()
-            nextchar = s[end:end + 1]
+        
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end = _w(s, end + 1).end()
+                nextchar = s[end]
+        except IndexError:
+            nextchar = ''
         end += 1
 
         if nextchar == '}':
@@ -214,73 +183,67 @@ def JSONObject((s, end), context, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
             raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
 
         try:
-            if s[end] in _ws:
+            nextchar = s[end]
+            if nextchar in _ws:
                 end += 1
-                if s[end] in _ws:
-                    end = _w(s, end).end()
+                nextchar = s[end]
+                if nextchar in _ws:
+                    end = _w(s, end + 1).end()
+                    nextchar = s[end]
         except IndexError:
-            pass
+            nextchar = ''
 
-        nextchar = s[end:end + 1]
         end += 1
         if nextchar != '"':
             raise ValueError(errmsg("Expecting property name", s, end - 1))
 
-    object_hook = getattr(context, 'object_hook', None)
+    object_hook = context.object_hook
     if object_hook is not None:
         pairs = object_hook(pairs)
     return pairs, end
-pattern(r'{')(JSONObject)
-
 
 def JSONArray((s, end), context, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
     values = []
     nextchar = s[end:end + 1]
     if nextchar in _ws:
-        end = _w(s, end).end()
+        end = _w(s, end + 1).end()
         nextchar = s[end:end + 1]
     # Look-ahead for trivial empty array
     if nextchar == ']':
         return values, end + 1
-    scan_once = JSONScanner
+    scan_once = context.scan_once
+    _append = values.append
     while True:
         try:
             value, end = scan_once(s, end, context)
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
-        values.append(value)
+        _append(value)
         nextchar = s[end:end + 1]
         if nextchar in _ws:
-            end = _w(s, end).end()
+            end = _w(s, end + 1).end()
             nextchar = s[end:end + 1]
         end += 1
         if nextchar == ']':
             break
-        if nextchar != ',':
+        elif nextchar != ',':
             raise ValueError(errmsg("Expecting , delimiter", s, end))
         
         try:
             if s[end] in _ws:
                 end += 1
                 if s[end] in _ws:
-                    end = _w(s, end).end()
+                    end = _w(s, end + 1).end()
         except IndexError:
             pass
 
     return values, end
-pattern(r'\[')(JSONArray)
-
-
-ANYTHING = [
-    JSONObject,
-    JSONArray,
-    JSONString,
-    JSONConstant,
-    JSONNumber,
-]
-
-JSONScanner = make_scanner(ANYTHING)
 
+_LEXICON = dict(
+    object=JSONObject,
+    array=JSONArray,
+    string=scanstring,
+)
 
 class JSONDecoder(object):
     """
@@ -340,16 +303,17 @@ class JSONDecoder(object):
         for JSON integers (e.g. float).
 
         ``parse_constant``, if specified, will be called with one of the
-        following strings: -Infinity, Infinity, NaN, null, true, false.
+        following strings: -Infinity, Infinity, NaN.
         This can be used to raise an exception if invalid JSON numbers
         are encountered.
         """
         self.encoding = encoding
         self.object_hook = object_hook
-        self.parse_float = parse_float
-        self.parse_int = parse_int
-        self.parse_constant = parse_constant
+        self.parse_float = parse_float or float
+        self.parse_int = parse_int or int
+        self.parse_constant = parse_constant or _CONSTANTS.__getitem__
         self.strict = strict
+        self.scan_once = make_scanner(_LEXICON)
 
     def decode(self, s, _w=WHITESPACE.match):
         """
@@ -374,7 +338,7 @@ class JSONDecoder(object):
         idx = kw.get('idx', 0)
         context = kw.get('context', self)
         try:
-            obj, end = JSONScanner(s, idx, context)
+            obj, end = self.scan_once(s, idx, context)
         except StopIteration:
             raise ValueError("No JSON object could be decoded")
         return obj, end