summaryrefslogtreecommitdiff
path: root/django/utils/simplejson/decoder.py
diff options
context:
space:
mode:
Diffstat (limited to 'django/utils/simplejson/decoder.py')
-rw-r--r--django/utils/simplejson/decoder.py272
1 files changed, 137 insertions, 135 deletions
diff --git a/django/utils/simplejson/decoder.py b/django/utils/simplejson/decoder.py
index cef9fc002b..5a845ccfdc 100644
--- a/django/utils/simplejson/decoder.py
+++ b/django/utils/simplejson/decoder.py
@@ -1,20 +1,17 @@
-"""
-Implementation of JSONDecoder
+"""Implementation of JSONDecoder
"""
import re
import sys
+import struct
+
+from django.utils.simplejson.scanner import make_scanner
+c_scanstring = None
-from django.utils.simplejson.scanner import Scanner, pattern
-try:
- from django.utils.simplejson._speedups import scanstring as c_scanstring
-except ImportError:
- pass
+__all__ = ['JSONDecoder']
FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
def _floatconstants():
- import struct
- import sys
_BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
if sys.byteorder != 'big':
_BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
@@ -34,6 +31,7 @@ def linecol(doc, pos):
def errmsg(msg, doc, pos, end=None):
+ # Note that this function is called from _speedups
lineno, colno = linecol(doc, pos)
if end is None:
return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
@@ -46,35 +44,8 @@ _CONSTANTS = {
'-Infinity': NegInf,
'Infinity': PosInf,
'NaN': NaN,
- 'true': True,
- 'false': False,
- 'null': None,
}
-def JSONConstant(match, context, c=_CONSTANTS):
- s = match.group(0)
- fn = getattr(context, 'parse_constant', None)
- if fn is None:
- rval = c[s]
- else:
- rval = fn(s)
- return rval, None
-pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
-
-
-def JSONNumber(match, context):
- match = JSONNumber.regex.match(match.string, *match.span())
- integer, frac, exp = match.groups()
- if frac or exp:
- fn = getattr(context, 'parse_float', None) or float
- res = fn(integer + (frac or '') + (exp or ''))
- else:
- fn = getattr(context, 'parse_int', None) or int
- res = fn(integer)
- return res, None
-pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
-
-
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': u'"', '\\': u'\\', '/': u'/',
@@ -84,6 +55,14 @@ BACKSLASH = {
DEFAULT_ENCODING = "utf-8"
def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
+ """Scan the string s for a JSON string. End is the index of the
+ character in s after the quote that started the JSON string.
+ Unescapes all valid JSON string escape sequences and raises ValueError
+ on attempt to decode an invalid string. If strict is False then literal
+ control characters are allowed in the string.
+
+ Returns a tuple of the decoded string and the index of the character in s
+ after the end quote."""
if encoding is None:
encoding = DEFAULT_ENCODING
chunks = []
@@ -96,15 +75,19 @@ def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHU
errmsg("Unterminated string starting at", s, begin))
end = chunk.end()
content, terminator = chunk.groups()
+ # Content is contains zero or more unescaped string characters
if content:
if not isinstance(content, unicode):
content = unicode(content, encoding)
_append(content)
+ # Terminator is the end of string, a literal control character,
+ # or a backslash denoting that an escape sequence follows
if terminator == '"':
break
elif terminator != '\\':
if strict:
- raise ValueError(errmsg("Invalid control character %r at", s, end))
+ msg = "Invalid control character %r at" % (terminator,)
+ raise ValueError(msg, s, end)
else:
_append(terminator)
continue
@@ -113,142 +96,162 @@ def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHU
except IndexError:
raise ValueError(
errmsg("Unterminated string starting at", s, begin))
+ # If not a unicode escape sequence, must be in the lookup table
if esc != 'u':
try:
- m = _b[esc]
+ char = _b[esc]
except KeyError:
raise ValueError(
errmsg("Invalid \\escape: %r" % (esc,), s, end))
end += 1
else:
+ # Unicode escape sequence
esc = s[end + 1:end + 5]
next_end = end + 5
- msg = "Invalid \\uXXXX escape"
- try:
- if len(esc) != 4:
- raise ValueError
- uni = int(esc, 16)
- if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
- msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
- if not s[end + 5:end + 7] == '\\u':
- raise ValueError
- esc2 = s[end + 7:end + 11]
- if len(esc2) != 4:
- raise ValueError
- uni2 = int(esc2, 16)
- uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
- next_end += 6
- m = unichr(uni)
- except ValueError:
+ if len(esc) != 4:
+ msg = "Invalid \\uXXXX escape"
raise ValueError(errmsg(msg, s, end))
+ uni = int(esc, 16)
+ # Check for surrogate pair on UCS-4 systems
+ if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
+ msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
+ if not s[end + 5:end + 7] == '\\u':
+ raise ValueError(errmsg(msg, s, end))
+ esc2 = s[end + 7:end + 11]
+ if len(esc2) != 4:
+ raise ValueError(errmsg(msg, s, end))
+ uni2 = int(esc2, 16)
+ uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
+ next_end += 6
+ char = unichr(uni)
end = next_end
- _append(m)
+ # Append the unescaped character
+ _append(char)
return u''.join(chunks), end
-# Use speedup
-try:
- scanstring = c_scanstring
-except NameError:
- scanstring = py_scanstring
-
-def JSONString(match, context):
- encoding = getattr(context, 'encoding', None)
- strict = getattr(context, 'strict', True)
- return scanstring(match.string, match.end(), encoding, strict)
-pattern(r'"')(JSONString)
+# Use speedup if available
+scanstring = c_scanstring or py_scanstring
+WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+WHITESPACE_STR = ' \t\n\r'
-WHITESPACE = re.compile(r'\s*', FLAGS)
-
-def JSONObject(match, context, _w=WHITESPACE.match):
+def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
pairs = {}
- s = match.string
- end = _w(s, match.end()).end()
+ # Use a slice to prevent IndexError from being raised, the following
+ # check will raise a more specific ValueError if the string is empty
nextchar = s[end:end + 1]
- # Trivial empty object
- if nextchar == '}':
- return pairs, end + 1
+ # Normally we expect nextchar == '"'
if nextchar != '"':
- raise ValueError(errmsg("Expecting property name", s, end))
+ if nextchar in _ws:
+ end = _w(s, end).end()
+ nextchar = s[end:end + 1]
+ # Trivial empty object
+ if nextchar == '}':
+ return pairs, end + 1
+ elif nextchar != '"':
+ raise ValueError(errmsg("Expecting property name", s, end))
end += 1
- encoding = getattr(context, 'encoding', None)
- strict = getattr(context, 'strict', True)
- iterscan = JSONScanner.iterscan
while True:
key, end = scanstring(s, end, encoding, strict)
- end = _w(s, end).end()
+
+ # To skip some function call overhead we optimize the fast paths where
+ # the JSON key separator is ": " or just ":".
if s[end:end + 1] != ':':
- raise ValueError(errmsg("Expecting : delimiter", s, end))
- end = _w(s, end + 1).end()
+ end = _w(s, end).end()
+ if s[end:end + 1] != ':':
+ raise ValueError(errmsg("Expecting : delimiter", s, end))
+
+ end += 1
+
try:
- value, end = iterscan(s, idx=end, context=context).next()
+ if s[end] in _ws:
+ end += 1
+ if s[end] in _ws:
+ end = _w(s, end + 1).end()
+ except IndexError:
+ pass
+
+ try:
+ value, end = scan_once(s, end)
except StopIteration:
raise ValueError(errmsg("Expecting object", s, end))
pairs[key] = value
- end = _w(s, end).end()
- nextchar = s[end:end + 1]
+
+ try:
+ nextchar = s[end]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end]
+ except IndexError:
+ nextchar = ''
end += 1
+
if nextchar == '}':
break
- if nextchar != ',':
+ elif nextchar != ',':
raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
- end = _w(s, end).end()
- nextchar = s[end:end + 1]
+
+ try:
+ nextchar = s[end]
+ if nextchar in _ws:
+ end += 1
+ nextchar = s[end]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end]
+ except IndexError:
+ nextchar = ''
+
end += 1
if nextchar != '"':
raise ValueError(errmsg("Expecting property name", s, end - 1))
- object_hook = getattr(context, 'object_hook', None)
+
if object_hook is not None:
pairs = object_hook(pairs)
return pairs, end
-pattern(r'{')(JSONObject)
-
-def JSONArray(match, context, _w=WHITESPACE.match):
+def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
values = []
- s = match.string
- end = _w(s, match.end()).end()
- # Look-ahead for trivial empty array
nextchar = s[end:end + 1]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end:end + 1]
+ # Look-ahead for trivial empty array
if nextchar == ']':
return values, end + 1
- iterscan = JSONScanner.iterscan
+ _append = values.append
while True:
try:
- value, end = iterscan(s, idx=end, context=context).next()
+ value, end = scan_once(s, end)
except StopIteration:
raise ValueError(errmsg("Expecting object", s, end))
- values.append(value)
- end = _w(s, end).end()
+ _append(value)
nextchar = s[end:end + 1]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end:end + 1]
end += 1
if nextchar == ']':
break
- if nextchar != ',':
+ elif nextchar != ',':
raise ValueError(errmsg("Expecting , delimiter", s, end))
- end = _w(s, end).end()
- return values, end
-pattern(r'\[')(JSONArray)
+ try:
+ if s[end] in _ws:
+ end += 1
+ if s[end] in _ws:
+ end = _w(s, end + 1).end()
+ except IndexError:
+ pass
-ANYTHING = [
- JSONObject,
- JSONArray,
- JSONString,
- JSONConstant,
- JSONNumber,
-]
-
-JSONScanner = Scanner(ANYTHING)
-
+ return values, end
class JSONDecoder(object):
- """
- Simple JSON <http://json.org> decoder
+ """Simple JSON <http://json.org> decoder
Performs the following translations in decoding by default:
-
+
+---------------+-------------------+
| JSON | Python |
+===============+===================+
@@ -271,18 +274,15 @@ class JSONDecoder(object):
It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
their corresponding ``float`` values, which is outside the JSON spec.
- """
- _scanner = Scanner(ANYTHING)
- __all__ = ['__init__', 'decode', 'raw_decode']
+ """
def __init__(self, encoding=None, object_hook=None, parse_float=None,
parse_int=None, parse_constant=None, strict=True):
- """
- ``encoding`` determines the encoding used to interpret any ``str``
+ """``encoding`` determines the encoding used to interpret any ``str``
objects decoded by this instance (utf-8 by default). It has no
effect when decoding ``unicode`` objects.
-
+
Note that currently only encodings that are a superset of ASCII work,
strings of other encodings should be passed in as ``unicode``.
@@ -302,21 +302,26 @@ class JSONDecoder(object):
for JSON integers (e.g. float).
``parse_constant``, if specified, will be called with one of the
- following strings: -Infinity, Infinity, NaN, null, true, false.
+ following strings: -Infinity, Infinity, NaN.
This can be used to raise an exception if invalid JSON numbers
are encountered.
+
"""
self.encoding = encoding
self.object_hook = object_hook
- self.parse_float = parse_float
- self.parse_int = parse_int
- self.parse_constant = parse_constant
+ self.parse_float = parse_float or float
+ self.parse_int = parse_int or int
+ self.parse_constant = parse_constant or _CONSTANTS.__getitem__
self.strict = strict
+ self.parse_object = JSONObject
+ self.parse_array = JSONArray
+ self.parse_string = scanstring
+ self.scan_once = make_scanner(self)
def decode(self, s, _w=WHITESPACE.match):
- """
- Return the Python representation of ``s`` (a ``str`` or ``unicode``
+ """Return the Python representation of ``s`` (a ``str`` or ``unicode``
instance containing a JSON document)
+
"""
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
end = _w(s, end).end()
@@ -324,20 +329,17 @@ class JSONDecoder(object):
raise ValueError(errmsg("Extra data", s, end, len(s)))
return obj
- def raw_decode(self, s, **kw):
- """
- Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
+ def raw_decode(self, s, idx=0):
+ """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
with a JSON document) and return a 2-tuple of the Python
representation and the index in ``s`` where the document ended.
This can be used to decode a JSON document from a string that may
have extraneous data at the end.
+
"""
- kw.setdefault('context', self)
try:
- obj, end = self._scanner.iterscan(s, **kw).next()
+ obj, end = self.scan_once(s, idx)
except StopIteration:
raise ValueError("No JSON object could be decoded")
return obj, end
-
-__all__ = ['JSONDecoder']
generated by cgit v1.3 (git 2.53.0) at 2026-05-02 03:28:59 +0000