queryset-refactor: Merged from trunk up to [6689].

git-svn-id: http://code.djangoproject.com/svn/django/branches/queryset-refactor@6690 bcc190cf-cafb-0310-a4f2-bffc1f526a37

7 files changed, 207 insertions, 30 deletions

diff --git a/django/utils/datastructures.py b/django/utils/datastructures.py
index e0835b2cfc..549aa3f183 100644
--- a/django/utils/datastructures.py
+++ b/django/utils/datastructures.py
@@ -62,12 +62,10 @@ class SortedDict(dict):
         else:
             self.keyOrder = [key for key, value in data]
 
-    def __deepcopy__(self,memo):
+    def __deepcopy__(self, memo):
         from copy import deepcopy
-        obj = self.__class__()
-        for k, v in self.items():
-            obj[k] = deepcopy(v, memo)
-        return obj
+        return self.__class__([(key, deepcopy(value, memo))
+                               for key, value in self.iteritems()])
 
     def __setitem__(self, key, value):
         dict.__setitem__(self, key, value)
diff --git a/django/utils/dateformat.py b/django/utils/dateformat.py
index 9a296c3f2d..bfca46b3d5 100644
--- a/django/utils/dateformat.py
+++ b/django/utils/dateformat.py
@@ -170,7 +170,7 @@ class DateFormat(TimeFormat):
         return u"%+03d%02d" % (seconds // 3600, (seconds // 60) % 60)
 
     def r(self):
-        "RFC 822 formatted date; e.g. 'Thu, 21 Dec 2000 16:01:07 +0200'"
+        "RFC 2822 formatted date; e.g. 'Thu, 21 Dec 2000 16:01:07 +0200'"
         return self.format('D, j M Y H:i:s O')
 
     def S(self):
diff --git a/django/utils/encoding.py b/django/utils/encoding.py
index 2bd1ef6563..2ab0db7432 100644
--- a/django/utils/encoding.py
+++ b/django/utils/encoding.py
@@ -1,7 +1,19 @@
 import types
 import urllib
 import datetime
+
 from django.utils.functional import Promise
+from django.utils.safestring import SafeData, mark_safe
+
+class DjangoUnicodeDecodeError(UnicodeDecodeError):
+    def __init__(self, obj, *args):
+        self.obj = obj
+        UnicodeDecodeError.__init__(self, *args)
+
+    def __str__(self):
+        original = UnicodeDecodeError.__str__(self)
+        return '%s. You passed in %r (%s)' % (original, self.obj,
+                type(self.obj))
 
 class StrAndUnicode(object):
     """
@@ -33,13 +45,19 @@ def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
     """
     if strings_only and isinstance(s, (types.NoneType, int, long, datetime.datetime, datetime.date, datetime.time, float)):
         return s
-    if not isinstance(s, basestring,):
-        if hasattr(s, '__unicode__'):
-            s = unicode(s)
-        else:
-            s = unicode(str(s), encoding, errors)
-    elif not isinstance(s, unicode):
-        s = unicode(s, encoding, errors)
+    try:
+        if not isinstance(s, basestring,):
+            if hasattr(s, '__unicode__'):
+                s = unicode(s)
+            else:
+                s = unicode(str(s), encoding, errors)
+        elif not isinstance(s, unicode):
+            # Note: We use .decode() here, instead of unicode(s, encoding,
+            # errors), so that if s is a SafeString, it ends up being a
+            # SafeUnicode at the end.
+            s = s.decode(encoding, errors)
+    except UnicodeDecodeError, e:
+        raise DjangoUnicodeDecodeError(s, *e.args)
     return s
 
 def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
diff --git a/django/utils/html.py b/django/utils/html.py
index ebd04d1b3c..cb786db1e4 100644
--- a/django/utils/html.py
+++ b/django/utils/html.py
@@ -3,8 +3,10 @@
 import re
 import string
 
+from django.utils.safestring import SafeData, mark_safe
 from django.utils.encoding import force_unicode
 from django.utils.functional import allow_lazy
+from django.utils.http import urlquote
 
 # Configuration for urlize() function
 LEADING_PUNCTUATION  = ['(', '<', '&lt;']
@@ -27,16 +29,28 @@ del x # Temporary variable
 
 def escape(html):
     "Return the given HTML with ampersands, quotes and carets encoded."
-    return force_unicode(html).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#39;')
+    return mark_safe(force_unicode(html).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&#39;'))
 escape = allow_lazy(escape, unicode)
 
-def linebreaks(value):
-    "Convert newlines into <p> and <br />s."
+def conditional_escape(html):
+    """
+    Similar to escape(), except that it doesn't operate on pre-escaped strings.
+    """
+    if isinstance(html, SafeData):
+        return html
+    else:
+        return escape(html)
+
+def linebreaks(value, autoescape=False):
+    "Converts newlines into <p> and <br />s"
     value = re.sub(r'\r\n|\r|\n', '\n', force_unicode(value)) # normalize newlines
     paras = re.split('\n{2,}', value)
-    paras = [u'<p>%s</p>' % p.strip().replace('\n', '<br />') for p in paras]
+    if autoescape:
+        paras = [u'<p>%s</p>' % escape(p.strip()).replace('\n', '<br />') for p in paras]
+    else:
+        paras = [u'<p>%s</p>' % p.strip().replace('\n', '<br />') for p in paras]
     return u'\n\n'.join(paras)
-linebreaks = allow_lazy(linebreaks, unicode)
+linebreaks = allow_lazy(linebreaks, unicode) 
 
 def strip_tags(value):
     "Return the given HTML with all tags stripped."
@@ -58,7 +72,7 @@ def fix_ampersands(value):
     return unencoded_ampersands_re.sub('&amp;', force_unicode(value))
 fix_ampersands = allow_lazy(fix_ampersands, unicode)
 
-def urlize(text, trim_url_limit=None, nofollow=False):
+def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
     """
     Convert any URLs in text into clickable links.
 
@@ -72,24 +86,40 @@ def urlize(text, trim_url_limit=None, nofollow=False):
     If nofollow is True, the URLs in link text will get a rel="nofollow"
     attribute.
     """
-    trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x
+    if autoescape:
+        trim_url = lambda x, limit=trim_url_limit: conditional_escape(limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x)
+    else:
+        trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x
+    safe_input = isinstance(text, SafeData)
     words = word_split_re.split(force_unicode(text))
     nofollow_attr = nofollow and ' rel="nofollow"' or ''
     for i, word in enumerate(words):
         match = punctuation_re.match(word)
         if match:
             lead, middle, trail = match.groups()
+            if safe_input:
+                middle = mark_safe(middle)
             if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and \
                     len(middle) > 0 and middle[0] in string.letters + string.digits and \
                     (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))):
-                middle = '<a href="http://%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
+                middle = '<a href="http://%s"%s>%s</a>' % (
+                        urlquote(middle, safe='/&=:;#?+'),  nofollow_attr,
+                        trim_url(middle))
             if middle.startswith('http://') or middle.startswith('https://'):
-                middle = '<a href="%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
-            if '@' in middle and not middle.startswith('www.') and not ':' in middle \
-                and simple_email_re.match(middle):
+                middle = '<a href="%s"%s>%s</a>' % (
+                        urlquote(middle, safe='/&=:;#?+'), nofollow_attr,
+                        trim_url(middle))
+            if '@' in middle and not middle.startswith('www.') and \
+                    not ':' in middle and simple_email_re.match(middle):
                 middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
             if lead + middle + trail != word:
                 words[i] = lead + middle + trail
+            elif autoescape and not safe_input:
+                words[i] = escape(word)
+        elif safe_input:
+            words[i] = mark_safe(word)
+        elif autoescape:
+            words[i] = escape(word)
     return u''.join(words)
 urlize = allow_lazy(urlize, unicode)
 
diff --git a/django/utils/safestring.py b/django/utils/safestring.py
new file mode 100644
index 0000000000..f6f4c1eb2f
--- /dev/null
+++ b/django/utils/safestring.py
@@ -0,0 +1,122 @@
+"""
+Functions for working with "safe strings": strings that can be displayed safely
+without further escaping in HTML. Marking something as a "safe string" means
+that the producer of the string has already turned characters that should not
+be interpreted by the HTML engine (e.g. '<') into the appropriate entities.
+"""
+from django.utils.functional import curry, Promise
+
+class EscapeData(object):
+    pass
+
+class EscapeString(str, EscapeData):
+    """
+    A string that should be HTML-escaped when output.
+    """
+    pass
+
+class EscapeUnicode(unicode, EscapeData):
+    """
+    A unicode object that should be HTML-escaped when output.
+    """
+    pass
+
+class SafeData(object):
+    pass
+
+class SafeString(str, SafeData):
+    """
+    A string subclass that has been specifically marked as "safe" (requires no
+    further escaping) for HTML output purposes.
+    """
+    def __add__(self, rhs):
+        """
+        Concatenating a safe string with another safe string or safe unicode
+        object is safe. Otherwise, the result is no longer safe.
+        """
+        if isinstance(rhs, SafeUnicode):
+            return SafeUnicode(self + rhs)
+        elif isinstance(rhs, SafeString):
+            return SafeString(self + rhs)
+        else:
+            return super(SafeString, self).__add__(rhs)
+
+    def __str__(self):
+        return self
+
+    def _proxy_method(self, *args, **kwargs):
+        """
+        Wrap a call to a normal unicode method up so that we return safe
+        results. The method that is being wrapped is passed in the 'method'
+        argument.
+        """
+        method = kwargs.pop('method')
+        data = method(self, *args, **kwargs)
+        if isinstance(data, str):
+            return SafeString(data)
+        else:
+            return SafeUnicode(data)
+
+    decode = curry(_proxy_method, method = str.decode)
+
+class SafeUnicode(unicode, SafeData):
+    """
+    A unicode subclass that has been specifically marked as "safe" for HTML
+    output purposes.
+    """
+    def __add__(self, rhs):
+        """
+        Concatenating a safe unicode object with another safe string or safe
+        unicode object is safe. Otherwise, the result is no longer safe.
+        """
+        if isinstance(rhs, SafeData):
+            return SafeUnicode(self + rhs)
+        else:
+            return super(SafeUnicode, self).__add__(rhs)
+
+    def _proxy_method(self, *args, **kwargs):
+        """
+        Wrap a call to a normal unicode method up so that we return safe
+        results. The method that is being wrapped is passed in the 'method'
+        argument.
+        """
+        method = kwargs.pop('method')
+        data = method(self, *args, **kwargs)
+        if isinstance(data, str):
+            return SafeString(data)
+        else:
+            return SafeUnicode(data)
+
+    encode = curry(_proxy_method, method = unicode.encode)
+
+def mark_safe(s):
+    """
+    Explicitly mark a string as safe for (HTML) output purposes. The returned
+    object can be used everywhere a string or unicode object is appropriate.
+
+    Can be called multiple times on a single string.
+    """
+    if isinstance(s, SafeData):
+        return s
+    if isinstance(s, str) or (isinstance(s, Promise) and s._delegate_str):
+        return SafeString(s)
+    if isinstance(s, (unicode, Promise)):
+        return SafeUnicode(s)
+    return SafeString(str(s))
+
+def mark_for_escaping(s):
+    """
+    Explicitly mark a string as requiring HTML escaping upon output. Has no
+    effect on SafeData subclasses.
+
+    Can be called multiple times on a single string (the resulting escaping is
+    only applied once).
+    """
+    if isinstance(s, (SafeData, EscapeData)):
+        return s
+    if isinstance(s, str) or (isinstance(s, Promise) and s._delegate_str):
+        return EscapeString(s)
+    if isinstance(s, (unicode, Promise)):
+        return EscapeUnicode(s)
+    return EscapeString(str(s))
+
diff --git a/django/utils/translation/trans_null.py b/django/utils/translation/trans_null.py
index 0b62907e44..98c6de6197 100644
--- a/django/utils/translation/trans_null.py
+++ b/django/utils/translation/trans_null.py
@@ -4,6 +4,7 @@
 
 from django.conf import settings
 from django.utils.encoding import force_unicode
+from django.utils.safestring import mark_safe, SafeData
 
 def ngettext(singular, plural, number):
     if number == 1: return singular
@@ -31,7 +32,10 @@ TECHNICAL_ID_MAP = {
 }
 
 def gettext(message):
-    return TECHNICAL_ID_MAP.get(message, message)
+    result = TECHNICAL_ID_MAP.get(message, message)
+    if isinstance(message, SafeData):
+        return mark_safe(result)
+    return result
 
 def ugettext(message):
     return force_unicode(gettext(message))
diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py
index 98b4769031..c95c842a4f 100644
--- a/django/utils/translation/trans_real.py
+++ b/django/utils/translation/trans_real.py
@@ -8,6 +8,7 @@ import gettext as gettext_module
 from cStringIO import StringIO
 
 from django.utils.encoding import force_unicode
+from django.utils.safestring import mark_safe, SafeData
 
 try:
     import threading
@@ -271,11 +272,15 @@ def do_translate(message, translation_function):
     global _default, _active
     t = _active.get(currentThread(), None)
     if t is not None:
-        return getattr(t, translation_function)(message)
-    if _default is None:
-        from django.conf import settings
-        _default = translation(settings.LANGUAGE_CODE)
-    return getattr(_default, translation_function)(message)
+        result = getattr(t, translation_function)(message)
+    else:
+        if _default is None:
+            from django.conf import settings
+            _default = translation(settings.LANGUAGE_CODE)
+        result = getattr(_default, translation_function)(message)
+    if isinstance(message, SafeData):
+        return mark_safe(result)
+    return result
 
 def gettext(message):
     return do_translate(message, 'gettext')