gis: Made necessary modifications for unicode, manage refactor, backend refactor and merged 5584-6000 via svnmerge from [repos:django/trunk trunk].

git-svn-id: http://code.djangoproject.com/svn/django/branches/gis@6018 bcc190cf-cafb-0310-a4f2-bffc1f526a37

1 files changed, 95 insertions, 24 deletions

diff --git a/django/http/__init__.py b/django/http/__init__.py
index 1b2abe5049..7cd47481dc 100644
--- a/django/http/__init__.py
+++ b/django/http/__init__.py
@@ -1,8 +1,9 @@
 import os
 from Cookie import SimpleCookie
 from pprint import pformat
-from urllib import urlencode, quote
-from django.utils.datastructures import MultiValueDict
+from urllib import urlencode
+from django.utils.datastructures import MultiValueDict, FileDict
+from django.utils.encoding import smart_str, iri_to_uri, force_unicode
 
 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
 
@@ -17,6 +18,10 @@ class Http404(Exception):
 
 class HttpRequest(object):
     "A basic HTTP request"
+
+    # The encoding used in GET/POST dicts. None means use default setting.
+    _encoding = None
+
     def __init__(self):
         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
         self.path = ''
@@ -42,14 +47,31 @@ class HttpRequest(object):
     def is_secure(self):
         return os.environ.get("HTTPS") == "on"
 
+    def _set_encoding(self, val):
+        """
+        Sets the encoding used for GET/POST accesses. If the GET or POST
+        dictionary has already been created, it is removed and recreated on the
+        next access (so that it is decoded correctly).
+        """
+        self._encoding = val
+        if hasattr(self, '_get'):
+            del self._get
+        if hasattr(self, '_post'):
+            del self._post
+
+    def _get_encoding(self):
+        return self._encoding
+
+    encoding = property(_get_encoding, _set_encoding)
+
 def parse_file_upload(header_dict, post_data):
-    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+    "Returns a tuple of (POST QueryDict, FILES MultiValueDict)"
     import email, email.Message
     from cgi import parse_header
     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
     raw_message += '\r\n\r\n' + post_data
     msg = email.message_from_string(raw_message)
-    POST = MultiValueDict()
+    POST = QueryDict('', mutable=True)
     FILES = MultiValueDict()
     for submessage in msg.get_payload():
         if submessage and isinstance(submessage, email.Message.Message):
@@ -62,25 +84,38 @@ def parse_file_upload(header_dict, post_data):
                 if not name_dict['filename'].strip():
                     continue
                 # IE submits the full path, so trim everything but the basename.
-                # (We can't use os.path.basename because it expects Linux paths.)
+                # (We can't use os.path.basename because that uses the server's
+                # directory separator, which may not be the same as the
+                # client's one.)
                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
-                FILES.appendlist(name_dict['name'], {
+                FILES.appendlist(name_dict['name'], FileDict({
                     'filename': filename,
                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
                     'content': submessage.get_payload(),
-                })
+                }))
             else:
                 POST.appendlist(name_dict['name'], submessage.get_payload())
     return POST, FILES
 
 class QueryDict(MultiValueDict):
-    """A specialized MultiValueDict that takes a query string when initialized.
-    This is immutable unless you create a copy of it."""
-    def __init__(self, query_string, mutable=False):
+    """
+    A specialized MultiValueDict that takes a query string when initialized.
+    This is immutable unless you create a copy of it.
+
+    Values retrieved from this class are converted from the given encoding
+    (DEFAULT_CHARSET by default) to unicode.
+    """
+    def __init__(self, query_string, mutable=False, encoding=None):
         MultiValueDict.__init__(self)
+        if not encoding:
+            # *Important*: do not import settings any earlier because of note
+            # in core.handlers.modpython.
+            from django.conf import settings
+            encoding = settings.DEFAULT_CHARSET
+        self.encoding = encoding
         self._mutable = True
         for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True
-            self.appendlist(key, value)
+            self.appendlist(force_unicode(key, encoding, errors='replace'), force_unicode(value, encoding, errors='replace'))
         self._mutable = mutable
 
     def _assert_mutable(self):
@@ -89,6 +124,8 @@ class QueryDict(MultiValueDict):
 
     def __setitem__(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
@@ -111,15 +148,27 @@ class QueryDict(MultiValueDict):
 
     def setlist(self, key, list_):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        list_ = [str_to_unicode(elt, self.encoding) for elt in list_]
         MultiValueDict.setlist(self, key, list_)
 
+    def setlistdefault(self, key, default_list=()):
+        self._assert_mutable()
+        if key not in self:
+            self.setlist(key, default_list)
+        return MultiValueDict.getlist(self, key)
+
     def appendlist(self, key, value):
         self._assert_mutable()
+        key = str_to_unicode(key, self.encoding)
+        value = str_to_unicode(value, self.encoding)
         MultiValueDict.appendlist(self, key, value)
 
     def update(self, other_dict):
         self._assert_mutable()
-        MultiValueDict.update(self, other_dict)
+        f = lambda s: str_to_unicode(s, self.encoding)
+        d = dict([(f(k), f(v)) for k, v in other_dict.items()])
+        MultiValueDict.update(self, d)
 
     def pop(self, key, *args):
         self._assert_mutable()
@@ -133,9 +182,11 @@ class QueryDict(MultiValueDict):
         self._assert_mutable()
         MultiValueDict.clear(self)
 
-    def setdefault(self, *args):
+    def setdefault(self, key, default=None):
         self._assert_mutable()
-        return MultiValueDict.setdefault(self, *args)
+        key = str_to_unicode(key, self.encoding)
+        default = str_to_unicode(default, self.encoding)
+        return MultiValueDict.setdefault(self, key, default)
 
     def copy(self):
         "Returns a mutable copy of this object."
@@ -144,7 +195,8 @@ class QueryDict(MultiValueDict):
     def urlencode(self):
         output = []
         for k, list_ in self.lists():
-            output.extend([urlencode({k: v}) for v in list_])
+            k = smart_str(k, self.encoding)
+            output.extend([urlencode({k: smart_str(v, self.encoding)}) for v in list_])
         return '&'.join(output)
 
 def parse_cookie(cookie):
@@ -162,18 +214,22 @@ class HttpResponse(object):
 
     status_code = 200
 
-    def __init__(self, content='', mimetype=None, status=None):
+    def __init__(self, content='', mimetype=None, status=None,
+            content_type=None):
         from django.conf import settings
         self._charset = settings.DEFAULT_CHARSET
-        if not mimetype:
-            mimetype = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, settings.DEFAULT_CHARSET)
+        if mimetype:
+            content_type = mimetype     # For backwards compatibility
+        if not content_type:
+            content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE,
+                    settings.DEFAULT_CHARSET)
         if not isinstance(content, basestring) and hasattr(content, '__iter__'):
             self._container = content
             self._is_string = False
         else:
             self._container = [content]
             self._is_string = True
-        self.headers = {'Content-Type': mimetype}
+        self.headers = {'Content-Type': content_type}
         self.cookies = SimpleCookie()
         if status:
             self.status_code = status
@@ -221,9 +277,7 @@ class HttpResponse(object):
         self.cookies[key]['max-age'] = 0
 
     def _get_content(self):
-        content = ''.join(self._container)
-        if isinstance(content, unicode):
-            content = content.encode(self._charset)
+        content = smart_str(''.join(self._container), self._charset)
         return content
 
     def _set_content(self, value):
@@ -266,14 +320,14 @@ class HttpResponseRedirect(HttpResponse):
 
     def __init__(self, redirect_to):
         HttpResponse.__init__(self)
-        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
+        self['Location'] = iri_to_uri(redirect_to)
 
 class HttpResponsePermanentRedirect(HttpResponse):
     status_code = 301
 
     def __init__(self, redirect_to):
         HttpResponse.__init__(self)
-        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
+        self['Location'] = iri_to_uri(redirect_to)
 
 class HttpResponseNotModified(HttpResponse):
     status_code = 304
@@ -312,3 +366,20 @@ def get_host(request):
     if not host:
         host = request.META.get('HTTP_HOST', '')
     return host
+
+# It's neither necessary nor appropriate to use
+# django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus,
+# this slightly more restricted function.
+def str_to_unicode(s, encoding):
+    """
+    Convert basestring objects to unicode, using the given encoding. Illegaly
+    encoded input characters are replaced with Unicode "unknown" codepoint
+    (\ufffd).
+
+    Returns any non-basestring objects without change.
+    """
+    if isinstance(s, str):
+        return unicode(s, encoding, 'replace')
+    else:
+        return s
+