Fixed #2070: refactored Django's file upload capabilities.

A description of the new features can be found in the new [http://www.djangoproject.com/documentation/upload_handing/ upload handling documentation]; the executive summary is that Django will now happily handle uploads of large files without issues.

This changes the representation of uploaded files from dictionaries to bona fide objects; see BackwardsIncompatibleChanges for details.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@7814 bcc190cf-cafb-0310-a4f2-bffc1f526a37

14 files changed, 309 insertions, 61 deletions

diff --git a/tests/regressiontests/bug639/tests.py b/tests/regressiontests/bug639/tests.py
index f9596d06cb..2726dec897 100644
--- a/tests/regressiontests/bug639/tests.py
+++ b/tests/regressiontests/bug639/tests.py
@@ -9,6 +9,7 @@ import unittest
 from regressiontests.bug639.models import Photo
 from django.http import QueryDict
 from django.utils.datastructures import MultiValueDict
+from django.core.files.uploadedfile import SimpleUploadedFile
 
 class Bug639Test(unittest.TestCase):
         
@@ -21,12 +22,8 @@ class Bug639Test(unittest.TestCase):
         
         # Fake a request query dict with the file
         qd = QueryDict("title=Testing&image=", mutable=True)
-        qd["image_file"] = {
-            "filename" : "test.jpg",
-            "content-type" : "image/jpeg",
-            "content" : img
-        }
-        
+        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
+
         manip = Photo.AddManipulator()
         manip.do_html2python(qd)
         p = manip.save(qd)
@@ -39,4 +36,4 @@ class Bug639Test(unittest.TestCase):
         Make sure to delete the "uploaded" file to avoid clogging /tmp.
         """
         p = Photo.objects.get()
-        os.unlink(p.get_image_filename())
\ No newline at end of file
+        os.unlink(p.get_image_filename())
diff --git a/tests/regressiontests/datastructures/tests.py b/tests/regressiontests/datastructures/tests.py
index d6141b09ce..62c57bc019 100644
--- a/tests/regressiontests/datastructures/tests.py
+++ b/tests/regressiontests/datastructures/tests.py
@@ -117,14 +117,25 @@ Init from sequence of tuples
 >>> d['person']['2']['firstname']
 ['Adrian']
 
-### FileDict ################################################################
-
->>> d = FileDict({'content': 'once upon a time...'})
->>> repr(d)
-"{'content': '<omitted>'}"
->>> d = FileDict({'other-key': 'once upon a time...'})
+### ImmutableList ################################################################
+>>> d = ImmutableList(range(10))
+>>> d.sort()
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
+    raise AttributeError, self.warning
+AttributeError: ImmutableList object is immutable.
 >>> repr(d)
-"{'other-key': 'once upon a time...'}"
+'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
+>>> d = ImmutableList(range(10), warning="Object is immutable!")
+>>> d[1]
+1
+>>> d[1] = 'test'
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
+    raise AttributeError, self.warning
+AttributeError: Object is immutable!
 
 ### DictWrapper #############################################################
 
diff --git a/tests/regressiontests/file_uploads/__init__.py b/tests/regressiontests/file_uploads/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/regressiontests/file_uploads/__init__.py
diff --git a/tests/regressiontests/file_uploads/models.py b/tests/regressiontests/file_uploads/models.py
new file mode 100644
index 0000000000..2d5607b2a7
--- /dev/null
+++ b/tests/regressiontests/file_uploads/models.py
@@ -0,0 +1,2 @@
+# This file unintentionally left blank.
+# Oops.
\ No newline at end of file
diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py
new file mode 100644
index 0000000000..8992298470
--- /dev/null
+++ b/tests/regressiontests/file_uploads/tests.py
@@ -0,0 +1,158 @@
+import os
+import sha
+import tempfile
+from django.test import TestCase, client
+from django.utils import simplejson
+
+class FileUploadTests(TestCase):
+    def test_simple_upload(self):
+        post_data = {
+            'name': 'Ringo',
+            'file_field': open(__file__),
+        }
+        response = self.client.post('/file_uploads/upload/', post_data)
+        self.assertEqual(response.status_code, 200)
+
+    def test_large_upload(self):
+        tdir = tempfile.gettempdir()
+        
+        file1 = tempfile.NamedTemporaryFile(suffix=".file1", dir=tdir)
+        file1.write('a' * (2 ** 21))
+        file1.seek(0)
+
+        file2 = tempfile.NamedTemporaryFile(suffix=".file2", dir=tdir)
+        file2.write('a' * (10 * 2 ** 20))
+        file2.seek(0)
+
+        # This file contains chinese symbols for a name.
+        file3 = open(os.path.join(tdir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg'), 'w+b')
+        file3.write('b' * (2 ** 10))
+        file3.seek(0)
+
+        post_data = {
+            'name': 'Ringo',
+            'file_field1': open(file1.name),
+            'file_field2': open(file2.name),
+            'file_unicode': file3,
+            }
+
+        for key in post_data.keys():
+            try:
+                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
+                post_data[key].seek(0)
+            except AttributeError:
+                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
+
+        response = self.client.post('/file_uploads/verify/', post_data)
+
+        try:
+            os.unlink(file3.name)
+        except:
+            pass
+
+        self.assertEqual(response.status_code, 200)
+    
+    def test_dangerous_file_names(self):
+        """Uploaded file names should be sanitized before ever reaching the view."""
+        # This test simulates possible directory traversal attacks by a
+        # malicious uploader We have to do some monkeybusiness here to construct 
+        # a malicious payload with an invalid file name (containing os.sep or
+        # os.pardir). This similar to what an attacker would need to do when
+        # trying such an attack.
+        scary_file_names = [
+            "/tmp/hax0rd.txt",          # Absolute path, *nix-style.
+            "C:\\Windows\\hax0rd.txt",  # Absolute path, win-syle.
+            "C:/Windows/hax0rd.txt",    # Absolute path, broken-style.
+            "\\tmp\\hax0rd.txt",        # Absolute path, broken in a different way.
+            "/tmp\\hax0rd.txt",         # Absolute path, broken by mixing.
+            "subdir/hax0rd.txt",        # Descendant path, *nix-style.
+            "subdir\\hax0rd.txt",       # Descendant path, win-style.
+            "sub/dir\\hax0rd.txt",      # Descendant path, mixed.
+            "../../hax0rd.txt",         # Relative path, *nix-style.
+            "..\\..\\hax0rd.txt",       # Relative path, win-style.
+            "../..\\hax0rd.txt"         # Relative path, mixed.
+        ]
+        
+        payload = []
+        for i, name in enumerate(scary_file_names):
+            payload.extend([
+                '--' + client.BOUNDARY,
+                'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name),
+                'Content-Type: application/octet-stream',
+                '',
+                'You got pwnd.'
+            ])
+        payload.extend([
+            '--' + client.BOUNDARY + '--',
+            '',
+        ])
+        
+        payload = "\r\n".join(payload)
+        r = {
+            'CONTENT_LENGTH': len(payload),
+            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
+            'PATH_INFO':      "/file_uploads/echo/",
+            'REQUEST_METHOD': 'POST',
+            'wsgi.input':     client.FakePayload(payload),
+        }
+        response = self.client.request(**r)
+
+        # The filenames should have been sanitized by the time it got to the view.
+        recieved = simplejson.loads(response.content)
+        for i, name in enumerate(scary_file_names):
+            got = recieved["file%s" % i]
+            self.assertEqual(got, "hax0rd.txt")
+            
+    def test_filename_overflow(self):
+        """File names over 256 characters (dangerous on some platforms) get fixed up."""
+        name = "%s.txt" % ("f"*500)
+        payload = "\r\n".join([
+            '--' + client.BOUNDARY,
+            'Content-Disposition: form-data; name="file"; filename="%s"' % name,
+            'Content-Type: application/octet-stream',
+            '',
+            'Oops.'
+            '--' + client.BOUNDARY + '--',
+            '',
+        ])
+        r = {
+            'CONTENT_LENGTH': len(payload),
+            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
+            'PATH_INFO':      "/file_uploads/echo/",
+            'REQUEST_METHOD': 'POST',
+            'wsgi.input':     client.FakePayload(payload),
+        }
+        got = simplejson.loads(self.client.request(**r).content)
+        self.assert_(len(got['file']) < 256, "Got a long file name (%s characters)." % len(got['file']))
+        
+    def test_custom_upload_handler(self):
+        # A small file (under the 5M quota)                
+        smallfile = tempfile.NamedTemporaryFile()
+        smallfile.write('a' * (2 ** 21))
+
+        # A big file (over the quota)
+        bigfile = tempfile.NamedTemporaryFile()
+        bigfile.write('a' * (10 * 2 ** 20))
+                
+        # Small file posting should work.
+        response = self.client.post('/file_uploads/quota/', {'f': open(smallfile.name)})
+        got = simplejson.loads(response.content)
+        self.assert_('f' in got)
+        
+        # Large files don't go through.
+        response = self.client.post("/file_uploads/quota/", {'f': open(bigfile.name)})
+        got = simplejson.loads(response.content)
+        self.assert_('f' not in got)
+        
+    def test_broken_custom_upload_handler(self):
+        f = tempfile.NamedTemporaryFile()
+        f.write('a' * (2 ** 21))
+        
+        # AttributeError: You cannot alter upload handlers after the upload has been processed.
+        self.assertRaises(
+            AttributeError,
+            self.client.post,
+            '/file_uploads/quota/broken/', 
+            {'f': open(f.name)}
+        )        
+        
\ No newline at end of file
diff --git a/tests/regressiontests/file_uploads/uploadhandler.py b/tests/regressiontests/file_uploads/uploadhandler.py
new file mode 100644
index 0000000000..54f82f626c
--- /dev/null
+++ b/tests/regressiontests/file_uploads/uploadhandler.py
@@ -0,0 +1,26 @@
+"""
+Upload handlers to test the upload API.
+"""
+
+from django.core.files.uploadhandler import FileUploadHandler, StopUpload
+
+class QuotaUploadHandler(FileUploadHandler):
+    """
+    This test upload handler terminates the connection if more than a quota
+    (5MB) is uploaded.
+    """
+    
+    QUOTA = 5 * 2**20 # 5 MB
+    
+    def __init__(self, request=None):
+        super(QuotaUploadHandler, self).__init__(request)
+        self.total_upload = 0
+        
+    def receive_data_chunk(self, raw_data, start):
+        self.total_upload += len(raw_data)
+        if self.total_upload >= self.QUOTA:
+            raise StopUpload(connection_reset=True)
+        return raw_data
+            
+    def file_complete(self, file_size):
+        return None
\ No newline at end of file
diff --git a/tests/regressiontests/file_uploads/urls.py b/tests/regressiontests/file_uploads/urls.py
new file mode 100644
index 0000000000..529bee312d
--- /dev/null
+++ b/tests/regressiontests/file_uploads/urls.py
@@ -0,0 +1,10 @@
+from django.conf.urls.defaults import *
+import views
+
+urlpatterns = patterns('',
+    (r'^upload/$',          views.file_upload_view),
+    (r'^verify/$',          views.file_upload_view_verify),
+    (r'^echo/$',            views.file_upload_echo),
+    (r'^quota/$',           views.file_upload_quota),
+    (r'^quota/broken/$',    views.file_upload_quota_broken),
+)
diff --git a/tests/regressiontests/file_uploads/views.py b/tests/regressiontests/file_uploads/views.py
new file mode 100644
index 0000000000..833cf90531
--- /dev/null
+++ b/tests/regressiontests/file_uploads/views.py
@@ -0,0 +1,70 @@
+import os
+import sha
+from django.core.files.uploadedfile import UploadedFile
+from django.http import HttpResponse, HttpResponseServerError
+from django.utils import simplejson
+from uploadhandler import QuotaUploadHandler
+
+def file_upload_view(request):
+    """
+    Check that a file upload can be updated into the POST dictionary without
+    going pear-shaped.
+    """
+    form_data = request.POST.copy()
+    form_data.update(request.FILES)
+    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
+        # If a file is posted, the dummy client should only post the file name,
+        # not the full path.
+        if os.path.dirname(form_data['file_field'].file_name) != '':
+            return HttpResponseServerError()            
+        return HttpResponse('')
+    else:
+        return HttpResponseServerError()
+
+def file_upload_view_verify(request):
+    """
+    Use the sha digest hash to verify the uploaded contents.
+    """
+    form_data = request.POST.copy()
+    form_data.update(request.FILES)
+
+    # Check to see if unicode names worked out.
+    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
+        return HttpResponseServerError()
+
+    for key, value in form_data.items():
+        if key.endswith('_hash'):
+            continue
+        if key + '_hash' not in form_data:
+            continue
+        submitted_hash = form_data[key + '_hash']
+        if isinstance(value, UploadedFile):
+            new_hash = sha.new(value.read()).hexdigest()
+        else:
+            new_hash = sha.new(value).hexdigest()
+        if new_hash != submitted_hash:
+            return HttpResponseServerError()
+
+    return HttpResponse('')
+
+def file_upload_echo(request):
+    """
+    Simple view to echo back info about uploaded files for tests.
+    """
+    r = dict([(k, f.file_name) for k, f in request.FILES.items()])
+    return HttpResponse(simplejson.dumps(r))
+    
+def file_upload_quota(request):
+    """
+    Dynamically add in an upload handler.
+    """
+    request.upload_handlers.insert(0, QuotaUploadHandler())
+    return file_upload_echo(request)
+        
+def file_upload_quota_broken(request):
+    """
+    You can't change handlers after reading FILES; this view shouldn't work.
+    """
+    response = file_upload_echo(request)
+    request.upload_handlers.insert(0, QuotaUploadHandler())
+    return response
\ No newline at end of file
diff --git a/tests/regressiontests/forms/error_messages.py b/tests/regressiontests/forms/error_messages.py
index 381282f121..580326f894 100644
--- a/tests/regressiontests/forms/error_messages.py
+++ b/tests/regressiontests/forms/error_messages.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 tests = r"""
 >>> from django.newforms import *
+>>> from django.core.files.uploadedfile import SimpleUploadedFile
 
 # CharField ###################################################################
 
@@ -214,11 +215,11 @@ ValidationError: [u'REQUIRED']
 Traceback (most recent call last):
 ...
 ValidationError: [u'INVALID']
->>> f.clean({})
+>>> f.clean(SimpleUploadedFile('name', None))
 Traceback (most recent call last):
 ...
-ValidationError: [u'MISSING']
->>> f.clean({'filename': 'name', 'content':''})
+ValidationError: [u'EMPTY FILE']
+>>> f.clean(SimpleUploadedFile('name', ''))
 Traceback (most recent call last):
 ...
 ValidationError: [u'EMPTY FILE']
diff --git a/tests/regressiontests/forms/fields.py b/tests/regressiontests/forms/fields.py
index c9f3efdbda..4725c3ecf3 100644
--- a/tests/regressiontests/forms/fields.py
+++ b/tests/regressiontests/forms/fields.py
@@ -2,6 +2,7 @@
 tests = r"""
 >>> from django.newforms import *
 >>> from django.newforms.widgets import RadioFieldRenderer
+>>> from django.core.files.uploadedfile import SimpleUploadedFile
 >>> import datetime
 >>> import time
 >>> import re
@@ -770,17 +771,17 @@ ValidationError: [u'This field is required.']
 >>> f.clean(None, 'files/test2.pdf')
 'files/test2.pdf'
 
->>> f.clean({})
+>>> f.clean(SimpleUploadedFile('', ''))
 Traceback (most recent call last):
 ...
-ValidationError: [u'No file was submitted.']
+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
 
->>> f.clean({}, '')
+>>> f.clean(SimpleUploadedFile('', ''), '')
 Traceback (most recent call last):
 ...
-ValidationError: [u'No file was submitted.']
+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
 
->>> f.clean({}, 'files/test3.pdf')
+>>> f.clean(None, 'files/test3.pdf')
 'files/test3.pdf'
 
 >>> f.clean('some content that is not a file')
@@ -788,20 +789,20 @@ Traceback (most recent call last):
 ...
 ValidationError: [u'No file was submitted. Check the encoding type on the form.']
 
->>> f.clean({'filename': 'name', 'content': None})
+>>> f.clean(SimpleUploadedFile('name', None))
 Traceback (most recent call last):
 ...
 ValidationError: [u'The submitted file is empty.']
 
->>> f.clean({'filename': 'name', 'content': ''})
+>>> f.clean(SimpleUploadedFile('name', ''))
 Traceback (most recent call last):
 ...
 ValidationError: [u'The submitted file is empty.']
 
->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
 <class 'django.newforms.fields.UploadedFile'>
 
->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
 <class 'django.newforms.fields.UploadedFile'>
 
 # URLField ##################################################################
diff --git a/tests/regressiontests/forms/forms.py b/tests/regressiontests/forms/forms.py
index 7fc206de4c..041fa4054c 100644
--- a/tests/regressiontests/forms/forms.py
+++ b/tests/regressiontests/forms/forms.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 tests = r"""
 >>> from django.newforms import *
+>>> from django.core.files.uploadedfile import SimpleUploadedFile
 >>> import datetime
 >>> import time
 >>> import re
@@ -1465,7 +1466,7 @@ not request.POST.
 >>> print f
 <tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
 
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
 >>> print f
 <tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
 
@@ -1473,7 +1474,7 @@ not request.POST.
 >>> print f
 <tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
 
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
 >>> print f
 <tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
 >>> f.is_valid()
diff --git a/tests/regressiontests/test_client_regress/models.py b/tests/regressiontests/test_client_regress/models.py
index 37e81668b6..1eb55e312e 100644
--- a/tests/regressiontests/test_client_regress/models.py
+++ b/tests/regressiontests/test_client_regress/models.py
@@ -6,6 +6,7 @@ from django.test import Client, TestCase
 from django.core.urlresolvers import reverse
 from django.core.exceptions import SuspiciousOperation
 import os
+import sha
 
 class AssertContainsTests(TestCase):
     def test_contains(self):
@@ -240,16 +241,6 @@ class AssertFormErrorTests(TestCase):
         except AssertionError, e:
             self.assertEqual(str(e), "The form 'form' in context 0 does not contain the non-field error 'Some error.' (actual errors: )")
 
-class FileUploadTests(TestCase):
-    def test_simple_upload(self):
-        fd = open(os.path.join(os.path.dirname(__file__), "views.py"))
-        post_data = {
-            'name': 'Ringo',
-            'file_field': fd,
-        }
-        response = self.client.post('/test_client_regress/file_upload/', post_data)
-        self.assertEqual(response.status_code, 200)
-
 class LoginTests(TestCase):
     fixtures = ['testdata']
 
@@ -269,7 +260,6 @@ class LoginTests(TestCase):
         # default client.
         self.assertRedirects(response, "http://testserver/test_client_regress/get_view/")
 
-
 class URLEscapingTests(TestCase):
     def test_simple_argument_get(self):
         "Get a view that has a simple string argument"
diff --git a/tests/regressiontests/test_client_regress/urls.py b/tests/regressiontests/test_client_regress/urls.py
index dc26d1260a..12f6afacf3 100644
--- a/tests/regressiontests/test_client_regress/urls.py
+++ b/tests/regressiontests/test_client_regress/urls.py
@@ -3,7 +3,6 @@ import views
 
 urlpatterns = patterns('',
     (r'^no_template_view/$', views.no_template_view),
-    (r'^file_upload/$', views.file_upload_view),
     (r'^staff_only/$', views.staff_only_view),
     (r'^get_view/$', views.get_view),
     url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
diff --git a/tests/regressiontests/test_client_regress/views.py b/tests/regressiontests/test_client_regress/views.py
index 9632c17284..d703c82124 100644
--- a/tests/regressiontests/test_client_regress/views.py
+++ b/tests/regressiontests/test_client_regress/views.py
@@ -1,36 +1,18 @@
-import os
-
 from django.contrib.auth.decorators import login_required
-from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
+from django.http import HttpResponse, HttpResponseRedirect
 from django.core.exceptions import SuspiciousOperation
 
 def no_template_view(request):
     "A simple view that expects a GET request, and returns a rendered template"
     return HttpResponse("No template used. Sample content: twice once twice. Content ends.")
 
-def file_upload_view(request):
-    """
-    Check that a file upload can be updated into the POST dictionary without
-    going pear-shaped.
-    """
-    form_data = request.POST.copy()
-    form_data.update(request.FILES)
-    if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
-        # If a file is posted, the dummy client should only post the file name,
-        # not the full path.
-        if os.path.dirname(form_data['file_field']['filename']) != '':
-            return HttpResponseServerError()            
-        return HttpResponse('')
-    else:
-        return HttpResponseServerError()
-
 def staff_only_view(request):
     "A view that can only be visited by staff. Non staff members get an exception"
     if request.user.is_staff:
         return HttpResponse('')
     else:
         raise SuspiciousOperation()
-    
+
 def get_view(request):
     "A simple login protected view"
     return HttpResponse("Hello world")
@@ -51,4 +33,4 @@ def view_with_argument(request, name):
 def login_protected_redirect_view(request):
     "A view that redirects all requests to the GET view"
     return HttpResponseRedirect('/test_client_regress/get_view/')
-login_protected_redirect_view = login_required(login_protected_redirect_view)
\ No newline at end of file
+login_protected_redirect_view = login_required(login_protected_redirect_view)