File storage refactoring, adding far more flexibility to Django's file handling. The new files.txt document has details of the new features.

This is a backwards-incompatible change; consult BackwardsIncompatibleChanges for details.

Fixes #3567, #3621, #4345, #5361, #5655, #7415.

Many thanks to Marty Alchin who did the vast majority of this work.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@8244 bcc190cf-cafb-0310-a4f2-bffc1f526a37

11 files changed, 147 insertions, 29 deletions

diff --git a/tests/regressiontests/admin_widgets/models.py b/tests/regressiontests/admin_widgets/models.py
index 584d973c83..544806f819 100644
--- a/tests/regressiontests/admin_widgets/models.py
+++ b/tests/regressiontests/admin_widgets/models.py
@@ -1,6 +1,7 @@
 
 from django.conf import settings
 from django.db import models
+from django.core.files.storage import default_storage
 
 class Member(models.Model):
     name = models.CharField(max_length=100)
@@ -18,6 +19,7 @@ class Band(models.Model):
 class Album(models.Model):
     band = models.ForeignKey(Band)
     name = models.CharField(max_length=100)
+    cover_art = models.ImageField(upload_to='albums')
     
     def __unicode__(self):
         return self.name
@@ -46,12 +48,12 @@ HTML escaped.
 >>> print conditional_escape(w.render('test', datetime(2007, 12, 1, 9, 30)))
 <p class="datetime">Date: <input value="2007-12-01" type="text" class="vDateField" name="test_0" size="10" /><br />Time: <input value="09:30:00" type="text" class="vTimeField" name="test_1" size="8" /></p>
 
->>> w = AdminFileWidget()
->>> print conditional_escape(w.render('test', 'test'))
-Currently: <a target="_blank" href="%(MEDIA_URL)stest">test</a> <br />Change: <input type="file" name="test" />
-
 >>> band = Band.objects.create(pk=1, name='Linkin Park')
->>> album = band.album_set.create(name='Hybrid Theory')
+>>> album = band.album_set.create(name='Hybrid Theory', cover_art=r'albums\hybrid_theory.jpg')
+
+>>> w = AdminFileWidget()
+>>> print conditional_escape(w.render('test', album.cover_art))
+Currently: <a target="_blank" href="%(STORAGE_URL)salbums/hybrid_theory.jpg">albums\hybrid_theory.jpg</a> <br />Change: <input type="file" name="test" />
 
 >>> rel = Album._meta.get_field('band').rel
 >>> w = ForeignKeyRawIdWidget(rel)
@@ -81,5 +83,5 @@ True
 
 """ % {
     'ADMIN_MEDIA_PREFIX': settings.ADMIN_MEDIA_PREFIX,
-    'MEDIA_URL': settings.MEDIA_URL,
+    'STORAGE_URL': default_storage.url(''),
 }}
diff --git a/tests/regressiontests/bug639/models.py b/tests/regressiontests/bug639/models.py
index fc241aba8c..22dd440fc3 100644
--- a/tests/regressiontests/bug639/models.py
+++ b/tests/regressiontests/bug639/models.py
@@ -1,16 +1,20 @@
 import tempfile
+
 from django.db import models
+from django.core.files.storage import FileSystemStorage
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
 
 class Photo(models.Model):
     title = models.CharField(max_length=30)
-    image = models.FileField(upload_to=tempfile.gettempdir())
+    image = models.FileField(storage=temp_storage, upload_to='tests')
     
     # Support code for the tests; this keeps track of how many times save() gets
     # called on each instance.
     def __init__(self, *args, **kwargs):
-       super(Photo, self).__init__(*args, **kwargs)
-       self._savecount = 0
+        super(Photo, self).__init__(*args, **kwargs)
+        self._savecount = 0
     
     def save(self):
         super(Photo, self).save()
-        self._savecount +=1
\ No newline at end of file
+        self._savecount += 1
diff --git a/tests/regressiontests/bug639/tests.py b/tests/regressiontests/bug639/tests.py
index 2726dec897..69e4a3ba3b 100644
--- a/tests/regressiontests/bug639/tests.py
+++ b/tests/regressiontests/bug639/tests.py
@@ -36,4 +36,4 @@ class Bug639Test(unittest.TestCase):
         Make sure to delete the "uploaded" file to avoid clogging /tmp.
         """
         p = Photo.objects.get()
-        os.unlink(p.get_image_filename())
+        p.image.delete(save=False)
diff --git a/tests/regressiontests/file_storage/__init__.py b/tests/regressiontests/file_storage/__init__.py
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/tests/regressiontests/file_storage/__init__.py
@@ -0,0 +1 @@
+
diff --git a/tests/regressiontests/file_storage/models.py b/tests/regressiontests/file_storage/models.py
new file mode 100644
index 0000000000..8cd8b9e56c
--- /dev/null
+++ b/tests/regressiontests/file_storage/models.py
@@ -0,0 +1,44 @@
+import os
+import tempfile
+from django.db import models
+from django.core.files.storage import FileSystemStorage
+from django.core.files.base import ContentFile
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
+
+# Test for correct behavior of width_field/height_field.
+# Of course, we can't run this without PIL.
+
+try:
+    # Checking for the existence of Image is enough for CPython, but
+    # for PyPy, you need to check for the underlying modules
+    import Image, _imaging
+except ImportError:
+    Image = None
+
+# If we have PIL, do these tests
+if Image:
+    class Person(models.Model):
+        name = models.CharField(max_length=50)
+        mugshot = models.ImageField(storage=temp_storage, upload_to='tests', 
+                                    height_field='mug_height', 
+                                    width_field='mug_width')
+        mug_height = models.PositiveSmallIntegerField()
+        mug_width = models.PositiveSmallIntegerField()
+        
+    __test__ = {'API_TESTS': """
+
+>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png"), 'rb').read()
+>>> p = Person(name="Joe")
+>>> p.mugshot.save("mug", ContentFile(image_data))
+>>> p.mugshot.width
+16
+>>> p.mugshot.height
+16
+>>> p.mug_height
+16
+>>> p.mug_width
+16
+
+"""}
+    
\ No newline at end of file
diff --git a/tests/regressiontests/file_storage/test.png b/tests/regressiontests/file_storage/test.png
new file mode 100644
index 0000000000..4f17cd075d
--- /dev/null
+++ b/tests/regressiontests/file_storage/test.png
diff --git a/tests/regressiontests/file_storage/tests.py b/tests/regressiontests/file_storage/tests.py
new file mode 100644
index 0000000000..a4503d9805
--- /dev/null
+++ b/tests/regressiontests/file_storage/tests.py
@@ -0,0 +1,66 @@
+"""
+Tests for the file storage mechanism
+
+>>> import tempfile
+>>> from django.core.files.storage import FileSystemStorage
+>>> from django.core.files.base import ContentFile
+
+>>> temp_storage = FileSystemStorage(location=tempfile.gettempdir())
+
+# Standard file access options are available, and work as expected.
+
+>>> temp_storage.exists('storage_test')
+False
+>>> file = temp_storage.open('storage_test', 'w')
+>>> file.write('storage contents')
+>>> file.close()
+
+>>> temp_storage.exists('storage_test')
+True
+>>> file = temp_storage.open('storage_test', 'r')
+>>> file.read()
+'storage contents'
+>>> file.close()
+
+>>> temp_storage.delete('storage_test')
+>>> temp_storage.exists('storage_test')
+False
+
+# Files can only be accessed if they're below the specified location.
+
+>>> temp_storage.exists('..')
+Traceback (most recent call last):
+...
+SuspiciousOperation: Attempted access to '..' denied.
+>>> temp_storage.open('/etc/passwd')
+Traceback (most recent call last):
+  ...
+SuspiciousOperation: Attempted access to '/etc/passwd' denied.
+
+# Custom storage systems can be created to customize behavior
+
+>>> class CustomStorage(FileSystemStorage):
+...     def get_available_name(self, name):
+...         # Append numbers to duplicate files rather than underscores, like Trac
+...
+...         parts = name.split('.')
+...         basename, ext = parts[0], parts[1:]
+...         number = 2
+...
+...         while self.exists(name):
+...             name = '.'.join([basename, str(number)] + ext)
+...             number += 1
+...
+...         return name
+>>> custom_storage = CustomStorage(tempfile.gettempdir())
+
+>>> first = custom_storage.save('custom_storage', ContentFile('custom contents'))
+>>> first
+u'custom_storage'
+>>> second = custom_storage.save('custom_storage', ContentFile('more contents'))
+>>> second
+u'custom_storage.2'
+
+>>> custom_storage.delete(first)
+>>> custom_storage.delete(second)
+"""
diff --git a/tests/regressiontests/file_uploads/models.py b/tests/regressiontests/file_uploads/models.py
index 3701750afe..9d020509af 100644
--- a/tests/regressiontests/file_uploads/models.py
+++ b/tests/regressiontests/file_uploads/models.py
@@ -1,9 +1,10 @@
 import tempfile
 import os
 from django.db import models
+from django.core.files.storage import FileSystemStorage
 
-UPLOAD_ROOT = tempfile.mkdtemp()
-UPLOAD_TO = os.path.join(UPLOAD_ROOT, 'test_upload')
+temp_storage = FileSystemStorage(tempfile.mkdtemp())
+UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')
 
 class FileModel(models.Model):
-    testfile = models.FileField(upload_to=UPLOAD_TO)
+    testfile = models.FileField(storage=temp_storage, upload_to='test_upload')
diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py
index dd6b7c4181..7c8b53ea89 100644
--- a/tests/regressiontests/file_uploads/tests.py
+++ b/tests/regressiontests/file_uploads/tests.py
@@ -9,7 +9,7 @@ from django.test import TestCase, client
 from django.utils import simplejson
 from django.utils.hashcompat import sha_constructor
 
-from models import FileModel, UPLOAD_ROOT, UPLOAD_TO
+from models import FileModel, temp_storage, UPLOAD_TO
 
 class FileUploadTests(TestCase):
     def test_simple_upload(self):
@@ -194,22 +194,22 @@ class DirectoryCreationTests(unittest.TestCase):
     """
     def setUp(self):
         self.obj = FileModel()
-        if not os.path.isdir(UPLOAD_ROOT):
-            os.makedirs(UPLOAD_ROOT)
+        if not os.path.isdir(temp_storage.location):
+            os.makedirs(temp_storage.location)
 
     def tearDown(self):
-        os.chmod(UPLOAD_ROOT, 0700)
-        shutil.rmtree(UPLOAD_ROOT)
+        os.chmod(temp_storage.location, 0700)
+        shutil.rmtree(temp_storage.location)
 
     def test_readonly_root(self):
         """Permission errors are not swallowed"""
-        os.chmod(UPLOAD_ROOT, 0500)
+        os.chmod(temp_storage.location, 0500)
         try:
-            self.obj.save_testfile_file('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+            self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
         except OSError, err:
             self.assertEquals(err.errno, errno.EACCES)
-        except:
-            self.fail("OSError [Errno %s] not raised" % errno.EACCES)
+        except Exception, err:
+            self.fail("OSError [Errno %s] not raised." % errno.EACCES)
 
     def test_not_a_directory(self):
         """The correct IOError is raised when the upload directory name exists but isn't a directory"""
@@ -217,11 +217,11 @@ class DirectoryCreationTests(unittest.TestCase):
         fd = open(UPLOAD_TO, 'w')
         fd.close()
         try:
-            self.obj.save_testfile_file('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+            self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
         except IOError, err:
             # The test needs to be done on a specific string as IOError
             # is raised even without the patch (just not early enough)
             self.assertEquals(err.args[0],
-                              "%s exists and is not a directory" % UPLOAD_TO)
+                              "%s exists and is not a directory." % UPLOAD_TO)
         except:
             self.fail("IOError not raised")
diff --git a/tests/regressiontests/serializers_regress/models.py b/tests/regressiontests/serializers_regress/models.py
index 7d3f9d3b1d..4e2fbb1ee2 100644
--- a/tests/regressiontests/serializers_regress/models.py
+++ b/tests/regressiontests/serializers_regress/models.py
@@ -157,8 +157,8 @@ class DecimalPKData(models.Model):
 class EmailPKData(models.Model):
     data = models.EmailField(primary_key=True)
 
-class FilePKData(models.Model):
-    data = models.FileField(primary_key=True, upload_to='/foo/bar')
+# class FilePKData(models.Model):
+#    data = models.FileField(primary_key=True, upload_to='/foo/bar')
 
 class FilePathPKData(models.Model):
     data = models.FilePathField(primary_key=True)
diff --git a/tests/regressiontests/serializers_regress/tests.py b/tests/regressiontests/serializers_regress/tests.py
index f990d57a17..7a38af4cf9 100644
--- a/tests/regressiontests/serializers_regress/tests.py
+++ b/tests/regressiontests/serializers_regress/tests.py
@@ -144,7 +144,7 @@ test_data = [
     (data_obj, 41, EmailData, None),
     (data_obj, 42, EmailData, ""),
     (data_obj, 50, FileData, 'file:///foo/bar/whiz.txt'),
-    (data_obj, 51, FileData, None),
+#     (data_obj, 51, FileData, None),
     (data_obj, 52, FileData, ""),
     (data_obj, 60, FilePathData, "/foo/bar/whiz.txt"),
     (data_obj, 61, FilePathData, None),
@@ -242,7 +242,7 @@ The end."""),
 #     (pk_obj, 620, DatePKData, datetime.date(2006,6,16)),
 #     (pk_obj, 630, DateTimePKData, datetime.datetime(2006,6,16,10,42,37)),
     (pk_obj, 640, EmailPKData, "hovercraft@example.com"),
-    (pk_obj, 650, FilePKData, 'file:///foo/bar/whiz.txt'),
+#     (pk_obj, 650, FilePKData, 'file:///foo/bar/whiz.txt'),
     (pk_obj, 660, FilePathPKData, "/foo/bar/whiz.txt"),
     (pk_obj, 670, DecimalPKData, decimal.Decimal('12.345')),
     (pk_obj, 671, DecimalPKData, decimal.Decimal('-12.345')),