File storage refactoring, adding far more flexibility to Django's file handling. The new files.txt document has details of the new features.

This is a backwards-incompatible change; consult BackwardsIncompatibleChanges for details.

Fixes #3567, #3621, #4345, #5361, #5655, #7415.

Many thanks to Marty Alchin who did the vast majority of this work.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@8244 bcc190cf-cafb-0310-a4f2-bffc1f526a37

3 files changed, 156 insertions, 32 deletions

diff --git a/tests/modeltests/files/__init__.py b/tests/modeltests/files/__init__.py
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/tests/modeltests/files/__init__.py
@@ -0,0 +1 @@
+
diff --git a/tests/modeltests/files/models.py b/tests/modeltests/files/models.py
new file mode 100644
index 0000000000..a2ee5a7256
--- /dev/null
+++ b/tests/modeltests/files/models.py
@@ -0,0 +1,118 @@
+"""
+42. Storing files according to a custom storage system
+
+FileField and its variations can take a "storage" argument to specify how and
+where files should be stored.
+"""
+
+import tempfile
+
+from django.db import models
+from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
+from django.core.cache import cache
+
+temp_storage = FileSystemStorage(location=tempfile.gettempdir())
+
+# Write out a file to be used as default content
+temp_storage.save('tests/default.txt', ContentFile('default content'))
+
+class Storage(models.Model):
+    def custom_upload_to(self, filename):
+        return 'foo'
+
+    def random_upload_to(self, filename):
+        # This returns a different result each time,
+        # to make sure it only gets called once.
+        import random
+        return '%s/%s' % (random.randint(100, 999), filename)
+
+    normal = models.FileField(storage=temp_storage, upload_to='tests')
+    custom = models.FileField(storage=temp_storage, upload_to=custom_upload_to)
+    random = models.FileField(storage=temp_storage, upload_to=random_upload_to)
+    default = models.FileField(storage=temp_storage, upload_to='tests', default='tests/default.txt')
+
+__test__ = {'API_TESTS':"""
+# An object without a file has limited functionality.
+
+>>> obj1 = Storage()
+>>> obj1.normal
+<FieldFile: None>
+>>> obj1.normal.size
+Traceback (most recent call last):
+...
+ValueError: The 'normal' attribute has no file associated with it.
+
+# Saving a file enables full functionality.
+
+>>> obj1.normal.save('django_test.txt', ContentFile('content'))
+>>> obj1.normal
+<FieldFile: tests/django_test.txt>
+>>> obj1.normal.size
+7
+>>> obj1.normal.read()
+'content'
+
+# Files can be read in a little at a time, if necessary.
+
+>>> obj1.normal.open()
+>>> obj1.normal.read(3)
+'con'
+>>> obj1.normal.read()
+'tent'
+>>> '-'.join(obj1.normal.chunks(chunk_size=2))
+'co-nt-en-t'
+
+# Save another file with the same name.
+
+>>> obj2 = Storage()
+>>> obj2.normal.save('django_test.txt', ContentFile('more content'))
+>>> obj2.normal
+<FieldFile: tests/django_test_.txt>
+>>> obj2.normal.size
+12
+
+# Push the objects into the cache to make sure they pickle properly
+
+>>> cache.set('obj1', obj1)
+>>> cache.set('obj2', obj2)
+>>> cache.get('obj2').normal
+<FieldFile: tests/django_test_.txt>
+
+# Deleting an object deletes the file it uses, if there are no other objects
+# still using that file.
+
+>>> obj2.delete()
+>>> obj2.normal.save('django_test.txt', ContentFile('more content'))
+>>> obj2.normal
+<FieldFile: tests/django_test_.txt>
+
+# Default values allow an object to access a single file.
+
+>>> obj3 = Storage.objects.create()
+>>> obj3.default
+<FieldFile: tests/default.txt>
+>>> obj3.default.read()
+'default content'
+
+# But it shouldn't be deleted, even if there are no more objects using it.
+
+>>> obj3.delete()
+>>> obj3 = Storage()
+>>> obj3.default.read()
+'default content'
+
+# Verify the fix for #5655, making sure the directory is only determined once.
+
+>>> obj4 = Storage()
+>>> obj4.random.save('random_file', ContentFile('random content'))
+>>> obj4.random
+<FieldFile: .../random_file>
+
+# Clean up the temporary files.
+
+>>> obj1.normal.delete()
+>>> obj2.normal.delete()
+>>> obj3.default.delete()
+>>> obj4.random.delete()
+"""}
diff --git a/tests/modeltests/model_forms/models.py b/tests/modeltests/model_forms/models.py
index be2a8ba835..3463cb7554 100644
--- a/tests/modeltests/model_forms/models.py
+++ b/tests/modeltests/model_forms/models.py
@@ -11,6 +11,9 @@ import os
 import tempfile
 
 from django.db import models
+from django.core.files.storage import FileSystemStorage
+
+temp_storage = FileSystemStorage(tempfile.gettempdir())
 
 ARTICLE_STATUS = (
     (1, 'Draft'),
@@ -60,7 +63,7 @@ class PhoneNumber(models.Model):
 
 class TextFile(models.Model):
     description = models.CharField(max_length=20)
-    file = models.FileField(upload_to=tempfile.gettempdir())
+    file = models.FileField(storage=temp_storage, upload_to='tests')
 
     def __unicode__(self):
         return self.description
@@ -73,9 +76,9 @@ class ImageFile(models.Model):
         # for PyPy, you need to check for the underlying modules
         # If PIL is not available, this test is equivalent to TextFile above.
         import Image, _imaging
-        image = models.ImageField(upload_to=tempfile.gettempdir())
+        image = models.ImageField(storage=temp_storage, upload_to='tests')
     except ImportError:
-        image = models.FileField(upload_to=tempfile.gettempdir())
+        image = models.FileField(storage=temp_storage, upload_to='tests')
 
     def __unicode__(self):
         return self.description
@@ -786,6 +789,8 @@ u'Assistance'
 
 # FileField ###################################################################
 
+# File forms.
+
 >>> class TextFileForm(ModelForm):
 ...     class Meta:
 ...         model = TextFile
@@ -808,9 +813,9 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
 >>> f.is_valid()
@@ -819,7 +824,7 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
 # Edit an instance that already has the file defined in the model. This will not
 # save the file again, but leave it exactly as it is.
@@ -828,13 +833,13 @@ u'...test1.txt'
 >>> f.is_valid()
 True
 >>> f.cleaned_data['file']
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 >>> instance = f.save()
 >>> instance.file
-u'...test1.txt'
+<FieldFile: tests/test1.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 # Override the file by uploading a new one.
 
@@ -843,20 +848,20 @@ u'...test1.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test2.txt'
+<FieldFile: tests/test2.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')})
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test2.txt'
+<FieldFile: tests/test2.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 
 >>> instance.delete()
 
@@ -868,17 +873,17 @@ u'...test2.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-''
+<FieldFile: None>
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test3.txt'
+<FieldFile: tests/test3.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 >>> instance.delete()
 
 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')})
@@ -886,10 +891,10 @@ u'...test3.txt'
 True
 >>> instance = f.save()
 >>> instance.file
-u'...test3.txt'
+<FieldFile: tests/test3.txt>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_file_filename())
+>>> instance.file.delete()
 >>> instance.delete()
 
 # ImageField ###################################################################
@@ -911,10 +916,10 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 
 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
 >>> f.is_valid()
@@ -923,7 +928,7 @@ True
 <class 'django.core.files.uploadedfile.SimpleUploadedFile'>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Edit an instance that already has the image defined in the model. This will not
 # save the image again, but leave it exactly as it is.
@@ -932,14 +937,14 @@ u'...test.png'
 >>> f.is_valid()
 True
 >>> f.cleaned_data['image']
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 >>> instance = f.save()
 >>> instance.image
-u'...test.png'
+<ImageFieldFile: tests/test.png>
 
 # Delete the current image since this is not done by Django.
 
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 
 # Override the file by uploading a new one.
 
@@ -948,10 +953,10 @@ u'...test.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test2.png'
+<ImageFieldFile: tests/test2.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)})
@@ -959,10 +964,10 @@ u'...test2.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test2.png'
+<ImageFieldFile: tests/test2.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 # Test the non-required ImageField
@@ -973,17 +978,17 @@ u'...test2.png'
 True
 >>> instance = f.save()
 >>> instance.image
-''
+<ImageFieldFile: None>
 
 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
 >>> f.is_valid()
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test3.png'
+<ImageFieldFile: tests/test3.png>
 
 # Delete the current file since this is not done by Django.
->>> os.unlink(instance.get_image_filename())
+>>> instance.image.delete()
 >>> instance.delete()
 
 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)})
@@ -991,7 +996,7 @@ u'...test3.png'
 True
 >>> instance = f.save()
 >>> instance.image
-u'...test3.png'
+<ImageFieldFile: tests/test3.png>
 >>> instance.delete()
 
 # Media on a ModelForm ########################################################