File storage refactoring, adding far more flexibility to Django's file handling. The new files.txt document has details of the new features.

This is a backwards-incompatible change; consult BackwardsIncompatibleChanges for details.

Fixes #3567, #3621, #4345, #5361, #5655, #7415.

Many thanks to Marty Alchin who did the vast majority of this work.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@8244 bcc190cf-cafb-0310-a4f2-bffc1f526a37

5 files changed, 428 insertions, 57 deletions

diff --git a/django/core/files/__init__.py b/django/core/files/__init__.py
index e69de29bb2..0c3ef57af8 100644
--- a/django/core/files/__init__.py
+++ b/django/core/files/__init__.py
@@ -0,0 +1 @@
+from django.core.files.base import File
diff --git a/django/core/files/base.py b/django/core/files/base.py
new file mode 100644
index 0000000000..69739d6488
--- /dev/null
+++ b/django/core/files/base.py
@@ -0,0 +1,169 @@
+import os
+
+from django.utils.encoding import smart_str, smart_unicode
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+class File(object):
+    DEFAULT_CHUNK_SIZE = 64 * 2**10
+
+    def __init__(self, file):
+        self.file = file
+        self._name = file.name
+        self._mode = file.mode
+        self._closed = False
+
+    def __str__(self):
+        return smart_str(self.name or '')
+
+    def __unicode__(self):
+        return smart_unicode(self.name or u'')
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self or "None")
+
+    def __nonzero__(self):
+        return not not self.name
+
+    def __len__(self):
+        return self.size
+
+    def _get_name(self):
+        return self._name
+    name = property(_get_name)
+
+    def _get_mode(self):
+        return self._mode
+    mode = property(_get_mode)
+
+    def _get_closed(self):
+        return self._closed
+    closed = property(_get_closed)
+
+    def _get_size(self):
+        if not hasattr(self, '_size'):
+            if hasattr(self.file, 'size'):
+                self._size = self.file.size
+            elif os.path.exists(self.file.name):
+                self._size = os.path.getsize(self.file.name)
+            else:
+                raise AttributeError("Unable to determine the file's size.")
+        return self._size
+
+    def _set_size(self, size):
+        self._size = size
+
+    size = property(_get_size, _set_size)
+
+    def chunks(self, chunk_size=None):
+        """
+        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
+        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
+        """
+        if not chunk_size:
+            chunk_size = self.__class__.DEFAULT_CHUNK_SIZE
+
+        if hasattr(self, 'seek'):
+            self.seek(0)
+        # Assume the pointer is at zero...
+        counter = self.size
+
+        while counter > 0:
+            yield self.read(chunk_size)
+            counter -= chunk_size
+
+    def multiple_chunks(self, chunk_size=None):
+        """
+        Returns ``True`` if you can expect multiple chunks.
+
+        NB: If a particular file representation is in memory, subclasses should
+        always return ``False`` -- there's no good reason to read from memory in
+        chunks.
+        """
+        if not chunk_size:
+            chunk_size = self.DEFAULT_CHUNK_SIZE
+        return self.size > chunk_size
+
+    def xreadlines(self):
+        return iter(self)
+
+    def readlines(self):
+        return list(self.xreadlines())
+
+    def __iter__(self):
+        # Iterate over this file-like object by newlines
+        buffer_ = None
+        for chunk in self.chunks():
+            chunk_buffer = StringIO(chunk)
+
+            for line in chunk_buffer:
+                if buffer_:
+                    line = buffer_ + line
+                    buffer_ = None
+
+                # If this is the end of a line, yield
+                # otherwise, wait for the next round
+                if line[-1] in ('\n', '\r'):
+                    yield line
+                else:
+                    buffer_ = line
+
+        if buffer_ is not None:
+            yield buffer_
+
+    def open(self, mode=None):
+        if not self.closed:
+            self.seek(0)
+        elif os.path.exists(self.file.name):
+            self.file = open(self.file.name, mode or self.file.mode)
+        else:
+            raise ValueError("The file cannot be reopened.")
+
+    def seek(self, position):
+        self.file.seek(position)
+
+    def tell(self):
+        return self.file.tell()
+
+    def read(self, num_bytes=None):
+        if num_bytes is None:
+            return self.file.read()
+        return self.file.read(num_bytes)
+
+    def write(self, content):
+        if not self.mode.startswith('w'):
+            raise IOError("File was not opened with write access.")
+        self.file.write(content)
+
+    def flush(self):
+        if not self.mode.startswith('w'):
+            raise IOError("File was not opened with write access.")
+        self.file.flush()
+
+    def close(self):
+        self.file.close()
+        self._closed = True
+
+class ContentFile(File):
+    """
+    A File-like object that takes just raw content, rather than an actual file.
+    """
+    def __init__(self, content):
+        self.file = StringIO(content or '')
+        self.size = len(content or '')
+        self.file.seek(0)
+        self._closed = False
+
+    def __str__(self):
+        return 'Raw content'
+
+    def __nonzero__(self):
+        return True
+
+    def open(self, mode=None):
+        if self._closed:
+            self._closed = False
+        self.seek(0)
diff --git a/django/core/files/images.py b/django/core/files/images.py
new file mode 100644
index 0000000000..3fa5013027
--- /dev/null
+++ b/django/core/files/images.py
@@ -0,0 +1,42 @@
+"""
+Utility functions for handling images.
+
+Requires PIL, as you might imagine.
+"""
+
+from PIL import ImageFile as PIL
+from django.core.files import File
+
+class ImageFile(File):
+    """
+    A mixin for use alongside django.core.files.base.File, which provides
+    additional features for dealing with images.
+    """
+    def _get_width(self):
+        return self._get_image_dimensions()[0]
+    width = property(_get_width)
+
+    def _get_height(self):
+        return self._get_image_dimensions()[1]
+    height = property(_get_height)
+
+    def _get_image_dimensions(self):
+        if not hasattr(self, '_dimensions_cache'):
+            self._dimensions_cache = get_image_dimensions(self)
+        return self._dimensions_cache
+
+def get_image_dimensions(file_or_path):
+    """Returns the (width, height) of an image, given an open file or a path."""
+    p = PIL.Parser()
+    if hasattr(file_or_path, 'read'):
+        file = file_or_path
+    else:
+        file = open(file_or_path, 'rb')
+    while 1:
+        data = file.read(1024)
+        if not data:
+            break
+        p.feed(data)
+        if p.image:
+            return p.image.size
+    return None
diff --git a/django/core/files/storage.py b/django/core/files/storage.py
new file mode 100644
index 0000000000..351cc03bb5
--- /dev/null
+++ b/django/core/files/storage.py
@@ -0,0 +1,214 @@
+import os
+import urlparse
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
+from django.utils.encoding import force_unicode, smart_str
+from django.utils.text import force_unicode, get_valid_filename
+from django.utils._os import safe_join
+from django.core.files import locks, File
+
+__all__ = ('Storage', 'FileSystemStorage', 'DefaultStorage', 'default_storage')
+
+class Storage(object):
+    """
+    A base storage class, providing some default behaviors that all other
+    storage systems can inherit or override, as necessary.
+    """
+
+    # The following methods represent a public interface to private methods.
+    # These shouldn't be overridden by subclasses unless absolutely necessary.
+
+    def open(self, name, mode='rb', mixin=None):
+        """
+        Retrieves the specified file from storage, using the optional mixin
+        class to customize what features are available on the File returned.
+        """
+        file = self._open(name, mode)
+        if mixin:
+            # Add the mixin as a parent class of the File returned from storage.
+            file.__class__ = type(mixin.__name__, (mixin, file.__class__), {})
+        return file
+
+    def save(self, name, content):
+        """
+        Saves new content to the file specified by name. The content should be a
+        proper File object, ready to be read from the beginning.
+        """
+        # Check for old-style usage. Warn here first since there are multiple
+        # locations where we need to support both new and old usage.
+        if isinstance(content, basestring):
+            import warnings
+            warnings.warn(
+                message = "Representing files as strings is deprecated." \
+                          "Use django.core.files.base.ContentFile instead.",
+                category = DeprecationWarning,
+                stacklevel = 2
+            )
+            from django.core.files.base import ContentFile
+            content = ContentFile(content)
+
+        # Get the proper name for the file, as it will actually be saved.
+        if name is None:
+            name = content.name
+        name = self.get_available_name(name)
+
+        self._save(name, content)
+
+        # Store filenames with forward slashes, even on Windows
+        return force_unicode(name.replace('\\', '/'))
+
+    # These methods are part of the public API, with default implementations.
+
+    def get_valid_name(self, name):
+        """
+        Returns a filename, based on the provided filename, that's suitable for
+        use in the target storage system.
+        """
+        return get_valid_filename(name)
+
+    def get_available_name(self, name):
+        """
+        Returns a filename that's free on the target storage system, and
+        available for new content to be written to.
+        """
+        # If the filename already exists, keep adding an underscore to the name
+        # of the file until the filename doesn't exist.
+        while self.exists(name):
+            try:
+                dot_index = name.rindex('.')
+            except ValueError: # filename has no dot
+                name += '_'
+            else:
+                name = name[:dot_index] + '_' + name[dot_index:]
+        return name
+
+    def path(self, name):
+        """
+        Returns a local filesystem path where the file can be retrieved using
+        Python's built-in open() function. Storage systems that can't be
+        accessed using open() should *not* implement this method.
+        """
+        raise NotImplementedError("This backend doesn't support absolute paths.")
+
+    # The following methods form the public API for storage systems, but with
+    # no default implementations. Subclasses must implement *all* of these.
+
+    def delete(self, name):
+        """
+        Deletes the specified file from the storage system.
+        """
+        raise NotImplementedError()
+
+    def exists(self, name):
+        """
+        Returns True if a file referened by the given name already exists in the
+        storage system, or False if the name is available for a new file.
+        """
+        raise NotImplementedError()
+
+    def listdir(self, path):
+        """
+        Lists the contents of the specified path, returning a 2-tuple of lists;
+        the first item being directories, the second item being files.
+        """
+        raise NotImplementedError()
+
+    def size(self, name):
+        """
+        Returns the total size, in bytes, of the file specified by name.
+        """
+        raise NotImplementedError()
+
+    def url(self, name):
+        """
+        Returns an absolute URL where the file's contents can be accessed
+        directly by a web browser.
+        """
+        raise NotImplementedError()
+
+class FileSystemStorage(Storage):
+    """
+    Standard filesystem storage
+    """
+
+    def __init__(self, location=settings.MEDIA_ROOT, base_url=settings.MEDIA_URL):
+        self.location = os.path.abspath(location)
+        self.base_url = base_url
+
+    def _open(self, name, mode='rb'):
+        return File(open(self.path(name), mode))
+
+    def _save(self, name, content):
+        full_path = self.path(name)
+
+        directory = os.path.dirname(full_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        elif not os.path.isdir(directory):
+            raise IOError("%s exists and is not a directory." % directory)
+            
+        if hasattr(content, 'temporary_file_path'):
+            # This file has a file path that we can move.
+            file_move_safe(content.temporary_file_path(), full_path)
+            content.close()
+        else:
+            # This is a normal uploadedfile that we can stream.
+            fp = open(full_path, 'wb')
+            locks.lock(fp, locks.LOCK_EX)
+            for chunk in content.chunks():
+                fp.write(chunk)
+            locks.unlock(fp)
+            fp.close()
+
+    def delete(self, name):
+        name = self.path(name)
+        # If the file exists, delete it from the filesystem.
+        if os.path.exists(name):
+            os.remove(name)
+
+    def exists(self, name):
+        return os.path.exists(self.path(name))
+
+    def listdir(self, path):
+        path = self.path(path)
+        directories, files = [], []
+        for entry in os.listdir(path):
+            if os.path.isdir(os.path.join(path, entry)):
+                directories.append(entry)
+            else:
+                files.append(entry)
+        return directories, files
+
+    def path(self, name):
+        try:
+            path = safe_join(self.location, name)
+        except ValueError:
+            raise SuspiciousOperation("Attempted access to '%s' denied." % name)
+        return os.path.normpath(path)
+
+    def size(self, name):
+        return os.path.getsize(self.path(name))
+
+    def url(self, name):
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
+
+def get_storage_class(import_path):
+    try:
+        dot = import_path.rindex('.')
+    except ValueError:
+        raise ImproperlyConfigured("%s isn't a storage module." % import_path)
+    module, classname = import_path[:dot], import_path[dot+1:]
+    try:
+        mod = __import__(module, {}, {}, [''])
+    except ImportError, e:
+        raise ImproperlyConfigured('Error importing storage module %s: "%s"' % (module, e))
+    try:
+        return getattr(mod, classname)
+    except AttributeError:
+        raise ImproperlyConfigured('Storage module "%s" does not define a "%s" class.' % (module, classname))
+
+DefaultStorage = get_storage_class(settings.DEFAULT_FILE_STORAGE)
+default_storage = DefaultStorage()
diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py
index 5e81e968cd..a5a12930e3 100644
--- a/django/core/files/uploadedfile.py
+++ b/django/core/files/uploadedfile.py
@@ -10,6 +10,7 @@ except ImportError:
     from StringIO import StringIO
 
 from django.conf import settings
+from django.core.files.base import File
 
 from django.core.files import temp as tempfile
 
@@ -39,7 +40,7 @@ def deprecated_property(old, new, readonly=False):
     else:
         return property(getter, setter)
 
-class UploadedFile(object):
+class UploadedFile(File):
     """
     A abstract uploaded file (``TemporaryUploadedFile`` and
     ``InMemoryUploadedFile`` are the built-in concrete subclasses).
@@ -76,23 +77,6 @@ class UploadedFile(object):
 
     name = property(_get_name, _set_name)
 
-    def chunks(self, chunk_size=None):
-        """
-        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
-        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
-        """
-        if not chunk_size:
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
-
-        if hasattr(self, 'seek'):
-            self.seek(0)
-        # Assume the pointer is at zero...
-        counter = self.size
-
-        while counter > 0:
-            yield self.read(chunk_size)
-            counter -= chunk_size
-
     # Deprecated properties
     filename = deprecated_property(old="filename", new="name")
     file_name = deprecated_property(old="file_name", new="name")
@@ -108,18 +92,6 @@ class UploadedFile(object):
         return self.read()
     data = property(_get_data)
 
-    def multiple_chunks(self, chunk_size=None):
-        """
-        Returns ``True`` if you can expect multiple chunks.
-
-        NB: If a particular file representation is in memory, subclasses should
-        always return ``False`` -- there's no good reason to read from memory in
-        chunks.
-        """
-        if not chunk_size:
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
-        return self.size > chunk_size
-
     # Abstract methods; subclasses *must* define read() and probably should
     # define open/close.
     def read(self, num_bytes=None):
@@ -131,33 +103,6 @@ class UploadedFile(object):
     def close(self):
         pass
 
-    def xreadlines(self):
-        return self
-
-    def readlines(self):
-        return list(self.xreadlines())
-
-    def __iter__(self):
-        # Iterate over this file-like object by newlines
-        buffer_ = None
-        for chunk in self.chunks():
-            chunk_buffer = StringIO(chunk)
-
-            for line in chunk_buffer:
-                if buffer_:
-                    line = buffer_ + line
-                    buffer_ = None
-
-                # If this is the end of a line, yield
-                # otherwise, wait for the next round
-                if line[-1] in ('\n', '\r'):
-                    yield line
-                else:
-                    buffer_ = line
-
-        if buffer_ is not None:
-            yield buffer_
-
     # Backwards-compatible support for uploaded-files-as-dictionaries.
     def __getitem__(self, key):
         warnings.warn(