Fixed #26207 -- Replaced dynamic classes with non-data descriptors for deferred instance loading.

12 files changed, 67 insertions, 156 deletions

diff --git a/django/apps/config.py b/django/apps/config.py
index 6a0c8d8e57..edd7a48a6f 100644
--- a/django/apps/config.py
+++ b/django/apps/config.py
@@ -165,8 +165,7 @@ class AppConfig(object):
             raise LookupError(
                 "App '%s' doesn't have a '%s' model." % (self.label, model_name))
 
-    def get_models(self, include_auto_created=False,
-                   include_deferred=False, include_swapped=False):
+    def get_models(self, include_auto_created=False, include_swapped=False):
         """
         Returns an iterable of models.
 
@@ -182,8 +181,6 @@ class AppConfig(object):
         """
         self.check_models_ready()
         for model in self.models.values():
-            if model._deferred and not include_deferred:
-                continue
             if model._meta.auto_created and not include_auto_created:
                 continue
             if model._meta.swapped and not include_swapped:
diff --git a/django/apps/registry.py b/django/apps/registry.py
index 166e6adb97..7ed6e5d468 100644
--- a/django/apps/registry.py
+++ b/django/apps/registry.py
@@ -156,8 +156,7 @@ class Apps(object):
 
     # This method is performance-critical at least for Django's test suite.
     @lru_cache.lru_cache(maxsize=None)
-    def get_models(self, include_auto_created=False,
-                   include_deferred=False, include_swapped=False):
+    def get_models(self, include_auto_created=False, include_swapped=False):
         """
         Returns a list of all installed models.
 
@@ -174,8 +173,7 @@ class Apps(object):
 
         result = []
         for app_config in self.app_configs.values():
-            result.extend(list(app_config.get_models(
-                include_auto_created, include_deferred, include_swapped)))
+            result.extend(list(app_config.get_models(include_auto_created, include_swapped)))
         return result
 
     def get_model(self, app_label, model_name=None):
diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py
index 6a212ab684..97869d7aca 100644
--- a/django/contrib/contenttypes/models.py
+++ b/django/contrib/contenttypes/models.py
@@ -27,8 +27,6 @@ class ContentTypeManager(models.Manager):
     def _get_opts(self, model, for_concrete_model):
         if for_concrete_model:
             model = model._meta.concrete_model
-        elif model._deferred:
-            model = model._meta.proxy_for_model
         return model._meta
 
     def _get_from_cache(self, opts):
diff --git a/django/contrib/gis/db/models/proxy.py b/django/contrib/gis/db/models/proxy.py
index 1dbcaee26a..340e76f4bf 100644
--- a/django/contrib/gis/db/models/proxy.py
+++ b/django/contrib/gis/db/models/proxy.py
@@ -5,10 +5,11 @@ objects corresponding to geographic model fields.
 
 Thanks to Robert Coup for providing this functionality (see #4322).
 """
+from django.db.models.query_utils import DeferredAttribute
 from django.utils import six
 
 
-class SpatialProxy(object):
+class SpatialProxy(DeferredAttribute):
     def __init__(self, klass, field):
         """
         Proxy initializes on the given Geometry or Raster class (not an instance)
@@ -16,6 +17,7 @@ class SpatialProxy(object):
         """
         self._field = field
         self._klass = klass
+        super(SpatialProxy, self).__init__(field.attname, klass)
 
     def __get__(self, instance, cls=None):
         """
@@ -29,7 +31,10 @@ class SpatialProxy(object):
             return self
 
         # Getting the value of the field.
-        geo_value = instance.__dict__[self._field.attname]
+        try:
+            geo_value = instance.__dict__[self._field.attname]
+        except KeyError:
+            geo_value = super(SpatialProxy, self).__get__(instance, cls)
 
         if isinstance(geo_value, self._klass):
             geo_obj = geo_value
diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py
index 94a0272729..7a61999f15 100644
--- a/django/core/serializers/python.py
+++ b/django/core/serializers/python.py
@@ -37,8 +37,7 @@ class Serializer(base.Serializer):
         self._current = None
 
     def get_dump_object(self, obj):
-        model = obj._meta.proxy_for_model if obj._deferred else obj.__class__
-        data = OrderedDict([('model', force_text(model._meta))])
+        data = OrderedDict([('model', force_text(obj._meta))])
         if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
             data["pk"] = force_text(obj._get_pk_val(), strings_only=True)
         data['fields'] = self._current
diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py
index 3d38217612..b8e2f72734 100644
--- a/django/core/serializers/xml_serializer.py
+++ b/django/core/serializers/xml_serializer.py
@@ -52,8 +52,7 @@ class Serializer(base.Serializer):
             raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj))
 
         self.indent(1)
-        model = obj._meta.proxy_for_model if obj._deferred else obj.__class__
-        attrs = OrderedDict([("model", smart_text(model._meta))])
+        attrs = OrderedDict([("model", smart_text(obj._meta))])
         if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'):
             obj_pk = obj._get_pk_val()
             if obj_pk is not None:
diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py
index 354985297e..958b122d9f 100644
--- a/django/db/models/__init__.py
+++ b/django/db/models/__init__.py
@@ -19,7 +19,7 @@ from django.db.models.query import (  # NOQA
 )
 
 # Imports that would create circular imports if sorted
-from django.db.models.base import Model  # NOQA isort:skip
+from django.db.models.base import DEFERRED, Model  # NOQA isort:skip
 from django.db.models.fields.related import (  # NOQA isort:skip
     ForeignKey, ForeignObject, OneToOneField, ManyToManyField,
     ManyToOneRel, ManyToManyRel, OneToOneRel,
diff --git a/django/db/models/base.py b/django/db/models/base.py
index 1c454f9b0a..94031d76a9 100644
--- a/django/db/models/base.py
+++ b/django/db/models/base.py
@@ -27,12 +27,11 @@ from django.db.models.fields.related import (
 from django.db.models.manager import ensure_default_manager
 from django.db.models.options import Options
 from django.db.models.query import Q
-from django.db.models.query_utils import (
-    DeferredAttribute, deferred_class_factory,
-)
 from django.db.models.utils import make_model_tuple
 from django.utils import six
-from django.utils.encoding import force_str, force_text
+from django.utils.encoding import (
+    force_str, force_text, python_2_unicode_compatible,
+)
 from django.utils.functional import curry
 from django.utils.six.moves import zip
 from django.utils.text import capfirst, get_text_list
@@ -40,6 +39,17 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.version import get_version
 
 
+@python_2_unicode_compatible
+class Deferred(object):
+    def __repr__(self):
+        return str('<Deferred field>')
+
+    def __str__(self):
+        return str('<Deferred field>')
+
+DEFERRED = Deferred()
+
+
 def subclass_exception(name, parents, module, attached_to=None):
     """
     Create exception subclass. Used by ModelBase below.
@@ -353,7 +363,6 @@ class ModelState(object):
 
 
 class Model(six.with_metaclass(ModelBase)):
-    _deferred = False
 
     def __init__(self, *args, **kwargs):
         signals.pre_init.send(sender=self.__class__, args=args, kwargs=kwargs)
@@ -377,11 +386,15 @@ class Model(six.with_metaclass(ModelBase)):
             # is *not* consumed. We rely on this, so don't change the order
             # without changing the logic.
             for val, field in zip(args, fields_iter):
+                if val is DEFERRED:
+                    continue
                 setattr(self, field.attname, val)
         else:
             # Slower, kwargs-ready version.
             fields_iter = iter(self._meta.fields)
             for val, field in zip(args, fields_iter):
+                if val is DEFERRED:
+                    continue
                 setattr(self, field.attname, val)
                 kwargs.pop(field.name, None)
                 # Maintain compatibility with existing calls.
@@ -393,13 +406,8 @@ class Model(six.with_metaclass(ModelBase)):
 
         for field in fields_iter:
             is_related_object = False
-            # This slightly odd construct is so that we can access any
-            # data-descriptor object (DeferredAttribute) without triggering its
-            # __get__ method.
-            if (field.attname not in kwargs and
-                    (isinstance(self.__class__.__dict__.get(field.attname), DeferredAttribute) or
-                        field.column is None)):
-                # This field will be populated on request.
+            # Virtual field
+            if field.attname not in kwargs and field.column is None:
                 continue
             if kwargs:
                 if isinstance(field.remote_field, ForeignObjectRel):
@@ -435,15 +443,18 @@ class Model(six.with_metaclass(ModelBase)):
                 # field.name instead of field.attname (e.g. "user" instead of
                 # "user_id") so that the object gets properly cached (and type
                 # checked) by the RelatedObjectDescriptor.
-                setattr(self, field.name, rel_obj)
+                if rel_obj is not DEFERRED:
+                    setattr(self, field.name, rel_obj)
             else:
-                setattr(self, field.attname, val)
+                if val is not DEFERRED:
+                    setattr(self, field.attname, val)
 
         if kwargs:
             for prop in list(kwargs):
                 try:
                     if isinstance(getattr(self.__class__, prop), property):
-                        setattr(self, prop, kwargs[prop])
+                        if kwargs[prop] is not DEFERRED:
+                            setattr(self, prop, kwargs[prop])
                         del kwargs[prop]
                 except AttributeError:
                     pass
@@ -454,10 +465,11 @@ class Model(six.with_metaclass(ModelBase)):
 
     @classmethod
     def from_db(cls, db, field_names, values):
-        if cls._deferred:
-            new = cls(**dict(zip(field_names, values)))
-        else:
-            new = cls(*values)
+        if len(values) != len(cls._meta.concrete_fields):
+            values = list(values)
+            values.reverse()
+            values = [values.pop() if f.attname in field_names else DEFERRED for f in cls._meta.concrete_fields]
+        new = cls(*values)
         new._state.adding = False
         new._state.db = db
         return new
@@ -501,17 +513,8 @@ class Model(six.with_metaclass(ModelBase)):
         """
         data = self.__dict__
         data[DJANGO_VERSION_PICKLE_KEY] = get_version()
-        if not self._deferred:
-            class_id = self._meta.app_label, self._meta.object_name
-            return model_unpickle, (class_id, [], simple_class_factory), data
-        defers = []
-        for field in self._meta.fields:
-            if isinstance(self.__class__.__dict__.get(field.attname),
-                          DeferredAttribute):
-                defers.append(field.attname)
-        model = self._meta.proxy_for_model
-        class_id = model._meta.app_label, model._meta.object_name
-        return (model_unpickle, (class_id, defers, deferred_class_factory), data)
+        class_id = self._meta.app_label, self._meta.object_name
+        return model_unpickle, (class_id,), data
 
     def __setstate__(self, state):
         msg = None
@@ -547,7 +550,7 @@ class Model(six.with_metaclass(ModelBase)):
         """
         return {
             f.attname for f in self._meta.concrete_fields
-            if isinstance(self.__class__.__dict__.get(f.attname), DeferredAttribute)
+            if f.attname not in self.__dict__
         }
 
     def refresh_from_db(self, using=None, fields=None, **kwargs):
@@ -574,18 +577,14 @@ class Model(six.with_metaclass(ModelBase)):
                     'are not allowed in fields.' % LOOKUP_SEP)
 
         db = using if using is not None else self._state.db
-        if self._deferred:
-            non_deferred_model = self._meta.proxy_for_model
-        else:
-            non_deferred_model = self.__class__
-        db_instance_qs = non_deferred_model._default_manager.using(db).filter(pk=self.pk)
+        db_instance_qs = self.__class__._default_manager.using(db).filter(pk=self.pk)
 
         # Use provided fields, if not set then reload all non-deferred fields.
+        deferred_fields = self.get_deferred_fields()
         if fields is not None:
             fields = list(fields)
             db_instance_qs = db_instance_qs.only(*fields)
-        elif self._deferred:
-            deferred_fields = self.get_deferred_fields()
+        elif deferred_fields:
             fields = [f.attname for f in self._meta.concrete_fields
                       if f.attname not in deferred_fields]
             db_instance_qs = db_instance_qs.only(*fields)
@@ -664,6 +663,7 @@ class Model(six.with_metaclass(ModelBase)):
         if force_insert and (force_update or update_fields):
             raise ValueError("Cannot force both insert and updating in model saving.")
 
+        deferred_fields = self.get_deferred_fields()
         if update_fields is not None:
             # If update_fields is empty, skip the save. We do also check for
             # no-op saves later on for inheritance cases. This bailout is
@@ -690,17 +690,11 @@ class Model(six.with_metaclass(ModelBase)):
 
         # If saving to the same database, and this model is deferred, then
         # automatically do a "update_fields" save on the loaded fields.
-        elif not force_insert and self._deferred and using == self._state.db:
+        elif not force_insert and deferred_fields and using == self._state.db:
             field_names = set()
             for field in self._meta.concrete_fields:
                 if not field.primary_key and not hasattr(field, 'through'):
                     field_names.add(field.attname)
-            deferred_fields = [
-                f.attname for f in self._meta.fields
-                if (f.attname not in self.__dict__ and
-                    isinstance(self.__class__.__dict__[f.attname], DeferredAttribute))
-            ]
-
             loaded_fields = field_names.difference(deferred_fields)
             if loaded_fields:
                 update_fields = frozenset(loaded_fields)
@@ -1662,14 +1656,7 @@ def make_foreign_order_accessors(model, related_model):
 ########
 
 
-def simple_class_factory(model, attrs):
-    """
-    Needed for dynamic classes.
-    """
-    return model
-
-
-def model_unpickle(model_id, attrs, factory):
+def model_unpickle(model_id):
     """
     Used to unpickle Model subclasses with deferred fields.
     """
@@ -1678,8 +1665,7 @@ def model_unpickle(model_id, attrs, factory):
     else:
         # Backwards compat - the model was cached directly in earlier versions.
         model = model_id
-    cls = factory(model, attrs)
-    return cls.__new__(cls)
+    return model.__new__(model)
 model_unpickle.__safe_for_unpickle__ = True
 
 
diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
index 447556cb8c..ec6cf3cf29 100644
--- a/django/db/models/fields/__init__.py
+++ b/django/db/models/fields/__init__.py
@@ -20,7 +20,7 @@ from django.core import checks, exceptions, validators
 # purposes.
 from django.core.exceptions import FieldDoesNotExist  # NOQA
 from django.db import connection, connections, router
-from django.db.models.query_utils import RegisterLookupMixin
+from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin
 from django.utils import six, timezone
 from django.utils.datastructures import DictWrapper
 from django.utils.dateparse import (
@@ -504,10 +504,6 @@ class Field(RegisterLookupMixin):
             # class self.__class__, then update its dict with self.__dict__
             # values - so, this is very close to normal pickle.
             return _empty, (self.__class__,), self.__dict__
-        if self.model._deferred:
-            # Deferred model will not be found from the app registry. This
-            # could be fixed by reconstructing the deferred model on unpickle.
-            raise RuntimeError("Fields of deferred models can't be reduced")
         return _load_field, (self.model._meta.app_label, self.model._meta.object_name,
                              self.name)
 
@@ -696,6 +692,12 @@ class Field(RegisterLookupMixin):
             cls._meta.add_field(self, private=True)
         else:
             cls._meta.add_field(self)
+        if self.column:
+            # Don't override classmethods with the descriptor. This means that
+            # if you have a classmethod and a field with the same name, then
+            # such fields can't be deferred (we don't have a check for this).
+            if not getattr(cls, self.attname, None):
+                setattr(cls, self.attname, DeferredAttribute(self.attname, cls))
         if self.choices:
             setattr(cls, 'get_%s_display' % self.name,
                     curry(cls._get_FIELD_display, field=self))
diff --git a/django/db/models/query.py b/django/db/models/query.py
index 61c52167c7..03d12cfa08 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -19,7 +19,7 @@ from django.db.models.deletion import Collector
 from django.db.models.expressions import Date, DateTime, F
 from django.db.models.fields import AutoField
 from django.db.models.query_utils import (
-    InvalidQuery, Q, check_rel_lookup_compatibility, deferred_class_factory,
+    InvalidQuery, Q, check_rel_lookup_compatibility,
 )
 from django.db.models.sql.constants import CURSOR
 from django.utils import six, timezone
@@ -60,11 +60,6 @@ class ModelIterable(BaseIterable):
         model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
         init_list = [f[0].target.attname
                      for f in select[model_fields_start:model_fields_end]]
-        if len(init_list) != len(model_cls._meta.concrete_fields):
-            init_set = set(init_list)
-            skip = [f.attname for f in model_cls._meta.concrete_fields
-                    if f.attname not in init_set]
-            model_cls = deferred_class_factory(model_cls, skip)
         related_populators = get_related_populators(klass_info, select, db)
         for row in compiler.results_iter(results):
             obj = model_cls.from_db(db, init_list, row[model_fields_start:model_fields_end])
@@ -1254,9 +1249,7 @@ class RawQuerySet(object):
             if skip:
                 if self.model._meta.pk.attname in skip:
                     raise InvalidQuery('Raw query must include the primary key')
-                model_cls = deferred_class_factory(self.model, skip)
-            else:
-                model_cls = self.model
+            model_cls = self.model
             fields = [self.model_fields.get(c) for c in self.columns]
             converters = compiler.get_converters([
                 f.get_col(f.model._meta.db_table) if f else None for f in fields
@@ -1718,7 +1711,7 @@ class RelatedPopulator(object):
                 return [row[row_pos] for row_pos in pos_list]
             self.reorder_for_init = reorder_for_init
 
-        self.model_cls = self.get_deferred_cls(klass_info, self.init_list)
+        self.model_cls = klass_info['model']
         self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)
         self.related_populators = get_related_populators(klass_info, select, self.db)
         field = klass_info['field']
@@ -1732,17 +1725,6 @@ class RelatedPopulator(object):
             if field.unique:
                 self.reverse_cache_name = field.remote_field.get_cache_name()
 
-    def get_deferred_cls(self, klass_info, init_list):
-        model_cls = klass_info['model']
-        if len(init_list) != len(model_cls._meta.concrete_fields):
-            init_set = set(init_list)
-            skip = [
-                f.attname for f in model_cls._meta.concrete_fields
-                if f.attname not in init_set
-            ]
-            model_cls = deferred_class_factory(model_cls, skip)
-        return model_cls
-
     def populate(self, row, from_obj):
         if self.reorder_for_init:
             obj_data = self.reorder_for_init(row)
diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py
index e2a20d3cf7..b71cd5649c 100644
--- a/django/db/models/query_utils.py
+++ b/django/db/models/query_utils.py
@@ -11,7 +11,6 @@ import inspect
 from collections import namedtuple
 
 from django.core.exceptions import FieldDoesNotExist
-from django.db.backends import utils
 from django.db.models.constants import LOOKUP_SEP
 from django.utils import tree
 
@@ -97,10 +96,9 @@ class DeferredAttribute(object):
         Retrieves and caches the value from the datastore on the first lookup.
         Returns the cached value.
         """
-        non_deferred_model = instance._meta.proxy_for_model
-        opts = non_deferred_model._meta
-
-        assert instance is not None
+        if instance is None:
+            return self
+        opts = instance._meta
         data = instance.__dict__
         if data.get(self.field_name, self) is self:
             # self.field_name is the attname of the field, but only() takes the
@@ -119,13 +117,6 @@ class DeferredAttribute(object):
             data[self.field_name] = val
         return data[self.field_name]
 
-    def __set__(self, instance, value):
-        """
-        Deferred loading attributes can be set normally (which means there will
-        never be a database lookup involved.
-        """
-        instance.__dict__[self.field_name] = value
-
     def _check_parent_chain(self, instance, name):
         """
         Check if the field value can be fetched from a parent field already
@@ -230,48 +221,6 @@ def select_related_descend(field, restricted, requested, load_fields, reverse=Fa
     return True
 
 
-# This function is needed because data descriptors must be defined on a class
-# object, not an instance, to have any effect.
-
-def deferred_class_factory(model, attrs):
-    """
-    Returns a class object that is a copy of "model" with the specified "attrs"
-    being replaced with DeferredAttribute objects. The "pk_value" ties the
-    deferred attributes to a particular instance of the model.
-    """
-    if not attrs:
-        return model
-    opts = model._meta
-    # Never create deferred models based on deferred model
-    if model._deferred:
-        # Deferred models are proxies for the non-deferred model. We never
-        # create chains of defers => proxy_for_model is the non-deferred
-        # model.
-        model = opts.proxy_for_model
-    # The app registry wants a unique name for each model, otherwise the new
-    # class won't be created (we get an exception). Therefore, we generate
-    # the name using the passed in attrs. It's OK to reuse an existing class
-    # object if the attrs are identical.
-    name = "%s_Deferred_%s" % (model.__name__, '_'.join(sorted(attrs)))
-    name = utils.truncate_name(name, 80, 32)
-
-    try:
-        return opts.apps.get_model(model._meta.app_label, name)
-
-    except LookupError:
-
-        class Meta:
-            proxy = True
-            apps = opts.apps
-            app_label = opts.app_label
-
-        overrides = {attr: DeferredAttribute(attr, model) for attr in attrs}
-        overrides["Meta"] = Meta
-        overrides["__module__"] = model.__module__
-        overrides["_deferred"] = True
-        return type(str(name), (model,), overrides)
-
-
 def refs_expression(lookup_parts, annotations):
     """
     A helper method to check if the lookup_parts contains references
diff --git a/django/views/generic/detail.py b/django/views/generic/detail.py
index 5cd6b506de..2b67ef5e5c 100644
--- a/django/views/generic/detail.py
+++ b/django/views/generic/detail.py
@@ -89,8 +89,6 @@ class SingleObjectMixin(ContextMixin):
         if self.context_object_name:
             return self.context_object_name
         elif isinstance(obj, models.Model):
-            if obj._deferred:
-                obj = obj._meta.proxy_for_model
             return obj._meta.model_name
         else:
             return None
@@ -152,8 +150,6 @@ class SingleObjectTemplateResponseMixin(TemplateResponseMixin):
             # only use this if the object in question is a model.
             if isinstance(self.object, models.Model):
                 object_meta = self.object._meta
-                if self.object._deferred:
-                    object_meta = self.object._meta.proxy_for_model._meta
                 names.append("%s/%s%s.html" % (
                     object_meta.app_label,
                     object_meta.model_name,