Fixed #12663 -- Formalized the Model._meta API for retrieving fields.

Thanks to Russell Keith-Magee for mentoring this Google Summer of
Code 2014 project and everyone else who helped with the patch!

3 files changed, 59 insertions, 29 deletions

diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index e8948cb9e1..1c0b99e897 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -298,7 +298,12 @@ class SQLCompiler(object):
         # be used by local fields.
         seen_models = {None: start_alias}
 
-        for field, model in opts.get_concrete_fields_with_model():
+        for field in opts.concrete_fields:
+            model = field.model._meta.concrete_model
+            # A proxy model will have a different model and concrete_model. We
+            # will assign None if the field belongs to this model.
+            if model == opts.model:
+                model = None
             if from_parent and model is not None and issubclass(from_parent, model):
                 # Avoid loading data for already loaded parents.
                 continue
@@ -601,10 +606,10 @@ class SQLCompiler(object):
         connections to the root model).
         """
         def _get_field_choices():
-            direct_choices = (f.name for (f, _) in opts.get_fields_with_model() if f.rel)
+            direct_choices = (f.name for f in opts.fields if f.is_relation)
             reverse_choices = (
                 f.field.related_query_name()
-                for f in opts.get_all_related_objects() if f.field.unique
+                for f in opts.related_objects if f.field.unique
             )
             return chain(direct_choices, reverse_choices)
 
@@ -628,12 +633,13 @@ class SQLCompiler(object):
             else:
                 restricted = False
 
-        for f, model in opts.get_fields_with_model():
+        for f in opts.fields:
+            field_model = f.model._meta.concrete_model
             fields_found.add(f.name)
 
             if restricted:
                 next = requested.get(f.name, {})
-                if not f.rel:
+                if not f.is_relation:
                     # If a non-related field is used like a relation,
                     # or if a single non-relational field is given.
                     if next or (cur_depth == 1 and f.name in requested):
@@ -647,10 +653,6 @@ class SQLCompiler(object):
             else:
                 next = False
 
-            # The get_fields_with_model() returns None for fields that live
-            # in the field's local model. So, for those fields we want to use
-            # the f.model - that is the field's local model.
-            field_model = model or f.model
             if not select_related_descend(f, restricted, requested,
                                           only_load.get(field_model)):
                 continue
@@ -666,9 +668,9 @@ class SQLCompiler(object):
 
         if restricted:
             related_fields = [
-                (o.field, o.model)
-                for o in opts.get_all_related_objects()
-                if o.field.unique
+                (o.field, o.related_model)
+                for o in opts.related_objects
+                if o.field.unique and not o.many_to_many
             ]
             for f, model in related_fields:
                 if not select_related_descend(f, restricted, requested,
@@ -760,7 +762,7 @@ class SQLCompiler(object):
                     if self.query.select:
                         fields = [f.field for f in self.query.select]
                     elif self.query.default_cols:
-                        fields = self.query.get_meta().concrete_fields
+                        fields = list(self.query.get_meta().concrete_fields)
                     else:
                         fields = []
 
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index eb16419091..1d1dbd8162 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -11,6 +11,7 @@ from itertools import count, product
 
 from collections import Mapping, OrderedDict
 import copy
+from itertools import chain
 import warnings
 
 from django.core.exceptions import FieldDoesNotExist, FieldError
@@ -33,6 +34,13 @@ from django.utils.tree import Node
 __all__ = ['Query', 'RawQuery']
 
 
+def get_field_names_from_opts(opts):
+    return set(chain.from_iterable(
+        (f.name, f.attname) if f.concrete else (f.name,)
+        for f in opts.get_fields()
+    ))
+
+
 class RawQuery(object):
     """
     A single raw SQL query
@@ -593,9 +601,9 @@ class Query(object):
             opts = orig_opts
             for name in parts[:-1]:
                 old_model = cur_model
-                source = opts.get_field_by_name(name)[0]
+                source = opts.get_field(name)
                 if is_reverse_o2o(source):
-                    cur_model = source.model
+                    cur_model = source.related_model
                 else:
                     cur_model = source.rel.to
                 opts = cur_model._meta
@@ -605,8 +613,11 @@ class Query(object):
                 if not is_reverse_o2o(source):
                     must_include[old_model].add(source)
                 add_to_dict(must_include, cur_model, opts.pk)
-            field, model, _, _ = opts.get_field_by_name(parts[-1])
-            if model is None:
+            field = opts.get_field(parts[-1])
+            is_reverse_object = field.auto_created and not field.concrete
+            model = field.related_model if is_reverse_object else field.model
+            model = model._meta.concrete_model
+            if model == opts.model:
                 model = cur_model
             if not is_reverse_o2o(field):
                 add_to_dict(seen, model, field)
@@ -618,10 +629,11 @@ class Query(object):
             # models.
             workset = {}
             for model, values in six.iteritems(seen):
-                for field, m in model._meta.get_fields_with_model():
+                for field in model._meta.fields:
                     if field in values:
                         continue
-                    add_to_dict(workset, m or model, field)
+                    m = field.model._meta.concrete_model
+                    add_to_dict(workset, m, field)
             for model, values in six.iteritems(must_include):
                 # If we haven't included a model in workset, we don't add the
                 # corresponding must_include fields for that model, since an
@@ -934,8 +946,9 @@ class Query(object):
         root_alias = self.tables[0]
         seen = {None: root_alias}
 
-        for field, model in opts.get_fields_with_model():
-            if model not in seen:
+        for field in opts.fields:
+            model = field.model._meta.concrete_model
+            if model is not opts.model and model not in seen:
                 self.join_parent_model(opts, model, root_alias, seen)
         self.included_inherited_models = seen
 
@@ -1368,7 +1381,19 @@ class Query(object):
             if name == 'pk':
                 name = opts.pk.name
             try:
-                field, model, _, _ = opts.get_field_by_name(name)
+                field = opts.get_field(name)
+
+                # Fields that contain one-to-many relations with a generic
+                # model (like a GenericForeignKey) cannot generate reverse
+                # relations and therefore cannot be used for reverse querying.
+                if field.is_relation and not field.related_model:
+                    raise FieldError(
+                        "Field %r does not generate an automatic reverse "
+                        "relation and therefore cannot be used for reverse "
+                        "querying. If it is a GenericForeignKey, consider "
+                        "adding a GenericRelation." % name
+                    )
+                model = field.model._meta.concrete_model
             except FieldDoesNotExist:
                 # is it an annotation?
                 if self._annotations and name in self._annotations:
@@ -1382,14 +1407,15 @@ class Query(object):
                 # one step.
                 pos -= 1
                 if pos == -1 or fail_on_missing:
-                    available = opts.get_all_field_names() + list(self.annotation_select)
+                    field_names = list(get_field_names_from_opts(opts))
+                    available = sorted(field_names + list(self.annotation_select))
                     raise FieldError("Cannot resolve keyword %r into field. "
                                      "Choices are: %s" % (name, ", ".join(available)))
                 break
             # Check if we need any joins for concrete inheritance cases (the
             # field lives in parent, but we are currently in one of its
             # children)
-            if model:
+            if model is not opts.model:
                 # The field lives on a base class of the current model.
                 # Skip the chain of proxy to the concrete proxied model
                 proxied_model = opts.concrete_model
@@ -1432,7 +1458,7 @@ class Query(object):
         return path, final_field, targets, names[pos + 1:]
 
     def raise_field_error(self, opts, name):
-        available = opts.get_all_field_names() + list(self.annotation_select)
+        available = list(get_field_names_from_opts(opts)) + list(self.annotation_select)
         raise FieldError("Cannot resolve keyword %r into field. "
                          "Choices are: %s" % (name, ", ".join(available)))
 
@@ -1693,7 +1719,7 @@ class Query(object):
                 # from the model on which the lookup failed.
                 raise
             else:
-                names = sorted(opts.get_all_field_names() + list(self.extra)
+                names = sorted(list(get_field_names_from_opts(opts)) + list(self.extra)
                                + list(self.annotation_select))
                 raise FieldError("Cannot resolve keyword %r into field. "
                                  "Choices are: %s" % (name, ", ".join(names)))
diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py
index 12bde13bf3..bae9f11c23 100644
--- a/django/db/models/sql/subqueries.py
+++ b/django/db/models/sql/subqueries.py
@@ -120,13 +120,15 @@ class UpdateQuery(Query):
         """
         values_seq = []
         for name, val in six.iteritems(values):
-            field, model, direct, m2m = self.get_meta().get_field_by_name(name)
-            if not direct or m2m:
+            field = self.get_meta().get_field(name)
+            direct = not (field.auto_created and not field.concrete) or not field.concrete
+            model = field.model._meta.concrete_model
+            if not direct or (field.is_relation and field.many_to_many):
                 raise FieldError(
                     'Cannot update model field %r (only non-relations and '
                     'foreign keys permitted).' % field
                 )
-            if model:
+            if model is not self.get_meta().model:
                 self.add_related_update(model, field, val)
                 continue
             values_seq.append((field, model, val))