newforms-admin: Merged from trunk up to [7808]. Fixed #7519, #7573

git-svn-id: http://code.djangoproject.com/svn/django/branches/newforms-admin@7809 bcc190cf-cafb-0310-a4f2-bffc1f526a37

3 files changed, 165 insertions, 51 deletions

diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index e8d10bc55b..6c06609969 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -7,6 +7,7 @@ databases). The abstraction barrier only works one way: this module has to know
 all about the internals of models in order to get the information it needs.
 """
 
+import datetime
 from copy import deepcopy
 
 from django.utils.tree import Node
@@ -14,9 +15,10 @@ from django.utils.datastructures import SortedDict
 from django.dispatch import dispatcher
 from django.db import connection
 from django.db.models import signals
+from django.db.models.fields import FieldDoesNotExist
+from django.db.models.query_utils import select_related_descend
 from django.db.models.sql.where import WhereNode, EverythingNode, AND, OR
 from django.db.models.sql.datastructures import Count
-from django.db.models.fields import FieldDoesNotExist
 from django.core.exceptions import FieldError
 from datastructures import EmptyResultSet, Empty, MultiJoin
 from constants import *
@@ -56,6 +58,7 @@ class Query(object):
         self.start_meta = None
         self.select_fields = []
         self.related_select_fields = []
+        self.dupe_avoidance = {}
 
         # SQL-related attributes
         self.select = []
@@ -164,6 +167,7 @@ class Query(object):
         obj.start_meta = self.start_meta
         obj.select_fields = self.select_fields[:]
         obj.related_select_fields = self.related_select_fields[:]
+        obj.dupe_avoidance = self.dupe_avoidance.copy()
         obj.select = self.select[:]
         obj.tables = self.tables[:]
         obj.where = deepcopy(self.where)
@@ -214,7 +218,7 @@ class Query(object):
         obj.select_related = False
         obj.related_select_cols = []
         obj.related_select_fields = []
-        if obj.distinct and len(obj.select) > 1:
+        if len(obj.select) > 1:
             obj = self.clone(CountQuery, _query=obj, where=self.where_class(),
                     distinct=False)
             obj.select = []
@@ -362,10 +366,21 @@ class Query(object):
                 item.relabel_aliases(change_map)
                 self.select.append(item)
         self.select_fields = rhs.select_fields[:]
-        self.extra_select = rhs.extra_select.copy()
-        self.extra_tables = rhs.extra_tables
-        self.extra_where = rhs.extra_where
-        self.extra_params = rhs.extra_params
+
+        if connector == OR:
+            # It would be nice to be able to handle this, but the queries don't
+            # really make sense (or return consistent value sets). Not worth
+            # the extra complexity when you can write a real query instead.
+            if self.extra_select and rhs.extra_select:
+                raise ValueError("When merging querysets using 'or', you "
+                        "cannot have extra(select=...) on both sides.")
+            if self.extra_where and rhs.extra_where:
+                raise ValueError("When merging querysets using 'or', you "
+                        "cannot have extra(where=...) on both sides.")
+        self.extra_select.update(rhs.extra_select)
+        self.extra_tables += rhs.extra_tables
+        self.extra_where += rhs.extra_where
+        self.extra_params += rhs.extra_params
 
         # Ordering uses the 'rhs' ordering, unless it has none, in which case
         # the current ordering is used.
@@ -439,28 +454,39 @@ class Query(object):
         self._select_aliases = aliases
         return result
 
-    def get_default_columns(self, with_aliases=False, col_aliases=None):
+    def get_default_columns(self, with_aliases=False, col_aliases=None,
+            start_alias=None, opts=None, as_pairs=False):
         """
         Computes the default columns for selecting every field in the base
         model.
 
         Returns a list of strings, quoted appropriately for use in SQL
-        directly, as well as a set of aliases used in the select statement.
+        directly, as well as a set of aliases used in the select statement (if
+        'as_pairs' is True, returns a list of (alias, col_name) pairs instead
+        of strings as the first component and None as the second component).
         """
         result = []
-        table_alias = self.tables[0]
-        root_pk = self.model._meta.pk.column
+        if opts is None:
+            opts = self.model._meta
+        if start_alias:
+            table_alias = start_alias
+        else:
+            table_alias = self.tables[0]
+        root_pk = opts.pk.column
         seen = {None: table_alias}
         qn = self.quote_name_unless_alias
         qn2 = self.connection.ops.quote_name
         aliases = set()
-        for field, model in self.model._meta.get_fields_with_model():
+        for field, model in opts.get_fields_with_model():
             try:
                 alias = seen[model]
             except KeyError:
                 alias = self.join((table_alias, model._meta.db_table,
                         root_pk, model._meta.pk.column))
                 seen[model] = alias
+            if as_pairs:
+                result.append((alias, field.column))
+                continue
             if with_aliases and field.column in col_aliases:
                 c_alias = 'Col%d' % len(col_aliases)
                 result.append('%s.%s AS %s' % (qn(alias),
@@ -473,6 +499,8 @@ class Query(object):
                 aliases.add(r)
                 if with_aliases:
                     col_aliases.add(field.column)
+        if as_pairs:
+            return result, None
         return result, aliases
 
     def get_from_clause(self):
@@ -609,6 +637,11 @@ class Query(object):
                 alias, False)
         alias = joins[-1]
         col = target.column
+        if not field.rel:
+            # To avoid inadvertent trimming of a necessary alias, use the
+            # refcount to show that we are referencing a non-relation field on
+            # the model.
+            self.ref_alias(alias)
 
         # Must use left outer joins for nullable fields.
         for join in joins:
@@ -829,8 +862,8 @@ class Query(object):
 
         if reuse and always_create and table in self.table_map:
             # Convert the 'reuse' to case to be "exclude everything but the
-            # reusable set for this table".
-            exclusions = set(self.table_map[table]).difference(reuse)
+            # reusable set, minus exclusions, for this table".
+            exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions))
             always_create = False
         t_ident = (lhs_table, table, lhs_col, col)
         if not always_create:
@@ -865,7 +898,8 @@ class Query(object):
         return alias
 
     def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
-            used=None, requested=None, restricted=None, nullable=None):
+            used=None, requested=None, restricted=None, nullable=None,
+            dupe_set=None):
         """
         Fill in the information needed for a select_related query. The current
         depth is measured as the number of connections away from the root model
@@ -875,6 +909,7 @@ class Query(object):
         if not restricted and self.max_depth and cur_depth > self.max_depth:
             # We've recursed far enough; bail out.
             return
+
         if not opts:
             opts = self.get_meta()
             root_alias = self.get_initial_alias()
@@ -882,6 +917,10 @@ class Query(object):
             self.related_select_fields = []
         if not used:
             used = set()
+        if dupe_set is None:
+            dupe_set = set()
+        orig_dupe_set = dupe_set
+        orig_used = used
 
         # Setup for the case when only particular related fields should be
         # included in the related selection.
@@ -893,9 +932,10 @@ class Query(object):
                 restricted = False
 
         for f, model in opts.get_fields_with_model():
-            if (not f.rel or (restricted and f.name not in requested) or
-                    (not restricted and f.null) or f.rel.parent_link):
+            if not select_related_descend(f, restricted, requested):
                 continue
+            dupe_set = orig_dupe_set.copy()
+            used = orig_used.copy()
             table = f.rel.to._meta.db_table
             if nullable or f.null:
                 promote = True
@@ -906,18 +946,32 @@ class Query(object):
                 alias = root_alias
                 for int_model in opts.get_base_chain(model):
                     lhs_col = int_opts.parents[int_model].column
+                    dedupe = lhs_col in opts.duplicate_targets
+                    if dedupe:
+                        used.update(self.dupe_avoidance.get(id(opts), lhs_col),
+                                ())
+                        dupe_set.add((opts, lhs_col))
                     int_opts = int_model._meta
                     alias = self.join((alias, int_opts.db_table, lhs_col,
                             int_opts.pk.column), exclusions=used,
                             promote=promote)
+                    for (dupe_opts, dupe_col) in dupe_set:
+                        self.update_dupe_avoidance(dupe_opts, dupe_col, alias)
             else:
                 alias = root_alias
+
+            dedupe = f.column in opts.duplicate_targets
+            if dupe_set or dedupe:
+                used.update(self.dupe_avoidance.get((id(opts), f.column), ()))
+                if dedupe:
+                    dupe_set.add((opts, f.column))
+
             alias = self.join((alias, table, f.column,
                     f.rel.get_related_field().column), exclusions=used,
                     promote=promote)
             used.add(alias)
-            self.related_select_cols.extend([(alias, f2.column)
-                    for f2 in f.rel.to._meta.fields])
+            self.related_select_cols.extend(self.get_default_columns(
+                start_alias=alias, opts=f.rel.to._meta, as_pairs=True)[0])
             self.related_select_fields.extend(f.rel.to._meta.fields)
             if restricted:
                 next = requested.get(f.name, {})
@@ -927,8 +981,10 @@ class Query(object):
                 new_nullable = f.null
             else:
                 new_nullable = None
+            for dupe_opts, dupe_col in dupe_set:
+                self.update_dupe_avoidance(dupe_opts, dupe_col, alias)
             self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1,
-                    used, next, restricted, new_nullable)
+                    used, next, restricted, new_nullable, dupe_set)
 
     def add_filter(self, filter_expr, connector=AND, negate=False, trim=False,
             can_reuse=None):
@@ -1048,7 +1104,19 @@ class Query(object):
                 # that's harmless.
                 self.promote_alias(table)
 
-        self.where.add((alias, col, field, lookup_type, value), connector)
+        # To save memory and copying time, convert the value from the Python
+        # object to the actual value used in the SQL query.
+        if field:
+            params = field.get_db_prep_lookup(lookup_type, value)
+        else:
+            params = Field().get_db_prep_lookup(lookup_type, value)
+        if isinstance(value, datetime.datetime):
+            annotation = datetime.datetime
+        else:
+            annotation = bool(value)
+
+        self.where.add((alias, col, field.db_type(), lookup_type, annotation,
+            params), connector)
 
         if negate:
             for alias in join_list:
@@ -1058,7 +1126,8 @@ class Query(object):
                     for alias in join_list:
                         if self.alias_map[alias][JOIN_TYPE] == self.LOUTER:
                             j_col = self.alias_map[alias][RHS_JOIN_COL]
-                            entry = Node([(alias, j_col, None, 'isnull', True)])
+                            entry = Node([(alias, j_col, None, 'isnull', True,
+                                    [True])])
                             entry.negate()
                             self.where.add(entry, AND)
                             break
@@ -1066,7 +1135,7 @@ class Query(object):
                     # Leaky abstraction artifact: We have to specifically
                     # exclude the "foo__in=[]" case from this handling, because
                     # it's short-circuited in the Where class.
-                    entry = Node([(alias, col, field, 'isnull', True)])
+                    entry = Node([(alias, col, None, 'isnull', True, [True])])
                     entry.negate()
                     self.where.add(entry, AND)
 
@@ -1114,7 +1183,9 @@ class Query(object):
         (which gives the table we are joining to), 'alias' is the alias for the
         table we are joining to. If dupe_multis is True, any many-to-many or
         many-to-one joins will always create a new alias (necessary for
-        disjunctive filters).
+        disjunctive filters). If can_reuse is not None, it's a list of aliases
+        that can be reused in these joins (nothing else can be reused in this
+        case).
 
         Returns the final field involved in the join, the target database
         column (used for any 'where' constraint), the final 'opts' value and the
@@ -1122,7 +1193,14 @@ class Query(object):
         """
         joins = [alias]
         last = [0]
+        dupe_set = set()
+        exclusions = set()
         for pos, name in enumerate(names):
+            try:
+                exclusions.add(int_alias)
+            except NameError:
+                pass
+            exclusions.add(alias)
             last.append(len(joins))
             if name == 'pk':
                 name = opts.pk.name
@@ -1141,6 +1219,7 @@ class Query(object):
                     names = opts.get_all_field_names()
                     raise FieldError("Cannot resolve keyword %r into field. "
                             "Choices are: %s" % (name, ", ".join(names)))
+
             if not allow_many and (m2m or not direct):
                 for alias in joins:
                     self.unref_alias(alias)
@@ -1150,12 +1229,27 @@ class Query(object):
                 alias_list = []
                 for int_model in opts.get_base_chain(model):
                     lhs_col = opts.parents[int_model].column
+                    dedupe = lhs_col in opts.duplicate_targets
+                    if dedupe:
+                        exclusions.update(self.dupe_avoidance.get(
+                                (id(opts), lhs_col), ()))
+                        dupe_set.add((opts, lhs_col))
                     opts = int_model._meta
                     alias = self.join((alias, opts.db_table, lhs_col,
-                            opts.pk.column), exclusions=joins)
+                            opts.pk.column), exclusions=exclusions)
                     joins.append(alias)
+                    exclusions.add(alias)
+                    for (dupe_opts, dupe_col) in dupe_set:
+                        self.update_dupe_avoidance(dupe_opts, dupe_col, alias)
             cached_data = opts._join_cache.get(name)
             orig_opts = opts
+            dupe_col = direct and field.column or field.field.column
+            dedupe = dupe_col in opts.duplicate_targets
+            if dupe_set or dedupe:
+                if dedupe:
+                    dupe_set.add((opts, dupe_col))
+                exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col),
+                        ()))
 
             if direct:
                 if m2m:
@@ -1177,9 +1271,11 @@ class Query(object):
                                 target)
 
                     int_alias = self.join((alias, table1, from_col1, to_col1),
-                            dupe_multis, joins, nullable=True, reuse=can_reuse)
+                            dupe_multis, exclusions, nullable=True,
+                            reuse=can_reuse)
                     alias = self.join((int_alias, table2, from_col2, to_col2),
-                            dupe_multis, joins, nullable=True, reuse=can_reuse)
+                            dupe_multis, exclusions, nullable=True,
+                            reuse=can_reuse)
                     joins.extend([int_alias, alias])
                 elif field.rel:
                     # One-to-one or many-to-one field
@@ -1195,7 +1291,7 @@ class Query(object):
                                 opts, target)
 
                     alias = self.join((alias, table, from_col, to_col),
-                            exclusions=joins, nullable=field.null)
+                            exclusions=exclusions, nullable=field.null)
                     joins.append(alias)
                 else:
                     # Non-relation fields.
@@ -1223,9 +1319,11 @@ class Query(object):
                                 target)
 
                     int_alias = self.join((alias, table1, from_col1, to_col1),
-                            dupe_multis, joins, nullable=True, reuse=can_reuse)
+                            dupe_multis, exclusions, nullable=True,
+                            reuse=can_reuse)
                     alias = self.join((int_alias, table2, from_col2, to_col2),
-                            dupe_multis, joins, nullable=True, reuse=can_reuse)
+                            dupe_multis, exclusions, nullable=True,
+                            reuse=can_reuse)
                     joins.extend([int_alias, alias])
                 else:
                     # One-to-many field (ForeignKey defined on the target model)
@@ -1243,14 +1341,34 @@ class Query(object):
                                 opts, target)
 
                     alias = self.join((alias, table, from_col, to_col),
-                            dupe_multis, joins, nullable=True, reuse=can_reuse)
+                            dupe_multis, exclusions, nullable=True,
+                            reuse=can_reuse)
                     joins.append(alias)
 
+            for (dupe_opts, dupe_col) in dupe_set:
+                try:
+                    self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias)
+                except NameError:
+                    self.update_dupe_avoidance(dupe_opts, dupe_col, alias)
+
         if pos != len(names) - 1:
             raise FieldError("Join on field %r not permitted." % name)
 
         return field, target, opts, joins, last
 
+    def update_dupe_avoidance(self, opts, col, alias):
+        """
+        For a column that is one of multiple pointing to the same table, update
+        the internal data structures to note that this alias shouldn't be used
+        for those other columns.
+        """
+        ident = id(opts)
+        for name in opts.duplicate_targets[col]:
+            try:
+                self.dupe_avoidance[ident, name].add(alias)
+            except KeyError:
+                self.dupe_avoidance[ident, name] = set([alias])
+
     def split_exclude(self, filter_expr, prefix):
         """
         When doing an exclude against any kind of N-to-many relation, we need
diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py
index 0bb741d706..d8c5b074ec 100644
--- a/django/db/models/sql/subqueries.py
+++ b/django/db/models/sql/subqueries.py
@@ -49,7 +49,7 @@ class DeleteQuery(Query):
                 for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
                     where = self.where_class()
                     where.add((None, related.field.m2m_reverse_name(),
-                            related.field, 'in',
+                            related.field.db_type(), 'in', True,
                             pk_list[offset : offset+GET_ITERATOR_CHUNK_SIZE]),
                             AND)
                     self.do_query(related.field.m2m_db_table(), where)
@@ -59,11 +59,11 @@ class DeleteQuery(Query):
             if isinstance(f, generic.GenericRelation):
                 from django.contrib.contenttypes.models import ContentType
                 field = f.rel.to._meta.get_field(f.content_type_field_name)
-                w1.add((None, field.column, field, 'exact',
-                        ContentType.objects.get_for_model(cls).id), AND)
+                w1.add((None, field.column, field.db_type(), 'exact', True,
+                        [ContentType.objects.get_for_model(cls).id]), AND)
             for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
                 where = self.where_class()
-                where.add((None, f.m2m_column_name(), f, 'in',
+                where.add((None, f.m2m_column_name(), f.db_type(), 'in', True,
                         pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]),
                         AND)
                 if w1:
@@ -81,7 +81,7 @@ class DeleteQuery(Query):
         for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
             where = self.where_class()
             field = self.model._meta.pk
-            where.add((None, field.column, field, 'in',
+            where.add((None, field.column, field.db_type(), 'in', True,
                     pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), AND)
             self.do_query(self.model._meta.db_table, where)
 
@@ -204,7 +204,7 @@ class UpdateQuery(Query):
         for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
             self.where = self.where_class()
             f = self.model._meta.pk
-            self.where.add((None, f.column, f, 'in',
+            self.where.add((None, f.column, f.db_type(), 'in', True,
                     pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]),
                     AND)
             self.values = [(related_field.column, None, '%s')]
diff --git a/django/db/models/sql/where.py b/django/db/models/sql/where.py
index 14e54487a3..bf45bceb4b 100644
--- a/django/db/models/sql/where.py
+++ b/django/db/models/sql/where.py
@@ -21,8 +21,9 @@ class WhereNode(tree.Node):
     the correct SQL).
 
     The children in this tree are usually either Q-like objects or lists of
-    [table_alias, field_name, field_class, lookup_type, value]. However, a
-    child could also be any class with as_sql() and relabel_aliases() methods.
+    [table_alias, field_name, db_type, lookup_type, value_annotation,
+    params]. However, a child could also be any class with as_sql() and
+    relabel_aliases() methods.
     """
     default = AND
 
@@ -88,29 +89,24 @@ class WhereNode(tree.Node):
 
     def make_atom(self, child, qn):
         """
-        Turn a tuple (table_alias, field_name, field_class, lookup_type, value)
-        into valid SQL.
+        Turn a tuple (table_alias, field_name, db_type, lookup_type,
+        value_annot, params) into valid SQL.
 
         Returns the string for the SQL fragment and the parameters to use for
         it.
         """
-        table_alias, name, field, lookup_type, value = child
+        table_alias, name, db_type, lookup_type, value_annot, params = child
         if table_alias:
             lhs = '%s.%s' % (qn(table_alias), qn(name))
         else:
             lhs = qn(name)
-        db_type = field and field.db_type() or None
         field_sql = connection.ops.field_cast_sql(db_type) % lhs
 
-        if isinstance(value, datetime.datetime):
+        if value_annot is datetime.datetime:
             cast_sql = connection.ops.datetime_cast_sql()
         else:
             cast_sql = '%s'
 
-        if field:
-            params = field.get_db_prep_lookup(lookup_type, value)
-        else:
-            params = Field().get_db_prep_lookup(lookup_type, value)
         if isinstance(params, QueryWrapper):
             extra, params = params.data
         else:
@@ -123,11 +119,11 @@ class WhereNode(tree.Node):
                     connection.operators[lookup_type] % cast_sql), params)
 
         if lookup_type == 'in':
-            if not value:
+            if not value_annot:
                 raise EmptyResultSet
             if extra:
                 return ('%s IN %s' % (field_sql, extra), params)
-            return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(value))),
+            return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(params))),
                     params)
         elif lookup_type in ('range', 'year'):
             return ('%s BETWEEN %%s and %%s' % field_sql, params)
@@ -135,8 +131,8 @@ class WhereNode(tree.Node):
             return ('%s = %%s' % connection.ops.date_extract_sql(lookup_type,
                     field_sql), params)
         elif lookup_type == 'isnull':
-            return ('%s IS %sNULL' % (field_sql, (not value and 'NOT ' or '')),
-                    params)
+            return ('%s IS %sNULL' % (field_sql,
+                (not value_annot and 'NOT ' or '')), ())
         elif lookup_type == 'search':
             return (connection.ops.fulltext_search_sql(field_sql), params)
         elif lookup_type in ('regex', 'iregex'):