diff options
| author | Josh Smeaton <josh.smeaton@gmail.com> | 2013-12-26 00:13:18 +1100 |
|---|---|---|
| committer | Marc Tamlyn <marc.tamlyn@gmail.com> | 2014-11-15 14:00:43 +0000 |
| commit | f59fd15c4928caf3dfcbd50f6ab47be409a43b01 (patch) | |
| tree | fe4a04d98359e1ffcbfe991303eb97d9a8e16afc /django | |
| parent | 39e3ef88c237e3f4cedc89cd36494a6d3f490812 (diff) | |
Fixed #14030 -- Allowed annotations to accept all expressions
Diffstat (limited to 'django')
28 files changed, 1006 insertions, 663 deletions
diff --git a/django/contrib/contenttypes/fields.py b/django/contrib/contenttypes/fields.py index 201016ae6e..cc115b01aa 100644 --- a/django/contrib/contenttypes/fields.py +++ b/django/contrib/contenttypes/fields.py @@ -10,7 +10,7 @@ from django.db.models import signals, FieldDoesNotExist, DO_NOTHING from django.db.models.base import ModelBase from django.db.models.fields.related import ForeignObject, ForeignObjectRel from django.db.models.related import PathInfo -from django.db.models.sql.datastructures import Col +from django.db.models.expressions import Col from django.contrib.contenttypes.models import ContentType from django.utils.encoding import smart_text, python_2_unicode_compatible diff --git a/django/contrib/gis/db/backends/base.py b/django/contrib/gis/db/backends/base.py index c6e48ce0ef..0d2a9db870 100644 --- a/django/contrib/gis/db/backends/base.py +++ b/django/contrib/gis/db/backends/base.py @@ -186,7 +186,7 @@ class BaseSpatialOperations(object): """ raise NotImplementedError('Distance operations not available on this spatial backend.') - def get_geom_placeholder(self, f, value): + def get_geom_placeholder(self, f, value, qn): """ Returns the placeholder for the given geometry field with the given value. Depending on the spatial backend, the placeholder may contain a @@ -195,16 +195,6 @@ class BaseSpatialOperations(object): """ raise NotImplementedError('subclasses of BaseSpatialOperations must provide a geo_db_placeholder() method') - def get_expression_column(self, evaluator): - """ - Helper method to return the quoted column string from the evaluator - for its expression. - """ - for expr, col_tup in evaluator.cols: - if expr is evaluator.expression: - return '%s.%s' % tuple(map(self.quote_name, col_tup)) - raise Exception("Could not find the column for the expression.") - # Spatial SQL Construction def spatial_aggregate_sql(self, agg): raise NotImplementedError('Aggregate support not implemented for this spatial backend.') diff --git a/django/contrib/gis/db/backends/mysql/operations.py b/django/contrib/gis/db/backends/mysql/operations.py index bcbd634fd8..191e2c8956 100644 --- a/django/contrib/gis/db/backends/mysql/operations.py +++ b/django/contrib/gis/db/backends/mysql/operations.py @@ -35,14 +35,14 @@ class MySQLOperations(DatabaseOperations, BaseSpatialOperations): def geo_db_type(self, f): return f.geom_type - def get_geom_placeholder(self, f, value): + def get_geom_placeholder(self, f, value, qn): """ The placeholder here has to include MySQL's WKT constructor. Because MySQL does not support spatial transformations, there is no need to modify the placeholder based on the contents of the given value. """ - if hasattr(value, 'expression'): - placeholder = self.get_expression_column(value) + if hasattr(value, 'as_sql'): + placeholder, _ = qn.compile(value) else: placeholder = '%s(%%s)' % self.from_text return placeholder diff --git a/django/contrib/gis/db/backends/oracle/operations.py b/django/contrib/gis/db/backends/oracle/operations.py index d4671e6cd6..aa002d3b82 100644 --- a/django/contrib/gis/db/backends/oracle/operations.py +++ b/django/contrib/gis/db/backends/oracle/operations.py @@ -9,7 +9,7 @@ """ import re -from django.db.backends.oracle.base import DatabaseOperations +from django.db.backends.oracle.base import DatabaseOperations, Database from django.contrib.gis.db.backends.base import BaseSpatialOperations from django.contrib.gis.db.backends.oracle.adapter import OracleSpatialAdapter from django.contrib.gis.db.backends.utils import SpatialOperator @@ -145,9 +145,11 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): else: return None - def convert_geom(self, clob, geo_field): - if clob: - return Geometry(clob.read(), geo_field.srid) + def convert_geom(self, value, geo_field): + if value: + if isinstance(value, Database.LOB): + value = value.read() + return Geometry(value, geo_field.srid) else: return None @@ -184,7 +186,7 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): return [dist_param] - def get_geom_placeholder(self, f, value): + def get_geom_placeholder(self, f, value, qn): """ Provides a proper substitution value for Geometries that are not in the SRID of the field. Specifically, this routine will substitute in the @@ -196,14 +198,15 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): def transform_value(val, srid): return val.srid != srid - if hasattr(value, 'expression'): + if hasattr(value, 'as_sql'): if transform_value(value, f.srid): placeholder = '%s(%%s, %s)' % (self.transform, f.srid) else: placeholder = '%s' # No geometry value used for F expression, substitute in # the column name instead. - return placeholder % self.get_expression_column(value) + sql, _ = qn.compile(value) + return placeholder % sql else: if transform_value(value, f.srid): return '%s(SDO_GEOMETRY(%%s, %s), %s)' % (self.transform, value.srid, f.srid) @@ -219,9 +222,9 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): if agg_name == 'union': agg_name += 'agg' if agg.is_extent: - sql_template = '%(function)s(%(field)s)' + sql_template = '%(function)s(%(expressions)s)' else: - sql_template = '%(function)s(SDOAGGRTYPE(%(field)s,%(tolerance)s))' + sql_template = '%(function)s(SDOAGGRTYPE(%(expressions)s,%(tolerance)s))' sql_function = getattr(self, agg_name) return self.select % sql_template, sql_function diff --git a/django/contrib/gis/db/backends/postgis/operations.py b/django/contrib/gis/db/backends/postgis/operations.py index 690194fa9e..d78b081950 100644 --- a/django/contrib/gis/db/backends/postgis/operations.py +++ b/django/contrib/gis/db/backends/postgis/operations.py @@ -22,7 +22,7 @@ class PostGISOperator(SpatialOperator): super(PostGISOperator, self).__init__(**kwargs) def as_sql(self, connection, lookup, *args): - if lookup.lhs.source.geography and not self.geography: + if lookup.lhs.output_field.geography and not self.geography: raise ValueError('PostGIS geography does not support the "%s" ' 'function/operator.' % (self.func or self.op,)) return super(PostGISOperator, self).as_sql(connection, lookup, *args) @@ -32,7 +32,7 @@ class PostGISDistanceOperator(PostGISOperator): sql_template = '%(func)s(%(lhs)s, %(rhs)s) %(op)s %%s' def as_sql(self, connection, lookup, template_params, sql_params): - if not lookup.lhs.source.geography and lookup.lhs.source.geodetic(connection): + if not lookup.lhs.output_field.geography and lookup.lhs.output_field.geodetic(connection): sql_template = self.sql_template if len(lookup.rhs) == 3 and lookup.rhs[-1] == 'spheroid': template_params.update({'op': self.op, 'func': 'ST_Distance_Spheroid'}) @@ -215,7 +215,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): Converts the geometry returned from PostGIS aggretates. """ if hex: - return Geometry(hex) + return Geometry(hex, srid=geo_field.srid) else: return None @@ -284,7 +284,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): else: return [dist_param] - def get_geom_placeholder(self, f, value): + def get_geom_placeholder(self, f, value, qn): """ Provides a proper substitution value for Geometries that are not in the SRID of the field. Specifically, this routine will substitute in the @@ -296,11 +296,12 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): # Adding Transform() to the SQL placeholder. placeholder = '%s(%%s, %s)' % (self.transform, f.srid) - if hasattr(value, 'expression'): + if hasattr(value, 'as_sql'): # If this is an F expression, then we don't really want # a placeholder and instead substitute in the column # of the expression. - placeholder = placeholder % self.get_expression_column(value) + sql, _ = qn.compile(value) + placeholder = placeholder % sql return placeholder @@ -375,7 +376,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): agg_name = agg_name.lower() if agg_name == 'union': agg_name += 'agg' - sql_template = '%(function)s(%(field)s)' + sql_template = '%(function)s(%(expressions)s)' sql_function = getattr(self, agg_name) return sql_template, sql_function diff --git a/django/contrib/gis/db/backends/spatialite/operations.py b/django/contrib/gis/db/backends/spatialite/operations.py index d7a4912fbe..4ec98c402d 100644 --- a/django/contrib/gis/db/backends/spatialite/operations.py +++ b/django/contrib/gis/db/backends/spatialite/operations.py @@ -178,7 +178,7 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations): dist_param = value return [dist_param] - def get_geom_placeholder(self, f, value): + def get_geom_placeholder(self, f, value, qn): """ Provides a proper substitution value for Geometries that are not in the SRID of the field. Specifically, this routine will substitute in the @@ -186,14 +186,15 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations): """ def transform_value(value, srid): return not (value is None or value.srid == srid) - if hasattr(value, 'expression'): + if hasattr(value, 'as_sql'): if transform_value(value, f.srid): placeholder = '%s(%%s, %s)' % (self.transform, f.srid) else: placeholder = '%s' # No geometry value used for F expression, substitute in # the column name instead. - return placeholder % self.get_expression_column(value) + sql, _ = qn.compile(value) + return placeholder % sql else: if transform_value(value, f.srid): # Adding Transform() to the SQL placeholder. @@ -255,7 +256,7 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations): agg_name = agg_name.lower() if agg_name == 'union': agg_name += 'agg' - sql_template = self.select % '%(function)s(%(field)s)' + sql_template = self.select % '%(function)s(%(expressions)s)' sql_function = getattr(self, agg_name) return sql_template, sql_function diff --git a/django/contrib/gis/db/models/aggregates.py b/django/contrib/gis/db/models/aggregates.py index 43e9d1a0ae..0cf0a8b266 100644 --- a/django/contrib/gis/db/models/aggregates.py +++ b/django/contrib/gis/db/models/aggregates.py @@ -1,23 +1,66 @@ -from django.db.models import Aggregate +from django.db.models.aggregates import Aggregate +from django.contrib.gis.db.models.fields import GeometryField, ExtentField __all__ = ['Collect', 'Extent', 'Extent3D', 'MakeLine', 'Union'] -class Collect(Aggregate): +class GeoAggregate(Aggregate): + template = None + function = None + is_extent = False + + def as_sql(self, compiler, connection): + if connection.ops.oracle: + if not hasattr(self, 'tolerance'): + self.tolerance = 0.05 + self.extra['tolerance'] = self.tolerance + + template, function = connection.ops.spatial_aggregate_sql(self) + if template is None: + template = '%(function)s(%(expressions)s)' + self.extra['template'] = self.extra.get('template', template) + self.extra['function'] = self.extra.get('function', function) + return super(GeoAggregate, self).as_sql(compiler, connection) + + def prepare(self, query=None, allow_joins=True, reuse=None, summarize=False): + c = super(GeoAggregate, self).prepare(query, allow_joins, reuse, summarize) + if not isinstance(self.expressions[0].output_field, GeometryField): + raise ValueError('Geospatial aggregates only allowed on geometry fields.') + return c + + def convert_value(self, value, connection): + return connection.ops.convert_geom(value, self.output_field) + + +class Collect(GeoAggregate): name = 'Collect' -class Extent(Aggregate): +class Extent(GeoAggregate): name = 'Extent' + is_extent = '2D' + + def __init__(self, expression, **extra): + super(Extent, self).__init__(expression, output_field=ExtentField(), **extra) + + def convert_value(self, value, connection): + return connection.ops.convert_extent(value) -class Extent3D(Aggregate): +class Extent3D(GeoAggregate): name = 'Extent3D' + is_extent = '3D' + + def __init__(self, expression, **extra): + super(Extent3D, self).__init__(expression, output_field=ExtentField(), **extra) + + def convert_value(self, value, connection): + return connection.ops.convert_extent3d(value) -class MakeLine(Aggregate): +class MakeLine(GeoAggregate): name = 'MakeLine' -class Union(Aggregate): +class Union(GeoAggregate): name = 'Union' diff --git a/django/contrib/gis/db/models/fields.py b/django/contrib/gis/db/models/fields.py index c538c18d63..1d64a06be2 100644 --- a/django/contrib/gis/db/models/fields.py +++ b/django/contrib/gis/db/models/fields.py @@ -1,5 +1,5 @@ from django.db.models.fields import Field -from django.db.models.sql.expressions import SQLEvaluator +from django.db.models.expressions import ExpressionNode from django.utils.translation import ugettext_lazy as _ from django.contrib.gis import forms from django.contrib.gis.db.models.lookups import gis_lookups @@ -165,7 +165,7 @@ class GeometryField(Field): returning to the caller. """ value = super(GeometryField, self).get_prep_value(value) - if isinstance(value, SQLEvaluator): + if isinstance(value, ExpressionNode): return value elif isinstance(value, (tuple, list)): geom = value[0] @@ -197,7 +197,7 @@ class GeometryField(Field): return geom def from_db_value(self, value, connection): - if value: + if value and not isinstance(value, Geometry): value = Geometry(value) return value @@ -259,7 +259,7 @@ class GeometryField(Field): pass else: params += value[1:] - elif isinstance(value, SQLEvaluator): + elif isinstance(value, ExpressionNode): params = [] else: params = [connection.ops.Adapter(value)] @@ -282,12 +282,12 @@ class GeometryField(Field): else: return connection.ops.Adapter(self.get_prep_value(value)) - def get_placeholder(self, value, connection): + def get_placeholder(self, value, qn, connection): """ Returns the placeholder for the geometry column for the given value. """ - return connection.ops.get_geom_placeholder(self, value) + return connection.ops.get_geom_placeholder(self, value, qn) for klass in gis_lookups.values(): @@ -335,3 +335,12 @@ class GeometryCollectionField(GeometryField): geom_type = 'GEOMETRYCOLLECTION' form_class = forms.GeometryCollectionField description = _("Geometry collection") + + +class ExtentField(Field): + "Used as a return value from an extent aggregate" + + description = _("Extent Aggregate Field") + + def get_internal_type(self): + return "ExtentField" diff --git a/django/contrib/gis/db/models/lookups.py b/django/contrib/gis/db/models/lookups.py index d3b53b32f0..889237751a 100644 --- a/django/contrib/gis/db/models/lookups.py +++ b/django/contrib/gis/db/models/lookups.py @@ -4,7 +4,7 @@ import re from django.db.models.constants import LOOKUP_SEP from django.db.models.fields import FieldDoesNotExist from django.db.models.lookups import Lookup -from django.db.models.sql.expressions import SQLEvaluator +from django.db.models.expressions import ExpressionNode, Col from django.utils import six gis_lookups = {} @@ -68,18 +68,19 @@ class GISLookup(Lookup): rhs, rhs_params = super(GISLookup, self).process_rhs(qn, connection) geom = self.rhs - if isinstance(self.rhs, SQLEvaluator): + if isinstance(self.rhs, Col): # Make sure the F Expression destination field exists, and # set an `srid` attribute with the same as that of the # destination. - geo_fld = self._check_geo_field(self.rhs.opts, self.rhs.expression.name) - if not geo_fld: + geo_fld = self.rhs.output_field + if not hasattr(geo_fld, 'srid'): raise ValueError('No geographic field found in expression.') self.rhs.srid = geo_fld.srid + elif isinstance(self.rhs, ExpressionNode): + raise ValueError('Complex expressions not supported for GeometryField') elif isinstance(self.rhs, (list, tuple)): geom = self.rhs[0] - - rhs = connection.ops.get_geom_placeholder(self.lhs.source, geom) + rhs = connection.ops.get_geom_placeholder(self.lhs.output_field, geom, qn) return rhs, rhs_params def as_sql(self, qn, connection): diff --git a/django/contrib/gis/db/models/query.py b/django/contrib/gis/db/models/query.py index cbc1536b8a..f2e7657850 100644 --- a/django/contrib/gis/db/models/query.py +++ b/django/contrib/gis/db/models/query.py @@ -530,7 +530,7 @@ class GeoQuerySet(QuerySet): # transformation SQL. geom = geo_field.get_prep_value(settings['procedure_args'][name]) params = geo_field.get_db_prep_lookup('contains', geom, connection=connection) - geom_placeholder = geo_field.get_placeholder(geom, connection) + geom_placeholder = geo_field.get_placeholder(geom, None, connection) # Replacing the procedure format with that of any needed # transformation SQL. diff --git a/django/contrib/gis/db/models/sql/aggregates.py b/django/contrib/gis/db/models/sql/aggregates.py index c0a7d894eb..c3943eb9f6 100644 --- a/django/contrib/gis/db/models/sql/aggregates.py +++ b/django/contrib/gis/db/models/sql/aggregates.py @@ -6,12 +6,15 @@ from django.contrib.gis.db.models.fields import GeometryField __all__ = ['Collect', 'Extent', 'Extent3D', 'MakeLine', 'Union'] + aggregates.__all__ +warnings.warn( + "django.contrib.gis.db.models.sql.aggregates is deprecated. Use " + "django.contrib.gis.db.models.aggregates instead.", + RemovedInDjango20Warning, stacklevel=2) + + class GeoAggregate(Aggregate): # Default SQL template for spatial aggregates. - sql_template = '%(function)s(%(field)s)' - - # Conversion class, if necessary. - conversion_class = None + sql_template = '%(function)s(%(expressions)s)' # Flags for indicating the type of the aggregate. is_extent = False @@ -45,7 +48,7 @@ class GeoAggregate(Aggregate): substitutions = { 'function': sql_function, - 'field': field_name + 'expressions': field_name } substitutions.update(self.extra) diff --git a/django/contrib/gis/db/models/sql/compiler.py b/django/contrib/gis/db/models/sql/compiler.py index 05491ef930..915f33fc2b 100644 --- a/django/contrib/gis/db/models/sql/compiler.py +++ b/django/contrib/gis/db/models/sql/compiler.py @@ -70,8 +70,8 @@ class GeoSQLCompiler(compiler.SQLCompiler): aliases.update(new_aliases) max_name_length = self.connection.ops.max_name_length() - for alias, aggregate in self.query.aggregate_select.items(): - agg_sql, agg_params = aggregate.as_sql(qn, self.connection) + for alias, annotation in self.query.annotation_select.items(): + agg_sql, agg_params = self.compile(annotation) if alias is None: result.append(agg_sql) else: diff --git a/django/contrib/gis/db/models/sql/query.py b/django/contrib/gis/db/models/sql/query.py index 7e7715c682..9c071b85fc 100644 --- a/django/contrib/gis/db/models/sql/query.py +++ b/django/contrib/gis/db/models/sql/query.py @@ -4,7 +4,7 @@ from django.db.models.sql.constants import QUERY_TERMS from django.contrib.gis.db.models.fields import GeometryField from django.contrib.gis.db.models.lookups import GISLookup -from django.contrib.gis.db.models.sql import aggregates as gis_aggregates +from django.contrib.gis.db.models import aggregates as gis_aggregates from django.contrib.gis.db.models.sql.conversion import GeomField @@ -14,7 +14,6 @@ class GeoQuery(sql.Query): """ # Overriding the valid query terms. query_terms = QUERY_TERMS | set(GeometryField.class_lookups.keys()) - aggregates_module = gis_aggregates compiler = 'GeoSQLCompiler' @@ -40,28 +39,12 @@ class GeoQuery(sql.Query): # Remove any aggregates marked for reduction from the subquery # and move them to the outer AggregateQuery. connection = connections[using] - for alias, aggregate in self.aggregate_select.items(): - if isinstance(aggregate, gis_aggregates.GeoAggregate): - if not getattr(aggregate, 'is_extent', False) or connection.ops.oracle: + for alias, annotation in self.annotation_select.items(): + if isinstance(annotation, gis_aggregates.GeoAggregate): + if not getattr(annotation, 'is_extent', False) or connection.ops.oracle: self.extra_select_fields[alias] = GeomField() return super(GeoQuery, self).get_aggregation(using, force_subq) - def resolve_aggregate(self, value, aggregate, connection): - """ - Overridden from GeoQuery's normalize to handle the conversion of - GeoAggregate objects. - """ - if isinstance(aggregate, self.aggregates_module.GeoAggregate): - if aggregate.is_extent: - if aggregate.is_extent == '3D': - return connection.ops.convert_extent3d(value) - else: - return connection.ops.convert_extent(value) - else: - return connection.ops.convert_geom(value, aggregate.source) - else: - return super(GeoQuery, self).resolve_aggregate(value, aggregate, connection) - # Private API utilities, subject to change. def _geo_field(self, field_name=None): """ diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 349fae0253..4799576ba9 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -20,8 +20,7 @@ from django.db.backends.sqlite3.client import DatabaseClient from django.db.backends.sqlite3.creation import DatabaseCreation from django.db.backends.sqlite3.introspection import DatabaseIntrospection from django.db.backends.sqlite3.schema import DatabaseSchemaEditor -from django.db.models import fields -from django.db.models.sql import aggregates +from django.db.models import fields, aggregates from django.utils.dateparse import parse_date, parse_datetime, parse_time from django.utils.encoding import force_text from django.utils.functional import cached_property @@ -163,8 +162,7 @@ class DatabaseOperations(BaseDatabaseOperations): bad_fields = (fields.DateField, fields.DateTimeField, fields.TimeField) bad_aggregates = (aggregates.Sum, aggregates.Avg, aggregates.Variance, aggregates.StdDev) - if (isinstance(aggregate.source, bad_fields) and - isinstance(aggregate, bad_aggregates)): + if aggregate.refs_field(bad_aggregates, bad_fields): raise NotImplementedError( 'You cannot use Sum, Avg, StdDev and Variance aggregations ' 'on date/time fields in sqlite3 ' diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py index c054542e3e..6fe8fe8aae 100644 --- a/django/db/models/__init__.py +++ b/django/db/models/__init__.py @@ -4,7 +4,7 @@ import warnings from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured # NOQA from django.db.models.query import Q, QuerySet, Prefetch # NOQA -from django.db.models.expressions import F # NOQA +from django.db.models.expressions import ExpressionNode, F, Value, Func # NOQA from django.db.models.manager import Manager # NOQA from django.db.models.base import Model # NOQA from django.db.models.aggregates import * # NOQA diff --git a/django/db/models/aggregates.py b/django/db/models/aggregates.py index e31d228aa5..c68378c7da 100644 --- a/django/db/models/aggregates.py +++ b/django/db/models/aggregates.py @@ -1,94 +1,152 @@ """ Classes to represent the definitions of aggregate functions. """ -from django.db.models.constants import LOOKUP_SEP +from django.core.exceptions import FieldError +from django.db.models.expressions import Func, Value +from django.db.models.fields import IntegerField, FloatField __all__ = [ 'Aggregate', 'Avg', 'Count', 'Max', 'Min', 'StdDev', 'Sum', 'Variance', ] -def refs_aggregate(lookup_parts, aggregates): - """ - A little helper method to check if the lookup_parts contains references - to the given aggregates set. Because the LOOKUP_SEP is contained in the - default annotation names we must check each prefix of the lookup_parts - for match. - """ - for n in range(len(lookup_parts) + 1): - level_n_lookup = LOOKUP_SEP.join(lookup_parts[0:n]) - if level_n_lookup in aggregates: - return aggregates[level_n_lookup], lookup_parts[n:] - return False, () +class Aggregate(Func): + contains_aggregate = True + name = None + def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False): + assert len(self.source_expressions) == 1 + c = super(Aggregate, self).resolve_expression(query, allow_joins, reuse, summarize) + if c.source_expressions[0].contains_aggregate and not summarize: + name = self.source_expressions[0].name + raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % ( + c.name, name, name)) + c._patch_aggregate(query) # backward-compatibility support + return c -class Aggregate(object): - """ - Default Aggregate definition. - """ - def __init__(self, lookup, **extra): - """Instantiate a new aggregate. + def refs_field(self, aggregate_types, field_types): + try: + return (isinstance(self, aggregate_types) and + isinstance(self.input_field._output_field_or_none, field_types)) + except FieldError: + # Sometimes we don't know the input_field's output type (for example, + # doing Sum(F('datetimefield') + F('datefield'), output_type=DateTimeField()) + # is OK, but the Expression(F('datetimefield') + F('datefield')) doesn't + # have any output field. + return False - * lookup is the field on which the aggregate operates. - * extra is a dictionary of additional data to provide for the - aggregate definition + @property + def input_field(self): + return self.source_expressions[0] - Also utilizes the class variables: - * name, the identifier for this aggregate function. - """ - self.lookup = lookup - self.extra = extra + @property + def default_alias(self): + if hasattr(self.source_expressions[0], 'name'): + return '%s__%s' % (self.source_expressions[0].name, self.name.lower()) + raise TypeError("Complex expressions require an alias") + + def get_group_by_cols(self): + return [] - def _default_alias(self): - return '%s__%s' % (self.lookup, self.name.lower()) - default_alias = property(_default_alias) + def _patch_aggregate(self, query): + """ + Helper method for patching 3rd party aggregates that do not yet support + the new way of subclassing. This method should be removed in 2.0 - def add_to_query(self, query, alias, col, source, is_summary): - """Add the aggregate to the nominated query. + add_to_query(query, alias, col, source, is_summary) will be defined on + legacy aggregates which, in turn, instantiates the SQL implementation of + the aggregate. In all the cases found, the general implementation of + add_to_query looks like: - This method is used to convert the generic Aggregate definition into a - backend-specific definition. + def add_to_query(self, query, alias, col, source, is_summary): + klass = SQLImplementationAggregate + aggregate = klass(col, source=source, is_summary=is_summary, **self.extra) + query.aggregates[alias] = aggregate - * query is the backend-specific query instance to which the aggregate - is to be added. - * col is a column reference describing the subject field - of the aggregate. It can be an alias, or a tuple describing - a table and column name. - * source is the underlying field or aggregate definition for - the column reference. If the aggregate is not an ordinal or - computed type, this reference is used to determine the coerced - output type of the aggregate. - * is_summary is a boolean that is set True if the aggregate is a - summary value rather than an annotation. + By supplying a known alias, we can get the SQLAggregate out of the + aggregates dict, and use the sql_function and sql_template attributes + to patch *this* aggregate. """ - klass = getattr(query.aggregates_module, self.name) - aggregate = klass(col, source=source, is_summary=is_summary, **self.extra) - query.aggregates[alias] = aggregate + if not hasattr(self, 'add_to_query') or self.function is not None: + return + + placeholder_alias = "_XXXXXXXX_" + self.add_to_query(query, placeholder_alias, None, None, None) + sql_aggregate = query.aggregates.pop(placeholder_alias) + if 'sql_function' not in self.extra and hasattr(sql_aggregate, 'sql_function'): + self.extra['function'] = sql_aggregate.sql_function + + if hasattr(sql_aggregate, 'sql_template'): + self.extra['template'] = sql_aggregate.sql_template class Avg(Aggregate): + function = 'AVG' name = 'Avg' + def __init__(self, expression, **extra): + super(Avg, self).__init__(expression, output_field=FloatField(), **extra) + + def convert_value(self, value, connection): + if value is None: + return value + return float(value) + class Count(Aggregate): + function = 'COUNT' name = 'Count' + template = '%(function)s(%(distinct)s%(expressions)s)' + + def __init__(self, expression, distinct=False, **extra): + if expression == '*': + expression = Value(expression) + expression._output_field = IntegerField() + super(Count, self).__init__( + expression, distinct='DISTINCT ' if distinct else '', output_field=IntegerField(), **extra) + + def convert_value(self, value, connection): + if value is None: + return 0 + return int(value) class Max(Aggregate): + function = 'MAX' name = 'Max' class Min(Aggregate): + function = 'MIN' name = 'Min' class StdDev(Aggregate): name = 'StdDev' + def __init__(self, expression, sample=False, **extra): + self.function = 'STDDEV_SAMP' if sample else 'STDDEV_POP' + super(StdDev, self).__init__(expression, output_field=FloatField(), **extra) + + def convert_value(self, value, connection): + if value is None: + return value + return float(value) + class Sum(Aggregate): + function = 'SUM' name = 'Sum' class Variance(Aggregate): name = 'Variance' + + def __init__(self, expression, sample=False, **extra): + self.function = 'VAR_SAMP' if sample else 'VAR_POP' + super(Variance, self).__init__(expression, output_field=FloatField(), **extra) + + def convert_value(self, value, connection): + if value is None: + return value + return float(value) diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 5484ca7f47..22a5e3ab1e 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -1,14 +1,20 @@ +import copy import datetime -from django.db.models.aggregates import refs_aggregate +from django.core.exceptions import FieldError +from django.db.backends import utils as backend_utils +from django.db.models import fields from django.db.models.constants import LOOKUP_SEP -from django.utils import tree +from django.db.models.query_utils import refs_aggregate +from django.utils.functional import cached_property -class ExpressionNode(tree.Node): +class CombinableMixin(object): """ - Base class for all query expressions. + Provides the ability to combine one or two objects with + some connector. For example F('foo') + F('bar'). """ + # Arithmetic connectors ADD = '+' SUB = '-' @@ -25,44 +31,17 @@ class ExpressionNode(tree.Node): BITAND = '&' BITOR = '|' - def __init__(self, children=None, connector=None, negated=False): - if children is not None and len(children) > 1 and connector is None: - raise TypeError('You have to specify a connector.') - super(ExpressionNode, self).__init__(children, connector, negated) - def _combine(self, other, connector, reversed, node=None): if isinstance(other, datetime.timedelta): - return DateModifierNode([self, other], connector) - - if reversed: - obj = ExpressionNode([other], connector) - obj.add(node or self, connector) - else: - obj = node or ExpressionNode([self], connector) - obj.add(other, connector) - return obj - - def contains_aggregate(self, existing_aggregates): - if self.children: - return any(child.contains_aggregate(existing_aggregates) - for child in self.children - if hasattr(child, 'contains_aggregate')) - else: - return refs_aggregate(self.name.split(LOOKUP_SEP), - existing_aggregates) - - def prepare_database_save(self, unused): - return self - - ################### - # VISITOR METHODS # - ################### + return DateModifierNode(self, connector, other) - def prepare(self, evaluator, query, allow_joins): - return evaluator.prepare_node(self, query, allow_joins) + if not hasattr(other, 'resolve_expression'): + # everything must be resolvable to an expression + other = Value(other) - def evaluate(self, evaluator, qn, connection): - return evaluator.evaluate_node(self, qn, connection) + if reversed: + return Expression(other, connector, self) + return Expression(self, connector, other) ############# # OPERATORS # @@ -137,27 +116,240 @@ class ExpressionNode(tree.Node): ) -class F(ExpressionNode): +class ExpressionNode(CombinableMixin): """ - An expression representing the value of the given field. + Base class for all query expressions. """ - def __init__(self, name): - super(F, self).__init__(None, None, False) - self.name = name - def __deepcopy__(self, memodict): - obj = super(F, self).__deepcopy__(memodict) - obj.name = self.name - return obj + # aggregate specific fields + is_summary = False + + def __init__(self, output_field=None): + self._output_field = output_field + + def get_source_expressions(self): + return [] + + def set_source_expressions(self, exprs): + assert len(exprs) == 0 + + def as_sql(self, compiler, connection): + """ + Responsible for returning a (sql, [params]) tuple to be included + in the current query. + + Different backends can provide their own implementation, by + providing an `as_{vendor}` method and patching the Expression: + + ``` + def override_as_sql(self, compiler, connection): + # custom logic + return super(ExpressionNode, self).as_sql(compiler, connection) + setattr(ExpressionNode, 'as_' + connection.vendor, override_as_sql) + ``` - def prepare(self, evaluator, query, allow_joins): - return evaluator.prepare_leaf(self, query, allow_joins) + Arguments: + * compiler: the query compiler responsible for generating the query. + Must have a compile method, returning a (sql, [params]) tuple. + Calling compiler(value) will return a quoted `value`. - def evaluate(self, evaluator, qn, connection): - return evaluator.evaluate_leaf(self, qn, connection) + * connection: the database connection used for the current query. + Returns: (sql, params) + Where `sql` is a string containing ordered sql parameters to be + replaced with the elements of the list `params`. + """ + raise NotImplementedError("Subclasses must implement as_sql()") + + @cached_property + def contains_aggregate(self): + for expr in self.get_source_expressions(): + if expr and expr.contains_aggregate: + return True + return False + + def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False): + """ + Provides the chance to do any preprocessing or validation before being + added to the query. + + Arguments: + * query: the backend query implementation + * allow_joins: boolean allowing or denying use of joins + in this query + * reuse: a set of reusable joins for multijoins + * summarize: a terminal aggregate clause + + Returns: an ExpressionNode to be added to the query. + """ + c = self.copy() + c.is_summary = summarize + return c + + def _prepare(self): + """ + Hook used by Field.get_prep_lookup() to do custom preparation. + """ + return self -class DateModifierNode(ExpressionNode): + @property + def field(self): + return self.output_field + + @cached_property + def output_field(self): + """ + Returns the output type of this expressions. + """ + if self._output_field_or_none is None: + raise FieldError("Cannot resolve expression type, unknown output_field") + return self._output_field_or_none + + @cached_property + def _output_field_or_none(self): + """ + Returns the output field of this expression, or None if no output type + can be resolved. Note that the 'output_field' property will raise + FieldError if no type can be resolved, but this attribute allows for + None values. + """ + if self._output_field is None: + self._resolve_output_field() + return self._output_field + + def _resolve_output_field(self): + """ + Attempts to infer the output type of the expression. If the output + fields of all source fields match then we can simply infer the same + type here. + """ + if self._output_field is None: + sources = self.get_source_fields() + num_sources = len(sources) + if num_sources == 0: + self._output_field = None + else: + self._output_field = sources[0] + for source in sources: + if source is not None and not isinstance(self._output_field, source.__class__): + raise FieldError( + "Expression contains mixed types. You must set output_field") + + def convert_value(self, value, connection): + """ + Expressions provide their own converters because users have the option + of manually specifying the output_field which may be a different type + from the one the database returns. + """ + field = self.output_field + internal_type = field.get_internal_type() + if value is None: + return value + elif internal_type == 'FloatField': + return float(value) + elif internal_type.endswith('IntegerField'): + return int(value) + elif internal_type == 'DecimalField': + return backend_utils.typecast_decimal(field.format_number(value)) + return value + + def get_lookup(self, lookup): + return self.output_field.get_lookup(lookup) + + def get_transform(self, name): + return self.output_field.get_transform(name) + + def relabeled_clone(self, change_map): + clone = self.copy() + clone.set_source_expressions( + [e.relabeled_clone(change_map) for e in self.get_source_expressions()]) + return clone + + def copy(self): + c = copy.copy(self) + c.copied = True + return c + + def refs_aggregate(self, existing_aggregates): + """ + Does this expression contain a reference to some of the + existing aggregates? If so, returns the aggregate and also + the lookup parts that *weren't* found. So, if + exsiting_aggregates = {'max_id': Max('id')} + self.name = 'max_id' + queryset.filter(max_id__range=[10,100]) + then this method will return Max('id') and those parts of the + name that weren't found. In this case `max_id` is found and the range + portion is returned as ('range',). + """ + for node in self.get_source_expressions(): + agg, lookup = node.refs_aggregate(existing_aggregates) + if agg: + return agg, lookup + return False, () + + def refs_field(self, aggregate_types, field_types): + """ + Helper method for check_aggregate_support on backends + """ + return any( + node.refs_field(aggregate_types, field_types) + for node in self.get_source_expressions()) + + def prepare_database_save(self, field): + return self + + def get_group_by_cols(self): + cols = [] + for source in self.get_source_expressions(): + cols.extend(source.get_group_by_cols()) + return cols + + def get_source_fields(self): + """ + Returns the underlying field types used by this + aggregate. + """ + return [e._output_field_or_none for e in self.get_source_expressions()] + + +class Expression(ExpressionNode): + + def __init__(self, lhs, connector, rhs, output_field=None): + super(Expression, self).__init__(output_field=output_field) + self.connector = connector + self.lhs = lhs + self.rhs = rhs + + def get_source_expressions(self): + return [self.lhs, self.rhs] + + def set_source_expressions(self, exprs): + self.lhs, self.rhs = exprs + + def as_sql(self, compiler, connection): + expressions = [] + expression_params = [] + sql, params = compiler.compile(self.lhs) + expressions.append(sql) + expression_params.extend(params) + sql, params = compiler.compile(self.rhs) + expressions.append(sql) + expression_params.extend(params) + # order of precedence + expression_wrapper = '(%s)' + sql = connection.ops.combine_expression(self.connector, expressions) + return expression_wrapper % sql, expression_params + + def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False): + c = self.copy() + c.is_summary = summarize + c.lhs = c.lhs.resolve_expression(query, allow_joins, reuse, summarize) + c.rhs = c.rhs.resolve_expression(query, allow_joins, reuse, summarize) + return c + + +class DateModifierNode(Expression): """ Node that implements the following syntax: filter(end_date__gt=F('start_date') + datetime.timedelta(days=3, seconds=200)) @@ -183,14 +375,195 @@ class DateModifierNode(ExpressionNode): Only adding and subtracting timedeltas is supported, attempts to use other operations raise a TypeError. """ - def __init__(self, children, connector, negated=False): - if len(children) != 2: - raise TypeError('Must specify a node and a timedelta.') - if not isinstance(children[1], datetime.timedelta): - raise TypeError('Second child must be a timedelta.') + def __init__(self, lhs, connector, rhs): + if not isinstance(rhs, datetime.timedelta): + raise TypeError('rhs must be a timedelta.') if connector not in (self.ADD, self.SUB): raise TypeError('Connector must be + or -, not %s' % connector) - super(DateModifierNode, self).__init__(children, connector, negated) + super(DateModifierNode, self).__init__(lhs, connector, Value(rhs)) + + def as_sql(self, compiler, connection): + timedelta = self.rhs.value + sql, params = compiler.compile(self.lhs) + if (timedelta.days == timedelta.seconds == timedelta.microseconds == 0): + return sql, params + return connection.ops.date_interval_sql(sql, self.connector, timedelta), params + + +class F(CombinableMixin): + """ + An object capable of resolving references to existing query objects. + """ + def __init__(self, name): + """ + Arguments: + * name: the name of the field this expression references + """ + self.name = name + + def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False): + return query.resolve_ref(self.name, allow_joins, reuse, summarize) + + def refs_aggregate(self, existing_aggregates): + return refs_aggregate(self.name.split(LOOKUP_SEP), existing_aggregates) + + +class Func(ExpressionNode): + """ + A SQL function call. + """ + function = None + template = '%(function)s(%(expressions)s)' + arg_joiner = ', ' + + def __init__(self, *expressions, **extra): + output_field = extra.pop('output_field', None) + super(Func, self).__init__(output_field=output_field) + self.source_expressions = self._parse_expressions(*expressions) + self.extra = extra + + def get_source_expressions(self): + return self.source_expressions + + def set_source_expressions(self, exprs): + self.source_expressions = exprs + + def _parse_expressions(self, *expressions): + return [ + arg if hasattr(arg, 'resolve_expression') else F(arg) + for arg in expressions + ] + + def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False): + c = self.copy() + c.is_summary = summarize + for pos, arg in enumerate(c.source_expressions): + c.source_expressions[pos] = arg.resolve_expression(query, allow_joins, reuse, summarize) + return c + + def as_sql(self, compiler, connection, function=None, template=None): + sql_parts = [] + params = [] + for arg in self.source_expressions: + arg_sql, arg_params = compiler.compile(arg) + sql_parts.append(arg_sql) + params.extend(arg_params) + if function is None: + self.extra['function'] = self.extra.get('function', self.function) + else: + self.extra['function'] = function + self.extra['expressions'] = self.extra['field'] = self.arg_joiner.join(sql_parts) + template = template or self.extra.get('template', self.template) + return template % self.extra, params + + def copy(self): + copy = super(Func, self).copy() + copy.source_expressions = self.source_expressions[:] + copy.extra = self.extra.copy() + return copy + + +class Value(ExpressionNode): + """ + Represents a wrapped value as a node within an expression + """ + def __init__(self, value, output_field=None): + """ + Arguments: + * value: the value this expression represents. The value will be + added into the sql parameter list and properly quoted. + + * output_field: an instance of the model field type that this + expression will return, such as IntegerField() or CharField(). + """ + super(Value, self).__init__(output_field=output_field) + self.value = value + + def as_sql(self, compiler, connection): + return '%s', [self.value] + + +class Col(ExpressionNode): + def __init__(self, alias, target, source=None): + if source is None: + source = target + super(Col, self).__init__(output_field=source) + self.alias, self.target = alias, target + + def as_sql(self, qn, connection): + return "%s.%s" % (qn(self.alias), qn(self.target.column)), [] + + def relabeled_clone(self, relabels): + return self.__class__(relabels.get(self.alias, self.alias), self.target, self.output_field) + + def get_group_by_cols(self): + return [(self.alias, self.target.column)] + + +class Ref(ExpressionNode): + """ + Reference to column alias of the query. For example, Ref('sum_cost') in + qs.annotate(sum_cost=Sum('cost')) query. + """ + def __init__(self, refs, source): + super(Ref, self).__init__() + self.source = source + self.refs = refs + + def get_source_expressions(self): + return [self.source] + + def set_source_expressions(self, exprs): + self.source, = exprs + + def relabeled_clone(self, relabels): + return self + + def as_sql(self, compiler, connection): + return "%s" % compiler(self.refs), [] + + def get_group_by_cols(self): + return [(None, self.refs)] + + +class Date(ExpressionNode): + """ + Add a date selection column. + """ + def __init__(self, col, lookup_type): + super(Date, self).__init__(output_field=fields.DateField()) + self.col = col + self.lookup_type = lookup_type + + def get_source_expressions(self): + return [self.col] + + def set_source_expressions(self, exprs): + self.col, = self.exprs + + def as_sql(self, qn, connection): + sql, params = self.col.as_sql(qn, connection) + assert not(params) + return connection.ops.date_trunc_sql(self.lookup_type, sql), [] + + +class DateTime(ExpressionNode): + """ + Add a datetime selection column. + """ + def __init__(self, col, lookup_type, tzname): + super(DateTime, self).__init__(output_field=fields.DateTimeField()) + self.col = col + self.lookup_type = lookup_type + self.tzname = tzname + + def get_source_expressions(self): + return [self.col] + + def set_source_expressions(self, exprs): + self.col, = exprs - def evaluate(self, evaluator, qn, connection): - return evaluator.evaluate_date_modifier_node(self, qn, connection) + def as_sql(self, qn, connection): + sql, params = self.col.as_sql(qn, connection) + assert not(params) + return connection.ops.datetime_trunc_sql(self.lookup_type, sql, self.tzname) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 6eed25d96a..552f868de2 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -637,8 +637,6 @@ class Field(RegisterLookupMixin): """ Perform preliminary non-db specific lookup checks and conversions """ - if hasattr(value, 'prepare'): - return value.prepare() if hasattr(value, '_prepare'): return value._prepare() diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index b52d59394d..f7699f5152 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -13,7 +13,7 @@ from django.db.models.fields import (AutoField, Field, IntegerField, from django.db.models.lookups import IsNull from django.db.models.related import RelatedObject, PathInfo from django.db.models.query import QuerySet -from django.db.models.sql.datastructures import Col +from django.db.models.expressions import Col from django.utils.encoding import force_text, smart_text from django.utils import six from django.utils.translation import ugettext_lazy as _ diff --git a/django/db/models/query.py b/django/db/models/query.py index d35f54b0b5..a7474bbc45 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -154,8 +154,7 @@ class QuerySet(object): 2. sql/compiler.results_iter() - Returns one row at time. At this point the rows are still just tuples. In some cases the return values are converted to - Python values at this location (see resolve_columns(), - resolve_aggregate()). + Python values at this location. 3. self.iterator() - Responsible for turning the rows into model objects. """ @@ -241,7 +240,7 @@ class QuerySet(object): max_depth = self.query.max_depth extra_select = list(self.query.extra_select) - aggregate_select = list(self.query.aggregate_select) + annotation_select = list(self.query.annotation_select) only_load = self.query.get_loaded_field_names() fields = self.model._meta.concrete_fields @@ -282,7 +281,7 @@ class QuerySet(object): db = self.db compiler = self.query.get_compiler(using=db) index_start = len(extra_select) - aggregate_start = index_start + len(init_list) + annotation_start = index_start + len(init_list) if fill_cache: klass_info = get_klass_info(model_cls, max_depth=max_depth, @@ -290,18 +289,18 @@ class QuerySet(object): for row in compiler.results_iter(): if fill_cache: obj, _ = get_cached_row(row, index_start, db, klass_info, - offset=len(aggregate_select)) + offset=len(annotation_select)) else: - obj = model_cls.from_db(db, init_list, row[index_start:aggregate_start]) + obj = model_cls.from_db(db, init_list, row[index_start:annotation_start]) if extra_select: for i, k in enumerate(extra_select): setattr(obj, k, row[i]) - # Add the aggregates to the model - if aggregate_select: - for i, aggregate in enumerate(aggregate_select): - setattr(obj, aggregate, row[i + aggregate_start]) + # Add the annotations to the model + if annotation_select: + for i, annotation in enumerate(annotation_select): + setattr(obj, annotation, row[i + annotation_start]) # Add the known related objects to the model, if there are any if self._known_related_objects: @@ -330,13 +329,16 @@ class QuerySet(object): if self.query.distinct_fields: raise NotImplementedError("aggregate() + distinct(fields) not implemented.") for arg in args: + if not hasattr(arg, 'default_alias'): + raise TypeError("Complex aggregates require an alias") kwargs[arg.default_alias] = arg query = self.query.clone() force_subq = query.low_mark != 0 or query.high_mark is not None for (alias, aggregate_expr) in kwargs.items(): - query.add_aggregate(aggregate_expr, self.model, alias, - is_summary=True) + query.add_annotation(aggregate_expr, self.model, alias, is_summary=True) + if not query.annotations[alias].contains_aggregate: + raise TypeError("%s is not an aggregate expression" % alias) return query.get_aggregation(using=self.db, force_subq=force_subq) def count(self): @@ -787,33 +789,40 @@ class QuerySet(object): def annotate(self, *args, **kwargs): """ Return a query set in which the returned objects have been annotated - with data aggregated from related fields. + with extra data or aggregations. """ - aggrs = OrderedDict() # To preserve ordering of args + annotations = OrderedDict() # To preserve ordering of args for arg in args: - if arg.default_alias in kwargs: - raise ValueError("The named annotation '%s' conflicts with the " - "default name for another annotation." - % arg.default_alias) - aggrs[arg.default_alias] = arg - aggrs.update(kwargs) + try: + # we can't do an hasattr here because py2 returns False + # if default_alias exists but throws a TypeError + if arg.default_alias in kwargs: + raise ValueError("The named annotation '%s' conflicts with the " + "default name for another annotation." + % arg.default_alias) + except AttributeError: # default_alias + raise TypeError("Complex annotations require an alias") + annotations[arg.default_alias] = arg + annotations.update(kwargs) + obj = self._clone() names = getattr(self, '_fields', None) if names is None: names = set(self.model._meta.get_all_field_names()) - for aggregate in aggrs: - if aggregate in names: - raise ValueError("The annotation '%s' conflicts with a field on " - "the model." % aggregate) - obj = self._clone() - - obj._setup_aggregate_query(list(aggrs)) - - # Add the aggregates to the query - for (alias, aggregate_expr) in aggrs.items(): - obj.query.add_aggregate(aggregate_expr, self.model, alias, - is_summary=False) + # Add the annotations to the query + for alias, annotation in annotations.items(): + if alias in names: + raise ValueError("The annotation '%s' conflicts with a field on " + "the model." % alias) + obj.query.add_annotation(annotation, self.model, alias, is_summary=False) + # expressions need to be added to the query before we know if they contain aggregates + added_aggregates = [] + for alias, annotation in obj.query.annotations.items(): + if alias in annotations and annotation.contains_aggregate: + added_aggregates.append(alias) + if added_aggregates: + obj._setup_aggregate_query(list(added_aggregates)) return obj @@ -1096,9 +1105,9 @@ class ValuesQuerySet(QuerySet): # Purge any extra columns that haven't been explicitly asked for extra_names = list(self.query.extra_select) field_names = self.field_names - aggregate_names = list(self.query.aggregate_select) + annotation_names = list(self.query.annotation_select) - names = extra_names + field_names + aggregate_names + names = extra_names + field_names + annotation_names for row in self.query.get_compiler(self.db).results_iter(): yield dict(zip(names, row)) @@ -1122,9 +1131,9 @@ class ValuesQuerySet(QuerySet): if self._fields: self.extra_names = [] - self.aggregate_names = [] - if not self.query._extra and not self.query._aggregates: - # Short cut - if there are no extra or aggregates, then + self.annotation_names = [] + if not self.query._extra and not self.query._annotations: + # Short cut - if there are no extra or annotations, then # the values() clause must be just field names. self.field_names = list(self._fields) else: @@ -1136,22 +1145,22 @@ class ValuesQuerySet(QuerySet): # had selected previously. if self.query._extra and f in self.query._extra: self.extra_names.append(f) - elif f in self.query.aggregate_select: - self.aggregate_names.append(f) + elif f in self.query.annotation_select: + self.annotation_names.append(f) else: self.field_names.append(f) else: # Default to all fields. self.extra_names = None self.field_names = [f.attname for f in self.model._meta.concrete_fields] - self.aggregate_names = None + self.annotation_names = None self.query.select = [] if self.extra_names is not None: self.query.set_extra_mask(self.extra_names) self.query.add_fields(self.field_names, True) - if self.aggregate_names is not None: - self.query.set_aggregate_mask(self.aggregate_names) + if self.annotation_names is not None: + self.query.set_annotation_mask(self.annotation_names) def _clone(self, klass=None, setup=False, **kwargs): """ @@ -1164,7 +1173,7 @@ class ValuesQuerySet(QuerySet): c._fields = self._fields[:] c.field_names = self.field_names c.extra_names = self.extra_names - c.aggregate_names = self.aggregate_names + c.annotation_names = self.annotation_names if setup and hasattr(c, '_setup_query'): c._setup_query() return c @@ -1173,7 +1182,7 @@ class ValuesQuerySet(QuerySet): super(ValuesQuerySet, self)._merge_sanity_check(other) if (set(self.extra_names) != set(other.extra_names) or set(self.field_names) != set(other.field_names) or - self.aggregate_names != other.aggregate_names): + self.annotation_names != other.annotation_names): raise TypeError("Merging '%s' classes must involve the same values in each case." % self.__class__.__name__) @@ -1183,9 +1192,9 @@ class ValuesQuerySet(QuerySet): """ self.query.set_group_by() - if self.aggregate_names is not None: - self.aggregate_names.extend(aggregates) - self.query.set_aggregate_mask(self.aggregate_names) + if self.annotation_names is not None: + self.annotation_names.extend(aggregates) + self.query.set_annotation_mask(self.annotation_names) super(ValuesQuerySet, self)._setup_aggregate_query(aggregates) @@ -1231,7 +1240,7 @@ class ValuesListQuerySet(ValuesQuerySet): if self.flat and len(self._fields) == 1: for row in self.query.get_compiler(self.db).results_iter(): yield row[0] - elif not self.query.extra_select and not self.query.aggregate_select: + elif not self.query.extra_select and not self.query.annotation_select: for row in self.query.get_compiler(self.db).results_iter(): yield tuple(row) else: @@ -1240,14 +1249,14 @@ class ValuesListQuerySet(ValuesQuerySet): # the fields to match the order in self._fields. extra_names = list(self.query.extra_select) field_names = self.field_names - aggregate_names = list(self.query.aggregate_select) + annotation_names = list(self.query.annotation_select) - names = extra_names + field_names + aggregate_names + names = extra_names + field_names + annotation_names # If a field list has been specified, use it. Otherwise, use the - # full list of fields, including extras and aggregates. + # full list of fields, including extras and annotations. if self._fields: - fields = list(self._fields) + [f for f in aggregate_names if f not in self._fields] + fields = list(self._fields) + [f for f in annotation_names if f not in self._fields] else: fields = names diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py index a8699f6334..59cd453722 100644 --- a/django/db/models/query_utils.py +++ b/django/db/models/query_utils.py @@ -9,6 +9,7 @@ from __future__ import unicode_literals from django.apps import apps from django.db.backends import utils +from django.db.models.constants import LOOKUP_SEP from django.utils import six from django.utils import tree @@ -220,3 +221,17 @@ def deferred_class_factory(model, attrs): # The above function is also used to unpickle model instances with deferred # fields. deferred_class_factory.__safe_for_unpickling__ = True + + +def refs_aggregate(lookup_parts, aggregates): + """ + A little helper method to check if the lookup_parts contains references + to the given aggregates set. Because the LOOKUP_SEP is contained in the + default annotation names we must check each prefix of the lookup_parts + for a match. + """ + for n in range(len(lookup_parts) + 1): + level_n_lookup = LOOKUP_SEP.join(lookup_parts[0:n]) + if level_n_lookup in aggregates and aggregates[level_n_lookup].contains_aggregate: + return aggregates[level_n_lookup], lookup_parts[n:] + return False, () diff --git a/django/db/models/sql/aggregates.py b/django/db/models/sql/aggregates.py index 8274d43621..6ebf5fb966 100644 --- a/django/db/models/sql/aggregates.py +++ b/django/db/models/sql/aggregates.py @@ -2,15 +2,23 @@ Classes to represent the default SQL aggregate functions """ import copy +import warnings from django.db.models.fields import IntegerField, FloatField from django.db.models.lookups import RegisterLookupMixin +from django.utils.deprecation import RemovedInDjango20Warning from django.utils.functional import cached_property __all__ = ['Aggregate', 'Avg', 'Count', 'Max', 'Min', 'StdDev', 'Sum', 'Variance'] +warnings.warn( + "django.db.models.sql.aggregates is deprecated. Use " + "django.db.models.aggregates instead.", + RemovedInDjango20Warning, stacklevel=2) + + class Aggregate(RegisterLookupMixin): """ Default SQL Aggregate. diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 33fe343b5b..5f425a7543 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -4,12 +4,10 @@ from django.conf import settings from django.core.exceptions import FieldError from django.db.backends.utils import truncate_name from django.db.models.constants import LOOKUP_SEP -from django.db.models.expressions import ExpressionNode from django.db.models.query_utils import select_related_descend, QueryWrapper from django.db.models.sql.constants import (CURSOR, SINGLE, MULTI, NO_RESULTS, ORDER_DIR, GET_ITERATOR_CHUNK_SIZE, SelectInfo) from django.db.models.sql.datastructures import EmptyResultSet -from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.query import get_order_dir, Query from django.db.transaction import TransactionManagementError from django.db.utils import DatabaseError @@ -248,8 +246,8 @@ class SQLCompiler(object): aliases.update(new_aliases) max_name_length = self.connection.ops.max_name_length() - for alias, aggregate in self.query.aggregate_select.items(): - agg_sql, agg_params = self.compile(aggregate) + for alias, annotation in self.query.annotation_select.items(): + agg_sql, agg_params = self.compile(annotation) if alias is None: result.append(agg_sql) else: @@ -409,7 +407,7 @@ class SQLCompiler(object): group_by.append((str(field), [])) continue col, order = get_order_dir(field, asc) - if col in self.query.aggregate_select: + if col in self.query.annotation_select: result.append('%s %s' % (qn(col), order)) continue if '.' in field: @@ -718,25 +716,17 @@ class SQLCompiler(object): """ fields = None converters = None - has_aggregate_select = bool(self.query.aggregate_select) + has_annotation_select = bool(self.query.annotation_select) for rows in self.execute_sql(MULTI): for row in rows: - if has_aggregate_select: - loaded_fields = ( - self.query.get_loaded_field_names().get(self.query.model, set()) or - self.query.select - ) - aggregate_start = len(self.query.extra_select) + len(loaded_fields) - aggregate_end = aggregate_start + len(self.query.aggregate_select) if fields is None: # We only set this up here because # related_select_cols isn't populated until # execute_sql() has been called. - # We also include types of fields of related models that - # will be included via select_related() for the benefit - # of MySQL/MySQLdb when boolean fields are involved - # (#15040). + # If the field was deferred, exclude it from being passed + # into `get_converters` because it wasn't selected. + only_load = self.deferred_to_columns() # This code duplicates the logic for the order of fields # found in get_columns(). It would be nice to clean this up. @@ -746,30 +736,45 @@ class SQLCompiler(object): fields = self.query.get_meta().concrete_fields else: fields = [] - fields = fields + [f.field for f in self.query.related_select_cols] - # If the field was deferred, exclude it from being passed - # into `get_converters` because it wasn't selected. - only_load = self.deferred_to_columns() if only_load: - fields = [f for f in fields if f.model._meta.db_table not in only_load or - f.column in only_load[f.model._meta.db_table]] - if has_aggregate_select: - # pad None in to fields for aggregates - fields = fields[:aggregate_start] + [ - None for x in range(0, aggregate_end - aggregate_start) - ] + fields[aggregate_start:] + # strip deferred fields + fields = [ + f for f in fields if + f.model._meta.db_table not in only_load or + f.column in only_load[f.model._meta.db_table] + ] + + # annotations come before the related cols + if has_annotation_select: + # extra is always at the start of the field list + prepended_cols = len(self.query.extra_select) + annotation_start = len(fields) + prepended_cols + fields = fields + [ + anno.output_field for alias, anno in self.query.annotation_select.items()] + annotation_end = len(fields) + prepended_cols + + # add related fields + fields = fields + [ + # strip deferred + f.field for f in self.query.related_select_cols if + f.field.model._meta.db_table not in only_load or + f.field.column in only_load[f.field.model._meta.db_table] + ] + converters = self.get_converters(fields) + if has_annotation_select: + for (alias, annotation), position in zip( + self.query.annotation_select.items(), + range(annotation_start, annotation_end + 1)): + if position in converters: + # annotation conversions always run first + converters[position][1].insert(0, annotation.convert_value) + else: + converters[position] = ([], [annotation.convert_value], annotation.output_field) + if converters: row = self.apply_converters(row, converters) - - if has_aggregate_select: - row = tuple(row[:aggregate_start]) + tuple( - self.query.resolve_aggregate(value, aggregate, self.connection) - for (alias, aggregate), value - in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end]) - ) + tuple(row[aggregate_end:]) - yield row def has_results(self): @@ -878,7 +883,7 @@ class SQLInsertCompiler(SQLCompiler): elif hasattr(field, 'get_placeholder'): # Some fields (e.g. geo fields) need special munging before # they can be inserted. - return field.get_placeholder(val, self.connection) + return field.get_placeholder(val, self, self.connection) else: # Return the common case for the placeholder return '%s' @@ -985,8 +990,10 @@ class SQLUpdateCompiler(SQLCompiler): result.append('SET') values, update_params = [], [] for field, model, val in self.query.values: - if hasattr(val, 'prepare_database_save'): - if field.rel or isinstance(val, ExpressionNode): + if hasattr(val, 'resolve_expression'): + val = val.resolve_expression(self.query, allow_joins=False) + elif hasattr(val, 'prepare_database_save'): + if field.rel: val = val.prepare_database_save(field) else: raise TypeError("Database is trying to update a relational field " @@ -998,12 +1005,9 @@ class SQLUpdateCompiler(SQLCompiler): # Getting the placeholder for the field. if hasattr(field, 'get_placeholder'): - placeholder = field.get_placeholder(val, self.connection) + placeholder = field.get_placeholder(val, self, self.connection) else: placeholder = '%s' - - if hasattr(val, 'evaluate'): - val = SQLEvaluator(val, self.query, allow_joins=False) name = field.column if hasattr(val, 'as_sql'): sql, params = self.compile(val) @@ -1103,8 +1107,8 @@ class SQLAggregateCompiler(SQLCompiler): qn = self sql, params = [], [] - for aggregate in self.query.aggregate_select.values(): - agg_sql, agg_params = self.compile(aggregate) + for annotation in self.query.annotation_select.values(): + agg_sql, agg_params = self.compile(annotation) sql.append(agg_sql) params.extend(agg_params) sql = ', '.join(sql) diff --git a/django/db/models/sql/datastructures.py b/django/db/models/sql/datastructures.py index f9c9c259de..321451ac42 100644 --- a/django/db/models/sql/datastructures.py +++ b/django/db/models/sql/datastructures.py @@ -4,33 +4,6 @@ the SQL domain. """ -class Col(object): - def __init__(self, alias, target, source): - self.alias, self.target, self.source = alias, target, source - - def as_sql(self, qn, connection): - return "%s.%s" % (qn(self.alias), qn(self.target.column)), [] - - @property - def output_field(self): - return self.source - - def relabeled_clone(self, relabels): - return self.__class__(relabels.get(self.alias, self.alias), self.target, self.source) - - def get_group_by_cols(self): - return [(self.alias, self.target.column)] - - def get_lookup(self, name): - return self.output_field.get_lookup(name) - - def get_transform(self, name): - return self.output_field.get_transform(name) - - def prepare(self): - return self - - class EmptyResultSet(Exception): pass @@ -49,42 +22,3 @@ class MultiJoin(Exception): class Empty(object): pass - - -class Date(object): - """ - Add a date selection column. - """ - def __init__(self, col, lookup_type): - self.col = col - self.lookup_type = lookup_type - - def relabeled_clone(self, change_map): - return self.__class__((change_map.get(self.col[0], self.col[0]), self.col[1])) - - def as_sql(self, qn, connection): - if isinstance(self.col, (list, tuple)): - col = '%s.%s' % tuple(qn(c) for c in self.col) - else: - col = self.col - return connection.ops.date_trunc_sql(self.lookup_type, col), [] - - -class DateTime(object): - """ - Add a datetime selection column. - """ - def __init__(self, col, lookup_type, tzname): - self.col = col - self.lookup_type = lookup_type - self.tzname = tzname - - def relabeled_clone(self, change_map): - return self.__class__((change_map.get(self.col[0], self.col[0]), self.col[1])) - - def as_sql(self, qn, connection): - if isinstance(self.col, (list, tuple)): - col = '%s.%s' % tuple(qn(c) for c in self.col) - else: - col = self.col - return connection.ops.datetime_trunc_sql(self.lookup_type, col, self.tzname) diff --git a/django/db/models/sql/expressions.py b/django/db/models/sql/expressions.py deleted file mode 100644 index e15cc2642c..0000000000 --- a/django/db/models/sql/expressions.py +++ /dev/null @@ -1,119 +0,0 @@ -import copy - -from django.core.exceptions import FieldError -from django.db.models.constants import LOOKUP_SEP -from django.db.models.fields import FieldDoesNotExist - - -class SQLEvaluator(object): - def __init__(self, expression, query, allow_joins=True, reuse=None): - self.expression = expression - self.opts = query.get_meta() - self.reuse = reuse - self.cols = [] - self.expression.prepare(self, query, allow_joins) - - def relabeled_clone(self, change_map): - clone = copy.copy(self) - clone.cols = [] - for node, col in self.cols: - if hasattr(col, 'relabeled_clone'): - clone.cols.append((node, col.relabeled_clone(change_map))) - else: - clone.cols.append((node, - (change_map.get(col[0], col[0]), col[1]))) - return clone - - def get_group_by_cols(self): - cols = [] - for node, col in self.cols: - if hasattr(node, 'get_group_by_cols'): - cols.extend(node.get_group_by_cols()) - elif isinstance(col, tuple): - cols.append(col) - return cols - - def prepare(self): - return self - - def as_sql(self, qn, connection): - return self.expression.evaluate(self, qn, connection) - - ##################################################### - # Visitor methods for initial expression preparation # - ##################################################### - - def prepare_node(self, node, query, allow_joins): - for child in node.children: - if hasattr(child, 'prepare'): - child.prepare(self, query, allow_joins) - - def prepare_leaf(self, node, query, allow_joins): - if not allow_joins and LOOKUP_SEP in node.name: - raise FieldError("Joined field references are not permitted in this query") - - field_list = node.name.split(LOOKUP_SEP) - if node.name in query.aggregates: - self.cols.append((node, query.aggregate_select[node.name])) - else: - try: - _, sources, _, join_list, path = query.setup_joins( - field_list, query.get_meta(), query.get_initial_alias(), - can_reuse=self.reuse) - self._used_joins = join_list - targets, _, join_list = query.trim_joins(sources, join_list, path) - if self.reuse is not None: - self.reuse.update(join_list) - for t in targets: - self.cols.append((node, (join_list[-1], t.column))) - except FieldDoesNotExist: - raise FieldError("Cannot resolve keyword %r into field. " - "Choices are: %s" % (self.name, - [f.name for f in self.opts.fields])) - - ################################################## - # Visitor methods for final expression evaluation # - ################################################## - - def evaluate_node(self, node, qn, connection): - expressions = [] - expression_params = [] - for child in node.children: - if hasattr(child, 'evaluate'): - sql, params = child.evaluate(self, qn, connection) - else: - sql, params = '%s', (child,) - - if len(getattr(child, 'children', [])) > 1: - format = '(%s)' - else: - format = '%s' - - if sql: - expressions.append(format % sql) - expression_params.extend(params) - - return connection.ops.combine_expression(node.connector, expressions), expression_params - - def evaluate_leaf(self, node, qn, connection): - col = None - for n, c in self.cols: - if n is node: - col = c - break - if col is None: - raise ValueError("Given node not found") - if hasattr(col, 'as_sql'): - return col.as_sql(qn, connection) - else: - return '%s.%s' % (qn(col[0]), qn(col[1])), [] - - def evaluate_date_modifier_node(self, node, qn, connection): - timedelta = node.children.pop() - sql, params = self.evaluate_node(node, qn, connection) - node.children.append(timedelta) - - if (timedelta.days == timedelta.seconds == timedelta.microseconds == 0): - return sql, params - - return connection.ops.date_interval_sql(sql, node.connector, timedelta), params diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 856bc51f4f..a17cd62f29 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -14,20 +14,18 @@ import warnings from django.core.exceptions import FieldError from django.db import connections, DEFAULT_DB_ALIAS from django.db.models.constants import LOOKUP_SEP -from django.db.models.aggregates import refs_aggregate -from django.db.models.expressions import ExpressionNode +from django.db.models.expressions import Col, Ref from django.db.models.fields import FieldDoesNotExist -from django.db.models.query_utils import Q +from django.db.models.query_utils import Q, refs_aggregate from django.db.models.related import PathInfo -from django.db.models.sql import aggregates as base_aggregates_module +from django.db.models.aggregates import Count from django.db.models.sql.constants import (QUERY_TERMS, ORDER_DIR, SINGLE, ORDER_PATTERN, JoinInfo, SelectInfo) -from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin, Col -from django.db.models.sql.expressions import SQLEvaluator +from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode, ExtraWhere, AND, OR, EmptyWhere) from django.utils import six -from django.utils.deprecation import RemovedInDjango19Warning +from django.utils.deprecation import RemovedInDjango19Warning, RemovedInDjango20Warning from django.utils.encoding import force_text from django.utils.tree import Node @@ -49,7 +47,7 @@ class RawQuery(object): # the compiler can be used to process results. self.low_mark, self.high_mark = 0, None # Used for offset/limit self.extra_select = {} - self.aggregate_select = {} + self.annotation_select = {} def clone(self, using): return RawQuery(self.sql, using, params=self.params) @@ -97,7 +95,6 @@ class Query(object): alias_prefix = 'T' subq_aliases = frozenset([alias_prefix]) query_terms = QUERY_TERMS - aggregates_module = base_aggregates_module compiler = 'SQLCompiler' @@ -140,13 +137,13 @@ class Query(object): self.select_for_update_nowait = False self.select_related = False - # SQL aggregate-related attributes - # The _aggregates will be an OrderedDict when used. Due to the cost + # SQL annotation-related attributes + # The _annotations will be an OrderedDict when used. Due to the cost # of creating OrderedDict this attribute is created lazily (in - # self.aggregates property). - self._aggregates = None # Maps alias -> SQL aggregate function - self.aggregate_select_mask = None - self._aggregate_select_cache = None + # self.annotations property). + self._annotations = None # Maps alias -> Annotation Expression + self.annotation_select_mask = None + self._annotation_select_cache = None # Arbitrary maximum limit for select_related. Prevents infinite # recursion. Can be changed by the depth parameter to select_related(). @@ -155,7 +152,7 @@ class Query(object): # These are for extensions. The contents are more or less appended # verbatim to the appropriate clause. # The _extra attribute is an OrderedDict, lazily created similarly to - # .aggregates + # .annotations self._extra = None # Maps col_alias -> (col_sql, params). self.extra_select_mask = None self._extra_select_cache = None @@ -175,10 +172,17 @@ class Query(object): return self._extra @property + def annotations(self): + if self._annotations is None: + self._annotations = OrderedDict() + return self._annotations + + @property def aggregates(self): - if self._aggregates is None: - self._aggregates = OrderedDict() - return self._aggregates + warnings.warn( + "The aggregates property is deprecated. Use annotations instead.", + RemovedInDjango20Warning, stacklevel=2) + return self.annotations def __str__(self): """ @@ -203,7 +207,7 @@ class Query(object): memo[id(self)] = result return result - def prepare(self): + def _prepare(self): return self def get_compiler(self, using=None, connection=None): @@ -213,8 +217,8 @@ class Query(object): connection = connections[using] # Check that the compiler will be able to execute the query - for alias, aggregate in self.aggregate_select.items(): - connection.ops.check_aggregate_support(aggregate) + for alias, annotation in self.annotation_select.items(): + connection.ops.check_aggregate_support(annotation) return connection.ops.compiler(self.compiler)(self, connection, using) @@ -260,17 +264,17 @@ class Query(object): obj.select_for_update_nowait = self.select_for_update_nowait obj.select_related = self.select_related obj.related_select_cols = [] - obj._aggregates = self._aggregates.copy() if self._aggregates is not None else None - if self.aggregate_select_mask is None: - obj.aggregate_select_mask = None + obj._annotations = self._annotations.copy() if self._annotations is not None else None + if self.annotation_select_mask is None: + obj.annotation_select_mask = None else: - obj.aggregate_select_mask = self.aggregate_select_mask.copy() - # _aggregate_select_cache cannot be copied, as doing so breaks the - # (necessary) state in which both aggregates and - # _aggregate_select_cache point to the same underlying objects. + obj.annotation_select_mask = self.annotation_select_mask.copy() + # _annotation_select_cache cannot be copied, as doing so breaks the + # (necessary) state in which both annotations and + # _annotation_select_cache point to the same underlying objects. # It will get re-populated in the cloned queryset the next time it's # used. - obj._aggregate_select_cache = None + obj._annotation_select_cache = None obj.max_depth = self.max_depth obj._extra = self._extra.copy() if self._extra is not None else None if self.extra_select_mask is None: @@ -299,94 +303,84 @@ class Query(object): obj._setup_query() return obj - def resolve_aggregate(self, value, aggregate, connection): - """Resolve the value of aggregates returned by the database to - consistent (and reasonable) types. - - This is required because of the predisposition of certain backends - to return Decimal and long types when they are not needed. - """ - if value is None: - if aggregate.is_ordinal: - return 0 - # Return None as-is - return value - elif aggregate.is_ordinal: - # Any ordinal aggregate (e.g., count) returns an int - return int(value) - elif aggregate.is_computed: - # Any computed aggregate (e.g., avg) returns a float - return float(value) - else: - # Return value depends on the type of the field being processed. - backend_converters = connection.ops.get_db_converters(aggregate.field.get_internal_type()) - field_converters = aggregate.field.get_db_converters(connection) - for converter in backend_converters: - value = converter(value, aggregate.field) - for converter in field_converters: - value = converter(value, connection) - return value - def get_aggregation(self, using, force_subq=False): """ Returns the dictionary with the values of the existing aggregations. """ - if not self.aggregate_select: + if not self.annotation_select: return {} + # annotations must be forced into subquery + has_annotation = any( + annotation for alias, annotation + in self.annotation_select.items() + if not annotation.contains_aggregate) + # If there is a group by clause, aggregating does not add useful # information but retrieves only the first row. Aggregate # over the subquery instead. - if self.group_by is not None or force_subq: + if self.group_by is not None or force_subq or has_annotation: from django.db.models.sql.subqueries import AggregateQuery - query = AggregateQuery(self.model) - obj = self.clone() + outer_query = AggregateQuery(self.model) + inner_query = self.clone() if not force_subq: # In forced subq case the ordering and limits will likely # affect the results. - obj.clear_ordering(True) - obj.clear_limits() - obj.select_for_update = False - obj.select_related = False - obj.related_select_cols = [] + inner_query.clear_ordering(True) + inner_query.clear_limits() + inner_query.select_for_update = False + inner_query.select_related = False + inner_query.related_select_cols = [] - relabels = dict((t, 'subquery') for t in self.tables) + relabels = dict((t, 'subquery') for t in inner_query.tables) + relabels[None] = 'subquery' # Remove any aggregates marked for reduction from the subquery # and move them to the outer AggregateQuery. - for alias, aggregate in self.aggregate_select.items(): - if aggregate.is_summary: - query.aggregates[alias] = aggregate.relabeled_clone(relabels) - del obj.aggregate_select[alias] - + for alias, annotation in inner_query.annotation_select.items(): + if annotation.is_summary: + # The annotation is already referring the subquery alias, so we + # just need to move the annotation to the outer query. + outer_query.annotations[alias] = annotation.relabeled_clone(relabels) + del inner_query.annotation_select[alias] try: - query.add_subquery(obj, using) + outer_query.add_subquery(inner_query, using) except EmptyResultSet: return dict( (alias, None) - for alias in query.aggregate_select + for alias in outer_query.annotation_select ) else: - query = self + outer_query = self self.select = [] self.default_cols = False self._extra = {} self.remove_inherited_models() - query.clear_ordering(True) - query.clear_limits() - query.select_for_update = False - query.select_related = False - query.related_select_cols = [] - - result = query.get_compiler(using).execute_sql(SINGLE) + outer_query.clear_ordering(True) + outer_query.clear_limits() + outer_query.select_for_update = False + outer_query.select_related = False + outer_query.related_select_cols = [] + compiler = outer_query.get_compiler(using) + result = compiler.execute_sql(SINGLE) if result is None: - result = [None for q in query.aggregate_select.items()] + result = [None for q in outer_query.annotation_select.items()] + + fields = [annotation.output_field + for alias, annotation in outer_query.annotation_select.items()] + converters = compiler.get_converters(fields) + for position, (alias, annotation) in enumerate(outer_query.annotation_select.items()): + if position in converters: + converters[position][1].insert(0, annotation.convert_value) + else: + converters[position] = ([], [annotation.convert_value], annotation.output_field) + result = compiler.apply_converters(result, converters) return dict( - (alias, self.resolve_aggregate(val, aggregate, connection=connections[using])) - for (alias, aggregate), val - in zip(query.aggregate_select.items(), result) + (alias, val) + for (alias, annotation), val + in zip(outer_query.annotation_select.items(), result) ) def get_count(self, using): @@ -394,7 +388,7 @@ class Query(object): Performs a COUNT() query using the current filter constraints. """ obj = self.clone() - if len(self.select) > 1 or self.aggregate_select or (self.distinct and self.distinct_fields): + if len(self.select) > 1 or self.annotation_select or (self.distinct and self.distinct_fields): # If a select clause exists, then the query has already started to # specify the columns that are to be returned. # In this case, we need to use a subquery to evaluate the count. @@ -769,9 +763,9 @@ class Query(object): self.group_by = [relabel_column(col) for col in self.group_by] self.select = [SelectInfo(relabel_column(s.col), s.field) for s in self.select] - if self._aggregates: - self._aggregates = OrderedDict( - (key, relabel_column(col)) for key, col in self._aggregates.items()) + if self._annotations: + self._annotations = OrderedDict( + (key, relabel_column(col)) for key, col in self._annotations.items()) # 2. Rename the alias in the internal table/alias datastructures. for ident, aliases in self.join_map.items(): @@ -974,52 +968,18 @@ class Query(object): self.included_inherited_models = {} def add_aggregate(self, aggregate, model, alias, is_summary): + warnings.warn( + "add_aggregate() is deprecated. Use add_annotation() instead.", + RemovedInDjango20Warning, stacklevel=2) + self.add_annotation(aggregate, model, alias, is_summary) + + def add_annotation(self, annotation, model, alias, is_summary): """ - Adds a single aggregate expression to the Query + Adds a single annotation expression to the Query """ - opts = model._meta - field_list = aggregate.lookup.split(LOOKUP_SEP) - if len(field_list) == 1 and self._aggregates and aggregate.lookup in self.aggregates: - # Aggregate is over an annotation - field_name = field_list[0] - col = field_name - source = self.aggregates[field_name] - if not is_summary: - raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % ( - aggregate.name, field_name, field_name)) - elif ((len(field_list) > 1) or - (field_list[0] not in [i.name for i in opts.fields]) or - self.group_by is None or - not is_summary): - # If: - # - the field descriptor has more than one part (foo__bar), or - # - the field descriptor is referencing an m2m/m2o field, or - # - this is a reference to a model field (possibly inherited), or - # - this is an annotation over a model field - # then we need to explore the joins that are required. - - # Join promotion note - we must not remove any rows here, so use - # outer join if there isn't any existing join. - _, sources, opts, join_list, path = self.setup_joins( - field_list, opts, self.get_initial_alias()) - - # Process the join chain to see if it can be trimmed - targets, _, join_list = self.trim_joins(sources, join_list, path) - - col = targets[0].column - source = sources[0] - col = (join_list[-1], col) - else: - # The simplest cases. No joins required - - # just reference the provided column alias. - field_name = field_list[0] - source = opts.get_field(field_name) - col = field_name - # We want to have the alias in SELECT clause even if mask is set. - self.append_aggregate_mask([alias]) - - # Add the aggregate to the query - aggregate.add_to_query(self, alias, col=col, source=source, is_summary=is_summary) + annotation = annotation.resolve_expression(self, summarize=is_summary) + self.append_annotation_mask([alias]) + self.annotations[alias] = annotation def prepare_lookup_value(self, value, lookups, can_reuse): # Default lookup if none given is exact. @@ -1037,9 +997,8 @@ class Query(object): "Passing callable arguments to queryset is deprecated.", RemovedInDjango19Warning, stacklevel=2) value = value() - elif isinstance(value, ExpressionNode): - # If value is a query expression, evaluate it - value = SQLEvaluator(value, self, reuse=can_reuse) + elif hasattr(value, 'resolve_expression'): + value = value.resolve_expression(self, reuse=can_reuse) if hasattr(value, 'query') and hasattr(value.query, 'bump_prefix'): value = value._clone() value.query.bump_prefix(self) @@ -1061,8 +1020,8 @@ class Query(object): Solve the lookup type from the lookup (eg: 'foobar__id__icontains') """ lookup_splitted = lookup.split(LOOKUP_SEP) - if self._aggregates: - aggregate, aggregate_lookups = refs_aggregate(lookup_splitted, self.aggregates) + if self._annotations: + aggregate, aggregate_lookups = refs_aggregate(lookup_splitted, self.annotations) if aggregate: return aggregate_lookups, (), aggregate _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta()) @@ -1232,7 +1191,11 @@ class Query(object): lookup_type = lookups[-1] else: assert(len(targets) == 1) - col = Col(alias, targets[0], field) + if hasattr(targets[0], 'as_sql'): + # handle Expressions as annotations + col = targets[0] + else: + col = Col(alias, targets[0], field) condition = self.build_lookup(lookups, col, value) if not condition: # Backwards compat for custom lookups @@ -1278,12 +1241,12 @@ class Query(object): Returns whether or not all elements of this q_object need to be put together in the HAVING clause. """ - if not self._aggregates: + if not self._annotations: return False if not isinstance(obj, Node): - return (refs_aggregate(obj[0].split(LOOKUP_SEP), self.aggregates)[0] - or (hasattr(obj[1], 'contains_aggregate') - and obj[1].contains_aggregate(self.aggregates))) + return (refs_aggregate(obj[0].split(LOOKUP_SEP), self.annotations)[0] + or (hasattr(obj[1], 'refs_aggregate') + and obj[1].refs_aggregate(self.annotations)[0])) return any(self.need_having(c) for c in obj.children) def split_having_parts(self, q_object, negated=False): @@ -1390,13 +1353,21 @@ class Query(object): if name == 'pk': name = opts.pk.name try: - field, model, direct, m2m = opts.get_field_by_name(name) + field, model, _, _ = opts.get_field_by_name(name) except FieldDoesNotExist: + # is it an annotation? + if self._annotations and name in self._annotations: + field, model = self._annotations[name], None + if not field.contains_aggregate: + # Local non-relational field. + final_field = field + targets = (field,) + break # We didn't find the current field, so move position back # one step. pos -= 1 if pos == -1 or fail_on_missing: - available = opts.get_all_field_names() + list(self.aggregate_select) + available = opts.get_all_field_names() + list(self.annotation_select) raise FieldError("Cannot resolve keyword %r into field. " "Choices are: %s" % (name, ", ".join(available))) break @@ -1445,6 +1416,11 @@ class Query(object): break return path, final_field, targets, names[pos + 1:] + def raise_field_error(self, opts, name): + available = opts.get_all_field_names() + list(self.annotation_select) + raise FieldError("Cannot resolve keyword %r into field. " + "Choices are: %s" % (name, ", ".join(available))) + def setup_joins(self, names, opts, alias, can_reuse=None, allow_many=True): """ Compute the necessary table joins for the passage through the fields @@ -1519,6 +1495,29 @@ class Query(object): self.unref_alias(joins.pop()) return targets, joins[-1], joins + def resolve_ref(self, name, allow_joins, reuse, summarize): + if not allow_joins and LOOKUP_SEP in name: + raise FieldError("Joined field references are not permitted in this query") + if name in self.annotations: + if summarize: + return Ref(name, self.annotation_select[name]) + else: + return self.annotation_select[name] + else: + field_list = name.split(LOOKUP_SEP) + field, sources, opts, join_list, path = self.setup_joins( + field_list, self.get_meta(), + self.get_initial_alias(), reuse) + targets, _, join_list = self.trim_joins(sources, join_list, path) + if len(targets) > 1: + raise FieldError("Referencing multicolumn fields with F() objects " + "isn't supported") + if reuse is not None: + reuse.update(join_list) + col = Col(join_list[-1], targets[0], sources[0]) + col._used_joins = join_list + return col + def split_exclude(self, filter_expr, prefix, can_reuse, names_with_path): """ When doing an exclude against any kind of N-to-many relation, we need @@ -1633,7 +1632,7 @@ class Query(object): self.default_cols = False self.select_related = False self.set_extra_mask(()) - self.set_aggregate_mask(()) + self.set_annotation_mask(()) def clear_select_fields(self): """ @@ -1676,7 +1675,7 @@ class Query(object): raise else: names = sorted(opts.get_all_field_names() + list(self.extra) - + list(self.aggregate_select)) + + list(self.annotation_select)) raise FieldError("Cannot resolve keyword %r into field. " "Choices are: %s" % (name, ", ".join(names))) self.remove_inherited_models() @@ -1725,39 +1724,55 @@ class Query(object): for col, _ in self.select: self.group_by.append(col) + if self._annotations: + for alias, annotation in six.iteritems(self.annotations): + for col in annotation.get_group_by_cols(): + self.group_by.append(col) + def add_count_column(self): """ Converts the query to do count(...) or count(distinct(pk)) in order to get its size. """ + summarize = False if not self.distinct: if not self.select: - count = self.aggregates_module.Count('*', is_summary=True) + count = Count('*') + summarize = True else: assert len(self.select) == 1, \ "Cannot add count col with multiple cols in 'select': %r" % self.select - count = self.aggregates_module.Count(self.select[0].col) + col = self.select[0].col + if isinstance(col, (tuple, list)): + count = Count(col[1]) + else: + count = Count(col) + else: opts = self.get_meta() if not self.select: - count = self.aggregates_module.Count( - (self.join((None, opts.db_table, None)), opts.pk.column), - is_summary=True, distinct=True) + lookup = self.join((None, opts.db_table, None)), opts.pk.column + count = Count(lookup[1], distinct=True) + summarize = True else: # Because of SQL portability issues, multi-column, distinct # counts need a sub-query -- see get_count() for details. assert len(self.select) == 1, \ "Cannot add count col with multiple cols in 'select'." - - count = self.aggregates_module.Count(self.select[0].col, distinct=True) + col = self.select[0].col + if isinstance(col, (tuple, list)): + count = Count(col[1], distinct=True) + else: + count = Count(col, distinct=True) # Distinct handling is done in Count(), so don't do it at this # level. self.distinct = False # Set only aggregate to be the count column. - # Clear out the select cache to reflect the new unmasked aggregates. - self._aggregates = {None: count} - self.set_aggregate_mask(None) + # Clear out the select cache to reflect the new unmasked annotations. + count = count.resolve_expression(self, summarize=summarize) + self._annotations = {None: count} + self.set_annotation_mask(None) self.group_by = None def add_select_related(self, fields): @@ -1886,16 +1901,28 @@ class Query(object): target[model] = set(f.name for f in fields) def set_aggregate_mask(self, names): - "Set the mask of aggregates that will actually be returned by the SELECT" + warnings.warn( + "set_aggregate_mask() is deprecated. Use set_annotation_mask() instead.", + RemovedInDjango20Warning, stacklevel=2) + self.set_annotation_mask(names) + + def set_annotation_mask(self, names): + "Set the mask of annotations that will actually be returned by the SELECT" if names is None: - self.aggregate_select_mask = None + self.annotation_select_mask = None else: - self.aggregate_select_mask = set(names) - self._aggregate_select_cache = None + self.annotation_select_mask = set(names) + self._annotation_select_cache = None def append_aggregate_mask(self, names): - if self.aggregate_select_mask is not None: - self.set_aggregate_mask(set(names).union(self.aggregate_select_mask)) + warnings.warn( + "append_aggregate_mask() is deprecated. Use append_annotation_mask() instead.", + RemovedInDjango20Warning, stacklevel=2) + self.append_annotation_mask(names) + + def append_annotation_mask(self, names): + if self.annotation_select_mask is not None: + self.set_annotation_mask(set(names).union(self.annotation_select_mask)) def set_extra_mask(self, names): """ @@ -1910,24 +1937,31 @@ class Query(object): self._extra_select_cache = None @property - def aggregate_select(self): + def annotation_select(self): """The OrderedDict of aggregate columns that are not masked, and should be used in the SELECT clause. This result is cached for optimization purposes. """ - if self._aggregate_select_cache is not None: - return self._aggregate_select_cache - elif not self._aggregates: + if self._annotation_select_cache is not None: + return self._annotation_select_cache + elif not self._annotations: return {} - elif self.aggregate_select_mask is not None: - self._aggregate_select_cache = OrderedDict( - (k, v) for k, v in self.aggregates.items() - if k in self.aggregate_select_mask + elif self.annotation_select_mask is not None: + self._annotation_select_cache = OrderedDict( + (k, v) for k, v in self.annotations.items() + if k in self.annotation_select_mask ) - return self._aggregate_select_cache + return self._annotation_select_cache else: - return self.aggregates + return self.annotations + + @property + def aggregate_select(self): + warnings.warn( + "aggregate_select() is deprecated. Use annotation_select() instead.", + RemovedInDjango20Warning, stacklevel=2) + return self.annotation_select @property def extra_select(self): diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py index 2f0de5b80c..6f3f7358d3 100644 --- a/django/db/models/sql/subqueries.py +++ b/django/db/models/sql/subqueries.py @@ -7,9 +7,9 @@ from django.core.exceptions import FieldError from django.db import connections from django.db.models.query_utils import Q from django.db.models.constants import LOOKUP_SEP +from django.db.models.expressions import Date, DateTime, Col from django.db.models.fields import DateField, DateTimeField, FieldDoesNotExist from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, NO_RESULTS, SelectInfo -from django.db.models.sql.datastructures import Date, DateTime from django.db.models.sql.query import Query from django.utils import six from django.utils import timezone @@ -229,7 +229,7 @@ class DateQuery(Query): )) self._check_field(field) # overridden in DateTimeQuery alias = joins[-1] - select = self._get_select((alias, field.column), lookup_type) + select = self._get_select(Col(alias, field), lookup_type) self.clear_select_clause() self.select = [SelectInfo(select, None)] self.distinct = True diff --git a/django/db/models/sql/where.py b/django/db/models/sql/where.py index f65e593a3a..13815cb68c 100644 --- a/django/db/models/sql/where.py +++ b/django/db/models/sql/where.py @@ -10,7 +10,6 @@ import warnings from django.conf import settings from django.db.models.fields import DateTimeField, Field from django.db.models.sql.datastructures import EmptyResultSet, Empty -from django.db.models.sql.aggregates import Aggregate from django.utils.deprecation import RemovedInDjango19Warning from django.utils.six.moves import xrange from django.utils import timezone @@ -78,7 +77,7 @@ class WhereNode(tree.Node): else: value_annotation = bool(value) - if hasattr(obj, "prepare"): + if hasattr(obj, 'prepare'): value = obj.prepare(lookup_type, value) return (obj, lookup_type, value_annotation, value) @@ -187,11 +186,9 @@ class WhereNode(tree.Node): lvalue, params = lvalue.process(lookup_type, params_or_value, connection) except EmptyShortCircuit: raise EmptyResultSet - elif isinstance(lvalue, Aggregate): - params = lvalue.field.get_db_prep_lookup(lookup_type, params_or_value, connection) else: - raise TypeError("'make_atom' expects a Constraint or an Aggregate " - "as the first item of its 'child' argument.") + raise TypeError("'make_atom' expects a Constraint as the first " + "item of its 'child' argument.") if isinstance(lvalue, tuple): # A direct database column lookup. |