Fixed #18757, #14462, #21565 -- Reworked database-python type conversions

Complete rework of translating data values from database

Deprecation of SubfieldBase, removal of resolve_columns and
convert_values in favour of a more general converter based approach and
public API Field.from_db_value(). Now works seamlessly with aggregation,
.values() and raw queries.

Thanks to akaariai in particular for extensive advice and inspiration,
also to shaib, manfre and timograham for their reviews.

2 files changed, 67 insertions, 74 deletions

diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 82438355cb..ee5e319360 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -690,12 +690,34 @@ class SQLCompiler(object):
         self.query.deferred_to_data(columns, self.query.deferred_to_columns_cb)
         return columns
 
+    def get_converters(self, fields):
+        converters = {}
+        index_extra_select = len(self.query.extra_select)
+        for i, field in enumerate(fields):
+            if field:
+                backend_converters = self.connection.ops.get_db_converters(field.get_internal_type())
+                field_converters = field.get_db_converters(self.connection)
+                if backend_converters or field_converters:
+                    converters[index_extra_select + i] = (backend_converters, field_converters, field)
+        return converters
+
+    def apply_converters(self, row, converters):
+        row = list(row)
+        for pos, (backend_converters, field_converters, field) in converters.items():
+            value = row[pos]
+            for converter in backend_converters:
+                value = converter(value, field)
+            for converter in field_converters:
+                value = converter(value, self.connection)
+            row[pos] = value
+        return tuple(row)
+
     def results_iter(self):
         """
         Returns an iterator over the results from executing this query.
         """
-        resolve_columns = hasattr(self, 'resolve_columns')
         fields = None
+        converters = None
         has_aggregate_select = bool(self.query.aggregate_select)
         for rows in self.execute_sql(MULTI):
             for row in rows:
@@ -703,39 +725,40 @@ class SQLCompiler(object):
                     loaded_fields = self.query.get_loaded_field_names().get(self.query.model, set()) or self.query.select
                     aggregate_start = len(self.query.extra_select) + len(loaded_fields)
                     aggregate_end = aggregate_start + len(self.query.aggregate_select)
-                if resolve_columns:
-                    if fields is None:
-                        # We only set this up here because
-                        # related_select_cols isn't populated until
-                        # execute_sql() has been called.
+                if fields is None:
+                    # We only set this up here because
+                    # related_select_cols isn't populated until
+                    # execute_sql() has been called.
 
-                        # We also include types of fields of related models that
-                        # will be included via select_related() for the benefit
-                        # of MySQL/MySQLdb when boolean fields are involved
-                        # (#15040).
+                    # We also include types of fields of related models that
+                    # will be included via select_related() for the benefit
+                    # of MySQL/MySQLdb when boolean fields are involved
+                    # (#15040).
 
-                        # This code duplicates the logic for the order of fields
-                        # found in get_columns(). It would be nice to clean this up.
-                        if self.query.select:
-                            fields = [f.field for f in self.query.select]
-                        elif self.query.default_cols:
-                            fields = self.query.get_meta().concrete_fields
-                        else:
-                            fields = []
-                        fields = fields + [f.field for f in self.query.related_select_cols]
+                    # This code duplicates the logic for the order of fields
+                    # found in get_columns(). It would be nice to clean this up.
+                    if self.query.select:
+                        fields = [f.field for f in self.query.select]
+                    elif self.query.default_cols:
+                        fields = self.query.get_meta().concrete_fields
+                    else:
+                        fields = []
+                    fields = fields + [f.field for f in self.query.related_select_cols]
 
-                        # If the field was deferred, exclude it from being passed
-                        # into `resolve_columns` because it wasn't selected.
-                        only_load = self.deferred_to_columns()
-                        if only_load:
-                            fields = [f for f in fields if f.model._meta.db_table not in only_load or
-                                      f.column in only_load[f.model._meta.db_table]]
-                        if has_aggregate_select:
-                            # pad None in to fields for aggregates
-                            fields = fields[:aggregate_start] + [
-                                None for x in range(0, aggregate_end - aggregate_start)
-                            ] + fields[aggregate_start:]
-                    row = self.resolve_columns(row, fields)
+                    # If the field was deferred, exclude it from being passed
+                    # into `get_converters` because it wasn't selected.
+                    only_load = self.deferred_to_columns()
+                    if only_load:
+                        fields = [f for f in fields if f.model._meta.db_table not in only_load or
+                                  f.column in only_load[f.model._meta.db_table]]
+                    if has_aggregate_select:
+                        # pad None in to fields for aggregates
+                        fields = fields[:aggregate_start] + [
+                            None for x in range(0, aggregate_end - aggregate_start)
+                        ] + fields[aggregate_start:]
+                    converters = self.get_converters(fields)
+                if converters:
+                    row = self.apply_converters(row, converters)
 
                 if has_aggregate_select:
                     row = tuple(row[:aggregate_start]) + tuple(
@@ -1092,22 +1115,13 @@ class SQLDateCompiler(SQLCompiler):
         """
         Returns an iterator over the results from executing this query.
         """
-        resolve_columns = hasattr(self, 'resolve_columns')
-        if resolve_columns:
-            from django.db.models.fields import DateField
-            fields = [DateField()]
-        else:
-            from django.db.backends.utils import typecast_date
-            needs_string_cast = self.connection.features.needs_datetime_string_cast
+        from django.db.models.fields import DateField
+        converters = self.get_converters([DateField()])
 
         offset = len(self.query.extra_select)
         for rows in self.execute_sql(MULTI):
             for row in rows:
-                date = row[offset]
-                if resolve_columns:
-                    date = self.resolve_columns(row, fields)[offset]
-                elif needs_string_cast:
-                    date = typecast_date(str(date))
+                date = self.apply_converters(row, converters)[offset]
                 if isinstance(date, datetime.datetime):
                     date = date.date()
                 yield date
@@ -1118,22 +1132,13 @@ class SQLDateTimeCompiler(SQLCompiler):
         """
         Returns an iterator over the results from executing this query.
         """
-        resolve_columns = hasattr(self, 'resolve_columns')
-        if resolve_columns:
-            from django.db.models.fields import DateTimeField
-            fields = [DateTimeField()]
-        else:
-            from django.db.backends.utils import typecast_timestamp
-            needs_string_cast = self.connection.features.needs_datetime_string_cast
+        from django.db.models.fields import DateTimeField
+        converters = self.get_converters([DateTimeField()])
 
         offset = len(self.query.extra_select)
         for rows in self.execute_sql(MULTI):
             for row in rows:
-                datetime = row[offset]
-                if resolve_columns:
-                    datetime = self.resolve_columns(row, fields)[offset]
-                elif needs_string_cast:
-                    datetime = typecast_timestamp(str(datetime))
+                datetime = self.apply_converters(row, converters)[offset]
                 # Datetimes are artificially returned in UTC on databases that
                 # don't support time zone. Restore the zone used in the query.
                 if settings.USE_TZ:
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 9ec84d64cd..e6708715d3 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -54,15 +54,6 @@ class RawQuery(object):
     def clone(self, using):
         return RawQuery(self.sql, using, params=self.params)
 
-    def convert_values(self, value, field, connection):
-        """Convert the database-returned value into a type that is consistent
-        across database backends.
-
-        By default, this defers to the underlying backend operations, but
-        it can be overridden by Query classes for specific backends.
-        """
-        return connection.ops.convert_values(value, field)
-
     def get_columns(self):
         if self.cursor is None:
             self._execute_query()
@@ -308,15 +299,6 @@ class Query(object):
             obj._setup_query()
         return obj
 
-    def convert_values(self, value, field, connection):
-        """Convert the database-returned value into a type that is consistent
-        across database backends.
-
-        By default, this defers to the underlying backend operations, but
-        it can be overridden by Query classes for specific backends.
-        """
-        return connection.ops.convert_values(value, field)
-
     def resolve_aggregate(self, value, aggregate, connection):
         """Resolve the value of aggregates returned by the database to
         consistent (and reasonable) types.
@@ -337,7 +319,13 @@ class Query(object):
             return float(value)
         else:
             # Return value depends on the type of the field being processed.
-            return self.convert_values(value, aggregate.field, connection)
+            backend_converters = connection.ops.get_db_converters(aggregate.field.get_internal_type())
+            field_converters = aggregate.field.get_db_converters(connection)
+            for converter in backend_converters:
+                value = converter(value, aggregate.field)
+            for converter in field_converters:
+                value = converter(value, connection)
+            return value
 
     def get_aggregation(self, using, force_subq=False):
         """