Fixed #18757, #14462, #21565 -- Reworked database-python type conversions

Complete rework of translating data values from database

Deprecation of SubfieldBase, removal of resolve_columns and
convert_values in favour of a more general converter based approach and
public API Field.from_db_value(). Now works seamlessly with aggregation,
.values() and raw queries.

Thanks to akaariai in particular for extensive advice and inspiration,
also to shaib, manfre and timograham for their reviews.

8 files changed, 83 insertions, 26 deletions

diff --git a/tests/aggregation_regress/tests.py b/tests/aggregation_regress/tests.py
index 9fce51059b..4955573161 100644
--- a/tests/aggregation_regress/tests.py
+++ b/tests/aggregation_regress/tests.py
@@ -894,18 +894,6 @@ class AggregationTests(TestCase):
             lambda b: b.name
         )
 
-    def test_type_conversion(self):
-        # The database backend convert_values function should not try to covert
-        # CharFields to float. Refs #13844.
-        from django.db.models import CharField
-        from django.db import connection
-        testData = 'not_a_float_value'
-        testField = CharField()
-        self.assertEqual(
-            connection.ops.convert_values(testData, testField),
-            testData
-        )
-
     def test_annotate_joins(self):
         """
         Test that the base table's join isn't promoted to LOUTER. This could
diff --git a/tests/backends/tests.py b/tests/backends/tests.py
index a059c76b9d..89132f2a31 100644
--- a/tests/backends/tests.py
+++ b/tests/backends/tests.py
@@ -20,8 +20,6 @@ from django.db.backends.signals import connection_created
 from django.db.backends.postgresql_psycopg2 import version as pg_version
 from django.db.backends.utils import format_number, CursorWrapper
 from django.db.models import Sum, Avg, Variance, StdDev
-from django.db.models.fields import (AutoField, DateField, DateTimeField,
-    DecimalField, IntegerField, TimeField)
 from django.db.models.sql.constants import CURSOR
 from django.db.utils import ConnectionHandler
 from django.test import (TestCase, TransactionTestCase, override_settings,
@@ -133,16 +131,6 @@ class SQLiteTests(TestCase):
             self.assertRaises(NotImplementedError,
                 models.Item.objects.all().aggregate, aggregate('last_modified'))
 
-    def test_convert_values_to_handle_null_value(self):
-        from django.db.backends.sqlite3.base import DatabaseOperations
-        convert_values = DatabaseOperations(connection).convert_values
-        self.assertIsNone(convert_values(None, AutoField(primary_key=True)))
-        self.assertIsNone(convert_values(None, DateField()))
-        self.assertIsNone(convert_values(None, DateTimeField()))
-        self.assertIsNone(convert_values(None, DecimalField()))
-        self.assertIsNone(convert_values(None, IntegerField()))
-        self.assertIsNone(convert_values(None, TimeField()))
-
 
 @unittest.skipUnless(connection.vendor == 'postgresql', "Test only for PostgreSQL")
 class PostgreSQLTests(TestCase):
diff --git a/tests/custom_pk/fields.py b/tests/custom_pk/fields.py
index 2aa3bad963..1f3265952a 100644
--- a/tests/custom_pk/fields.py
+++ b/tests/custom_pk/fields.py
@@ -23,7 +23,7 @@ class MyWrapper(object):
         return self.value == other
 
 
-class MyAutoField(six.with_metaclass(models.SubfieldBase, models.CharField)):
+class MyAutoField(models.CharField):
 
     def __init__(self, *args, **kwargs):
         kwargs['max_length'] = 10
@@ -43,6 +43,11 @@ class MyAutoField(six.with_metaclass(models.SubfieldBase, models.CharField)):
             value = MyWrapper(value)
         return value
 
+    def from_db_value(self, value, connection):
+        if not value:
+            return
+        return MyWrapper(value)
+
     def get_db_prep_save(self, value, connection):
         if not value:
             return
diff --git a/tests/field_subclassing/models.py b/tests/field_subclassing/models.py
index 3ed465cd7f..59712fcab3 100644
--- a/tests/field_subclassing/models.py
+++ b/tests/field_subclassing/models.py
@@ -2,13 +2,24 @@
 Tests for field subclassing.
 """
 
+import warnings
+
 from django.db import models
 from django.utils.encoding import force_text
+from django.utils.deprecation import RemovedInDjango20Warning
 
 from .fields import Small, SmallField, SmallerField, JSONField
 from django.utils.encoding import python_2_unicode_compatible
 
 
+# Catch warning about subfieldbase  -- remove in Django 2.0
+warnings.filterwarnings(
+    'ignore',
+    'SubfieldBase has been deprecated. Use Field.from_db_value instead.',
+    RemovedInDjango20Warning
+)
+
+
 @python_2_unicode_compatible
 class MyModel(models.Model):
     name = models.CharField(max_length=10)
diff --git a/tests/from_db_value/__init__.py b/tests/from_db_value/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/from_db_value/__init__.py
diff --git a/tests/from_db_value/models.py b/tests/from_db_value/models.py
new file mode 100644
index 0000000000..4cc9e62168
--- /dev/null
+++ b/tests/from_db_value/models.py
@@ -0,0 +1,32 @@
+import decimal
+
+from django.db import models
+from django.utils.encoding import python_2_unicode_compatible
+
+
+class Cash(decimal.Decimal):
+    currency = 'USD'
+
+    def __str__(self):
+        s = super(Cash, self).__str__(self)
+        return '%s %s' % (s, self.currency)
+
+
+class CashField(models.DecimalField):
+    def __init__(self, **kwargs):
+        kwargs['max_digits'] = 20
+        kwargs['decimal_places'] = 2
+        super(CashField, self).__init__(**kwargs)
+
+    def from_db_value(self, value, connection):
+        cash = Cash(value)
+        cash.vendor = connection.vendor
+        return cash
+
+
+@python_2_unicode_compatible
+class CashModel(models.Model):
+    cash = CashField()
+
+    def __str__(self):
+        return str(self.cash)
diff --git a/tests/from_db_value/tests.py b/tests/from_db_value/tests.py
new file mode 100644
index 0000000000..69e22d4290
--- /dev/null
+++ b/tests/from_db_value/tests.py
@@ -0,0 +1,30 @@
+from django.db import connection
+from django.db.models import Max
+from django.test import TestCase
+
+from .models import CashModel, Cash
+
+
+class FromDBValueTest(TestCase):
+    def setUp(self):
+        CashModel.objects.create(cash='12.50')
+
+    def test_simple_load(self):
+        instance = CashModel.objects.get()
+        self.assertIsInstance(instance.cash, Cash)
+
+    def test_values(self):
+        values_list = CashModel.objects.values_list('cash', flat=True)
+        self.assertIsInstance(values_list[0], Cash)
+
+    def test_aggregation(self):
+        maximum = CashModel.objects.aggregate(m=Max('cash'))['m']
+        self.assertIsInstance(maximum, Cash)
+
+    def test_defer(self):
+        instance = CashModel.objects.defer('cash').get()
+        self.assertIsInstance(instance.cash, Cash)
+
+    def test_connection(self):
+        instance = CashModel.objects.get()
+        self.assertEqual(instance.cash.vendor, connection.vendor)
diff --git a/tests/serializers/models.py b/tests/serializers/models.py
index 691d87ea29..78674664d3 100644
--- a/tests/serializers/models.py
+++ b/tests/serializers/models.py
@@ -99,7 +99,7 @@ class Team(object):
         return "%s" % self.title
 
 
-class TeamField(six.with_metaclass(models.SubfieldBase, models.CharField)):
+class TeamField(models.CharField):
 
     def __init__(self):
         super(TeamField, self).__init__(max_length=100)
@@ -112,6 +112,9 @@ class TeamField(six.with_metaclass(models.SubfieldBase, models.CharField)):
             return value
         return Team(value)
 
+    def from_db_value(self, value, connection):
+        return Team(value)
+
     def value_to_string(self, obj):
         return self._get_val_from_obj(obj).to_string()