Fixed #24020 -- Refactored SQL compiler to use expressions

Refactored compiler SELECT, GROUP BY and ORDER BY generation.
While there, also refactored select_related() implementation
(get_cached_row() and get_klass_info() are now gone!).

Made get_db_converters() method work on expressions instead of
internal_type. This allows the backend converters to target
specific expressions if need be.

Added query.context, this can be used to set per-query state.

Also changed the signature of database converters. They now accept
context as an argument.

9 files changed, 44 insertions, 32 deletions

diff --git a/tests/aggregation/tests.py b/tests/aggregation/tests.py
index 917137cce1..ec19fcfd53 100644
--- a/tests/aggregation/tests.py
+++ b/tests/aggregation/tests.py
@@ -679,7 +679,7 @@ class BaseAggregateTestCase(TestCase):
                 # the only "ORDER BY" clause present in the query.
                 self.assertEqual(
                     re.findall(r'order by (\w+)', qstr),
-                    [', '.join(forced_ordering).lower()]
+                    [', '.join(f[1][0] for f in forced_ordering).lower()]
                 )
             else:
                 self.assertNotIn('order by', qstr)
diff --git a/tests/aggregation_regress/tests.py b/tests/aggregation_regress/tests.py
index de18372f9c..8d2a006137 100644
--- a/tests/aggregation_regress/tests.py
+++ b/tests/aggregation_regress/tests.py
@@ -490,9 +490,10 @@ class AggregationTests(TestCase):
 
         # Regression for #15709 - Ensure each group_by field only exists once
         # per query
-        qs = Book.objects.values('publisher').annotate(max_pages=Max('pages')).order_by()
-        grouping, gb_params = qs.query.get_compiler(qs.db).get_grouping([], [])
-        self.assertEqual(len(grouping), 1)
+        qstr = str(Book.objects.values('publisher').annotate(max_pages=Max('pages')).order_by().query)
+        # Check that there is just one GROUP BY clause (zero commas means at
+        # most one clause)
+        self.assertEqual(qstr[qstr.index('GROUP BY'):].count(', '), 0)
 
     def test_duplicate_alias(self):
         # Regression for #11256 - duplicating a default alias raises ValueError.
@@ -924,14 +925,11 @@ class AggregationTests(TestCase):
 
         # There should only be one GROUP BY clause, for the `id` column.
         # `name` and `age` should not be grouped on.
-        grouping, gb_params = results.query.get_compiler(using='default').get_grouping([], [])
-        self.assertEqual(len(grouping), 1)
-        assert 'id' in grouping[0]
-        assert 'name' not in grouping[0]
-        assert 'age' not in grouping[0]
-
-        # The query group_by property should also only show the `id`.
-        self.assertEqual(results.query.group_by, [('aggregation_regress_author', 'id')])
+        _, _, group_by = results.query.get_compiler(using='default').pre_sql_setup()
+        self.assertEqual(len(group_by), 1)
+        self.assertIn('id', group_by[0][0])
+        self.assertNotIn('name', group_by[0][0])
+        self.assertNotIn('age', group_by[0][0])
 
         # Ensure that we get correct results.
         self.assertEqual(
@@ -953,14 +951,11 @@ class AggregationTests(TestCase):
     def test_aggregate_duplicate_columns_only(self):
         # Works with only() too.
         results = Author.objects.only('id', 'name').annotate(num_contacts=Count('book_contact_set'))
-        grouping, gb_params = results.query.get_compiler(using='default').get_grouping([], [])
+        _, _, grouping = results.query.get_compiler(using='default').pre_sql_setup()
         self.assertEqual(len(grouping), 1)
-        assert 'id' in grouping[0]
-        assert 'name' not in grouping[0]
-        assert 'age' not in grouping[0]
-
-        # The query group_by property should also only show the `id`.
-        self.assertEqual(results.query.group_by, [('aggregation_regress_author', 'id')])
+        self.assertIn('id', grouping[0][0])
+        self.assertNotIn('name', grouping[0][0])
+        self.assertNotIn('age', grouping[0][0])
 
         # Ensure that we get correct results.
         self.assertEqual(
@@ -983,14 +978,11 @@ class AggregationTests(TestCase):
         # And select_related()
         results = Book.objects.select_related('contact').annotate(
             num_authors=Count('authors'))
-        grouping, gb_params = results.query.get_compiler(using='default').get_grouping([], [])
+        _, _, grouping = results.query.get_compiler(using='default').pre_sql_setup()
         self.assertEqual(len(grouping), 1)
-        assert 'id' in grouping[0]
-        assert 'name' not in grouping[0]
-        assert 'contact' not in grouping[0]
-
-        # The query group_by property should also only show the `id`.
-        self.assertEqual(results.query.group_by, [('aggregation_regress_book', 'id')])
+        self.assertIn('id', grouping[0][0])
+        self.assertNotIn('name', grouping[0][0])
+        self.assertNotIn('contact', grouping[0][0])
 
         # Ensure that we get correct results.
         self.assertEqual(
diff --git a/tests/custom_pk/fields.py b/tests/custom_pk/fields.py
index 1f3265952a..bf349545e5 100644
--- a/tests/custom_pk/fields.py
+++ b/tests/custom_pk/fields.py
@@ -43,7 +43,7 @@ class MyAutoField(models.CharField):
             value = MyWrapper(value)
         return value
 
-    def from_db_value(self, value, connection):
+    def from_db_value(self, value, connection, context):
         if not value:
             return
         return MyWrapper(value)
diff --git a/tests/defer_regress/models.py b/tests/defer_regress/models.py
index 58693c83db..a81b02058d 100644
--- a/tests/defer_regress/models.py
+++ b/tests/defer_regress/models.py
@@ -96,3 +96,11 @@ class Request(models.Model):
     request2 = models.CharField(default='request2', max_length=1000)
     request3 = models.CharField(default='request3', max_length=1000)
     request4 = models.CharField(default='request4', max_length=1000)
+
+
+class Base(models.Model):
+    text = models.TextField()
+
+
+class Derived(Base):
+    other_text = models.TextField()
diff --git a/tests/defer_regress/tests.py b/tests/defer_regress/tests.py
index f2a015a7a1..1222212d8d 100644
--- a/tests/defer_regress/tests.py
+++ b/tests/defer_regress/tests.py
@@ -12,7 +12,7 @@ from django.test import TestCase, override_settings
 from .models import (
     ResolveThis, Item, RelatedItem, Child, Leaf, Proxy, SimpleItem, Feature,
     ItemAndSimpleItem, OneToOneItem, SpecialFeature, Location, Request,
-    ProxyRelated,
+    ProxyRelated, Derived, Base,
 )
 
 
@@ -145,6 +145,15 @@ class DeferRegressionTest(TestCase):
             list(SimpleItem.objects.annotate(Count('feature')).only('name')),
             list)
 
+    def test_ticket_23270(self):
+        Derived.objects.create(text="foo", other_text="bar")
+        with self.assertNumQueries(1):
+            obj = Base.objects.select_related("derived").defer("text")[0]
+            self.assertIsInstance(obj.derived, Derived)
+            self.assertEqual("bar", obj.derived.other_text)
+            self.assertNotIn("text", obj.__dict__)
+            self.assertEqual(1, obj.derived.base_ptr_id)
+
     def test_only_and_defer_usage_on_proxy_models(self):
         # Regression for #15790 - only() broken for proxy models
         proxy = Proxy.objects.create(name="proxy", value=42)
diff --git a/tests/from_db_value/models.py b/tests/from_db_value/models.py
index 4cc9e62168..6a06c832ea 100644
--- a/tests/from_db_value/models.py
+++ b/tests/from_db_value/models.py
@@ -18,7 +18,7 @@ class CashField(models.DecimalField):
         kwargs['decimal_places'] = 2
         super(CashField, self).__init__(**kwargs)
 
-    def from_db_value(self, value, connection):
+    def from_db_value(self, value, connection, context):
         cash = Cash(value)
         cash.vendor = connection.vendor
         return cash
diff --git a/tests/queries/tests.py b/tests/queries/tests.py
index 900149b873..92e5982e10 100644
--- a/tests/queries/tests.py
+++ b/tests/queries/tests.py
@@ -3563,8 +3563,10 @@ class Ticket20955Tests(TestCase):
         # version's queries.
         task_get.creator.staffuser.staff
         task_get.owner.staffuser.staff
-        task_select_related = Task.objects.select_related(
-            'creator__staffuser__staff', 'owner__staffuser__staff').get(pk=task.pk)
+        qs = Task.objects.select_related(
+            'creator__staffuser__staff', 'owner__staffuser__staff')
+        self.assertEqual(str(qs.query).count(' JOIN '), 6)
+        task_select_related = qs.get(pk=task.pk)
         with self.assertNumQueries(0):
             self.assertEqual(task_select_related.creator.staffuser.staff,
                              task_get.creator.staffuser.staff)
diff --git a/tests/select_related_onetoone/tests.py b/tests/select_related_onetoone/tests.py
index a13ac7809a..a6c0f3a42b 100644
--- a/tests/select_related_onetoone/tests.py
+++ b/tests/select_related_onetoone/tests.py
@@ -83,6 +83,7 @@ class ReverseSelectRelatedTestCase(TestCase):
             stat = UserStat.objects.select_related('user', 'advanceduserstat').get(posts=200)
             self.assertEqual(stat.advanceduserstat.posts, 200)
             self.assertEqual(stat.user.username, 'bob')
+        with self.assertNumQueries(1):
             self.assertEqual(stat.advanceduserstat.user.username, 'bob')
 
     def test_nullable_relation(self):
diff --git a/tests/serializers/models.py b/tests/serializers/models.py
index 2d1ab9758b..b2864b1c71 100644
--- a/tests/serializers/models.py
+++ b/tests/serializers/models.py
@@ -112,7 +112,7 @@ class TeamField(models.CharField):
             return value
         return Team(value)
 
-    def from_db_value(self, value, connection):
+    def from_db_value(self, value, connection, context):
         return Team(value)
 
     def value_to_string(self, obj):