Fixed #25367 -- Allowed boolean expressions in QuerySet.filter() and exclude().

This allows using expressions that have an output_field that is a
BooleanField to be used directly in a queryset filters, or in the
When() clauses of a Case() expression.

Thanks Josh Smeaton, Tim Graham, Simon Charette, Mariusz Felisiak, and
Adam Johnson for reviews.

Co-Authored-By: NyanKiyoshi <hello@vanille.bid>

10 files changed, 184 insertions, 23 deletions

diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py
index 76abc6dcb2..a0c84a8ff4 100644
--- a/django/db/backends/base/operations.py
+++ b/django/db/backends/base/operations.py
@@ -581,6 +581,13 @@ class BaseDatabaseOperations:
         """
         pass
 
+    def conditional_expression_supported_in_where_clause(self, expression):
+        """
+        Return True, if the conditional expression is supported in the WHERE
+        clause.
+        """
+        return True
+
     def combine_expression(self, connector, sub_expressions):
         """
         Combine a list of subexpressions into a single expression, using
diff --git a/django/db/backends/oracle/operations.py b/django/db/backends/oracle/operations.py
index 3cad02fa41..95126d37be 100644
--- a/django/db/backends/oracle/operations.py
+++ b/django/db/backends/oracle/operations.py
@@ -6,6 +6,8 @@ from functools import lru_cache
 from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
 from django.db.backends.utils import strip_quotes, truncate_name
+from django.db.models.expressions import Exists, ExpressionWrapper
+from django.db.models.query_utils import Q
 from django.db.utils import DatabaseError
 from django.utils import timezone
 from django.utils.encoding import force_bytes, force_str
@@ -607,3 +609,14 @@ END;
         if fields:
             return self.connection.features.max_query_params // len(fields)
         return len(objs)
+
+    def conditional_expression_supported_in_where_clause(self, expression):
+        """
+        Oracle supports only EXISTS(...) or filters in the WHERE clause, others
+        must be compared with True.
+        """
+        if isinstance(expression, Exists):
+            return True
+        if isinstance(expression, ExpressionWrapper) and isinstance(expression.expression, Q):
+            return True
+        return False
diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py
index 1dd061c152..9429adf81c 100644
--- a/django/db/models/expressions.py
+++ b/django/db/models/expressions.py
@@ -90,6 +90,8 @@ class Combinable:
         return self._combine(other, self.POW, False)
 
     def __and__(self, other):
+        if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
+            return Q(self) & Q(other)
         raise NotImplementedError(
             "Use .bitand() and .bitor() for bitwise logical operations."
         )
@@ -104,6 +106,8 @@ class Combinable:
         return self._combine(other, self.BITRIGHTSHIFT, False)
 
     def __or__(self, other):
+        if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
+            return Q(self) | Q(other)
         raise NotImplementedError(
             "Use .bitand() and .bitor() for bitwise logical operations."
         )
@@ -246,6 +250,10 @@ class BaseExpression:
         return c
 
     @property
+    def conditional(self):
+        return isinstance(self.output_field, fields.BooleanField)
+
+    @property
     def field(self):
         return self.output_field
 
@@ -873,12 +881,17 @@ class ExpressionWrapper(Expression):
 
 class When(Expression):
     template = 'WHEN %(condition)s THEN %(result)s'
+    # This isn't a complete conditional expression, must be used in Case().
+    conditional = False
 
     def __init__(self, condition=None, then=None, **lookups):
         if lookups and condition is None:
             condition, lookups = Q(**lookups), None
         if condition is None or not getattr(condition, 'conditional', False) or lookups:
-            raise TypeError("__init__() takes either a Q object or lookups as keyword arguments")
+            raise TypeError(
+                'When() supports a Q object, a boolean expression, or lookups '
+                'as a condition.'
+            )
         if isinstance(condition, Q) and not condition:
             raise ValueError("An empty Q() can't be used as a When() condition.")
         super().__init__(output_field=None)
@@ -1090,6 +1103,7 @@ class Exists(Subquery):
 
 class OrderBy(BaseExpression):
     template = '%(expression)s %(ordering)s'
+    conditional = False
 
     def __init__(self, expression, descending=False, nulls_first=False, nulls_last=False):
         if nulls_first and nulls_last:
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 4ad1fb3f36..35f3c5e1ea 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -1229,6 +1229,16 @@ class Query(BaseExpression):
         """
         if isinstance(filter_expr, dict):
             raise FieldError("Cannot parse keyword query as dict")
+        if hasattr(filter_expr, 'resolve_expression') and getattr(filter_expr, 'conditional', False):
+            if connections[DEFAULT_DB_ALIAS].ops.conditional_expression_supported_in_where_clause(filter_expr):
+                condition = filter_expr.resolve_expression(self)
+            else:
+                # Expression is not supported in the WHERE clause, add
+                # comparison with True.
+                condition = self.build_lookup(['exact'], filter_expr.resolve_expression(self), True)
+            clause = self.where_class()
+            clause.add(condition, AND)
+            return clause, []
         arg, value = filter_expr
         if not arg:
             raise FieldError("Cannot parse keyword query %r" % arg)
diff --git a/docs/ref/models/conditional-expressions.txt b/docs/ref/models/conditional-expressions.txt
index f9e681f667..e88554dec6 100644
--- a/docs/ref/models/conditional-expressions.txt
+++ b/docs/ref/models/conditional-expressions.txt
@@ -42,9 +42,15 @@ We'll be using the following model in the subsequent examples::
 A ``When()`` object is used to encapsulate a condition and its result for use
 in the conditional expression. Using a ``When()`` object is similar to using
 the :meth:`~django.db.models.query.QuerySet.filter` method. The condition can
-be specified using :ref:`field lookups <field-lookups>` or
-:class:`~django.db.models.Q` objects. The result is provided using the ``then``
-keyword.
+be specified using :ref:`field lookups <field-lookups>`,
+:class:`~django.db.models.Q` objects, or :class:`~django.db.models.Expression`
+objects that have an ``output_field`` that is a
+:class:`~django.db.models.BooleanField`. The result is provided using the
+``then`` keyword.
+
+.. versionchanged:: 3.0
+
+    Support for boolean :class:`~django.db.models.Expression` was added.
 
 Some examples::
 
@@ -60,6 +66,12 @@ Some examples::
     >>> # Complex conditions can be created using Q objects
     >>> When(Q(name__startswith="John") | Q(name__startswith="Paul"),
     ...      then='name')
+    >>> # Condition can be created using boolean expressions.
+    >>> from django.db.models import Exists, OuterRef
+    >>> non_unique_account_type = Client.objects.filter(
+    ...     account_type=OuterRef('account_type'),
+    ... ).exclude(pk=OuterRef('pk')).values('pk')
+    >>> When(Exists(non_unique_account_type), then=Value('non unique'))
 
 Keep in mind that each of these values can be an expression.
 
@@ -158,9 +170,9 @@ registered more than a year ago::
 Advanced queries
 ================
 
-Conditional expressions can be used in annotations, aggregations, lookups, and
-updates. They can also be combined and nested with other expressions. This
-allows you to make powerful conditional queries.
+Conditional expressions can be used in annotations, aggregations, filters,
+lookups, and updates. They can also be combined and nested with other
+expressions. This allows you to make powerful conditional queries.
 
 Conditional update
 ------------------
@@ -236,3 +248,29 @@ On other databases, this is emulated using a ``CASE`` statement:
 
 The two SQL statements are functionally equivalent but the more explicit
 ``FILTER`` may perform better.
+
+Conditional filter
+------------------
+
+.. versionadded:: 3.0
+
+When a conditional expression returns a boolean value, it is possible to use it
+directly in filters. This means that it will not be added to the ``SELECT``
+columns, but you can still use it to filter results::
+
+    >>> non_unique_account_type = Client.objects.filter(
+    ...     account_type=OuterRef('account_type'),
+    ... ).exclude(pk=OuterRef('pk')).values('pk')
+    >>> Client.objects.filter(~Exists(non_unique_account_type))
+
+In SQL terms, that evaluates to:
+
+.. code-block:: sql
+
+    SELECT ...
+    FROM client c0
+    WHERE NOT EXISTS (
+      SELECT c1.id
+      FROM client c1
+      WHERE c1.account_type = c0.account_type AND NOT c1.id = c0.id
+    )
diff --git a/docs/ref/models/expressions.txt b/docs/ref/models/expressions.txt
index ccd75d62a9..16dabf8177 100644
--- a/docs/ref/models/expressions.txt
+++ b/docs/ref/models/expressions.txt
@@ -5,10 +5,11 @@ Query Expressions
 .. currentmodule:: django.db.models
 
 Query expressions describe a value or a computation that can be used as part of
-an update, create, filter, order by, annotation, or aggregate. There are a
-number of built-in expressions (documented below) that can be used to help you
-write queries. Expressions can be combined, or in some cases nested, to form
-more complex computations.
+an update, create, filter, order by, annotation, or aggregate. When an
+expression outputs a boolean value, it may be used directly in filters. There
+are a number of built-in expressions (documented below) that can be used to
+help you write queries. Expressions can be combined, or in some cases nested,
+to form more complex computations.
 
 Supported arithmetic
 ====================
@@ -69,6 +70,12 @@ Some examples
     CharField.register_lookup(Length)
     Company.objects.order_by('name__length')
 
+    # Boolean expression can be used directly in filters.
+    from django.db.models import Exists
+    Company.objects.filter(
+        Exists(Employee.objects.filter(company=OuterRef('pk'), salary__gt=10))
+    )
+
 Built-in Expressions
 ====================
 
@@ -626,22 +633,25 @@ degrade performance, it's automatically removed.
 
 You can query using ``NOT EXISTS`` with ``~Exists()``.
 
-Filtering on a ``Subquery`` expression
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Filtering on a ``Subquery()`` or ``Exists()`` expressions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-It's not possible to filter directly using ``Subquery`` and ``Exists``, e.g.::
+``Subquery()`` that returns a boolean value and ``Exists()`` may be used as a
+``condition`` in :class:`~django.db.models.expressions.When` expressions, or to
+directly filter a queryset::
 
+    >>> recent_comments = Comment.objects.filter(...)  # From above
     >>> Post.objects.filter(Exists(recent_comments))
-    ...
-    TypeError: 'Exists' object is not iterable
 
+This will ensure that the subquery will not be added to the ``SELECT`` columns,
+which may result in a better performance.
 
-You must filter on a subquery expression by first annotating the queryset
-and then filtering based on that annotation::
+.. versionchanged:: 3.0
 
-    >>> Post.objects.annotate(
-    ...     recent_comment=Exists(recent_comments),
-    ... ).filter(recent_comment=True)
+    In previous versions of Django, it was necessary to first annotate and then
+    filter against the annotation. This resulted in the annotated value always
+    being present in the query result, and often resulted in a query that took
+    more time to execute.
 
 Using aggregates within a ``Subquery`` expression
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt
index 722baeeeaa..f6ec8f8cc7 100644
--- a/docs/releases/3.0.txt
+++ b/docs/releases/3.0.txt
@@ -74,6 +74,13 @@ enable adding exclusion constraints on PostgreSQL. Constraints are added to
 models using the
 :attr:`Meta.constraints <django.db.models.Options.constraints>` option.
 
+Filter expressions
+------------------
+
+Expressions that outputs :class:`~django.db.models.BooleanField` may now be
+used directly in ``QuerySet`` filters, without having to first annotate and
+then filter against the annotation.
+
 Minor features
 --------------
 
diff --git a/tests/expressions/models.py b/tests/expressions/models.py
index 33f7850ac1..a81ba9e16b 100644
--- a/tests/expressions/models.py
+++ b/tests/expressions/models.py
@@ -34,6 +34,7 @@ class Company(models.Model):
         related_name='company_point_of_contact_set',
         null=True,
     )
+    based_in_eu = models.BooleanField(default=False)
 
     def __str__(self):
         return self.name
diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py
index eaa3471177..f50c634014 100644
--- a/tests/expressions/tests.py
+++ b/tests/expressions/tests.py
@@ -37,7 +37,7 @@ class BasicExpressionsTests(TestCase):
             ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10)
         )
         cls.foobar_ltd = Company.objects.create(
-            name="Foobar Ltd.", num_employees=3, num_chairs=4,
+            name="Foobar Ltd.", num_employees=3, num_chairs=4, based_in_eu=True,
             ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20)
         )
         cls.max = Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30)
@@ -83,6 +83,14 @@ class BasicExpressionsTests(TestCase):
             2,
         )
 
+    def test_filtering_on_q_that_is_boolean(self):
+        self.assertEqual(
+            Company.objects.filter(
+                ExpressionWrapper(Q(num_employees__gt=3), output_field=models.BooleanField())
+            ).count(),
+            2,
+        )
+
     def test_filter_inter_attribute(self):
         # We can filter on attribute relationships on same model obj, e.g.
         # find companies where the number of employees is greater
@@ -642,6 +650,56 @@ class BasicExpressionsTests(TestCase):
         with self.assertRaisesMessage(FieldError, "Cannot resolve keyword 'nope' into field."):
             list(Company.objects.filter(ceo__pk=F('point_of_contact__nope')))
 
+    def test_exists_in_filter(self):
+        inner = Company.objects.filter(ceo=OuterRef('pk')).values('pk')
+        qs1 = Employee.objects.filter(Exists(inner))
+        qs2 = Employee.objects.annotate(found=Exists(inner)).filter(found=True)
+        self.assertCountEqual(qs1, qs2)
+        self.assertFalse(Employee.objects.exclude(Exists(inner)).exists())
+        self.assertCountEqual(qs2, Employee.objects.exclude(~Exists(inner)))
+
+    def test_subquery_in_filter(self):
+        inner = Company.objects.filter(ceo=OuterRef('pk')).values('based_in_eu')
+        self.assertSequenceEqual(
+            Employee.objects.filter(Subquery(inner)),
+            [self.foobar_ltd.ceo],
+        )
+
+    def test_case_in_filter_if_boolean_output_field(self):
+        is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
+        is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
+        qs = Employee.objects.filter(
+            Case(
+                When(Exists(is_ceo), then=True),
+                When(Exists(is_poc), then=True),
+                default=False,
+                output_field=models.BooleanField(),
+            ),
+        )
+        self.assertSequenceEqual(qs, [self.example_inc.ceo, self.foobar_ltd.ceo, self.max])
+
+    def test_boolean_expression_combined(self):
+        is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
+        is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
+        self.gmbh.point_of_contact = self.max
+        self.gmbh.save()
+        self.assertSequenceEqual(
+            Employee.objects.filter(Exists(is_ceo) | Exists(is_poc)),
+            [self.example_inc.ceo, self.foobar_ltd.ceo, self.max],
+        )
+        self.assertSequenceEqual(
+            Employee.objects.filter(Exists(is_ceo) & Exists(is_poc)),
+            [self.max],
+        )
+        self.assertSequenceEqual(
+            Employee.objects.filter(Exists(is_ceo) & Q(salary__gte=30)),
+            [self.max],
+        )
+        self.assertSequenceEqual(
+            Employee.objects.filter(Exists(is_poc) | Q(salary__lt=15)),
+            [self.example_inc.ceo, self.max],
+        )
+
 
 class IterableLookupInnerExpressionsTests(TestCase):
     @classmethod
diff --git a/tests/expressions_case/tests.py b/tests/expressions_case/tests.py
index f1255cb8e9..e3f4775cf2 100644
--- a/tests/expressions_case/tests.py
+++ b/tests/expressions_case/tests.py
@@ -1327,7 +1327,10 @@ class CaseWhenTests(SimpleTestCase):
             Case(When(Q(pk__in=[])), object())
 
     def test_invalid_when_constructor_args(self):
-        msg = '__init__() takes either a Q object or lookups as keyword arguments'
+        msg = (
+            'When() supports a Q object, a boolean expression, or lookups as '
+            'a condition.'
+        )
         with self.assertRaisesMessage(TypeError, msg):
             When(condition=object())
         with self.assertRaisesMessage(TypeError, msg):