Fixed #33264 -- Made test runner return non-zero error code for unexpected successes.

6 files changed, 51 insertions, 2 deletions

diff --git a/django/test/runner.py b/django/test/runner.py
index 09ac4e142a..2e36514922 100644
--- a/django/test/runner.py
+++ b/django/test/runner.py
@@ -875,7 +875,7 @@ class DiscoverRunner:
         teardown_test_environment()
 
     def suite_result(self, suite, result, **kwargs):
-        return len(result.failures) + len(result.errors)
+        return len(result.failures) + len(result.errors) + len(result.unexpectedSuccesses)
 
     def _get_databases(self, suite):
         databases = {}
diff --git a/docs/releases/4.1.txt b/docs/releases/4.1.txt
index ff1c59c852..be028f0623 100644
--- a/docs/releases/4.1.txt
+++ b/docs/releases/4.1.txt
@@ -260,6 +260,9 @@ Miscellaneous
   :class:`~django.contrib.contenttypes.fields.GenericRelation` are now cached
   on the :class:`~django.db.models.Model` instance to which they belong.
 
+* The Django test runner now returns a non-zero error code for unexpected
+  successes from tests marked with :py:func:`unittest.expectedFailure`.
+
 .. _deprecated-features-4.1:
 
 Features deprecated in 4.1
diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt
index cc55224761..3c45d509ae 100644
--- a/docs/topics/testing/overview.txt
+++ b/docs/topics/testing/overview.txt
@@ -331,10 +331,15 @@ but it's pretty intuitive. You can consult the documentation of Python's
 :mod:`unittest` library for details.
 
 Note that the return code for the test-runner script is 1 for any number of
-failed and erroneous tests. If all the tests pass, the return code is 0. This
+failed tests (whether the failure was caused by an error, a failed assertion,
+or an unexpected success). If all the tests pass, the return code is 0. This
 feature is useful if you're using the test-runner script in a shell script and
 need to test for success or failure at that level.
 
+.. versionchanged:: 4.1
+
+    In older versions, the return code was 0 for unexpected successes.
+
 .. _speeding-up-tests-auth-hashers:
 
 Speeding up the tests
diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py
index 0012be5a7e..7fef5652df 100644
--- a/tests/test_runner/test_discover_runner.py
+++ b/tests/test_runner/test_discover_runner.py
@@ -644,6 +644,24 @@ class DiscoverRunnerTests(SimpleTestCase):
                     runner.log('log message', level)
                 self.assertEqual(cm.output, [expected])
 
+    def test_suite_result_with_failure(self):
+        cases = [
+            (1, 'FailureTestCase'),
+            (1, 'ErrorTestCase'),
+            (0, 'ExpectedFailureTestCase'),
+            (1, 'UnexpectedSuccessTestCase'),
+        ]
+        runner = DiscoverRunner(verbosity=0)
+        for expected_failures, testcase in cases:
+            with self.subTest(testcase=testcase):
+                suite = runner.build_suite([
+                    f'test_runner_apps.failures.tests_failures.{testcase}',
+                ])
+                with captured_stderr():
+                    result = runner.run_suite(suite)
+                failures = runner.suite_result(suite, result)
+                self.assertEqual(failures, expected_failures)
+
 
 class DiscoverRunnerGetDatabasesTests(SimpleTestCase):
     runner = DiscoverRunner(verbosity=2)
diff --git a/tests/test_runner_apps/failures/__init__.py b/tests/test_runner_apps/failures/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/test_runner_apps/failures/__init__.py
diff --git a/tests/test_runner_apps/failures/tests_failures.py b/tests/test_runner_apps/failures/tests_failures.py
new file mode 100644
index 0000000000..0483fcd0f1
--- /dev/null
+++ b/tests/test_runner_apps/failures/tests_failures.py
@@ -0,0 +1,23 @@
+from unittest import TestCase, expectedFailure
+
+
+class FailureTestCase(TestCase):
+    def test_sample(self):
+        self.assertEqual(0, 1)
+
+
+class ErrorTestCase(TestCase):
+    def test_sample(self):
+        raise Exception('test')
+
+
+class ExpectedFailureTestCase(TestCase):
+    @expectedFailure
+    def test_sample(self):
+        self.assertEqual(0, 1)
+
+
+class UnexpectedSuccessTestCase(TestCase):
+    @expectedFailure
+    def test_sample(self):
+        self.assertEqual(1, 1)