Fixed #33264 -- Made test runner return non-zero error code for unexpected successes.

3 files changed, 41 insertions, 0 deletions

diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py
index 0012be5a7e..7fef5652df 100644
--- a/tests/test_runner/test_discover_runner.py
+++ b/tests/test_runner/test_discover_runner.py
@@ -644,6 +644,24 @@ class DiscoverRunnerTests(SimpleTestCase):
                     runner.log('log message', level)
                 self.assertEqual(cm.output, [expected])
 
+    def test_suite_result_with_failure(self):
+        cases = [
+            (1, 'FailureTestCase'),
+            (1, 'ErrorTestCase'),
+            (0, 'ExpectedFailureTestCase'),
+            (1, 'UnexpectedSuccessTestCase'),
+        ]
+        runner = DiscoverRunner(verbosity=0)
+        for expected_failures, testcase in cases:
+            with self.subTest(testcase=testcase):
+                suite = runner.build_suite([
+                    f'test_runner_apps.failures.tests_failures.{testcase}',
+                ])
+                with captured_stderr():
+                    result = runner.run_suite(suite)
+                failures = runner.suite_result(suite, result)
+                self.assertEqual(failures, expected_failures)
+
 
 class DiscoverRunnerGetDatabasesTests(SimpleTestCase):
     runner = DiscoverRunner(verbosity=2)
diff --git a/tests/test_runner_apps/failures/__init__.py b/tests/test_runner_apps/failures/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/test_runner_apps/failures/__init__.py
diff --git a/tests/test_runner_apps/failures/tests_failures.py b/tests/test_runner_apps/failures/tests_failures.py
new file mode 100644
index 0000000000..0483fcd0f1
--- /dev/null
+++ b/tests/test_runner_apps/failures/tests_failures.py
@@ -0,0 +1,23 @@
+from unittest import TestCase, expectedFailure
+
+
+class FailureTestCase(TestCase):
+    def test_sample(self):
+        self.assertEqual(0, 1)
+
+
+class ErrorTestCase(TestCase):
+    def test_sample(self):
+        raise Exception('test')
+
+
+class ExpectedFailureTestCase(TestCase):
+    @expectedFailure
+    def test_sample(self):
+        self.assertEqual(0, 1)
+
+
+class UnexpectedSuccessTestCase(TestCase):
+    @expectedFailure
+    def test_sample(self):
+        self.assertEqual(1, 1)