Fixed #22996 -- Prevented crash with unencoded query string

Thanks Jorge Carleitao for the report and Aymeric Augustin, Tim Graham
for the reviews.

2 files changed, 26 insertions, 10 deletions

diff --git a/tests/handlers/tests.py b/tests/handlers/tests.py
index 178f42d8a9..ca2daa0f75 100644
--- a/tests/handlers/tests.py
+++ b/tests/handlers/tests.py
@@ -42,14 +42,30 @@ class HandlerTests(TestCase):
         self.assertEqual(response.status_code, 400)
 
     def test_non_ascii_query_string(self):
-        """Test that non-ASCII query strings are properly decoded (#20530)."""
+        """
+        Test that non-ASCII query strings are properly decoded (#20530, #22996).
+        """
         environ = RequestFactory().get('/').environ
-        raw_query_string = 'want=café'
-        if six.PY3:
-            raw_query_string = raw_query_string.encode('utf-8').decode('iso-8859-1')
-        environ['QUERY_STRING'] = raw_query_string
-        request = WSGIRequest(environ)
-        self.assertEqual(request.GET['want'], "café")
+        raw_query_strings = [
+            b'want=caf%C3%A9', # This is the proper way to encode 'café'
+            b'want=caf\xc3\xa9', # UA forgot to quote bytes
+            b'want=caf%E9', # UA quoted, but not in UTF-8
+            b'want=caf\xe9', # UA forgot to convert Latin-1 to UTF-8 and to quote (typical of MSIE)
+        ]
+        got = []
+        for raw_query_string in raw_query_strings:
+            if six.PY3:
+                # Simulate http.server.BaseHTTPRequestHandler.parse_request handling of raw request
+                environ['QUERY_STRING'] = str(raw_query_string, 'iso-8859-1')
+            else:
+                environ['QUERY_STRING'] = raw_query_string
+            request = WSGIRequest(environ)
+            got.append(request.GET['want'])
+        if six.PY2:
+            self.assertListEqual(got, ['café', 'café', 'café', 'café'])
+        else:
+            # On Python 3, %E9 is converted to the unicode replacement character by parse_qsl
+            self.assertListEqual(got, ['café', 'café', 'caf\ufffd', 'café'])
 
     def test_non_ascii_cookie(self):
         """Test that non-ASCII cookies set in JavaScript are properly decoded (#20557)."""
diff --git a/tests/httpwrappers/tests.py b/tests/httpwrappers/tests.py
index 556d65a0a8..0be5df3024 100644
--- a/tests/httpwrappers/tests.py
+++ b/tests/httpwrappers/tests.py
@@ -203,14 +203,14 @@ class QueryDictTests(unittest.TestCase):
         def test_invalid_input_encoding(self):
             """
             QueryDicts must be able to handle invalid input encoding (in this
-            case, bad UTF-8 encoding).
+            case, bad UTF-8 encoding), falling back to ISO-8859-1 decoding.
 
             This test doesn't apply under Python 3 because the URL is a string
             and not a bytestring.
             """
             q = QueryDict(str(b'foo=bar&foo=\xff'))
-            self.assertEqual(q['foo'], '\ufffd')
-            self.assertEqual(q.getlist('foo'), ['bar', '\ufffd'])
+            self.assertEqual(q['foo'], '\xff')
+            self.assertEqual(q.getlist('foo'), ['bar', '\xff'])
 
     def test_pickle(self):
         q = QueryDict()