Fixed #36293 -- Avoided buffering streaming responses in GZipMiddleware.

This avoids latency and/or blocking.

The example of streaming a CSV file was rewritten to employ batching for
greater efficiency in all layers (db, HTTP, etc.). The improved
performance from batching should outweigh the drag introduced by an
additional byte for each flush.

Co-authored-by: huoyinghui <huoyinghui@users.noreply.github.com>

4 files changed, 56 insertions, 10 deletions

diff --git a/django/utils/text.py b/django/utils/text.py
index cfe6ceca9e..d1306f9c6f 100644
--- a/django/utils/text.py
+++ b/django/utils/text.py
@@ -382,6 +382,7 @@ def compress_sequence(sequence, *, max_random_bytes=None):
         yield buf.read()
         for item in sequence:
             zfile.write(item)
+            zfile.flush()
             data = buf.read()
             if data:
                 yield data
@@ -398,6 +399,7 @@ async def acompress_sequence(sequence, *, max_random_bytes=None):
         yield buf.read()
         async for item in sequence:
             zfile.write(item)
+            zfile.flush()
             data = buf.read()
             if data:
                 yield data
diff --git a/docs/howto/outputting-csv.txt b/docs/howto/outputting-csv.txt
index c5ae7094d2..10de00503e 100644
--- a/docs/howto/outputting-csv.txt
+++ b/docs/howto/outputting-csv.txt
@@ -67,9 +67,12 @@ avoid a load balancer dropping a connection that might have otherwise timed out
 while the server was generating the response.
 
 In this example, we make full use of Python generators to efficiently handle
-the assembly and transmission of a large CSV file::
+the assembly and transmission of a large CSV file. Rows are batched together
+to reduce HTTP overhead and improve compression efficiency when used with
+:class:`~django.middleware.gzip.GZipMiddleware`::
 
     import csv
+    from itertools import batched
 
     from django.http import StreamingHttpResponse
 
@@ -92,8 +95,13 @@ the assembly and transmission of a large CSV file::
         rows = (["Row {}".format(idx), str(idx)] for idx in range(65536))
         pseudo_buffer = Echo()
         writer = csv.writer(pseudo_buffer)
+
+        def stream_batched_rows():
+            for batch in batched(rows, 100):
+                yield "".join(writer.writerow(row) for row in batch)
+
         return StreamingHttpResponse(
-            (writer.writerow(row) for row in rows),
+            stream_batched_rows(),
             content_type="text/csv",
             headers={"Content-Disposition": 'attachment; filename="somefilename.csv"'},
         )
diff --git a/tests/decorators/test_gzip.py b/tests/decorators/test_gzip.py
index 2d64c171f7..8cd0869b53 100644
--- a/tests/decorators/test_gzip.py
+++ b/tests/decorators/test_gzip.py
@@ -1,6 +1,6 @@
 from inspect import iscoroutinefunction
 
-from django.http import HttpRequest, HttpResponse
+from django.http import HttpRequest, HttpResponse, StreamingHttpResponse
 from django.test import SimpleTestCase
 from django.views.decorators.gzip import gzip_page
 
@@ -44,3 +44,33 @@ class GzipPageTests(SimpleTestCase):
         response = await async_view(request)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.get("Content-Encoding"), "gzip")
+
+    def test_streaming_response_yields_chunks_incrementally(self):
+        @gzip_page
+        def stream_view(request):
+            return StreamingHttpResponse(self.content.encode() for _ in range(5))
+
+        request = HttpRequest()
+        request.META["HTTP_ACCEPT_ENCODING"] = "gzip"
+        response = stream_view(request)
+        compressed_chunks = list(response)
+        # Each input chunk should produce compressed output, not buffer
+        # everything into a single chunk.
+        self.assertGreater(len(compressed_chunks), 2)
+
+    async def test_async_streaming_response_yields_chunks_incrementally(self):
+        @gzip_page
+        async def stream_view(request):
+            async def content():
+                for _ in range(5):
+                    yield self.content.encode()
+
+            return StreamingHttpResponse(content())
+
+        request = HttpRequest()
+        request.META["HTTP_ACCEPT_ENCODING"] = "gzip"
+        response = await stream_view(request)
+        compressed_chunks = [chunk async for chunk in response]
+        # Each input chunk should produce compressed output, not buffer
+        # everything into a single chunk.
+        self.assertGreater(len(compressed_chunks), 2)
diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py
index 50e205a254..101943957c 100644
--- a/tests/utils_tests/test_text.py
+++ b/tests/utils_tests/test_text.py
@@ -1,3 +1,4 @@
+import gzip
 import json
 import sys
 
@@ -404,13 +405,18 @@ class TestUtilsText(SimpleTestCase):
             text.get_valid_filename("$.$.$")
 
     def test_compress_sequence(self):
-        data = [{"key": i} for i in range(10)]
-        seq = list(json.JSONEncoder().iterencode(data))
-        seq = [s.encode() for s in seq]
-        actual_length = len(b"".join(seq))
-        out = text.compress_sequence(seq)
-        compressed_length = len(b"".join(out))
-        self.assertLess(compressed_length, actual_length)
+        data = [{"key": i} for i in range(100)]
+        seq = [s.encode() for s in json.JSONEncoder().iterencode(data)]
+        original = b"".join(seq)
+        batch_size = 256
+        batched_seq = (
+            original[i : i + batch_size] for i in range(0, len(original), batch_size)
+        )
+        compressed_chunks = list(text.compress_sequence(batched_seq))
+        out = b"".join(compressed_chunks)
+        self.assertEqual(gzip.decompress(out), original)
+        self.assertLess(len(out), len(original))
+        self.assertGreater(len(compressed_chunks), 2)
 
     def test_format_lazy(self):
         self.assertEqual("django/test", format_lazy("{}/{}", "django", lazystr("test")))