diff options
| -rw-r--r-- | django/utils/text.py | 2 | ||||
| -rw-r--r-- | docs/howto/outputting-csv.txt | 12 | ||||
| -rw-r--r-- | tests/decorators/test_gzip.py | 32 | ||||
| -rw-r--r-- | tests/utils_tests/test_text.py | 20 |
4 files changed, 56 insertions, 10 deletions
diff --git a/django/utils/text.py b/django/utils/text.py index cfe6ceca9e..d1306f9c6f 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -382,6 +382,7 @@ def compress_sequence(sequence, *, max_random_bytes=None): yield buf.read() for item in sequence: zfile.write(item) + zfile.flush() data = buf.read() if data: yield data @@ -398,6 +399,7 @@ async def acompress_sequence(sequence, *, max_random_bytes=None): yield buf.read() async for item in sequence: zfile.write(item) + zfile.flush() data = buf.read() if data: yield data diff --git a/docs/howto/outputting-csv.txt b/docs/howto/outputting-csv.txt index c5ae7094d2..10de00503e 100644 --- a/docs/howto/outputting-csv.txt +++ b/docs/howto/outputting-csv.txt @@ -67,9 +67,12 @@ avoid a load balancer dropping a connection that might have otherwise timed out while the server was generating the response. In this example, we make full use of Python generators to efficiently handle -the assembly and transmission of a large CSV file:: +the assembly and transmission of a large CSV file. Rows are batched together +to reduce HTTP overhead and improve compression efficiency when used with +:class:`~django.middleware.gzip.GZipMiddleware`:: import csv + from itertools import batched from django.http import StreamingHttpResponse @@ -92,8 +95,13 @@ the assembly and transmission of a large CSV file:: rows = (["Row {}".format(idx), str(idx)] for idx in range(65536)) pseudo_buffer = Echo() writer = csv.writer(pseudo_buffer) + + def stream_batched_rows(): + for batch in batched(rows, 100): + yield "".join(writer.writerow(row) for row in batch) + return StreamingHttpResponse( - (writer.writerow(row) for row in rows), + stream_batched_rows(), content_type="text/csv", headers={"Content-Disposition": 'attachment; filename="somefilename.csv"'}, ) diff --git a/tests/decorators/test_gzip.py b/tests/decorators/test_gzip.py index 2d64c171f7..8cd0869b53 100644 --- a/tests/decorators/test_gzip.py +++ b/tests/decorators/test_gzip.py @@ -1,6 +1,6 @@ from inspect import iscoroutinefunction -from django.http import HttpRequest, HttpResponse +from django.http import HttpRequest, HttpResponse, StreamingHttpResponse from django.test import SimpleTestCase from django.views.decorators.gzip import gzip_page @@ -44,3 +44,33 @@ class GzipPageTests(SimpleTestCase): response = await async_view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.get("Content-Encoding"), "gzip") + + def test_streaming_response_yields_chunks_incrementally(self): + @gzip_page + def stream_view(request): + return StreamingHttpResponse(self.content.encode() for _ in range(5)) + + request = HttpRequest() + request.META["HTTP_ACCEPT_ENCODING"] = "gzip" + response = stream_view(request) + compressed_chunks = list(response) + # Each input chunk should produce compressed output, not buffer + # everything into a single chunk. + self.assertGreater(len(compressed_chunks), 2) + + async def test_async_streaming_response_yields_chunks_incrementally(self): + @gzip_page + async def stream_view(request): + async def content(): + for _ in range(5): + yield self.content.encode() + + return StreamingHttpResponse(content()) + + request = HttpRequest() + request.META["HTTP_ACCEPT_ENCODING"] = "gzip" + response = await stream_view(request) + compressed_chunks = [chunk async for chunk in response] + # Each input chunk should produce compressed output, not buffer + # everything into a single chunk. + self.assertGreater(len(compressed_chunks), 2) diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py index 50e205a254..101943957c 100644 --- a/tests/utils_tests/test_text.py +++ b/tests/utils_tests/test_text.py @@ -1,3 +1,4 @@ +import gzip import json import sys @@ -404,13 +405,18 @@ class TestUtilsText(SimpleTestCase): text.get_valid_filename("$.$.$") def test_compress_sequence(self): - data = [{"key": i} for i in range(10)] - seq = list(json.JSONEncoder().iterencode(data)) - seq = [s.encode() for s in seq] - actual_length = len(b"".join(seq)) - out = text.compress_sequence(seq) - compressed_length = len(b"".join(out)) - self.assertLess(compressed_length, actual_length) + data = [{"key": i} for i in range(100)] + seq = [s.encode() for s in json.JSONEncoder().iterencode(data)] + original = b"".join(seq) + batch_size = 256 + batched_seq = ( + original[i : i + batch_size] for i in range(0, len(original), batch_size) + ) + compressed_chunks = list(text.compress_sequence(batched_seq)) + out = b"".join(compressed_chunks) + self.assertEqual(gzip.decompress(out), original) + self.assertLess(len(out), len(original)) + self.assertGreater(len(compressed_chunks), 2) def test_format_lazy(self): self.assertEqual("django/test", format_lazy("{}/{}", "django", lazystr("test"))) |