summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMariusz Felisiak <felisiak.mariusz@gmail.com>2020-06-17 07:59:40 +0200
committerGitHub <noreply@github.com>2020-06-17 07:59:40 +0200
commit78c811334c9e5477b86fd113fa2c4a261e167d15 (patch)
tree7964379bfacb87e28b5d2357e1a2c71127a7210c
parente36028fb9c0c3b6d402fef2cdabe06ebb90f6d3a (diff)
Refs #30190 -- Minor edits to JSONL serializer.
Follow up to e29637681be07606674cdccb47d1e53acb930f5b.
-rw-r--r--django/core/serializers/jsonl.py2
-rw-r--r--docs/topics/serialization.txt8
-rw-r--r--tests/serializers/test_jsonl.py147
3 files changed, 53 insertions, 104 deletions
diff --git a/django/core/serializers/jsonl.py b/django/core/serializers/jsonl.py
index ff0d9eb605..4b3e46ed8e 100644
--- a/django/core/serializers/jsonl.py
+++ b/django/core/serializers/jsonl.py
@@ -50,7 +50,7 @@ def Deserializer(stream_or_string, **options):
if not line.strip():
continue
try:
- yield list(PythonDeserializer([json.loads(line), ], **options))[0]
+ yield from PythonDeserializer([json.loads(line)], **options)
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
diff --git a/docs/topics/serialization.txt b/docs/topics/serialization.txt
index 7cc4905da3..9ef851368a 100644
--- a/docs/topics/serialization.txt
+++ b/docs/topics/serialization.txt
@@ -318,12 +318,12 @@ JSONL
.. versionadded:: 3.2
*JSONL* stands for *JSON Lines*. With this format, objects are separated by new
-lines, and each line contains a valid JSON object. JSONL serialized data look
+lines, and each line contains a valid JSON object. JSONL serialized data looks
like this::
- { "pk": "4b678b301dfd8a4e0dad910de3ae245b", "model": "sessions.session", "fields": { ... }}
- { "pk": "88bea72c02274f3c9bf1cb2bb8cee4fc", "model": "sessions.session", "fields": { ... }}
- { "pk": "9cf0e26691b64147a67e2a9f06ad7a53", "model": "sessions.session", "fields": { ... }}
+ {"pk": "4b678b301dfd8a4e0dad910de3ae245b", "model": "sessions.session", "fields": {...}}
+ {"pk": "88bea72c02274f3c9bf1cb2bb8cee4fc", "model": "sessions.session", "fields": {...}}
+ {"pk": "9cf0e26691b64147a67e2a9f06ad7a53", "model": "sessions.session", "fields": {...}}
JSONL can be useful for populating large databases, since the data can be
processed line by line, rather than being loaded into memory all at once.
diff --git a/tests/serializers/test_jsonl.py b/tests/serializers/test_jsonl.py
index 19557e6c29..e4f1a3e80f 100644
--- a/tests/serializers/test_jsonl.py
+++ b/tests/serializers/test_jsonl.py
@@ -15,32 +15,20 @@ from .tests import SerializersTestBase, SerializersTransactionTestBase
class JsonlSerializerTestCase(SerializersTestBase, TestCase):
serializer_name = "jsonl"
pkless_str = [
- """{
- "pk": null,
- "model": "serializers.category",
- "fields": {"name": "Reference"}
- }""",
- """{
- "model": "serializers.category",
- "fields": {"name": "Non-fiction"}
- }"""
+ '{"pk": null,"model": "serializers.category","fields": {"name": "Reference"}}',
+ '{"model": "serializers.category","fields": {"name": "Non-fiction"}}',
]
pkless_str = "\n".join([s.replace("\n", "") for s in pkless_str])
- mapping_ordering_str = """{
-"model": "serializers.article",
-"pk": %(article_pk)s,
-"fields": {
-"author": %(author_pk)s,
-"headline": "Poker has no place on ESPN",
-"pub_date": "2006-06-16T11:00:00",
-"categories": [
-%(first_category_pk)s,
-%(second_category_pk)s
-],
-"meta_data": []
-}
-}""".replace("\n", "") + "\n"
+ mapping_ordering_str = (
+ '{"model": "serializers.article","pk": %(article_pk)s,'
+ '"fields": {'
+ '"author": %(author_pk)s,'
+ '"headline": "Poker has no place on ESPN",'
+ '"pub_date": "2006-06-16T11:00:00",'
+ '"categories": [%(first_category_pk)s,%(second_category_pk)s],'
+ '"meta_data": []}}\n'
+ )
@staticmethod
def _validate_output(serial_str):
@@ -82,7 +70,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
s = serializers.jsonl.Serializer()
json_data = s.serialize(
- [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder
+ [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder,
)
self.assertIn('"fields": {"score": "1"}', json_data)
@@ -93,55 +81,40 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
def test_helpful_error_message_invalid_pk(self):
"""
- If there is an invalid primary key, the error message should contain
- the model associated with it.
+ If there is an invalid primary key, the error message contains the
+ model associated with it.
"""
- test_string = """{
- "pk": "badpk",
- "model": "serializers.player",
- "fields": {
- "name": "Bob",
- "rank": 1,
- "team": "Team"
- }
- }""".replace("\n", "")
+ test_string = (
+ '{"pk": "badpk","model": "serializers.player",'
+ '"fields": {"name": "Bob","rank": 1,"team": "Team"}}'
+ )
with self.assertRaisesMessage(DeserializationError, "(serializers.player:pk=badpk)"):
list(serializers.deserialize('jsonl', test_string))
def test_helpful_error_message_invalid_field(self):
"""
- If there is an invalid field value, the error message should contain
- the model associated with it.
+ If there is an invalid field value, the error message contains the
+ model associated with it.
"""
- test_string = """{
- "pk": "1",
- "model": "serializers.player",
- "fields": {
- "name": "Bob",
- "rank": "invalidint",
- "team": "Team"
- }
- }""".replace("\n", "")
+ test_string = (
+ '{"pk": "1","model": "serializers.player",'
+ '"fields": {"name": "Bob","rank": "invalidint","team": "Team"}}'
+ )
expected = "(serializers.player:pk=1) field_value was 'invalidint'"
with self.assertRaisesMessage(DeserializationError, expected):
list(serializers.deserialize('jsonl', test_string))
def test_helpful_error_message_for_foreign_keys(self):
"""
- Invalid foreign keys with a natural key should throw a helpful error
- message, such as what the failing key is.
+ Invalid foreign keys with a natural key throws a helpful error message,
+ such as what the failing key is.
"""
- test_string = """{
- "pk": 1,
- "model": "serializers.category",
- "fields": {
- "name": "Unknown foreign key",
- "meta_data": [
- "doesnotexist",
- "metadata"
- ]
- }
- }""".replace("\n", "")
+ test_string = (
+ '{"pk": 1, "model": "serializers.category",'
+ '"fields": {'
+ '"name": "Unknown foreign key",'
+ '"meta_data": ["doesnotexist","metadata"]}}'
+ )
key = ["doesnotexist", "metadata"]
expected = "(serializers.category:pk=1) field_value was '%r'" % key
with self.assertRaisesMessage(DeserializationError, expected):
@@ -149,7 +122,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
def test_helpful_error_message_for_many2many_non_natural(self):
"""
- Invalid many-to-many keys should throw a helpful error message.
+ Invalid many-to-many keys throws a helpful error message.
"""
test_strings = [
"""{
@@ -165,16 +138,12 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
"""{
"pk": 1,
"model": "serializers.author",
- "fields": {
- "name": "Agnes"
- }
+ "fields": {"name": "Agnes"}
}""",
"""{
"pk": 1,
"model": "serializers.category",
- "fields": {
- "name": "Reference"
- }
+ "fields": {"name": "Reference"}
}"""
]
test_string = "\n".join([s.replace("\n", "") for s in test_strings])
@@ -184,18 +153,14 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
def test_helpful_error_message_for_many2many_natural1(self):
"""
- Invalid many-to-many keys should throw a helpful error message.
- This tests the code path where one of a list of natural keys is invalid.
+ Invalid many-to-many keys throws a helpful error message where one of a
+ list of natural keys is invalid.
"""
test_strings = [
"""{
"pk": 1,
"model": "serializers.categorymetadata",
- "fields": {
- "kind": "author",
- "name": "meta1",
- "value": "Agnes"
- }
+ "fields": {"kind": "author","name": "meta1","value": "Agnes"}
}""",
"""{
"pk": 1,
@@ -214,9 +179,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
"""{
"pk": 1,
"model": "serializers.author",
- "fields": {
- "name": "Agnes"
- }
+ "fields": {"name": "Agnes"}
}"""
]
test_string = "\n".join([s.replace("\n", "") for s in test_strings])
@@ -228,9 +191,8 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
def test_helpful_error_message_for_many2many_natural2(self):
"""
- Invalid many-to-many keys should throw a helpful error message. This
- tests the code path where a natural many-to-many key has only a single
- value.
+ Invalid many-to-many keys throws a helpful error message where a
+ natural many-to-many key has only a single value.
"""
test_strings = [
"""{
@@ -246,18 +208,12 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
"""{
"pk": 1,
"model": "serializers.categorymetadata",
- "fields": {
- "kind": "author",
- "name": "meta1",
- "value": "Agnes"
- }
+ "fields": {"kind": "author","name": "meta1","value": "Agnes"}
}""",
"""{
"pk": 1,
"model": "serializers.author",
- "fields": {
- "name": "Agnes"
- }
+ "fields": {"name": "Agnes"}
}"""
]
test_string = "\n".join([s.replace("\n", "") for s in test_strings])
@@ -270,12 +226,9 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase):
"""
Not iterable many-to-many field value throws a helpful error message.
"""
- test_string = """{
- "pk": 1,
- "model": "serializers.m2mdata",
- "fields": {"data": null}
- }""".replace("\n", "")
-
+ test_string = (
+ '{"pk": 1,"model": "serializers.m2mdata","fields": {"data": null}}'
+ )
expected = "(serializers.m2mdata:pk=1) field_value was 'None'"
with self.assertRaisesMessage(DeserializationError, expected):
next(serializers.deserialize('jsonl', test_string, ignore=False))
@@ -297,16 +250,12 @@ class JsonSerializerTransactionTestCase(SerializersTransactionTestBase, Transact
"""{
"pk": 1,
"model": "serializers.category",
- "fields": {
- "name": "Reference"
- }
+ "fields": {"name": "Reference"}
}""",
"""{
"pk": 1,
"model": "serializers.author",
- "fields": {
- "name": "Agnes"
- }
+ "fields": {"name": "Agnes"}
}"""
]
fwd_ref_str = "\n".join([s.replace("\n", "") for s in fwd_ref_str])