diff options
| author | sage <laymonage@gmail.com> | 2019-06-09 07:56:37 +0700 |
|---|---|---|
| committer | Mariusz Felisiak <felisiak.mariusz@gmail.com> | 2020-05-08 07:23:31 +0200 |
| commit | 6789ded0a6ab797f0dcdfa6ad5d1cfa46e23abcd (patch) | |
| tree | 1de598fc92480c64835b60b6ddbb461c3cd2e864 /tests | |
| parent | f97f71f59249f1fbeebe84d4fc858d70fc456f7d (diff) | |
Fixed #12990, Refs #27694 -- Added JSONField model field.
Thanks to Adam Johnson, Carlton Gibson, Mariusz Felisiak, and Raphael
Michel for mentoring this Google Summer of Code 2019 project and
everyone else who helped with the patch.
Special thanks to Mads Jensen, Nick Pope, and Simon Charette for
extensive reviews.
Co-authored-by: Mariusz Felisiak <felisiak.mariusz@gmail.com>
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/backends/base/test_operations.py | 5 | ||||
| -rw-r--r-- | tests/forms_tests/field_tests/test_jsonfield.py | 110 | ||||
| -rw-r--r-- | tests/inspectdb/models.py | 11 | ||||
| -rw-r--r-- | tests/inspectdb/tests.py | 9 | ||||
| -rw-r--r-- | tests/invalid_models_tests/test_models.py | 36 | ||||
| -rw-r--r-- | tests/invalid_models_tests/test_ordinary_fields.py | 48 | ||||
| -rw-r--r-- | tests/model_fields/models.py | 31 | ||||
| -rw-r--r-- | tests/model_fields/test_jsonfield.py | 667 | ||||
| -rw-r--r-- | tests/postgres_tests/fields.py | 7 | ||||
| -rw-r--r-- | tests/postgres_tests/migrations/0002_create_test_models.py | 17 | ||||
| -rw-r--r-- | tests/postgres_tests/models.py | 10 | ||||
| -rw-r--r-- | tests/postgres_tests/test_bulk_update.py | 3 | ||||
| -rw-r--r-- | tests/postgres_tests/test_introspection.py | 6 | ||||
| -rw-r--r-- | tests/postgres_tests/test_json.py | 583 | ||||
| -rw-r--r-- | tests/postgres_tests/test_json_deprecation.py | 54 | ||||
| -rw-r--r-- | tests/queries/models.py | 7 | ||||
| -rw-r--r-- | tests/queries/test_bulk_update.py | 17 |
17 files changed, 996 insertions, 625 deletions
diff --git a/tests/backends/base/test_operations.py b/tests/backends/base/test_operations.py index 17ee616f94..0ef2be73b0 100644 --- a/tests/backends/base/test_operations.py +++ b/tests/backends/base/test_operations.py @@ -121,6 +121,11 @@ class SimpleDatabaseOperationTests(SimpleTestCase): with self.assertRaisesMessage(NotImplementedError, self.may_require_msg % 'datetime_extract_sql'): self.ops.datetime_extract_sql(None, None, None) + def test_json_cast_text_sql(self): + msg = self.may_require_msg % 'json_cast_text_sql' + with self.assertRaisesMessage(NotImplementedError, msg): + self.ops.json_cast_text_sql(None) + class DatabaseOperationTests(TestCase): def setUp(self): diff --git a/tests/forms_tests/field_tests/test_jsonfield.py b/tests/forms_tests/field_tests/test_jsonfield.py new file mode 100644 index 0000000000..892a23a6a7 --- /dev/null +++ b/tests/forms_tests/field_tests/test_jsonfield.py @@ -0,0 +1,110 @@ +import json +import uuid + +from django.core.serializers.json import DjangoJSONEncoder +from django.forms import ( + CharField, Form, JSONField, Textarea, TextInput, ValidationError, +) +from django.test import SimpleTestCase + + +class JSONFieldTest(SimpleTestCase): + def test_valid(self): + field = JSONField() + value = field.clean('{"a": "b"}') + self.assertEqual(value, {'a': 'b'}) + + def test_valid_empty(self): + field = JSONField(required=False) + value = field.clean('') + self.assertIsNone(value) + + def test_invalid(self): + field = JSONField() + with self.assertRaisesMessage(ValidationError, 'Enter a valid JSON.'): + field.clean('{some badly formed: json}') + + def test_prepare_value(self): + field = JSONField() + self.assertEqual(field.prepare_value({'a': 'b'}), '{"a": "b"}') + self.assertEqual(field.prepare_value(None), 'null') + self.assertEqual(field.prepare_value('foo'), '"foo"') + + def test_widget(self): + field = JSONField() + self.assertIsInstance(field.widget, Textarea) + + def test_custom_widget_kwarg(self): + field = JSONField(widget=TextInput) + self.assertIsInstance(field.widget, TextInput) + + def test_custom_widget_attribute(self): + """The widget can be overridden with an attribute.""" + class CustomJSONField(JSONField): + widget = TextInput + + field = CustomJSONField() + self.assertIsInstance(field.widget, TextInput) + + def test_converted_value(self): + field = JSONField(required=False) + tests = [ + '["a", "b", "c"]', + '{"a": 1, "b": 2}', + '1', + '1.5', + '"foo"', + 'true', + 'false', + 'null', + ] + for json_string in tests: + with self.subTest(json_string=json_string): + val = field.clean(json_string) + self.assertEqual(field.clean(val), val) + + def test_has_changed(self): + field = JSONField() + self.assertIs(field.has_changed({'a': True}, '{"a": 1}'), True) + self.assertIs(field.has_changed({'a': 1, 'b': 2}, '{"b": 2, "a": 1}'), False) + + def test_custom_encoder_decoder(self): + class CustomDecoder(json.JSONDecoder): + def __init__(self, object_hook=None, *args, **kwargs): + return super().__init__(object_hook=self.as_uuid, *args, **kwargs) + + def as_uuid(self, dct): + if 'uuid' in dct: + dct['uuid'] = uuid.UUID(dct['uuid']) + return dct + + value = {'uuid': uuid.UUID('{c141e152-6550-4172-a784-05448d98204b}')} + encoded_value = '{"uuid": "c141e152-6550-4172-a784-05448d98204b"}' + field = JSONField(encoder=DjangoJSONEncoder, decoder=CustomDecoder) + self.assertEqual(field.prepare_value(value), encoded_value) + self.assertEqual(field.clean(encoded_value), value) + + def test_formfield_disabled(self): + class JSONForm(Form): + json_field = JSONField(disabled=True) + + form = JSONForm({'json_field': '["bar"]'}, initial={'json_field': ['foo']}) + self.assertIn('["foo"]</textarea>', form.as_p()) + + def test_redisplay_wrong_input(self): + """ + Displaying a bound form (typically due to invalid input). The form + should not overquote JSONField inputs. + """ + class JSONForm(Form): + name = CharField(max_length=2) + json_field = JSONField() + + # JSONField input is valid, name is too long. + form = JSONForm({'name': 'xyz', 'json_field': '["foo"]'}) + self.assertNotIn('json_field', form.errors) + self.assertIn('["foo"]</textarea>', form.as_p()) + # Invalid JSONField. + form = JSONForm({'name': 'xy', 'json_field': '{"foo"}'}) + self.assertEqual(form.errors['json_field'], ['Enter a valid JSON.']) + self.assertIn('{"foo"}</textarea>', form.as_p()) diff --git a/tests/inspectdb/models.py b/tests/inspectdb/models.py index 8a48031b24..d0076ce94f 100644 --- a/tests/inspectdb/models.py +++ b/tests/inspectdb/models.py @@ -68,6 +68,17 @@ class ColumnTypes(models.Model): uuid_field = models.UUIDField() +class JSONFieldColumnType(models.Model): + json_field = models.JSONField() + null_json_field = models.JSONField(blank=True, null=True) + + class Meta: + required_db_features = { + 'can_introspect_json_field', + 'supports_json_field', + } + + class UniqueTogether(models.Model): field1 = models.IntegerField() field2 = models.CharField(max_length=10) diff --git a/tests/inspectdb/tests.py b/tests/inspectdb/tests.py index 6e3f4b8aa6..afe89e0dda 100644 --- a/tests/inspectdb/tests.py +++ b/tests/inspectdb/tests.py @@ -85,6 +85,15 @@ class InspectDBTestCase(TestCase): elif not connection.features.interprets_empty_strings_as_nulls: assertFieldType('uuid_field', "models.CharField(max_length=32)") + @skipUnlessDBFeature('can_introspect_json_field', 'supports_json_field') + def test_json_field(self): + out = StringIO() + call_command('inspectdb', 'inspectdb_jsonfieldcolumntype', stdout=out) + output = out.getvalue() + if not connection.features.interprets_empty_strings_as_nulls: + self.assertIn('json_field = models.JSONField()', output) + self.assertIn('null_json_field = models.JSONField(blank=True, null=True)', output) + def test_number_field_types(self): """Test introspection of various Django field types""" assertFieldType = self.make_field_type_asserter() diff --git a/tests/invalid_models_tests/test_models.py b/tests/invalid_models_tests/test_models.py index 5a1bb4cc7a..6c062b2990 100644 --- a/tests/invalid_models_tests/test_models.py +++ b/tests/invalid_models_tests/test_models.py @@ -5,7 +5,7 @@ from django.core.checks.model_checks import _check_lazy_references from django.db import connection, connections, models from django.db.models.functions import Lower from django.db.models.signals import post_init -from django.test import SimpleTestCase, TestCase +from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature from django.test.utils import isolate_apps, override_settings, register_lookup @@ -1350,6 +1350,40 @@ class OtherModelTests(SimpleTestCase): @isolate_apps('invalid_models_tests') +class JSONFieldTests(TestCase): + @skipUnlessDBFeature('supports_json_field') + def test_ordering_pointing_to_json_field_value(self): + class Model(models.Model): + field = models.JSONField() + + class Meta: + ordering = ['field__value'] + + self.assertEqual(Model.check(databases=self.databases), []) + + def test_check_jsonfield(self): + class Model(models.Model): + field = models.JSONField() + + error = Error( + '%s does not support JSONFields.' % connection.display_name, + obj=Model, + id='fields.E180', + ) + expected = [] if connection.features.supports_json_field else [error] + self.assertEqual(Model.check(databases=self.databases), expected) + + def test_check_jsonfield_required_db_features(self): + class Model(models.Model): + field = models.JSONField() + + class Meta: + required_db_features = {'supports_json_field'} + + self.assertEqual(Model.check(databases=self.databases), []) + + +@isolate_apps('invalid_models_tests') class ConstraintsTests(TestCase): def test_check_constraints(self): class Model(models.Model): diff --git a/tests/invalid_models_tests/test_ordinary_fields.py b/tests/invalid_models_tests/test_ordinary_fields.py index d263dc5cc9..a81f9eed90 100644 --- a/tests/invalid_models_tests/test_ordinary_fields.py +++ b/tests/invalid_models_tests/test_ordinary_fields.py @@ -3,7 +3,9 @@ import uuid from django.core.checks import Error, Warning as DjangoWarning from django.db import connection, models -from django.test import SimpleTestCase, TestCase, skipIfDBFeature +from django.test import ( + SimpleTestCase, TestCase, skipIfDBFeature, skipUnlessDBFeature, +) from django.test.utils import isolate_apps, override_settings from django.utils.functional import lazy from django.utils.timezone import now @@ -793,3 +795,47 @@ class UUIDFieldTests(TestCase): ) self.assertEqual(Model._meta.get_field('field').check(), []) + + +@isolate_apps('invalid_models_tests') +@skipUnlessDBFeature('supports_json_field') +class JSONFieldTests(TestCase): + def test_invalid_default(self): + class Model(models.Model): + field = models.JSONField(default={}) + + self.assertEqual(Model._meta.get_field('field').check(), [ + DjangoWarning( + msg=( + "JSONField default should be a callable instead of an " + "instance so that it's not shared between all field " + "instances." + ), + hint=( + 'Use a callable instead, e.g., use `dict` instead of `{}`.' + ), + obj=Model._meta.get_field('field'), + id='fields.E010', + ) + ]) + + def test_valid_default(self): + class Model(models.Model): + field = models.JSONField(default=dict) + + self.assertEqual(Model._meta.get_field('field').check(), []) + + def test_valid_default_none(self): + class Model(models.Model): + field = models.JSONField(default=None) + + self.assertEqual(Model._meta.get_field('field').check(), []) + + def test_valid_callable_default(self): + def callable_default(): + return {'it': 'works'} + + class Model(models.Model): + field = models.JSONField(default=callable_default) + + self.assertEqual(Model._meta.get_field('field').check(), []) diff --git a/tests/model_fields/models.py b/tests/model_fields/models.py index a7efe199ab..a11eb0ba44 100644 --- a/tests/model_fields/models.py +++ b/tests/model_fields/models.py @@ -1,3 +1,4 @@ +import json import os import tempfile import uuid @@ -7,6 +8,7 @@ from django.contrib.contenttypes.fields import ( ) from django.contrib.contenttypes.models import ContentType from django.core.files.storage import FileSystemStorage +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.db.models.fields.files import ImageFieldFile from django.utils.translation import gettext_lazy as _ @@ -332,6 +334,35 @@ if Image: width_field='headshot_width') +class CustomJSONDecoder(json.JSONDecoder): + def __init__(self, object_hook=None, *args, **kwargs): + return super().__init__(object_hook=self.as_uuid, *args, **kwargs) + + def as_uuid(self, dct): + if 'uuid' in dct: + dct['uuid'] = uuid.UUID(dct['uuid']) + return dct + + +class JSONModel(models.Model): + value = models.JSONField() + + class Meta: + required_db_features = {'supports_json_field'} + + +class NullableJSONModel(models.Model): + value = models.JSONField(blank=True, null=True) + value_custom = models.JSONField( + encoder=DjangoJSONEncoder, + decoder=CustomJSONDecoder, + null=True, + ) + + class Meta: + required_db_features = {'supports_json_field'} + + class AllFieldsModel(models.Model): big_integer = models.BigIntegerField() binary = models.BinaryField() diff --git a/tests/model_fields/test_jsonfield.py b/tests/model_fields/test_jsonfield.py new file mode 100644 index 0000000000..464cf163d4 --- /dev/null +++ b/tests/model_fields/test_jsonfield.py @@ -0,0 +1,667 @@ +import operator +import uuid +from unittest import mock, skipIf, skipUnless + +from django import forms +from django.core import serializers +from django.core.exceptions import ValidationError +from django.core.serializers.json import DjangoJSONEncoder +from django.db import ( + DataError, IntegrityError, NotSupportedError, OperationalError, connection, + models, +) +from django.db.models import Count, F, OuterRef, Q, Subquery, Transform, Value +from django.db.models.expressions import RawSQL +from django.db.models.fields.json import ( + KeyTextTransform, KeyTransform, KeyTransformFactory, + KeyTransformTextLookupMixin, +) +from django.db.models.functions import Cast +from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature +from django.test.utils import CaptureQueriesContext + +from .models import CustomJSONDecoder, JSONModel, NullableJSONModel + + +@skipUnlessDBFeature('supports_json_field') +class JSONFieldTests(TestCase): + def test_invalid_value(self): + msg = 'is not JSON serializable' + with self.assertRaisesMessage(TypeError, msg): + NullableJSONModel.objects.create(value={ + 'uuid': uuid.UUID('d85e2076-b67c-4ee7-8c3a-2bf5a2cc2475'), + }) + + def test_custom_encoder_decoder(self): + value = {'uuid': uuid.UUID('{d85e2076-b67c-4ee7-8c3a-2bf5a2cc2475}')} + obj = NullableJSONModel(value_custom=value) + obj.clean_fields() + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.value_custom, value) + + def test_db_check_constraints(self): + value = '{@!invalid json value 123 $!@#' + with mock.patch.object(DjangoJSONEncoder, 'encode', return_value=value): + with self.assertRaises((IntegrityError, DataError, OperationalError)): + NullableJSONModel.objects.create(value_custom=value) + + +class TestMethods(SimpleTestCase): + def test_deconstruct(self): + field = models.JSONField() + name, path, args, kwargs = field.deconstruct() + self.assertEqual(path, 'django.db.models.JSONField') + self.assertEqual(args, []) + self.assertEqual(kwargs, {}) + + def test_deconstruct_custom_encoder_decoder(self): + field = models.JSONField(encoder=DjangoJSONEncoder, decoder=CustomJSONDecoder) + name, path, args, kwargs = field.deconstruct() + self.assertEqual(kwargs['encoder'], DjangoJSONEncoder) + self.assertEqual(kwargs['decoder'], CustomJSONDecoder) + + def test_get_transforms(self): + @models.JSONField.register_lookup + class MyTransform(Transform): + lookup_name = 'my_transform' + field = models.JSONField() + transform = field.get_transform('my_transform') + self.assertIs(transform, MyTransform) + models.JSONField._unregister_lookup(MyTransform) + models.JSONField._clear_cached_lookups() + transform = field.get_transform('my_transform') + self.assertIsInstance(transform, KeyTransformFactory) + + def test_key_transform_text_lookup_mixin_non_key_transform(self): + transform = Transform('test') + msg = ( + 'Transform should be an instance of KeyTransform in order to use ' + 'this lookup.' + ) + with self.assertRaisesMessage(TypeError, msg): + KeyTransformTextLookupMixin(transform) + + +class TestValidation(SimpleTestCase): + def test_invalid_encoder(self): + msg = 'The encoder parameter must be a callable object.' + with self.assertRaisesMessage(ValueError, msg): + models.JSONField(encoder=DjangoJSONEncoder()) + + def test_invalid_decoder(self): + msg = 'The decoder parameter must be a callable object.' + with self.assertRaisesMessage(ValueError, msg): + models.JSONField(decoder=CustomJSONDecoder()) + + def test_validation_error(self): + field = models.JSONField() + msg = 'Value must be valid JSON.' + value = uuid.UUID('{d85e2076-b67c-4ee7-8c3a-2bf5a2cc2475}') + with self.assertRaisesMessage(ValidationError, msg): + field.clean({'uuid': value}, None) + + def test_custom_encoder(self): + field = models.JSONField(encoder=DjangoJSONEncoder) + value = uuid.UUID('{d85e2076-b67c-4ee7-8c3a-2bf5a2cc2475}') + field.clean({'uuid': value}, None) + + +class TestFormField(SimpleTestCase): + def test_formfield(self): + model_field = models.JSONField() + form_field = model_field.formfield() + self.assertIsInstance(form_field, forms.JSONField) + + def test_formfield_custom_encoder_decoder(self): + model_field = models.JSONField(encoder=DjangoJSONEncoder, decoder=CustomJSONDecoder) + form_field = model_field.formfield() + self.assertIs(form_field.encoder, DjangoJSONEncoder) + self.assertIs(form_field.decoder, CustomJSONDecoder) + + +class TestSerialization(SimpleTestCase): + test_data = ( + '[{"fields": {"value": %s}, ' + '"model": "model_fields.jsonmodel", "pk": null}]' + ) + test_values = ( + # (Python value, serialized value), + ({'a': 'b', 'c': None}, '{"a": "b", "c": null}'), + ('abc', '"abc"'), + ('{"a": "a"}', '"{\\"a\\": \\"a\\"}"'), + ) + + def test_dumping(self): + for value, serialized in self.test_values: + with self.subTest(value=value): + instance = JSONModel(value=value) + data = serializers.serialize('json', [instance]) + self.assertJSONEqual(data, self.test_data % serialized) + + def test_loading(self): + for value, serialized in self.test_values: + with self.subTest(value=value): + instance = list( + serializers.deserialize('json', self.test_data % serialized) + )[0].object + self.assertEqual(instance.value, value) + + +@skipUnlessDBFeature('supports_json_field') +class TestSaveLoad(TestCase): + def test_null(self): + obj = NullableJSONModel(value=None) + obj.save() + obj.refresh_from_db() + self.assertIsNone(obj.value) + + @skipUnlessDBFeature('supports_primitives_in_json_field') + def test_json_null_different_from_sql_null(self): + json_null = NullableJSONModel.objects.create(value=Value('null')) + json_null.refresh_from_db() + sql_null = NullableJSONModel.objects.create(value=None) + sql_null.refresh_from_db() + # 'null' is not equal to NULL in the database. + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value=Value('null')), + [json_null], + ) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value=None), + [json_null], + ) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__isnull=True), + [sql_null], + ) + # 'null' is equal to NULL in Python (None). + self.assertEqual(json_null.value, sql_null.value) + + @skipUnlessDBFeature('supports_primitives_in_json_field') + def test_primitives(self): + values = [ + True, + 1, + 1.45, + 'String', + '', + ] + for value in values: + with self.subTest(value=value): + obj = JSONModel(value=value) + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.value, value) + + def test_dict(self): + values = [ + {}, + {'name': 'John', 'age': 20, 'height': 180.3}, + {'a': True, 'b': {'b1': False, 'b2': None}}, + ] + for value in values: + with self.subTest(value=value): + obj = JSONModel.objects.create(value=value) + obj.refresh_from_db() + self.assertEqual(obj.value, value) + + def test_list(self): + values = [ + [], + ['John', 20, 180.3], + [True, [False, None]], + ] + for value in values: + with self.subTest(value=value): + obj = JSONModel.objects.create(value=value) + obj.refresh_from_db() + self.assertEqual(obj.value, value) + + def test_realistic_object(self): + value = { + 'name': 'John', + 'age': 20, + 'pets': [ + {'name': 'Kit', 'type': 'cat', 'age': 2}, + {'name': 'Max', 'type': 'dog', 'age': 1}, + ], + 'courses': [ + ['A1', 'A2', 'A3'], + ['B1', 'B2'], + ['C1'], + ], + } + obj = JSONModel.objects.create(value=value) + obj.refresh_from_db() + self.assertEqual(obj.value, value) + + +@skipUnlessDBFeature('supports_json_field') +class TestQuerying(TestCase): + @classmethod + def setUpTestData(cls): + cls.primitives = [True, False, 'yes', 7, 9.6] + values = [ + None, + [], + {}, + {'a': 'b', 'c': 14}, + { + 'a': 'b', + 'c': 14, + 'd': ['e', {'f': 'g'}], + 'h': True, + 'i': False, + 'j': None, + 'k': {'l': 'm'}, + 'n': [None], + }, + [1, [2]], + {'k': True, 'l': False}, + { + 'foo': 'bar', + 'baz': {'a': 'b', 'c': 'd'}, + 'bar': ['foo', 'bar'], + 'bax': {'foo': 'bar'}, + }, + ] + cls.objs = [ + NullableJSONModel.objects.create(value=value) + for value in values + ] + if connection.features.supports_primitives_in_json_field: + cls.objs.extend([ + NullableJSONModel.objects.create(value=value) + for value in cls.primitives + ]) + cls.raw_sql = '%s::jsonb' if connection.vendor == 'postgresql' else '%s' + + def test_exact(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__exact={}), + [self.objs[2]], + ) + + def test_exact_complex(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__exact={'a': 'b', 'c': 14}), + [self.objs[3]], + ) + + def test_isnull(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__isnull=True), + [self.objs[0]], + ) + + def test_ordering_by_transform(self): + objs = [ + NullableJSONModel.objects.create(value={'ord': 93, 'name': 'bar'}), + NullableJSONModel.objects.create(value={'ord': 22.1, 'name': 'foo'}), + NullableJSONModel.objects.create(value={'ord': -1, 'name': 'baz'}), + NullableJSONModel.objects.create(value={'ord': 21.931902, 'name': 'spam'}), + NullableJSONModel.objects.create(value={'ord': -100291029, 'name': 'eggs'}), + ] + query = NullableJSONModel.objects.filter(value__name__isnull=False).order_by('value__ord') + expected = [objs[4], objs[2], objs[3], objs[1], objs[0]] + mariadb = connection.vendor == 'mysql' and connection.mysql_is_mariadb + if mariadb or connection.vendor == 'oracle': + # MariaDB and Oracle return JSON values as strings. + expected = [objs[2], objs[4], objs[3], objs[1], objs[0]] + self.assertSequenceEqual(query, expected) + + def test_ordering_grouping_by_key_transform(self): + base_qs = NullableJSONModel.objects.filter(value__d__0__isnull=False) + for qs in ( + base_qs.order_by('value__d__0'), + base_qs.annotate(key=KeyTransform('0', KeyTransform('d', 'value'))).order_by('key'), + ): + self.assertSequenceEqual(qs, [self.objs[4]]) + qs = NullableJSONModel.objects.filter(value__isnull=False) + self.assertQuerysetEqual( + qs.filter(value__isnull=False).annotate( + key=KeyTextTransform('f', KeyTransform('1', KeyTransform('d', 'value'))), + ).values('key').annotate(count=Count('key')).order_by('count'), + [(None, 0), ('g', 1)], + operator.itemgetter('key', 'count'), + ) + + @skipIf(connection.vendor == 'oracle', "Oracle doesn't support grouping by LOBs, see #24096.") + def test_ordering_grouping_by_count(self): + qs = NullableJSONModel.objects.filter( + value__isnull=False, + ).values('value__d__0').annotate(count=Count('value__d__0')).order_by('count') + self.assertQuerysetEqual(qs, [1, 11], operator.itemgetter('count')) + + def test_key_transform_raw_expression(self): + expr = RawSQL(self.raw_sql, ['{"x": "bar"}']) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__foo=KeyTransform('x', expr)), + [self.objs[7]], + ) + + def test_nested_key_transform_raw_expression(self): + expr = RawSQL(self.raw_sql, ['{"x": {"y": "bar"}}']) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__foo=KeyTransform('y', KeyTransform('x', expr))), + [self.objs[7]], + ) + + def test_key_transform_expression(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__d__0__isnull=False).annotate( + key=KeyTransform('d', 'value'), + chain=KeyTransform('0', 'key'), + expr=KeyTransform('0', Cast('key', models.JSONField())), + ).filter(chain=F('expr')), + [self.objs[4]], + ) + + def test_nested_key_transform_expression(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__d__0__isnull=False).annotate( + key=KeyTransform('d', 'value'), + chain=KeyTransform('f', KeyTransform('1', 'key')), + expr=KeyTransform('f', KeyTransform('1', Cast('key', models.JSONField()))), + ).filter(chain=F('expr')), + [self.objs[4]], + ) + + def test_has_key(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__has_key='a'), + [self.objs[3], self.objs[4]], + ) + + def test_has_key_null_value(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__has_key='j'), + [self.objs[4]], + ) + + def test_has_key_deep(self): + tests = [ + (Q(value__baz__has_key='a'), self.objs[7]), + (Q(value__has_key=KeyTransform('a', KeyTransform('baz', 'value'))), self.objs[7]), + (Q(value__has_key=KeyTransform('c', KeyTransform('baz', 'value'))), self.objs[7]), + (Q(value__d__1__has_key='f'), self.objs[4]), + ( + Q(value__has_key=KeyTransform('f', KeyTransform('1', KeyTransform('d', 'value')))), + self.objs[4], + ) + ] + for condition, expected in tests: + with self.subTest(condition=condition): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(condition), + [expected], + ) + + def test_has_key_list(self): + obj = NullableJSONModel.objects.create(value=[{'a': 1}, {'b': 'x'}]) + tests = [ + Q(value__1__has_key='b'), + Q(value__has_key=KeyTransform('b', KeyTransform(1, 'value'))), + Q(value__has_key=KeyTransform('b', KeyTransform('1', 'value'))), + ] + for condition in tests: + with self.subTest(condition=condition): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(condition), + [obj], + ) + + def test_has_keys(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__has_keys=['a', 'c', 'h']), + [self.objs[4]], + ) + + def test_has_any_keys(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__has_any_keys=['c', 'l']), + [self.objs[3], self.objs[4], self.objs[6]], + ) + + def test_contains(self): + tests = [ + ({}, self.objs[2:5] + self.objs[6:8]), + ({'baz': {'a': 'b', 'c': 'd'}}, [self.objs[7]]), + ({'k': True, 'l': False}, [self.objs[6]]), + ({'d': ['e', {'f': 'g'}]}, [self.objs[4]]), + ([1, [2]], [self.objs[5]]), + ({'n': [None]}, [self.objs[4]]), + ({'j': None}, [self.objs[4]]), + ] + for value, expected in tests: + with self.subTest(value=value): + qs = NullableJSONModel.objects.filter(value__contains=value) + self.assertSequenceEqual(qs, expected) + + @skipUnlessDBFeature('supports_primitives_in_json_field') + def test_contains_primitives(self): + for value in self.primitives: + with self.subTest(value=value): + qs = NullableJSONModel.objects.filter(value__contains=value) + self.assertIs(qs.exists(), True) + + @skipIf( + connection.vendor == 'oracle', + "Oracle doesn't support contained_by lookup.", + ) + def test_contained_by(self): + qs = NullableJSONModel.objects.filter(value__contained_by={'a': 'b', 'c': 14, 'h': True}) + self.assertSequenceEqual(qs, self.objs[2:4]) + + @skipUnless( + connection.vendor == 'oracle', + "Oracle doesn't support contained_by lookup.", + ) + def test_contained_by_unsupported(self): + msg = 'contained_by lookup is not supported on Oracle.' + with self.assertRaisesMessage(NotSupportedError, msg): + NullableJSONModel.objects.filter(value__contained_by={'a': 'b'}).get() + + def test_deep_values(self): + qs = NullableJSONModel.objects.values_list('value__k__l') + expected_objs = [(None,)] * len(self.objs) + expected_objs[4] = ('m',) + self.assertSequenceEqual(qs, expected_objs) + + @skipUnlessDBFeature('can_distinct_on_fields') + def test_deep_distinct(self): + query = NullableJSONModel.objects.distinct('value__k__l').values_list('value__k__l') + self.assertSequenceEqual(query, [('m',), (None,)]) + + def test_isnull_key(self): + # key__isnull=False works the same as has_key='key'. + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__a__isnull=True), + self.objs[:3] + self.objs[5:], + ) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__a__isnull=False), + [self.objs[3], self.objs[4]], + ) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__j__isnull=False), + [self.objs[4]], + ) + + def test_isnull_key_or_none(self): + obj = NullableJSONModel.objects.create(value={'a': None}) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(Q(value__a__isnull=True) | Q(value__a=None)), + self.objs[:3] + self.objs[5:] + [obj], + ) + + def test_none_key(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__j=None), + [self.objs[4]], + ) + + def test_none_key_exclude(self): + obj = NullableJSONModel.objects.create(value={'j': 1}) + if connection.vendor == 'oracle': + # Oracle supports filtering JSON objects with NULL keys, but the + # current implementation doesn't support it. + self.assertSequenceEqual( + NullableJSONModel.objects.exclude(value__j=None), + self.objs[1:4] + self.objs[5:] + [obj], + ) + else: + self.assertSequenceEqual(NullableJSONModel.objects.exclude(value__j=None), [obj]) + + def test_shallow_list_lookup(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__0=1), + [self.objs[5]], + ) + + def test_shallow_obj_lookup(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__a='b'), + [self.objs[3], self.objs[4]], + ) + + def test_obj_subquery_lookup(self): + qs = NullableJSONModel.objects.annotate( + field=Subquery(NullableJSONModel.objects.filter(pk=OuterRef('pk')).values('value')), + ).filter(field__a='b') + self.assertSequenceEqual(qs, [self.objs[3], self.objs[4]]) + + def test_deep_lookup_objs(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__k__l='m'), + [self.objs[4]], + ) + + def test_shallow_lookup_obj_target(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__k={'l': 'm'}), + [self.objs[4]], + ) + + def test_deep_lookup_array(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__1__0=2), + [self.objs[5]], + ) + + def test_deep_lookup_mixed(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__d__1__f='g'), + [self.objs[4]], + ) + + def test_deep_lookup_transform(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__c__gt=2), + [self.objs[3], self.objs[4]], + ) + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__c__gt=2.33), + [self.objs[3], self.objs[4]], + ) + self.assertIs(NullableJSONModel.objects.filter(value__c__lt=5).exists(), False) + + @skipIf( + connection.vendor == 'oracle', + 'Raises ORA-00600: internal error code on Oracle 18.', + ) + def test_usage_in_subquery(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter( + id__in=NullableJSONModel.objects.filter(value__c=14), + ), + self.objs[3:5], + ) + + def test_key_iexact(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__iexact='BaR').exists(), True) + self.assertIs(NullableJSONModel.objects.filter(value__foo__iexact='"BaR"').exists(), False) + + def test_key_contains(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__contains='ar').exists(), True) + + def test_key_icontains(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__icontains='Ar').exists(), True) + + def test_key_startswith(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__startswith='b').exists(), True) + + def test_key_istartswith(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__istartswith='B').exists(), True) + + def test_key_endswith(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__endswith='r').exists(), True) + + def test_key_iendswith(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__iendswith='R').exists(), True) + + def test_key_regex(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__regex=r'^bar$').exists(), True) + + def test_key_iregex(self): + self.assertIs(NullableJSONModel.objects.filter(value__foo__iregex=r'^bAr$').exists(), True) + + @skipUnless(connection.vendor == 'postgresql', 'kwargs are crafted for PostgreSQL.') + def test_key_sql_injection(self): + with CaptureQueriesContext(connection) as queries: + self.assertIs( + NullableJSONModel.objects.filter(**{ + """value__test' = '"a"') OR 1 = 1 OR ('d""": 'x', + }).exists(), + False, + ) + self.assertIn( + """."value" -> 'test'' = ''"a"'') OR 1 = 1 OR (''d') = '"x"' """, + queries[0]['sql'], + ) + + @skipIf(connection.vendor == 'postgresql', 'PostgreSQL uses operators not functions.') + def test_key_sql_injection_escape(self): + query = str(JSONModel.objects.filter(**{ + """value__test") = '"a"' OR 1 = 1 OR ("d""": 'x', + }).query) + self.assertIn('"test\\"', query) + self.assertIn('\\"d', query) + + def test_key_escape(self): + obj = NullableJSONModel.objects.create(value={'%total': 10}) + self.assertEqual(NullableJSONModel.objects.filter(**{'value__%total': 10}).get(), obj) + + def test_none_key_and_exact_lookup(self): + self.assertSequenceEqual( + NullableJSONModel.objects.filter(value__a='b', value__j=None), + [self.objs[4]], + ) + + def test_lookups_with_key_transform(self): + tests = ( + ('value__d__contains', 'e'), + ('value__baz__has_key', 'c'), + ('value__baz__has_keys', ['a', 'c']), + ('value__baz__has_any_keys', ['a', 'x']), + ('value__contains', KeyTransform('bax', 'value')), + ('value__has_key', KeyTextTransform('foo', 'value')), + ) + # contained_by lookup is not supported on Oracle. + if connection.vendor != 'oracle': + tests += ( + ('value__baz__contained_by', {'a': 'b', 'c': 'd', 'e': 'f'}), + ( + 'value__contained_by', + KeyTransform('x', RawSQL( + self.raw_sql, + ['{"x": {"a": "b", "c": 1, "d": "e"}}'], + )), + ), + ) + for lookup, value in tests: + with self.subTest(lookup=lookup): + self.assertIs(NullableJSONModel.objects.filter( + **{lookup: value}, + ).exists(), True) diff --git a/tests/postgres_tests/fields.py b/tests/postgres_tests/fields.py index 4ebc0ce7dc..a36c10c750 100644 --- a/tests/postgres_tests/fields.py +++ b/tests/postgres_tests/fields.py @@ -10,7 +10,7 @@ try: from django.contrib.postgres.fields import ( ArrayField, BigIntegerRangeField, CICharField, CIEmailField, CITextField, DateRangeField, DateTimeRangeField, DecimalRangeField, - HStoreField, IntegerRangeField, JSONField, + HStoreField, IntegerRangeField, ) from django.contrib.postgres.search import SearchVectorField except ImportError: @@ -26,10 +26,6 @@ except ImportError: }) return name, path, args, kwargs - class DummyJSONField(models.Field): - def __init__(self, encoder=None, **kwargs): - super().__init__(**kwargs) - ArrayField = DummyArrayField BigIntegerRangeField = models.Field CICharField = models.Field @@ -40,7 +36,6 @@ except ImportError: DecimalRangeField = models.Field HStoreField = models.Field IntegerRangeField = models.Field - JSONField = DummyJSONField SearchVectorField = models.Field diff --git a/tests/postgres_tests/migrations/0002_create_test_models.py b/tests/postgres_tests/migrations/0002_create_test_models.py index ee1463e1eb..cb5f4c6d3e 100644 --- a/tests/postgres_tests/migrations/0002_create_test_models.py +++ b/tests/postgres_tests/migrations/0002_create_test_models.py @@ -1,10 +1,9 @@ -from django.core.serializers.json import DjangoJSONEncoder from django.db import migrations, models from ..fields import ( ArrayField, BigIntegerRangeField, CICharField, CIEmailField, CITextField, DateRangeField, DateTimeRangeField, DecimalRangeField, EnumField, - HStoreField, IntegerRangeField, JSONField, SearchVectorField, + HStoreField, IntegerRangeField, SearchVectorField, ) from ..models import TagField @@ -60,7 +59,7 @@ class Migration(migrations.Migration): ('uuids', ArrayField(models.UUIDField(), size=None, default=list)), ('decimals', ArrayField(models.DecimalField(max_digits=5, decimal_places=2), size=None, default=list)), ('tags', ArrayField(TagField(), blank=True, null=True, size=None)), - ('json', ArrayField(JSONField(default={}), default=[])), + ('json', ArrayField(models.JSONField(default={}), default=[])), ('int_ranges', ArrayField(IntegerRangeField(), null=True, blank=True)), ('bigint_ranges', ArrayField(BigIntegerRangeField(), null=True, blank=True)), ], @@ -271,18 +270,6 @@ class Migration(migrations.Migration): bases=(models.Model,), ), migrations.CreateModel( - name='JSONModel', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('field', JSONField(null=True, blank=True)), - ('field_custom', JSONField(null=True, blank=True, encoder=DjangoJSONEncoder)), - ], - options={ - 'required_db_vendor': 'postgresql', - }, - bases=(models.Model,), - ), - migrations.CreateModel( name='ArrayEnumModel', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), diff --git a/tests/postgres_tests/models.py b/tests/postgres_tests/models.py index 24605954b2..464245fbab 100644 --- a/tests/postgres_tests/models.py +++ b/tests/postgres_tests/models.py @@ -1,10 +1,9 @@ -from django.core.serializers.json import DjangoJSONEncoder from django.db import models from .fields import ( ArrayField, BigIntegerRangeField, CICharField, CIEmailField, CITextField, DateRangeField, DateTimeRangeField, DecimalRangeField, EnumField, - HStoreField, IntegerRangeField, JSONField, SearchVectorField, + HStoreField, IntegerRangeField, SearchVectorField, ) @@ -68,7 +67,7 @@ class OtherTypesArrayModel(PostgreSQLModel): uuids = ArrayField(models.UUIDField(), default=list) decimals = ArrayField(models.DecimalField(max_digits=5, decimal_places=2), default=list) tags = ArrayField(TagField(), blank=True, null=True) - json = ArrayField(JSONField(default=dict), default=list) + json = ArrayField(models.JSONField(default=dict), default=list) int_ranges = ArrayField(IntegerRangeField(), blank=True, null=True) bigint_ranges = ArrayField(BigIntegerRangeField(), blank=True, null=True) @@ -150,11 +149,6 @@ class RangeLookupsModel(PostgreSQLModel): decimal_field = models.DecimalField(max_digits=5, decimal_places=2, blank=True, null=True) -class JSONModel(PostgreSQLModel): - field = JSONField(blank=True, null=True) - field_custom = JSONField(blank=True, null=True, encoder=DjangoJSONEncoder) - - class ArrayFieldSubclass(ArrayField): def __init__(self, *args, **kwargs): super().__init__(models.IntegerField()) diff --git a/tests/postgres_tests/test_bulk_update.py b/tests/postgres_tests/test_bulk_update.py index 6dd7036a9b..7fa2a6a7db 100644 --- a/tests/postgres_tests/test_bulk_update.py +++ b/tests/postgres_tests/test_bulk_update.py @@ -2,7 +2,7 @@ from datetime import date from . import PostgreSQLTestCase from .models import ( - HStoreModel, IntegerArrayModel, JSONModel, NestedIntegerArrayModel, + HStoreModel, IntegerArrayModel, NestedIntegerArrayModel, NullableIntegerArrayModel, OtherTypesArrayModel, RangesModel, ) @@ -17,7 +17,6 @@ class BulkSaveTests(PostgreSQLTestCase): test_data = [ (IntegerArrayModel, 'field', [], [1, 2, 3]), (NullableIntegerArrayModel, 'field', [1, 2, 3], None), - (JSONModel, 'field', {'a': 'b'}, {'c': 'd'}), (NestedIntegerArrayModel, 'field', [], [[1, 2, 3]]), (HStoreModel, 'field', {}, {1: 2}), (RangesModel, 'ints', None, NumericRange(lower=1, upper=10)), diff --git a/tests/postgres_tests/test_introspection.py b/tests/postgres_tests/test_introspection.py index 8ae5b80da1..50cb9b2828 100644 --- a/tests/postgres_tests/test_introspection.py +++ b/tests/postgres_tests/test_introspection.py @@ -19,12 +19,6 @@ class InspectDBTests(PostgreSQLTestCase): for field_output in field_outputs: self.assertIn(field_output, output) - def test_json_field(self): - self.assertFieldsInModel( - 'postgres_tests_jsonmodel', - ['field = django.contrib.postgres.fields.JSONField(blank=True, null=True)'], - ) - def test_range_fields(self): self.assertFieldsInModel( 'postgres_tests_rangesmodel', diff --git a/tests/postgres_tests/test_json.py b/tests/postgres_tests/test_json.py deleted file mode 100644 index 2ff765e918..0000000000 --- a/tests/postgres_tests/test_json.py +++ /dev/null @@ -1,583 +0,0 @@ -import datetime -import operator -import uuid -from decimal import Decimal - -from django.core import checks, exceptions, serializers -from django.core.serializers.json import DjangoJSONEncoder -from django.db import connection -from django.db.models import Count, F, OuterRef, Q, Subquery -from django.db.models.expressions import RawSQL -from django.db.models.functions import Cast -from django.forms import CharField, Form, widgets -from django.test.utils import CaptureQueriesContext, isolate_apps -from django.utils.html import escape - -from . import PostgreSQLSimpleTestCase, PostgreSQLTestCase -from .models import JSONModel, PostgreSQLModel - -try: - from django.contrib.postgres import forms - from django.contrib.postgres.fields import JSONField - from django.contrib.postgres.fields.jsonb import KeyTextTransform, KeyTransform -except ImportError: - pass - - -class TestModelMetaOrdering(PostgreSQLSimpleTestCase): - def test_ordering_by_json_field_value(self): - class TestJSONModel(JSONModel): - class Meta: - ordering = ['field__value'] - - self.assertEqual(TestJSONModel.check(), []) - - -class TestSaveLoad(PostgreSQLTestCase): - def test_null(self): - instance = JSONModel() - instance.save() - loaded = JSONModel.objects.get() - self.assertIsNone(loaded.field) - - def test_empty_object(self): - instance = JSONModel(field={}) - instance.save() - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field, {}) - - def test_empty_list(self): - instance = JSONModel(field=[]) - instance.save() - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field, []) - - def test_boolean(self): - instance = JSONModel(field=True) - instance.save() - loaded = JSONModel.objects.get() - self.assertIs(loaded.field, True) - - def test_string(self): - instance = JSONModel(field='why?') - instance.save() - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field, 'why?') - - def test_number(self): - instance = JSONModel(field=1) - instance.save() - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field, 1) - - def test_realistic_object(self): - obj = { - 'a': 'b', - 'c': 1, - 'd': ['e', {'f': 'g'}], - 'h': True, - 'i': False, - 'j': None, - } - instance = JSONModel(field=obj) - instance.save() - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field, obj) - - def test_custom_encoding(self): - """ - JSONModel.field_custom has a custom DjangoJSONEncoder. - """ - some_uuid = uuid.uuid4() - obj_before = { - 'date': datetime.date(2016, 8, 12), - 'datetime': datetime.datetime(2016, 8, 12, 13, 44, 47, 575981), - 'decimal': Decimal('10.54'), - 'uuid': some_uuid, - } - obj_after = { - 'date': '2016-08-12', - 'datetime': '2016-08-12T13:44:47.575', - 'decimal': '10.54', - 'uuid': str(some_uuid), - } - JSONModel.objects.create(field_custom=obj_before) - loaded = JSONModel.objects.get() - self.assertEqual(loaded.field_custom, obj_after) - - -class TestQuerying(PostgreSQLTestCase): - @classmethod - def setUpTestData(cls): - cls.objs = JSONModel.objects.bulk_create([ - JSONModel(field=None), - JSONModel(field=True), - JSONModel(field=False), - JSONModel(field='yes'), - JSONModel(field=7), - JSONModel(field=[]), - JSONModel(field={}), - JSONModel(field={ - 'a': 'b', - 'c': 1, - }), - JSONModel(field={ - 'a': 'b', - 'c': 1, - 'd': ['e', {'f': 'g'}], - 'h': True, - 'i': False, - 'j': None, - 'k': {'l': 'm'}, - }), - JSONModel(field=[1, [2]]), - JSONModel(field={ - 'k': True, - 'l': False, - }), - JSONModel(field={ - 'foo': 'bar', - 'baz': {'a': 'b', 'c': 'd'}, - 'bar': ['foo', 'bar'], - 'bax': {'foo': 'bar'}, - }), - ]) - - def test_exact(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__exact={}), - [self.objs[6]] - ) - - def test_exact_complex(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__exact={'a': 'b', 'c': 1}), - [self.objs[7]] - ) - - def test_isnull(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__isnull=True), - [self.objs[0]] - ) - - def test_ordering_by_transform(self): - objs = [ - JSONModel.objects.create(field={'ord': 93, 'name': 'bar'}), - JSONModel.objects.create(field={'ord': 22.1, 'name': 'foo'}), - JSONModel.objects.create(field={'ord': -1, 'name': 'baz'}), - JSONModel.objects.create(field={'ord': 21.931902, 'name': 'spam'}), - JSONModel.objects.create(field={'ord': -100291029, 'name': 'eggs'}), - ] - query = JSONModel.objects.filter(field__name__isnull=False).order_by('field__ord') - self.assertSequenceEqual(query, [objs[4], objs[2], objs[3], objs[1], objs[0]]) - - def test_ordering_grouping_by_key_transform(self): - base_qs = JSONModel.objects.filter(field__d__0__isnull=False) - for qs in ( - base_qs.order_by('field__d__0'), - base_qs.annotate(key=KeyTransform('0', KeyTransform('d', 'field'))).order_by('key'), - ): - self.assertSequenceEqual(qs, [self.objs[8]]) - qs = JSONModel.objects.filter(field__isnull=False) - self.assertQuerysetEqual( - qs.values('field__d__0').annotate(count=Count('field__d__0')).order_by('count'), - [1, 10], - operator.itemgetter('count'), - ) - self.assertQuerysetEqual( - qs.filter(field__isnull=False).annotate( - key=KeyTextTransform('f', KeyTransform('1', KeyTransform('d', 'field'))), - ).values('key').annotate(count=Count('key')).order_by('count'), - [(None, 0), ('g', 1)], - operator.itemgetter('key', 'count'), - ) - - def test_key_transform_raw_expression(self): - expr = RawSQL('%s::jsonb', ['{"x": "bar"}']) - self.assertSequenceEqual( - JSONModel.objects.filter(field__foo=KeyTransform('x', expr)), - [self.objs[-1]], - ) - - def test_key_transform_expression(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__d__0__isnull=False).annotate( - key=KeyTransform('d', 'field'), - chain=KeyTransform('0', 'key'), - expr=KeyTransform('0', Cast('key', JSONField())), - ).filter(chain=F('expr')), - [self.objs[8]], - ) - - def test_nested_key_transform_raw_expression(self): - expr = RawSQL('%s::jsonb', ['{"x": {"y": "bar"}}']) - self.assertSequenceEqual( - JSONModel.objects.filter(field__foo=KeyTransform('y', KeyTransform('x', expr))), - [self.objs[-1]], - ) - - def test_nested_key_transform_expression(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__d__0__isnull=False).annotate( - key=KeyTransform('d', 'field'), - chain=KeyTransform('f', KeyTransform('1', 'key')), - expr=KeyTransform('f', KeyTransform('1', Cast('key', JSONField()))), - ).filter(chain=F('expr')), - [self.objs[8]], - ) - - def test_deep_values(self): - query = JSONModel.objects.values_list('field__k__l') - self.assertSequenceEqual( - query, - [ - (None,), (None,), (None,), (None,), (None,), (None,), - (None,), (None,), ('m',), (None,), (None,), (None,), - ] - ) - - def test_deep_distinct(self): - query = JSONModel.objects.distinct('field__k__l').values_list('field__k__l') - self.assertSequenceEqual(query, [('m',), (None,)]) - - def test_isnull_key(self): - # key__isnull works the same as has_key='key'. - self.assertSequenceEqual( - JSONModel.objects.filter(field__a__isnull=True), - self.objs[:7] + self.objs[9:] - ) - self.assertSequenceEqual( - JSONModel.objects.filter(field__a__isnull=False), - [self.objs[7], self.objs[8]] - ) - - def test_none_key(self): - self.assertSequenceEqual(JSONModel.objects.filter(field__j=None), [self.objs[8]]) - - def test_none_key_exclude(self): - obj = JSONModel.objects.create(field={'j': 1}) - self.assertSequenceEqual(JSONModel.objects.exclude(field__j=None), [obj]) - - def test_isnull_key_or_none(self): - obj = JSONModel.objects.create(field={'a': None}) - self.assertSequenceEqual( - JSONModel.objects.filter(Q(field__a__isnull=True) | Q(field__a=None)), - self.objs[:7] + self.objs[9:] + [obj] - ) - - def test_contains(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__contains={'a': 'b'}), - [self.objs[7], self.objs[8]] - ) - - def test_contained_by(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__contained_by={'a': 'b', 'c': 1, 'h': True}), - [self.objs[6], self.objs[7]] - ) - - def test_has_key(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__has_key='a'), - [self.objs[7], self.objs[8]] - ) - - def test_has_keys(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__has_keys=['a', 'c', 'h']), - [self.objs[8]] - ) - - def test_has_any_keys(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__has_any_keys=['c', 'l']), - [self.objs[7], self.objs[8], self.objs[10]] - ) - - def test_shallow_list_lookup(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__0=1), - [self.objs[9]] - ) - - def test_shallow_obj_lookup(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__a='b'), - [self.objs[7], self.objs[8]] - ) - - def test_obj_subquery_lookup(self): - qs = JSONModel.objects.annotate( - value=Subquery(JSONModel.objects.filter(pk=OuterRef('pk')).values('field')), - ).filter(value__a='b') - self.assertSequenceEqual(qs, [self.objs[7], self.objs[8]]) - - def test_deep_lookup_objs(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__k__l='m'), - [self.objs[8]] - ) - - def test_shallow_lookup_obj_target(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__k={'l': 'm'}), - [self.objs[8]] - ) - - def test_deep_lookup_array(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__1__0=2), - [self.objs[9]] - ) - - def test_deep_lookup_mixed(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__d__1__f='g'), - [self.objs[8]] - ) - - def test_deep_lookup_transform(self): - self.assertSequenceEqual( - JSONModel.objects.filter(field__c__gt=1), - [] - ) - self.assertSequenceEqual( - JSONModel.objects.filter(field__c__lt=5), - [self.objs[7], self.objs[8]] - ) - - def test_usage_in_subquery(self): - self.assertSequenceEqual( - JSONModel.objects.filter(id__in=JSONModel.objects.filter(field__c=1)), - self.objs[7:9] - ) - - def test_iexact(self): - self.assertTrue(JSONModel.objects.filter(field__foo__iexact='BaR').exists()) - self.assertFalse(JSONModel.objects.filter(field__foo__iexact='"BaR"').exists()) - - def test_icontains(self): - self.assertFalse(JSONModel.objects.filter(field__foo__icontains='"bar"').exists()) - - def test_startswith(self): - self.assertTrue(JSONModel.objects.filter(field__foo__startswith='b').exists()) - - def test_istartswith(self): - self.assertTrue(JSONModel.objects.filter(field__foo__istartswith='B').exists()) - - def test_endswith(self): - self.assertTrue(JSONModel.objects.filter(field__foo__endswith='r').exists()) - - def test_iendswith(self): - self.assertTrue(JSONModel.objects.filter(field__foo__iendswith='R').exists()) - - def test_regex(self): - self.assertTrue(JSONModel.objects.filter(field__foo__regex=r'^bar$').exists()) - - def test_iregex(self): - self.assertTrue(JSONModel.objects.filter(field__foo__iregex=r'^bAr$').exists()) - - def test_key_sql_injection(self): - with CaptureQueriesContext(connection) as queries: - self.assertFalse( - JSONModel.objects.filter(**{ - """field__test' = '"a"') OR 1 = 1 OR ('d""": 'x', - }).exists() - ) - self.assertIn( - """."field" -> 'test'' = ''"a"'') OR 1 = 1 OR (''d') = '"x"' """, - queries[0]['sql'], - ) - - def test_lookups_with_key_transform(self): - tests = ( - ('field__d__contains', 'e'), - ('field__baz__contained_by', {'a': 'b', 'c': 'd', 'e': 'f'}), - ('field__baz__has_key', 'c'), - ('field__baz__has_keys', ['a', 'c']), - ('field__baz__has_any_keys', ['a', 'x']), - ('field__contains', KeyTransform('bax', 'field')), - ( - 'field__contained_by', - KeyTransform('x', RawSQL('%s::jsonb', ['{"x": {"a": "b", "c": 1, "d": "e"}}'])), - ), - ('field__has_key', KeyTextTransform('foo', 'field')), - ) - for lookup, value in tests: - with self.subTest(lookup=lookup): - self.assertTrue(JSONModel.objects.filter( - **{lookup: value}, - ).exists()) - - def test_key_escape(self): - obj = JSONModel.objects.create(field={'%total': 10}) - self.assertEqual(JSONModel.objects.filter(**{'field__%total': 10}).get(), obj) - - -@isolate_apps('postgres_tests') -class TestChecks(PostgreSQLSimpleTestCase): - - def test_invalid_default(self): - class MyModel(PostgreSQLModel): - field = JSONField(default={}) - - model = MyModel() - self.assertEqual(model.check(), [ - checks.Warning( - msg=( - "JSONField default should be a callable instead of an " - "instance so that it's not shared between all field " - "instances." - ), - hint='Use a callable instead, e.g., use `dict` instead of `{}`.', - obj=MyModel._meta.get_field('field'), - id='fields.E010', - ) - ]) - - def test_valid_default(self): - class MyModel(PostgreSQLModel): - field = JSONField(default=dict) - - model = MyModel() - self.assertEqual(model.check(), []) - - def test_valid_default_none(self): - class MyModel(PostgreSQLModel): - field = JSONField(default=None) - - model = MyModel() - self.assertEqual(model.check(), []) - - -class TestSerialization(PostgreSQLSimpleTestCase): - test_data = ( - '[{"fields": {"field": %s, "field_custom": null}, ' - '"model": "postgres_tests.jsonmodel", "pk": null}]' - ) - test_values = ( - # (Python value, serialized value), - ({'a': 'b', 'c': None}, '{"a": "b", "c": null}'), - ('abc', '"abc"'), - ('{"a": "a"}', '"{\\"a\\": \\"a\\"}"'), - ) - - def test_dumping(self): - for value, serialized in self.test_values: - with self.subTest(value=value): - instance = JSONModel(field=value) - data = serializers.serialize('json', [instance]) - self.assertJSONEqual(data, self.test_data % serialized) - - def test_loading(self): - for value, serialized in self.test_values: - with self.subTest(value=value): - instance = list(serializers.deserialize('json', self.test_data % serialized))[0].object - self.assertEqual(instance.field, value) - - -class TestValidation(PostgreSQLSimpleTestCase): - - def test_not_serializable(self): - field = JSONField() - with self.assertRaises(exceptions.ValidationError) as cm: - field.clean(datetime.timedelta(days=1), None) - self.assertEqual(cm.exception.code, 'invalid') - self.assertEqual(cm.exception.message % cm.exception.params, "Value must be valid JSON.") - - def test_custom_encoder(self): - with self.assertRaisesMessage(ValueError, "The encoder parameter must be a callable object."): - field = JSONField(encoder=DjangoJSONEncoder()) - field = JSONField(encoder=DjangoJSONEncoder) - self.assertEqual(field.clean(datetime.timedelta(days=1), None), datetime.timedelta(days=1)) - - -class TestFormField(PostgreSQLSimpleTestCase): - - def test_valid(self): - field = forms.JSONField() - value = field.clean('{"a": "b"}') - self.assertEqual(value, {'a': 'b'}) - - def test_valid_empty(self): - field = forms.JSONField(required=False) - value = field.clean('') - self.assertIsNone(value) - - def test_invalid(self): - field = forms.JSONField() - with self.assertRaises(exceptions.ValidationError) as cm: - field.clean('{some badly formed: json}') - self.assertEqual(cm.exception.messages[0], '“{some badly formed: json}” value must be valid JSON.') - - def test_formfield(self): - model_field = JSONField() - form_field = model_field.formfield() - self.assertIsInstance(form_field, forms.JSONField) - - def test_formfield_disabled(self): - class JsonForm(Form): - name = CharField() - jfield = forms.JSONField(disabled=True) - - form = JsonForm({'name': 'xyz', 'jfield': '["bar"]'}, initial={'jfield': ['foo']}) - self.assertIn('["foo"]</textarea>', form.as_p()) - - def test_prepare_value(self): - field = forms.JSONField() - self.assertEqual(field.prepare_value({'a': 'b'}), '{"a": "b"}') - self.assertEqual(field.prepare_value(None), 'null') - self.assertEqual(field.prepare_value('foo'), '"foo"') - - def test_redisplay_wrong_input(self): - """ - When displaying a bound form (typically due to invalid input), the form - should not overquote JSONField inputs. - """ - class JsonForm(Form): - name = CharField(max_length=2) - jfield = forms.JSONField() - - # JSONField input is fine, name is too long - form = JsonForm({'name': 'xyz', 'jfield': '["foo"]'}) - self.assertIn('["foo"]</textarea>', form.as_p()) - - # This time, the JSONField input is wrong - form = JsonForm({'name': 'xy', 'jfield': '{"foo"}'}) - # Appears once in the textarea and once in the error message - self.assertEqual(form.as_p().count(escape('{"foo"}')), 2) - - def test_widget(self): - """The default widget of a JSONField is a Textarea.""" - field = forms.JSONField() - self.assertIsInstance(field.widget, widgets.Textarea) - - def test_custom_widget_kwarg(self): - """The widget can be overridden with a kwarg.""" - field = forms.JSONField(widget=widgets.Input) - self.assertIsInstance(field.widget, widgets.Input) - - def test_custom_widget_attribute(self): - """The widget can be overridden with an attribute.""" - class CustomJSONField(forms.JSONField): - widget = widgets.Input - - field = CustomJSONField() - self.assertIsInstance(field.widget, widgets.Input) - - def test_already_converted_value(self): - field = forms.JSONField(required=False) - tests = [ - '["a", "b", "c"]', '{"a": 1, "b": 2}', '1', '1.5', '"foo"', - 'true', 'false', 'null', - ] - for json_string in tests: - val = field.clean(json_string) - self.assertEqual(field.clean(val), val) - - def test_has_changed(self): - field = forms.JSONField() - self.assertIs(field.has_changed({'a': True}, '{"a": 1}'), True) - self.assertIs(field.has_changed({'a': 1, 'b': 2}, '{"b": 2, "a": 1}'), False) diff --git a/tests/postgres_tests/test_json_deprecation.py b/tests/postgres_tests/test_json_deprecation.py new file mode 100644 index 0000000000..80deb0cb15 --- /dev/null +++ b/tests/postgres_tests/test_json_deprecation.py @@ -0,0 +1,54 @@ +try: + from django.contrib.postgres.fields import JSONField + from django.contrib.postgres.fields.jsonb import KeyTransform, KeyTextTransform + from django.contrib.postgres import forms +except ImportError: + pass + +from django.core.checks import Warning as DjangoWarning +from django.utils.deprecation import RemovedInDjango40Warning + +from . import PostgreSQLSimpleTestCase +from .models import PostgreSQLModel + + +class DeprecationTests(PostgreSQLSimpleTestCase): + def test_model_field_deprecation_message(self): + class PostgreSQLJSONModel(PostgreSQLModel): + field = JSONField() + + self.assertEqual(PostgreSQLJSONModel().check(), [ + DjangoWarning( + 'django.contrib.postgres.fields.JSONField is deprecated. ' + 'Support for it (except in historical migrations) will be ' + 'removed in Django 4.0.', + hint='Use django.db.models.JSONField instead.', + obj=PostgreSQLJSONModel._meta.get_field('field'), + id='fields.W904', + ), + ]) + + def test_form_field_deprecation_message(self): + msg = ( + 'django.contrib.postgres.forms.JSONField is deprecated in favor ' + 'of django.forms.JSONField.' + ) + with self.assertWarnsMessage(RemovedInDjango40Warning, msg): + forms.JSONField() + + def test_key_transform_deprecation_message(self): + msg = ( + 'django.contrib.postgres.fields.jsonb.KeyTransform is deprecated ' + 'in favor of django.db.models.fields.json.KeyTransform.' + ) + with self.assertWarnsMessage(RemovedInDjango40Warning, msg): + KeyTransform('foo', 'bar') + + def test_key_text_transform_deprecation_message(self): + msg = ( + 'django.contrib.postgres.fields.jsonb.KeyTextTransform is ' + 'deprecated in favor of ' + 'django.db.models.fields.json.KeyTextTransform.' + ) + with self.assertWarnsMessage(RemovedInDjango40Warning, msg): + KeyTextTransform('foo', 'bar') diff --git a/tests/queries/models.py b/tests/queries/models.py index fd994170dd..fc46205a79 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -747,3 +747,10 @@ class ReturningModel(models.Model): class NonIntegerPKReturningModel(models.Model): created = CreatedField(editable=False, primary_key=True) + + +class JSONFieldNullable(models.Model): + json_field = models.JSONField(blank=True, null=True) + + class Meta: + required_db_features = {'supports_json_field'} diff --git a/tests/queries/test_bulk_update.py b/tests/queries/test_bulk_update.py index e2e9a6147a..ec43c86691 100644 --- a/tests/queries/test_bulk_update.py +++ b/tests/queries/test_bulk_update.py @@ -3,11 +3,11 @@ import datetime from django.core.exceptions import FieldDoesNotExist from django.db.models import F from django.db.models.functions import Lower -from django.test import TestCase +from django.test import TestCase, skipUnlessDBFeature from .models import ( - Article, CustomDbColumn, CustomPk, Detail, Individual, Member, Note, - Number, Order, Paragraph, SpecialCategory, Tag, Valid, + Article, CustomDbColumn, CustomPk, Detail, Individual, JSONFieldNullable, + Member, Note, Number, Order, Paragraph, SpecialCategory, Tag, Valid, ) @@ -228,3 +228,14 @@ class BulkUpdateTests(TestCase): article.created = point_in_time Article.objects.bulk_update(articles, ['created']) self.assertCountEqual(Article.objects.filter(created=point_in_time), articles) + + @skipUnlessDBFeature('supports_json_field') + def test_json_field(self): + JSONFieldNullable.objects.bulk_create([ + JSONFieldNullable(json_field={'a': i}) for i in range(10) + ]) + objs = JSONFieldNullable.objects.all() + for obj in objs: + obj.json_field = {'c': obj.json_field['a'] + 1} + JSONFieldNullable.objects.bulk_update(objs, ['json_field']) + self.assertCountEqual(JSONFieldNullable.objects.filter(json_field__has_key='c'), objs) |
