Fixed #24604 -- Added JSONField to contrib.postgres.

11 files changed, 540 insertions, 3 deletions

diff --git a/django/contrib/postgres/fields/__init__.py b/django/contrib/postgres/fields/__init__.py
index bb2685eca1..9158f1e7cc 100644
--- a/django/contrib/postgres/fields/__init__.py
+++ b/django/contrib/postgres/fields/__init__.py
@@ -1,3 +1,4 @@
 from .array import *  # NOQA
 from .hstore import *  # NOQA
+from .jsonb import *  # NOQA
 from .ranges import *  # NOQA
diff --git a/django/contrib/postgres/fields/jsonb.py b/django/contrib/postgres/fields/jsonb.py
new file mode 100644
index 0000000000..eaf3e98d36
--- /dev/null
+++ b/django/contrib/postgres/fields/jsonb.py
@@ -0,0 +1,99 @@
+import json
+
+from psycopg2.extras import Json
+
+from django.contrib.postgres import forms, lookups
+from django.core import exceptions
+from django.db.models import Field, Transform
+from django.utils.translation import ugettext_lazy as _
+
+__all__ = ['JSONField']
+
+
+class JSONField(Field):
+    empty_strings_allowed = False
+    description = _('A JSON object')
+    default_error_messages = {
+        'invalid': _("Value must be valid JSON."),
+    }
+
+    def db_type(self, connection):
+        return 'jsonb'
+
+    def get_transform(self, name):
+        transform = super(JSONField, self).get_transform(name)
+        if transform:
+            return transform
+        return KeyTransformFactory(name)
+
+    def get_prep_value(self, value):
+        if value is not None:
+            return Json(value)
+        return value
+
+    def get_prep_lookup(self, lookup_type, value):
+        if lookup_type in ('has_key', 'has_keys', 'has_any_keys'):
+            return value
+        if isinstance(value, (dict, list)):
+            return Json(value)
+        return super(JSONField, self).get_prep_lookup(lookup_type, value)
+
+    def validate(self, value, model_instance):
+        super(JSONField, self).validate(value, model_instance)
+        try:
+            json.dumps(value)
+        except TypeError:
+            raise exceptions.ValidationError(
+                self.error_messages['invalid'],
+                code='invalid',
+                params={'value': value},
+            )
+
+    def value_to_string(self, obj):
+        value = self._get_val_from_obj(obj)
+        return value
+
+    def formfield(self, **kwargs):
+        defaults = {'form_class': forms.JSONField}
+        defaults.update(kwargs)
+        return super(JSONField, self).formfield(**defaults)
+
+
+JSONField.register_lookup(lookups.DataContains)
+JSONField.register_lookup(lookups.ContainedBy)
+JSONField.register_lookup(lookups.HasKey)
+JSONField.register_lookup(lookups.HasKeys)
+JSONField.register_lookup(lookups.HasAnyKeys)
+
+
+class KeyTransform(Transform):
+
+    def __init__(self, key_name, *args, **kwargs):
+        super(KeyTransform, self).__init__(*args, **kwargs)
+        self.key_name = key_name
+
+    def as_sql(self, compiler, connection):
+        key_transforms = [self.key_name]
+        previous = self.lhs
+        while isinstance(previous, KeyTransform):
+            key_transforms.insert(0, previous.key_name)
+            previous = previous.lhs
+        lhs, params = compiler.compile(previous)
+        if len(key_transforms) > 1:
+            return "{} #> %s".format(lhs), [key_transforms] + params
+        try:
+            int(self.key_name)
+        except ValueError:
+            lookup = "'%s'" % self.key_name
+        else:
+            lookup = "%s" % self.key_name
+        return "%s -> %s" % (lhs, lookup), params
+
+
+class KeyTransformFactory(object):
+
+    def __init__(self, key_name):
+        self.key_name = key_name
+
+    def __call__(self, *args, **kwargs):
+        return KeyTransform(self.key_name, *args, **kwargs)
diff --git a/django/contrib/postgres/forms/__init__.py b/django/contrib/postgres/forms/__init__.py
index bb2685eca1..9158f1e7cc 100644
--- a/django/contrib/postgres/forms/__init__.py
+++ b/django/contrib/postgres/forms/__init__.py
@@ -1,3 +1,4 @@
 from .array import *  # NOQA
 from .hstore import *  # NOQA
+from .jsonb import *  # NOQA
 from .ranges import *  # NOQA
diff --git a/django/contrib/postgres/forms/jsonb.py b/django/contrib/postgres/forms/jsonb.py
new file mode 100644
index 0000000000..8eefc14993
--- /dev/null
+++ b/django/contrib/postgres/forms/jsonb.py
@@ -0,0 +1,31 @@
+import json
+
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+__all__ = ['JSONField']
+
+
+class JSONField(forms.CharField):
+    default_error_messages = {
+        'invalid': _("'%(value)s' value must be valid JSON."),
+    }
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault('widget', forms.Textarea)
+        super(JSONField, self).__init__(**kwargs)
+
+    def to_python(self, value):
+        if value in self.empty_values:
+            return None
+        try:
+            return json.loads(value)
+        except ValueError:
+            raise forms.ValidationError(
+                self.error_messages['invalid'],
+                code='invalid',
+                params={'value': value},
+            )
+
+    def prepare_value(self, value):
+        return json.dumps(value)
diff --git a/docs/ref/contrib/postgres/fields.txt b/docs/ref/contrib/postgres/fields.txt
index 5313ac4092..cf92c294c9 100644
--- a/docs/ref/contrib/postgres/fields.txt
+++ b/docs/ref/contrib/postgres/fields.txt
@@ -450,6 +450,111 @@ using in conjunction with lookups on
     >>> Dog.objects.filter(data__values__contains=['collie'])
     [<Dog: Meg>]
 
+JSONField
+---------
+
+.. versionadded:: 1.9
+
+.. class:: JSONField(**options)
+
+    A field for storing JSON encoded data. In Python the data is represented in
+    its Python native format: dictionaries, lists, strings, numbers, booleans
+    and ``None``.
+
+.. note::
+
+    PostgreSQL has two native JSON based data types: ``json`` and ``jsonb``.
+    The main difference between them is how they are stored and how they can be
+    queried. PostgreSQL's ``json`` field is stored as the original string
+    representation of the JSON and must be decoded on the fly when queried
+    based on keys. The ``jsonb`` field is stored based on the actual structure
+    of the JSON which allows indexing. The trade-off is a small additional cost
+    on writing to the ``jsonb`` field. ``JSONField`` uses ``jsonb``.
+
+    **As a result, the usage of this field is only supported on PostgreSQL
+    versions at least 9.4**.
+
+Querying JSONField
+^^^^^^^^^^^^^^^^^^
+
+We will use the following example model::
+
+    from django.contrib.postgres.fields import JSONField
+    from django.db import models
+
+    class Dog(models.Model):
+        name = models.CharField(max_length=200)
+        data = JSONField()
+
+        def __str__(self):  # __unicode__ on Python 2
+            return self.name
+
+.. fieldlookup:: jsonfield.key
+
+Key, index, and path lookups
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To query based on a given dictionary key, simply use that key as the lookup
+name::
+
+    >>> Dog.objects.create(name='Rufus', data={
+    ...     'breed': 'labrador',
+    ...     'owner': {
+    ...         'name': 'Bob',
+    ...         'other_pets': [{
+    ...             'name': 'Fishy',
+    ...         }],
+    ...     },
+    ... })
+    >>> Dog.objects.create(name='Meg', data={'breed': 'collie'})
+
+    >>> Dog.objects.filter(data__breed='collie')
+    [<Dog: Meg>]
+
+Multiple keys can be chained together to form a path lookup::
+
+    >>> Dog.objects.filter(data__owner__name='Bob')
+    [<Dog: Rufus>]
+
+If the key is an integer, it will be interpreted as an index lookup in an
+array::
+
+    >>> Dog.objects.filter(data__owner__other_pets__0__name='Fishy')
+    [<Dog: Rufus>]
+
+If the key you wish to query by clashes with the name of another lookup, use
+the :lookup:`jsonfield.contains` lookup instead.
+
+If only one key or index is used, the SQL operator ``->`` is used. If multiple
+operators are used then the ``#>`` operator is used.
+
+.. warning::
+
+    Since any string could be a key in a JSON object, any lookup other than
+    those listed below will be interpreted as a key lookup. No errors are
+    raised. Be extra careful for typing mistakes, and always check your queries
+    work as you intend.
+
+Containment and key operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. fieldlookup:: jsonfield.contains
+.. fieldlookup:: jsonfield.contained_by
+.. fieldlookup:: jsonfield.has_key
+.. fieldlookup:: jsonfield.has_any_keys
+.. fieldlookup:: jsonfield.has_keys
+
+:class:`~django.contrib.postgres.fields.JSONField` shares lookups relating to
+containment and keys with :class:`~django.contrib.postgres.fields.HStoreField`.
+
+- :lookup:`contains <hstorefield.contains>` (accepts any JSON rather than
+  just a dictionary of strings)
+- :lookup:`contained_by <hstorefield.contained_by>` (accepts any JSON
+  rather than just a dictionary of strings)
+- :lookup:`has_key <hstorefield.has_key>`
+- :lookup:`has_any_keys <hstorefield.has_any_keys>`
+- :lookup:`has_keys <hstorefield.has_keys>`
+
 .. _range-fields:
 
 Range Fields
diff --git a/docs/ref/contrib/postgres/forms.txt b/docs/ref/contrib/postgres/forms.txt
index d1a15907ae..431cf9c802 100644
--- a/docs/ref/contrib/postgres/forms.txt
+++ b/docs/ref/contrib/postgres/forms.txt
@@ -155,6 +155,21 @@ HStoreField
         valid for a given field. This can be done using the
         :class:`~django.contrib.postgres.validators.KeysValidator`.
 
+JSONField
+---------
+
+.. class:: JSONField
+
+    A field which accepts JSON encoded data for a
+    :class:`~django.contrib.postgres.fields.JSONField`. It is represented by an
+    HTML ``<textarea>``.
+
+    .. admonition:: User friendly forms
+
+        ``JSONField`` is not particularly user friendly in most cases, however
+        it is a useful way to format data from a client-side widget for
+        submission to the server.
+
 Range Fields
 ------------
 
diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt
index f9bb9decfc..71960c5c3b 100644
--- a/docs/releases/1.9.txt
+++ b/docs/releases/1.9.txt
@@ -91,6 +91,7 @@ Minor features
 :mod:`django.contrib.postgres`
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+* Added :class:`~django.contrib.postgres.fields.JSONField`.
 * Added :doc:`/ref/contrib/postgres/aggregates`.
 
 :mod:`django.contrib.redirects`
diff --git a/tests/postgres_tests/fields.py b/tests/postgres_tests/fields.py
index e21ca1bfcb..0926175e1b 100644
--- a/tests/postgres_tests/fields.py
+++ b/tests/postgres_tests/fields.py
@@ -7,7 +7,7 @@ from django.db import models
 try:
     from django.contrib.postgres.fields import (
         ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
-        FloatRangeField, HStoreField, IntegerRangeField,
+        FloatRangeField, HStoreField, IntegerRangeField, JSONField,
     )
 except ImportError:
     class DummyArrayField(models.Field):
@@ -29,3 +29,4 @@ except ImportError:
     FloatRangeField = models.Field
     HStoreField = models.Field
     IntegerRangeField = models.Field
+    JSONField = models.Field
diff --git a/tests/postgres_tests/migrations/0002_create_test_models.py b/tests/postgres_tests/migrations/0002_create_test_models.py
index 106818a7c6..4eb2154f02 100644
--- a/tests/postgres_tests/migrations/0002_create_test_models.py
+++ b/tests/postgres_tests/migrations/0002_create_test_models.py
@@ -150,6 +150,19 @@ class Migration(migrations.Migration):
         ),
     ]
 
+    pg_94_operations = [
+        migrations.CreateModel(
+            name='JSONModel',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('field', JSONField(null=True, blank=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]
+
     def apply(self, project_state, schema_editor, collect_sql=False):
         try:
             PG_VERSION = schema_editor.connection.pg_version
@@ -158,4 +171,6 @@ class Migration(migrations.Migration):
         else:
             if PG_VERSION >= 90200:
                 self.operations = self.operations + self.pg_92_operations
+            if PG_VERSION >= 90400:
+                self.operations = self.operations + self.pg_94_operations
         return super(Migration, self).apply(project_state, schema_editor, collect_sql)
diff --git a/tests/postgres_tests/models.py b/tests/postgres_tests/models.py
index aafd529443..329a91c951 100644
--- a/tests/postgres_tests/models.py
+++ b/tests/postgres_tests/models.py
@@ -2,7 +2,7 @@ from django.db import connection, models
 
 from .fields import (
     ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
-    FloatRangeField, HStoreField, IntegerRangeField,
+    FloatRangeField, HStoreField, IntegerRangeField, JSONField,
 )
 
 
@@ -52,7 +52,7 @@ class TextFieldModel(models.Model):
     field = models.TextField()
 
 
-# Only create this model for databases which support it
+# Only create this model for postgres >= 9.2
 if connection.vendor == 'postgresql' and connection.pg_version >= 90200:
     class RangesModel(PostgreSQLModel):
         ints = IntegerRangeField(blank=True, null=True)
@@ -66,6 +66,16 @@ else:
         pass
 
 
+# Only create this model for postgres >= 9.4
+if connection.vendor == 'postgresql' and connection.pg_version >= 90400:
+    class JSONModel(models.Model):
+        field = JSONField(blank=True, null=True)
+else:
+    # create an object with this name so we don't have failing imports
+    class JSONModel(object):
+        pass
+
+
 class ArrayFieldSubclass(ArrayField):
     def __init__(self, *args, **kwargs):
         super(ArrayFieldSubclass, self).__init__(models.IntegerField())
diff --git a/tests/postgres_tests/test_json.py b/tests/postgres_tests/test_json.py
new file mode 100644
index 0000000000..b59abe7898
--- /dev/null
+++ b/tests/postgres_tests/test_json.py
@@ -0,0 +1,258 @@
+import datetime
+import unittest
+
+from django.core import exceptions, serializers
+from django.db import connection
+from django.test import TestCase
+
+from . import PostgresSQLTestCase
+from .models import JSONModel
+
+try:
+    from django.contrib.postgres import forms
+    from django.contrib.postgres.fields import JSONField
+except ImportError:
+    pass
+
+
+def skipUnlessPG94(test):
+    try:
+        PG_VERSION = connection.pg_version
+    except AttributeError:
+        PG_VERSION = 0
+    if PG_VERSION < 90400:
+        return unittest.skip('PostgreSQL >= 9.4 required')(test)
+    return test
+
+
+@skipUnlessPG94
+class TestSaveLoad(TestCase):
+    def test_null(self):
+        instance = JSONModel()
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, None)
+
+    def test_empty_object(self):
+        instance = JSONModel(field={})
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, {})
+
+    def test_empty_list(self):
+        instance = JSONModel(field=[])
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, [])
+
+    def test_boolean(self):
+        instance = JSONModel(field=True)
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, True)
+
+    def test_string(self):
+        instance = JSONModel(field='why?')
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, 'why?')
+
+    def test_number(self):
+        instance = JSONModel(field=1)
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, 1)
+
+    def test_realistic_object(self):
+        obj = {
+            'a': 'b',
+            'c': 1,
+            'd': ['e', {'f': 'g'}],
+            'h': True,
+            'i': False,
+            'j': None,
+        }
+        instance = JSONModel(field=obj)
+        instance.save()
+        loaded = JSONModel.objects.get()
+        self.assertEqual(loaded.field, obj)
+
+
+@skipUnlessPG94
+class TestQuerying(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.objs = [
+            JSONModel.objects.create(field=None),
+            JSONModel.objects.create(field=True),
+            JSONModel.objects.create(field=False),
+            JSONModel.objects.create(field='yes'),
+            JSONModel.objects.create(field=7),
+            JSONModel.objects.create(field=[]),
+            JSONModel.objects.create(field={}),
+            JSONModel.objects.create(field={
+                'a': 'b',
+                'c': 1,
+            }),
+            JSONModel.objects.create(field={
+                'a': 'b',
+                'c': 1,
+                'd': ['e', {'f': 'g'}],
+                'h': True,
+                'i': False,
+                'j': None,
+                'k': {'l': 'm'},
+            }),
+            JSONModel.objects.create(field=[1, [2]]),
+            JSONModel.objects.create(field={
+                'k': True,
+                'l': False,
+            }),
+        ]
+
+    def test_exact(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__exact={}),
+            [self.objs[6]]
+        )
+
+    def test_exact_complex(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__exact={'a': 'b', 'c': 1}),
+            [self.objs[7]]
+        )
+
+    def test_isnull(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__isnull=True),
+            [self.objs[0]]
+        )
+
+    def test_contains(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__contains={'a': 'b'}),
+            [self.objs[7], self.objs[8]]
+        )
+
+    def test_contained_by(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__contained_by={'a': 'b', 'c': 1, 'h': True}),
+            [self.objs[6], self.objs[7]]
+        )
+
+    def test_has_key(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__has_key='a'),
+            [self.objs[7], self.objs[8]]
+        )
+
+    def test_has_keys(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__has_keys=['a', 'c', 'h']),
+            [self.objs[8]]
+        )
+
+    def test_has_any_keys(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__has_any_keys=['c', 'l']),
+            [self.objs[7], self.objs[8], self.objs[10]]
+        )
+
+    def test_shallow_list_lookup(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__0=1),
+            [self.objs[9]]
+        )
+
+    def test_shallow_obj_lookup(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__a='b'),
+            [self.objs[7], self.objs[8]]
+        )
+
+    def test_deep_lookup_objs(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__k__l='m'),
+            [self.objs[8]]
+        )
+
+    def test_shallow_lookup_obj_target(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__k={'l': 'm'}),
+            [self.objs[8]]
+        )
+
+    def test_deep_lookup_array(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__1__0=2),
+            [self.objs[9]]
+        )
+
+    def test_deep_lookup_mixed(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__d__1__f='g'),
+            [self.objs[8]]
+        )
+
+    def test_deep_lookup_transform(self):
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__c__gt=1),
+            []
+        )
+        self.assertSequenceEqual(
+            JSONModel.objects.filter(field__c__lt=5),
+            [self.objs[7], self.objs[8]]
+        )
+
+
+@skipUnlessPG94
+class TestSerialization(TestCase):
+    test_data = '[{"fields": {"field": {"a": "b"}}, "model": "postgres_tests.jsonmodel", "pk": null}]'
+
+    def test_dumping(self):
+        instance = JSONModel(field={'a': 'b'})
+        data = serializers.serialize('json', [instance])
+        self.assertJSONEqual(data, self.test_data)
+
+    def test_loading(self):
+        instance = list(serializers.deserialize('json', self.test_data))[0].object
+        self.assertEqual(instance.field, {'a': 'b'})
+
+
+class TestValidation(PostgresSQLTestCase):
+
+    def test_not_serializable(self):
+        field = JSONField()
+        with self.assertRaises(exceptions.ValidationError) as cm:
+            field.clean(datetime.timedelta(days=1), None)
+        self.assertEqual(cm.exception.code, 'invalid')
+        self.assertEqual(cm.exception.message % cm.exception.params, "Value must be valid JSON.")
+
+
+class TestFormField(PostgresSQLTestCase):
+
+    def test_valid(self):
+        field = forms.JSONField()
+        value = field.clean('{"a": "b"}')
+        self.assertEqual(value, {'a': 'b'})
+
+    def test_valid_empty(self):
+        field = forms.JSONField(required=False)
+        value = field.clean('')
+        self.assertEqual(value, None)
+
+    def test_invalid(self):
+        field = forms.JSONField()
+        with self.assertRaises(exceptions.ValidationError) as cm:
+            field.clean('{some badly formed: json}')
+        self.assertEqual(cm.exception.messages[0], "'{some badly formed: json}' value must be valid JSON.")
+
+    def test_formfield(self):
+        model_field = JSONField()
+        form_field = model_field.formfield()
+        self.assertIsInstance(form_field, forms.JSONField)
+
+    def test_prepare_value(self):
+        field = forms.JSONField()
+        self.assertEqual(field.prepare_value({'a': 'b'}), '{"a": "b"}')
+        self.assertEqual(field.prepare_value(None), 'null')