Merge remote-tracking branch 'core/master' into schema-alteration

Conflicts:
	django/db/backends/oracle/base.py
	django/db/backends/postgresql_psycopg2/base.py
	django/db/models/signals.py
	tests/queries/tests.py

16 files changed, 120 insertions, 74 deletions

diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
index 4e8430aceb..6dd25e18f9 100644
--- a/django/conf/global_settings.py
+++ b/django/conf/global_settings.py
@@ -475,6 +475,7 @@ SESSION_SAVE_EVERY_REQUEST = False                      # Whether to save the se
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False                 # Whether a user's session cookie expires when the Web browser is closed.
 SESSION_ENGINE = 'django.contrib.sessions.backends.db'  # The module to store session data
 SESSION_FILE_PATH = None                                # Directory to store session files if using the file session module. If None, the backend will use a sensible default.
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer'  # class to serialize session data
 
 #########
 # CACHE #
diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py
index b475868598..805868306d 100644
--- a/django/contrib/admin/options.py
+++ b/django/contrib/admin/options.py
@@ -107,6 +107,7 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)):
         validator.validate(cls, model)
 
     def __init__(self):
+        self._orig_formfield_overrides = self.formfield_overrides
         overrides = FORMFIELD_FOR_DBFIELD_DEFAULTS.copy()
         overrides.update(self.formfield_overrides)
         self.formfield_overrides = overrides
@@ -123,6 +124,9 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)):
         # If the field specifies choices, we don't need to look for special
         # admin widgets - we just need to use a select widget of some kind.
         if db_field.choices:
+            # see #19303 for an explanation of self._orig_formfield_overrides
+            if db_field.__class__ in self._orig_formfield_overrides:
+                kwargs = dict(self._orig_formfield_overrides[db_field.__class__], **kwargs)
             return self.formfield_for_choice_field(db_field, request, **kwargs)
 
         # ForeignKey or ManyToManyFields
diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py
index 6c3c3e8511..ba109d1454 100644
--- a/django/contrib/admin/templatetags/admin_list.py
+++ b/django/contrib/admin/templatetags/admin_list.py
@@ -9,6 +9,7 @@ from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE,
     ORDER_VAR, PAGE_VAR, SEARCH_VAR)
 from django.contrib.admin.templatetags.admin_static import static
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.urlresolvers import NoReverseMatch
 from django.db import models
 from django.utils import formats
 from django.utils.html import escapejs, format_html
@@ -216,25 +217,36 @@ def items_for_result(cl, result, form):
         row_class = mark_safe(' class="%s"' % ' '.join(row_classes))
         # If list_display_links not defined, add the link tag to the first field
         if (first and not cl.list_display_links) or field_name in cl.list_display_links:
-            table_tag = {True:'th', False:'td'}[first]
+            table_tag = 'th' if first else 'td'
             first = False
-            url = cl.url_for_result(result)
-            url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url)
-            # Convert the pk to something that can be used in Javascript.
-            # Problem cases are long ints (23L) and non-ASCII strings.
-            if cl.to_field:
-                attr = str(cl.to_field)
+
+            # Display link to the result's change_view if the url exists, else
+            # display just the result's representation.
+            try:
+                url = cl.url_for_result(result)
+            except NoReverseMatch:
+                link_or_text = result_repr
             else:
-                attr = pk
-            value = result.serializable_value(attr)
-            result_id = escapejs(value)
-            yield format_html('<{0}{1}><a href="{2}"{3}>{4}</a></{5}>',
+                url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url)
+                # Convert the pk to something that can be used in Javascript.
+                # Problem cases are long ints (23L) and non-ASCII strings.
+                if cl.to_field:
+                    attr = str(cl.to_field)
+                else:
+                    attr = pk
+                value = result.serializable_value(attr)
+                result_id = escapejs(value)
+                link_or_text = format_html(
+                    '<a href="{0}"{1}>{2}</a>',
+                    url,
+                    format_html(' onclick="opener.dismissRelatedLookupPopup(window, &#39;{0}&#39;); return false;"', result_id)
+                        if cl.is_popup else '',
+                    result_repr)
+
+            yield format_html('<{0}{1}>{2}</{3}>',
                               table_tag,
                               row_class,
-                              url,
-                              format_html(' onclick="opener.dismissRelatedLookupPopup(window, &#39;{0}&#39;); return false;"', result_id)
-                                if cl.is_popup else '',
-                              result_repr,
+                              link_or_text,
                               table_tag)
         else:
             # By default the fields come from ModelAdmin.list_editable, but if we pull
diff --git a/django/contrib/messages/storage/session.py b/django/contrib/messages/storage/session.py
index 225dfda289..c3e293c22e 100644
--- a/django/contrib/messages/storage/session.py
+++ b/django/contrib/messages/storage/session.py
@@ -1,4 +1,8 @@
+import json
+
 from django.contrib.messages.storage.base import BaseStorage
+from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder
+from django.utils import six
 
 
 class SessionStorage(BaseStorage):
@@ -20,14 +24,23 @@ class SessionStorage(BaseStorage):
         always stores everything it is given, so return True for the
         all_retrieved flag.
         """
-        return self.request.session.get(self.session_key), True
+        return self.deserialize_messages(self.request.session.get(self.session_key)), True
 
     def _store(self, messages, response, *args, **kwargs):
         """
         Stores a list of messages to the request's session.
         """
         if messages:
-            self.request.session[self.session_key] = messages
+            self.request.session[self.session_key] = self.serialize_messages(messages)
         else:
             self.request.session.pop(self.session_key, None)
         return []
+
+    def serialize_messages(self, messages):
+        encoder = MessageEncoder(separators=(',', ':'))
+        return encoder.encode(messages)
+
+    def deserialize_messages(self, data):
+        if data and isinstance(data, six.string_types):
+            return json.loads(data, cls=MessageDecoder)
+        return data
diff --git a/django/contrib/messages/tests/base.py b/django/contrib/messages/tests/base.py
index 7011a5779a..5241436daf 100644
--- a/django/contrib/messages/tests/base.py
+++ b/django/contrib/messages/tests/base.py
@@ -61,6 +61,7 @@ class BaseTests(object):
             MESSAGE_TAGS    = '',
             MESSAGE_STORAGE = '%s.%s' % (self.storage_class.__module__,
                                          self.storage_class.__name__),
+            SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer',
         )
         self.settings_override.enable()
 
diff --git a/django/contrib/messages/tests/test_session.py b/django/contrib/messages/tests/test_session.py
index 2ce564b773..940e1c02d0 100644
--- a/django/contrib/messages/tests/test_session.py
+++ b/django/contrib/messages/tests/test_session.py
@@ -11,13 +11,13 @@ def set_session_data(storage, messages):
     Sets the messages into the backend request's session and remove the
     backend's loaded data cache.
     """
-    storage.request.session[storage.session_key] = messages
+    storage.request.session[storage.session_key] = storage.serialize_messages(messages)
     if hasattr(storage, '_loaded_data'):
         del storage._loaded_data
 
 
 def stored_session_messages_count(storage):
-    data = storage.request.session.get(storage.session_key, [])
+    data = storage.deserialize_messages(storage.request.session.get(storage.session_key, []))
     return len(data)
 
 
diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py
index 759d7ac7ad..7f5e958a60 100644
--- a/django/contrib/sessions/backends/base.py
+++ b/django/contrib/sessions/backends/base.py
@@ -3,11 +3,6 @@ from __future__ import unicode_literals
 import base64
 from datetime import datetime, timedelta
 import logging
-
-try:
-    from django.utils.six.moves import cPickle as pickle
-except ImportError:
-    import pickle
 import string
 
 from django.conf import settings
@@ -17,6 +12,7 @@ from django.utils.crypto import get_random_string
 from django.utils.crypto import salted_hmac
 from django.utils import timezone
 from django.utils.encoding import force_bytes, force_text
+from django.utils.module_loading import import_by_path
 
 from django.contrib.sessions.exceptions import SuspiciousSession
 
@@ -42,6 +38,7 @@ class SessionBase(object):
         self._session_key = session_key
         self.accessed = False
         self.modified = False
+        self.serializer = import_by_path(settings.SESSION_SERIALIZER)
 
     def __contains__(self, key):
         return key in self._session
@@ -86,21 +83,21 @@ class SessionBase(object):
         return salted_hmac(key_salt, value).hexdigest()
 
     def encode(self, session_dict):
-        "Returns the given session dictionary pickled and encoded as a string."
-        pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL)
-        hash = self._hash(pickled)
-        return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii')
+        "Returns the given session dictionary serialized and encoded as a string."
+        serialized = self.serializer().dumps(session_dict)
+        hash = self._hash(serialized)
+        return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii')
 
     def decode(self, session_data):
         encoded_data = base64.b64decode(force_bytes(session_data))
         try:
             # could produce ValueError if there is no ':'
-            hash, pickled = encoded_data.split(b':', 1)
-            expected_hash = self._hash(pickled)
+            hash, serialized = encoded_data.split(b':', 1)
+            expected_hash = self._hash(serialized)
             if not constant_time_compare(hash.decode(), expected_hash):
                 raise SuspiciousSession("Session data corrupted")
             else:
-                return pickle.loads(pickled)
+                return self.serializer().loads(serialized)
         except Exception as e:
             # ValueError, SuspiciousOperation, unpickling exceptions. If any of
             # these happen, just return an empty dictionary (an empty session).
diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py
index c2b7a3123f..77a6750ce4 100644
--- a/django/contrib/sessions/backends/signed_cookies.py
+++ b/django/contrib/sessions/backends/signed_cookies.py
@@ -1,26 +1,9 @@
-try:
-    from django.utils.six.moves import cPickle as pickle
-except ImportError:
-    import pickle
-
 from django.conf import settings
 from django.core import signing
 
 from django.contrib.sessions.backends.base import SessionBase
 
 
-class PickleSerializer(object):
-    """
-    Simple wrapper around pickle to be used in signing.dumps and
-    signing.loads.
-    """
-    def dumps(self, obj):
-        return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
-
-    def loads(self, data):
-        return pickle.loads(data)
-
-
 class SessionStore(SessionBase):
 
     def load(self):
@@ -31,7 +14,7 @@ class SessionStore(SessionBase):
         """
         try:
             return signing.loads(self.session_key,
-                serializer=PickleSerializer,
+                serializer=self.serializer,
                 # This doesn't handle non-default expiry dates, see #19201
                 max_age=settings.SESSION_COOKIE_AGE,
                 salt='django.contrib.sessions.backends.signed_cookies')
@@ -91,7 +74,7 @@ class SessionStore(SessionBase):
         session_cache = getattr(self, '_session_cache', {})
         return signing.dumps(session_cache, compress=True,
             salt='django.contrib.sessions.backends.signed_cookies',
-            serializer=PickleSerializer)
+            serializer=self.serializer)
 
     @classmethod
     def clear_expired(cls):
diff --git a/django/contrib/sessions/models.py b/django/contrib/sessions/models.py
index 0179c358b3..3a6e31152f 100644
--- a/django/contrib/sessions/models.py
+++ b/django/contrib/sessions/models.py
@@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _
 class SessionManager(models.Manager):
     def encode(self, session_dict):
         """
-        Returns the given session dictionary pickled and encoded as a string.
+        Returns the given session dictionary serialized and encoded as a string.
         """
         return SessionStore().encode(session_dict)
 
diff --git a/django/contrib/sessions/serializers.py b/django/contrib/sessions/serializers.py
new file mode 100644
index 0000000000..92a31c054b
--- /dev/null
+++ b/django/contrib/sessions/serializers.py
@@ -0,0 +1,20 @@
+from django.core.signing import JSONSerializer as BaseJSONSerializer
+try:
+    from django.utils.six.moves import cPickle as pickle
+except ImportError:
+    import pickle
+
+
+class PickleSerializer(object):
+    """
+    Simple wrapper around pickle to be used in signing.dumps and
+    signing.loads.
+    """
+    def dumps(self, obj):
+        return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
+
+    def loads(self, data):
+        return pickle.loads(data)
+
+
+JSONSerializer = BaseJSONSerializer
diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py
index f2a35c544e..4caefe938c 100644
--- a/django/contrib/sessions/tests.py
+++ b/django/contrib/sessions/tests.py
@@ -285,21 +285,25 @@ class SessionTestsMixin(object):
 
 
     def test_actual_expiry(self):
-        # Regression test for #19200
-        old_session_key = None
-        new_session_key = None
-        try:
-            self.session['foo'] = 'bar'
-            self.session.set_expiry(-timedelta(seconds=10))
-            self.session.save()
-            old_session_key = self.session.session_key
-            # With an expiry date in the past, the session expires instantly.
-            new_session = self.backend(self.session.session_key)
-            new_session_key = new_session.session_key
-            self.assertNotIn('foo', new_session)
-        finally:
-            self.session.delete(old_session_key)
-            self.session.delete(new_session_key)
+        # this doesn't work with JSONSerializer (serializing timedelta)
+        with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'):
+            self.session = self.backend()  # reinitialize after overriding settings
+
+            # Regression test for #19200
+            old_session_key = None
+            new_session_key = None
+            try:
+                self.session['foo'] = 'bar'
+                self.session.set_expiry(-timedelta(seconds=10))
+                self.session.save()
+                old_session_key = self.session.session_key
+                # With an expiry date in the past, the session expires instantly.
+                new_session = self.backend(self.session.session_key)
+                new_session_key = new_session.session_key
+                self.assertNotIn('foo', new_session)
+            finally:
+                self.session.delete(old_session_key)
+                self.session.delete(new_session_key)
 
 
 class DatabaseSessionTests(SessionTestsMixin, TestCase):
diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py
index a363c09001..fe6ac0e70c 100644
--- a/django/db/backends/oracle/base.py
+++ b/django/db/backends/oracle/base.py
@@ -96,6 +96,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     nulls_order_largest = True
     requires_literal_defaults = True
     connection_persists_old_columns = True
+    nulls_order_largest = True
 
 
 class DatabaseOperations(BaseDatabaseOperations):
diff --git a/django/db/models/base.py b/django/db/models/base.py
index 6a21544baf..d63017c438 100644
--- a/django/db/models/base.py
+++ b/django/db/models/base.py
@@ -459,14 +459,21 @@ class Model(six.with_metaclass(ModelBase)):
         return '%s object' % self.__class__.__name__
 
     def __eq__(self, other):
-        return (isinstance(other, Model) and
-                self._meta.concrete_model == other._meta.concrete_model and
-                self._get_pk_val() == other._get_pk_val())
+        if not isinstance(other, Model):
+            return False
+        if self._meta.concrete_model != other._meta.concrete_model:
+            return False
+        my_pk = self._get_pk_val()
+        if my_pk is None:
+            return self is other
+        return my_pk == other._get_pk_val()
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     def __hash__(self):
+        if self._get_pk_val() is None:
+            raise TypeError("Model instances without primary key value are unhashable")
         return hash(self._get_pk_val())
 
     def __reduce__(self):
diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index caaeaefa6e..6c8b850c84 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -664,9 +664,8 @@ class SQLCompiler(object):
                 # Use True here because we are looking at the _reverse_ side of
                 # the relation, which is always nullable.
                 new_nullable = True
-                table = model._meta.db_table
-                self.fill_related_selections(model._meta, table, cur_depth + 1,
-                    next, restricted, new_nullable)
+                self.fill_related_selections(model._meta, alias, cur_depth + 1,
+                                             next, restricted, new_nullable)
 
     def deferred_to_columns(self):
         """
diff --git a/django/forms/forms.py b/django/forms/forms.py
index ec51507981..d8d08e18fe 100644
--- a/django/forms/forms.py
+++ b/django/forms/forms.py
@@ -526,9 +526,9 @@ class BoundField(object):
         """
         contents = contents or self.label
         # Only add the suffix if the label does not end in punctuation.
+        label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix
         # Translators: If found as last label character, these punctuation
         # characters will prevent the default label_suffix to be appended to the label
-        label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix
         if label_suffix and contents and contents[-1] not in _(':?.!'):
             contents = format_html('{0}{1}', contents, label_suffix)
         widget = self.field.widget
diff --git a/django/forms/models.py b/django/forms/models.py
index a5b82e521d..4c6ee9c6ed 100644
--- a/django/forms/models.py
+++ b/django/forms/models.py
@@ -631,7 +631,11 @@ class BaseModelFormSet(BaseFormSet):
             seen_data = set()
             for form in valid_forms:
                 # get data for each field of each of unique_check
-                row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data])
+                row_data = (form.cleaned_data[field]
+                            for field in unique_check if field in form.cleaned_data)
+                # Reduce Model instances to their primary key values
+                row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d
+                                 for d in row_data)
                 if row_data and not None in row_data:
                     # if we've already seen it then we have a uniqueness failure
                     if row_data in seen_data: