From 96346ed5adf90849ac5ebd10d74377ed2e0c061c Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Tue, 20 Aug 2013 19:03:33 +0200 Subject: Fixed #20933 -- Allowed loaddata to load fixtures from relative paths. --- docs/howto/initial-data.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/howto/initial-data.txt b/docs/howto/initial-data.txt index b86aaa834e..70d1ae18a6 100644 --- a/docs/howto/initial-data.txt +++ b/docs/howto/initial-data.txt @@ -90,8 +90,8 @@ fixtures. You can set the :setting:`FIXTURE_DIRS` setting to a list of additional directories where Django should look. When running :djadmin:`manage.py loaddata `, you can also -specify an absolute path to a fixture file, which overrides searching -the usual directories. +specify a path to a fixture file, which overrides searching the usual +directories. .. seealso:: -- cgit v1.3 From f9d1d5dc1377cb21b39452b0897e7a79a3d02844 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Tue, 20 Aug 2013 22:17:26 -0300 Subject: Fixed #18967 -- Don't base64-encode message/rfc822 attachments. Thanks Michael Farrell for the report and his work on the fix. --- django/core/mail/message.py | 44 +++++++++++++++++++++++++++++++++++++++++--- docs/topics/email.txt | 14 ++++++++++++-- tests/mail/tests.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) (limited to 'docs') diff --git a/django/core/mail/message.py b/django/core/mail/message.py index db9023a0bb..9796e59260 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -4,11 +4,13 @@ import mimetypes import os import random import time -from email import charset as Charset, encoders as Encoders +from email import charset as Charset, encoders as Encoders, message_from_string from email.generator import Generator +from email.message import Message from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase +from email.mime.message import MIMEMessage from email.header import Header from email.utils import formatdate, getaddresses, formataddr, parseaddr @@ -118,6 +120,27 @@ def sanitize_address(addr, encoding): return formataddr((nm, addr)) +class SafeMIMEMessage(MIMEMessage): + + def __setitem__(self, name, val): + # message/rfc822 attachments must be ASCII + name, val = forbid_multi_line_headers(name, val, 'ascii') + MIMEMessage.__setitem__(self, name, val) + + def as_string(self, unixfrom=False): + """Return the entire formatted message as a string. + Optional `unixfrom' when True, means include the Unix From_ envelope + header. + + This overrides the default as_string() implementation to not mangle + lines that begin with 'From '. See bug #13433 for details. + """ + fp = six.StringIO() + g = Generator(fp, mangle_from_=False) + g.flatten(self, unixfrom=unixfrom) + return fp.getvalue() + + class SafeMIMEText(MIMEText): def __init__(self, text, subtype, charset): @@ -137,7 +160,7 @@ class SafeMIMEText(MIMEText): lines that begin with 'From '. See bug #13433 for details. """ fp = six.StringIO() - g = Generator(fp, mangle_from_ = False) + g = Generator(fp, mangle_from_=False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -161,7 +184,7 @@ class SafeMIMEMultipart(MIMEMultipart): lines that begin with 'From '. See bug #13433 for details. """ fp = six.StringIO() - g = Generator(fp, mangle_from_ = False) + g = Generator(fp, mangle_from_=False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -292,11 +315,26 @@ class EmailMessage(object): def _create_mime_attachment(self, content, mimetype): """ Converts the content, mimetype pair into a MIME attachment object. + + If the mimetype is message/rfc822, content may be an + email.Message or EmailMessage object, as well as a str. """ basetype, subtype = mimetype.split('/', 1) if basetype == 'text': encoding = self.encoding or settings.DEFAULT_CHARSET attachment = SafeMIMEText(content, subtype, encoding) + elif basetype == 'message' and subtype == 'rfc822': + # Bug #18967: per RFC2046 s5.2.1, message/rfc822 attachments + # must not be base64 encoded. + if isinstance(content, EmailMessage): + # convert content into an email.Message first + content = content.message() + elif not isinstance(content, Message): + # For compatibility with existing code, parse the message + # into a email.Message object if it is not one already. + content = message_from_string(content) + + attachment = SafeMIMEMessage(content, subtype) else: # Encode non-text attachments with base64. attachment = MIMEBase(basetype, subtype) diff --git a/docs/topics/email.txt b/docs/topics/email.txt index c007c2b856..ebbb0963f4 100644 --- a/docs/topics/email.txt +++ b/docs/topics/email.txt @@ -319,6 +319,18 @@ The class has the following methods: message.attach('design.png', img_data, 'image/png') + .. versionchanged:: 1.7 + + If you specify a ``mimetype`` of ``message/rfc822``, it will also accept + :class:`django.core.mail.EmailMessage` and :py:class:`email.message.Message`. + + In addition, ``message/rfc822`` attachments will no longer be + base64-encoded in violation of :rfc:`2046#section-5.2.1`, which can cause + issues with displaying the attachments in `Evolution`__ and `Thunderbird`__. + + __ https://bugzilla.gnome.org/show_bug.cgi?id=651197 + __ https://bugzilla.mozilla.org/show_bug.cgi?id=333880 + * ``attach_file()`` creates a new attachment using a file from your filesystem. Call it with the path of the file to attach and, optionally, the MIME type to use for the attachment. If the MIME type is omitted, it @@ -326,8 +338,6 @@ The class has the following methods: message.attach_file('/images/weather_map.png') -.. _DEFAULT_FROM_EMAIL: ../settings/#default-from-email - Sending alternative content types ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 0f85cc0c76..2ba428e359 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -331,6 +331,39 @@ class MailTests(TestCase): self.assertFalse(str('Content-Transfer-Encoding: quoted-printable') in s) self.assertTrue(str('Content-Transfer-Encoding: 8bit') in s) + def test_dont_base64_encode_message_rfc822(self): + # Ticket #18967 + # Shouldn't use base64 encoding for a child EmailMessage attachment. + # Create a child message first + child_msg = EmailMessage('Child Subject', 'Some body of child message', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + child_s = child_msg.message().as_string() + + # Now create a parent + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + + # Attach to parent as a string + parent_msg.attach(content=child_s, mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + + # Feature test: try attaching email.Message object directly to the mail. + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + parent_msg.attach(content=child_msg.message(), mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + + # Feature test: try attaching Django's EmailMessage object directly to the mail. + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + parent_msg.attach(content=child_msg, mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + class BaseEmailBackendTests(object): email_backend = None -- cgit v1.3 From 83e434a2c2910d9e0540ea693d5f65e6550240c1 Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Tue, 20 Aug 2013 23:22:25 -0400 Subject: Documentation - Noted that OneToOneField doesn't respect unique. Added OneToOneField to the list of model fields for which the unique argument isn't valid. (OneToOneFields are inherently unique, and if the user supplies a value for unique it is ignored / overwritten.) --- docs/ref/models/fields.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index a9673ce3d2..6ef487e90f 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -281,8 +281,8 @@ you try to save a model with a duplicate value in a :attr:`~Field.unique` field, a :exc:`django.db.IntegrityError` will be raised by the model's :meth:`~django.db.models.Model.save` method. -This option is valid on all field types except :class:`ManyToManyField` and -:class:`FileField`. +This option is valid on all field types except :class:`ManyToManyField`, +:class:`OneToOneField`, and :class:`FileField`. Note that when ``unique`` is ``True``, you don't need to specify :attr:`~Field.db_index`, because ``unique`` implies the creation of an index. -- cgit v1.3 From 082b0638ef05391fa3004463bd7f49721cde3c1d Mon Sep 17 00:00:00 2001 From: evildmp Date: Tue, 20 Aug 2013 22:56:39 +0200 Subject: Added myself to the committers list. --- AUTHORS | 2 +- docs/internals/committers.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'docs') diff --git a/AUTHORS b/AUTHORS index 059310d5c6..c343eeb67b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -46,6 +46,7 @@ The PRIMARY AUTHORS are (and/or have been): * Daniel Lindsley * Marc Tamlyn * Baptiste Mispelon + * Daniele Procida More information on the main contributors to Django can be found in docs/internals/committers.txt. @@ -478,7 +479,6 @@ answer newbie questions, and generally made Django that much better: polpak@yahoo.com Ross Poulton Mihai Preda - Daniele Procida Matthias Pronk Jyrki Pulliainen Thejaswi Puthraya diff --git a/docs/internals/committers.txt b/docs/internals/committers.txt index 6732f1561f..cc0a59e44a 100644 --- a/docs/internals/committers.txt +++ b/docs/internals/committers.txt @@ -537,6 +537,20 @@ Baptiste Mispelon .. _M2BPO: http://www.m2bpo.fr +`Daniele Procida`_ + Daniele works at Cardiff University `School of Medicine`_. He unexpectedly + became a Django developer on 29th April 2009. Since then he has relied + daily on Django's documentation, which has been a constant companion to + him. More recently he has been able to contribute back to the project by + helping improve the documentation itself. + + He is the author of `Arkestra`_ and `Don't be afraid to commit`_. + +.. _Daniele Procida: http://medicine.cf.ac.uk/person/mr-daniele-marco-procida/ +.. _School of Medicine: http://medicine.cf.ac.uk/ +.. _Arkestra: http://arkestra-project.org/ +.. _Don\'t be afraid to commit: https://dont-be-afraid-to-commit.readthedocs.org + Developers Emeritus =================== -- cgit v1.3 From bb011cf809359da3f717e35b5e70fec7897dd22f Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Wed, 21 Aug 2013 15:38:07 -0400 Subject: Documentation -- Corrected error about Model.full_clean() Although the ModelForm validation code was changed to call Model.full_clean(), the documentation still said otherwise. The offending phrase was removed. --- docs/ref/models/instances.txt | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'docs') diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 015393a408..6295eb0407 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -104,14 +104,9 @@ aren't present on your form from being validated since any errors raised could not be corrected by the user. Note that ``full_clean()`` will *not* be called automatically when you call -your model's :meth:`~Model.save()` method, nor as a result of -:class:`~django.forms.ModelForm` validation. In the case of -:class:`~django.forms.ModelForm` validation, :meth:`Model.clean_fields()`, -:meth:`Model.clean()`, and :meth:`Model.validate_unique()` are all called -individually. - -You'll need to call ``full_clean`` manually when you want to run one-step model -validation for your own manually created models. For example:: +your model's :meth:`~Model.save()` method. You'll need to call it manually +when you want to run one-step model validation for your own manually created +models. For example:: from django.core.exceptions import ValidationError try: -- cgit v1.3 From 297f5af222bde02a7cdd005da2e4b00ec81801de Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Thu, 22 Aug 2013 08:25:21 -0300 Subject: Made description of LANGUAGE_CODE setting more clear. --- docs/ref/settings.txt | 15 +++++++++++++-- docs/topics/i18n/translation.txt | 13 ++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) (limited to 'docs') diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 424f7d5795..738ae32cdb 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1290,11 +1290,22 @@ LANGUAGE_CODE Default: ``'en-us'`` -A string representing the language code for this installation. This should be -in standard :term:`language format`. For example, U.S. English +A string representing the language code for this installation. This should be in +standard :term:`language ID format `. For example, U.S. English is ``"en-us"``. See also the `list of language identifiers`_ and :doc:`/topics/i18n/index`. +:setting:`USE_I18N` must be active to this setting to have any effect. + +it serves two purposes: + +* If the locale middleware isn't in use, it decides which translation is served + to all users. +* If the locale middleware is active, it provides the fallback translation when + no translation exist for a given literal to the user preferred language. + +See :ref:`how-django-discovers-language-preference` for more details. + .. _list of language identifiers: http://www.i18nguy.com/unicode/language-identifiers.html .. setting:: LANGUAGE_COOKIE_NAME diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 6436e7dcf9..86f6637d77 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1550,14 +1550,17 @@ should be used -- installation-wide, for a particular user, or both. To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`. Django uses this language as the default translation -- the final attempt if no -other translator finds a translation. +better matching translation is found by one of the methods employed by the +locale middleware (see below). -If all you want to do is run Django with your native language, and a language -file is available for it, all you need to do is set :setting:`LANGUAGE_CODE`. +If all you want to do is run Django with your native language all you need to do +is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message +files ` and their compiled versions (``.mo``) exist. If you want to let each individual user specify which language he or she -prefers, use ``LocaleMiddleware``. ``LocaleMiddleware`` enables language -selection based on data from the request. It customizes content for each user. +prefers, the you also need to use use the ``LocaleMiddleware``. +``LocaleMiddleware`` enables language selection based on data from the request. +It customizes content for each user. To use ``LocaleMiddleware``, add ``'django.middleware.locale.LocaleMiddleware'`` to your :setting:`MIDDLEWARE_CLASSES` setting. Because middleware order -- cgit v1.3 From bac4d03ce68eae7fe13f1891ecdc015817c6d1d8 Mon Sep 17 00:00:00 2001 From: Marc Tamlyn Date: Tue, 20 Aug 2013 10:26:48 +0100 Subject: Fixed #20944 -- Removed inaccurate statement about View.dispatch(). --- docs/ref/class-based-views/base.txt | 4 ---- 1 file changed, 4 deletions(-) (limited to 'docs') diff --git a/docs/ref/class-based-views/base.txt b/docs/ref/class-based-views/base.txt index 0db1e15ea9..f0543e6095 100644 --- a/docs/ref/class-based-views/base.txt +++ b/docs/ref/class-based-views/base.txt @@ -79,10 +79,6 @@ View you can override the ``head()`` method. See :ref:`supporting-other-http-methods` for an example. - The default implementation also sets ``request``, ``args`` and - ``kwargs`` as instance variables, so any method on the view can know - the full details of the request that was made to invoke the view. - .. method:: http_method_not_allowed(request, *args, **kwargs) If the view was called with a HTTP method it doesn't support, this -- cgit v1.3 From 2e926b041c36d46d921acba516a262e21ddaa60d Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Thu, 22 Aug 2013 04:39:31 -0400 Subject: Documentation -- Clarified use of 'view' in test client introduction. --- docs/topics/testing/overview.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'docs') diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index 746c78f41b..89b43c42c5 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -328,7 +328,8 @@ Some of the things you can do with the test client are: everything from low-level HTTP (result headers and status codes) to page content. -* Test that the correct view is executed for a given URL. +* See the chain of redirects (if any) and check the URL and status code at + each step. * Test that a given request is rendered by a given Django template, with a template context that contains certain values. @@ -337,8 +338,8 @@ Note that the test client is not intended to be a replacement for Selenium_ or other "in-browser" frameworks. Django's test client has a different focus. In short: -* Use Django's test client to establish that the correct view is being - called and that the view is collecting the correct context data. +* Use Django's test client to establish that the correct template is being + rendered and that the template is passed the correct context data. * Use in-browser frameworks like Selenium_ to test *rendered* HTML and the *behavior* of Web pages, namely JavaScript functionality. Django also -- cgit v1.3 From 6af05e7a0f0e4604d6a67899acaa99d73ec0dfaa Mon Sep 17 00:00:00 2001 From: Anssi Kääriäinen Date: Wed, 14 Aug 2013 11:05:01 +0300 Subject: Fixed model.__eq__ and __hash__ for no pk value cases The __eq__ method now considers two instances without primary key value equal only when they have same id(). The __hash__ method raises TypeError for no primary key case. Fixed #18864, fixed #18250 Thanks to Tim Graham for docs review. --- django/db/models/base.py | 13 ++++++++++--- django/forms/models.py | 6 +++++- docs/ref/models/instances.txt | 19 +++++++++++++++++++ docs/releases/1.7.txt | 8 ++++++++ tests/basic/tests.py | 11 +++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) (limited to 'docs') diff --git a/django/db/models/base.py b/django/db/models/base.py index 3e2ae8d425..a5b0f188b4 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -459,14 +459,21 @@ class Model(six.with_metaclass(ModelBase)): return '%s object' % self.__class__.__name__ def __eq__(self, other): - return (isinstance(other, Model) and - self._meta.concrete_model == other._meta.concrete_model and - self._get_pk_val() == other._get_pk_val()) + if not isinstance(other, Model): + return False + if self._meta.concrete_model != other._meta.concrete_model: + return False + my_pk = self._get_pk_val() + if my_pk is None: + return self is other + return my_pk == other._get_pk_val() def __ne__(self, other): return not self.__eq__(other) def __hash__(self): + if self._get_pk_val() is None: + raise TypeError("Model instances without primary key value are unhashable") return hash(self._get_pk_val()) def __reduce__(self): diff --git a/django/forms/models.py b/django/forms/models.py index a5b82e521d..4c6ee9c6ed 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -631,7 +631,11 @@ class BaseModelFormSet(BaseFormSet): seen_data = set() for form in valid_forms: # get data for each field of each of unique_check - row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data]) + row_data = (form.cleaned_data[field] + for field in unique_check if field in form.cleaned_data) + # Reduce Model instances to their primary key values + row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d + for d in row_data) if row_data and not None in row_data: # if we've already seen it then we have a uniqueness failure if row_data in seen_data: diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 6295eb0407..da657a9a01 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -521,6 +521,25 @@ For example:: In previous versions only instances of the exact same class and same primary key value were considered equal. +``__hash__`` +------------ + +.. method:: Model.__hash__() + +The ``__hash__`` method is based on the instance's primary key value. It +is effectively hash(obj.pk). If the instance doesn't have a primary key +value then a ``TypeError`` will be raised (otherwise the ``__hash__`` +method would return different values before and after the instance is +saved, but changing the ``__hash__`` value of an instance `is forbidden +in Python`_). + +.. versionchanged:: 1.7 + + In previous versions instance's without primary key value were + hashable. + +.. _is forbidden in Python: http://docs.python.org/reference/datamodel.html#object.__hash__ + ``get_absolute_url`` -------------------- diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 774d9d3161..6480a2505f 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -199,6 +199,14 @@ Miscellaneous equal when primary keys match. Previously only instances of exact same class were considered equal on primary key match. +* The :meth:`django.db.models.Model.__eq__` method has changed such that + two ``Model`` instances without primary key values won't be considered + equal (unless they are the same instance). + +* The :meth:`django.db.models.Model.__hash__` will now raise ``TypeError`` + when called on an instance without a primary key value. This is done to + avoid mutable ``__hash__`` values in containers. + Features deprecated in 1.7 ========================== diff --git a/tests/basic/tests.py b/tests/basic/tests.py index 2b051621ef..611944902a 100644 --- a/tests/basic/tests.py +++ b/tests/basic/tests.py @@ -708,9 +708,20 @@ class ModelTest(TestCase): SelfRef.objects.get(selfref=sr) def test_eq(self): + self.assertEqual(Article(id=1), Article(id=1)) self.assertNotEqual(Article(id=1), object()) self.assertNotEqual(object(), Article(id=1)) + a = Article() + self.assertEqual(a, a) + self.assertNotEqual(Article(), a) + def test_hash(self): + # Value based on PK + self.assertEqual(hash(Article(id=1)), hash(1)) + with self.assertRaises(TypeError): + # No PK value -> unhashable (because save() would then change + # hash) + hash(Article()) class ConcurrentSaveTests(TransactionTestCase): -- cgit v1.3 From b0ce6fe656873825271bacb55e55474fc346c1c6 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 21 Aug 2013 20:12:19 -0400 Subject: Fixed #20922 -- Allowed customizing the serializer used by contrib.sessions Added settings.SESSION_SERIALIZER which is the import path of a serializer to use for sessions. Thanks apollo13, carljm, shaib, akaariai, charettes, and dstufft for reviews. --- django/conf/global_settings.py | 1 + django/contrib/messages/storage/session.py | 17 ++++- django/contrib/messages/tests/base.py | 1 + django/contrib/messages/tests/test_session.py | 4 +- django/contrib/sessions/backends/base.py | 21 +++--- django/contrib/sessions/backends/signed_cookies.py | 21 +----- django/contrib/sessions/models.py | 2 +- django/contrib/sessions/serializers.py | 20 +++++ django/contrib/sessions/tests.py | 34 +++++---- docs/ref/settings.txt | 24 +++++- docs/releases/1.6.txt | 23 ++++++ docs/topics/http/sessions.txt | 87 ++++++++++++++++++++-- tests/defer_regress/tests.py | 40 +++++----- 13 files changed, 218 insertions(+), 77 deletions(-) create mode 100644 django/contrib/sessions/serializers.py (limited to 'docs') diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 95deaa8d87..ab3cdab59e 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -475,6 +475,7 @@ SESSION_SAVE_EVERY_REQUEST = False # Whether to save the se SESSION_EXPIRE_AT_BROWSER_CLOSE = False # Whether a user's session cookie expires when the Web browser is closed. SESSION_ENGINE = 'django.contrib.sessions.backends.db' # The module to store session data SESSION_FILE_PATH = None # Directory to store session files if using the file session module. If None, the backend will use a sensible default. +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer' # class to serialize session data ######### # CACHE # diff --git a/django/contrib/messages/storage/session.py b/django/contrib/messages/storage/session.py index 225dfda289..c3e293c22e 100644 --- a/django/contrib/messages/storage/session.py +++ b/django/contrib/messages/storage/session.py @@ -1,4 +1,8 @@ +import json + from django.contrib.messages.storage.base import BaseStorage +from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder +from django.utils import six class SessionStorage(BaseStorage): @@ -20,14 +24,23 @@ class SessionStorage(BaseStorage): always stores everything it is given, so return True for the all_retrieved flag. """ - return self.request.session.get(self.session_key), True + return self.deserialize_messages(self.request.session.get(self.session_key)), True def _store(self, messages, response, *args, **kwargs): """ Stores a list of messages to the request's session. """ if messages: - self.request.session[self.session_key] = messages + self.request.session[self.session_key] = self.serialize_messages(messages) else: self.request.session.pop(self.session_key, None) return [] + + def serialize_messages(self, messages): + encoder = MessageEncoder(separators=(',', ':')) + return encoder.encode(messages) + + def deserialize_messages(self, data): + if data and isinstance(data, six.string_types): + return json.loads(data, cls=MessageDecoder) + return data diff --git a/django/contrib/messages/tests/base.py b/django/contrib/messages/tests/base.py index 7011a5779a..5241436daf 100644 --- a/django/contrib/messages/tests/base.py +++ b/django/contrib/messages/tests/base.py @@ -61,6 +61,7 @@ class BaseTests(object): MESSAGE_TAGS = '', MESSAGE_STORAGE = '%s.%s' % (self.storage_class.__module__, self.storage_class.__name__), + SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer', ) self.settings_override.enable() diff --git a/django/contrib/messages/tests/test_session.py b/django/contrib/messages/tests/test_session.py index 2ce564b773..940e1c02d0 100644 --- a/django/contrib/messages/tests/test_session.py +++ b/django/contrib/messages/tests/test_session.py @@ -11,13 +11,13 @@ def set_session_data(storage, messages): Sets the messages into the backend request's session and remove the backend's loaded data cache. """ - storage.request.session[storage.session_key] = messages + storage.request.session[storage.session_key] = storage.serialize_messages(messages) if hasattr(storage, '_loaded_data'): del storage._loaded_data def stored_session_messages_count(storage): - data = storage.request.session.get(storage.session_key, []) + data = storage.deserialize_messages(storage.request.session.get(storage.session_key, [])) return len(data) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 759d7ac7ad..7f5e958a60 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -3,11 +3,6 @@ from __future__ import unicode_literals import base64 from datetime import datetime, timedelta import logging - -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle import string from django.conf import settings @@ -17,6 +12,7 @@ from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone from django.utils.encoding import force_bytes, force_text +from django.utils.module_loading import import_by_path from django.contrib.sessions.exceptions import SuspiciousSession @@ -42,6 +38,7 @@ class SessionBase(object): self._session_key = session_key self.accessed = False self.modified = False + self.serializer = import_by_path(settings.SESSION_SERIALIZER) def __contains__(self, key): return key in self._session @@ -86,21 +83,21 @@ class SessionBase(object): return salted_hmac(key_salt, value).hexdigest() def encode(self, session_dict): - "Returns the given session dictionary pickled and encoded as a string." - pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL) - hash = self._hash(pickled) - return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii') + "Returns the given session dictionary serialized and encoded as a string." + serialized = self.serializer().dumps(session_dict) + hash = self._hash(serialized) + return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii') def decode(self, session_data): encoded_data = base64.b64decode(force_bytes(session_data)) try: # could produce ValueError if there is no ':' - hash, pickled = encoded_data.split(b':', 1) - expected_hash = self._hash(pickled) + hash, serialized = encoded_data.split(b':', 1) + expected_hash = self._hash(serialized) if not constant_time_compare(hash.decode(), expected_hash): raise SuspiciousSession("Session data corrupted") else: - return pickle.loads(pickled) + return self.serializer().loads(serialized) except Exception as e: # ValueError, SuspiciousOperation, unpickling exceptions. If any of # these happen, just return an empty dictionary (an empty session). diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py index c2b7a3123f..77a6750ce4 100644 --- a/django/contrib/sessions/backends/signed_cookies.py +++ b/django/contrib/sessions/backends/signed_cookies.py @@ -1,26 +1,9 @@ -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle - from django.conf import settings from django.core import signing from django.contrib.sessions.backends.base import SessionBase -class PickleSerializer(object): - """ - Simple wrapper around pickle to be used in signing.dumps and - signing.loads. - """ - def dumps(self, obj): - return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) - - def loads(self, data): - return pickle.loads(data) - - class SessionStore(SessionBase): def load(self): @@ -31,7 +14,7 @@ class SessionStore(SessionBase): """ try: return signing.loads(self.session_key, - serializer=PickleSerializer, + serializer=self.serializer, # This doesn't handle non-default expiry dates, see #19201 max_age=settings.SESSION_COOKIE_AGE, salt='django.contrib.sessions.backends.signed_cookies') @@ -91,7 +74,7 @@ class SessionStore(SessionBase): session_cache = getattr(self, '_session_cache', {}) return signing.dumps(session_cache, compress=True, salt='django.contrib.sessions.backends.signed_cookies', - serializer=PickleSerializer) + serializer=self.serializer) @classmethod def clear_expired(cls): diff --git a/django/contrib/sessions/models.py b/django/contrib/sessions/models.py index 0179c358b3..3a6e31152f 100644 --- a/django/contrib/sessions/models.py +++ b/django/contrib/sessions/models.py @@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _ class SessionManager(models.Manager): def encode(self, session_dict): """ - Returns the given session dictionary pickled and encoded as a string. + Returns the given session dictionary serialized and encoded as a string. """ return SessionStore().encode(session_dict) diff --git a/django/contrib/sessions/serializers.py b/django/contrib/sessions/serializers.py new file mode 100644 index 0000000000..92a31c054b --- /dev/null +++ b/django/contrib/sessions/serializers.py @@ -0,0 +1,20 @@ +from django.core.signing import JSONSerializer as BaseJSONSerializer +try: + from django.utils.six.moves import cPickle as pickle +except ImportError: + import pickle + + +class PickleSerializer(object): + """ + Simple wrapper around pickle to be used in signing.dumps and + signing.loads. + """ + def dumps(self, obj): + return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) + + def loads(self, data): + return pickle.loads(data) + + +JSONSerializer = BaseJSONSerializer diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index f2a35c544e..4caefe938c 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -285,21 +285,25 @@ class SessionTestsMixin(object): def test_actual_expiry(self): - # Regression test for #19200 - old_session_key = None - new_session_key = None - try: - self.session['foo'] = 'bar' - self.session.set_expiry(-timedelta(seconds=10)) - self.session.save() - old_session_key = self.session.session_key - # With an expiry date in the past, the session expires instantly. - new_session = self.backend(self.session.session_key) - new_session_key = new_session.session_key - self.assertNotIn('foo', new_session) - finally: - self.session.delete(old_session_key) - self.session.delete(new_session_key) + # this doesn't work with JSONSerializer (serializing timedelta) + with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'): + self.session = self.backend() # reinitialize after overriding settings + + # Regression test for #19200 + old_session_key = None + new_session_key = None + try: + self.session['foo'] = 'bar' + self.session.set_expiry(-timedelta(seconds=10)) + self.session.save() + old_session_key = self.session.session_key + # With an expiry date in the past, the session expires instantly. + new_session = self.backend(self.session.session_key) + new_session_key = new_session.session_key + self.assertNotIn('foo', new_session) + finally: + self.session.delete(old_session_key) + self.session.delete(new_session_key) class DatabaseSessionTests(SessionTestsMixin, TestCase): diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 738ae32cdb..0105b2ccf5 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2403,7 +2403,7 @@ SESSION_ENGINE Default: ``django.contrib.sessions.backends.db`` -Controls where Django stores session data. Valid values are: +Controls where Django stores session data. Included engines are: * ``'django.contrib.sessions.backends.db'`` * ``'django.contrib.sessions.backends.file'`` @@ -2446,6 +2446,28 @@ Whether to save the session data on every request. If this is ``False`` (default), then the session data will only be saved if it has been modified -- that is, if any of its dictionary values have been assigned or deleted. +.. setting:: SESSION_SERIALIZER + +SESSION_SERIALIZER +------------------ + +Default: ``'django.contrib.sessions.serializers.JSONSerializer'`` + +.. versionchanged:: 1.6 + + The default switched from + :class:`~django.contrib.sessions.serializers.PickleSerializer` to + :class:`~django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. + +Full import path of a serializer class to use for serializing session data. +Included serializers are: + +* ``'django.contrib.sessions.serializers.PickleSerializer'`` +* ``'django.contrib.sessions.serializers.JSONSerializer'`` + +See :ref:`session_serialization` for details, including a warning regarding +possible remote code execution when using +:class:`~django.contrib.sessions.serializers.PickleSerializer`. Sites ===== diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index c0f5c51194..556edddda1 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -727,6 +727,29 @@ the ``name`` argument so it doesn't conflict with the new url:: You can remove this url pattern after your app has been deployed with Django 1.6 for :setting:`PASSWORD_RESET_TIMEOUT_DAYS`. +Default session serialization switched to JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Historically, :mod:`django.contrib.sessions` used :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaulted to using :mod:`pickle` +in Django 1.5.3, but we've changed the default to JSON in 1.6. If you upgrade +and switch from pickle to JSON, sessions created before the upgrade will be +lost. While JSON serialization does not support all Python objects like +:mod:`pickle` does, we highly recommend using JSON-serialized sessions. See the +:ref:`session_serialization` documentation for more details. + Miscellaneous ~~~~~~~~~~~~~ diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 87b5972c2e..637991b3b5 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -128,8 +128,9 @@ and the :setting:`SECRET_KEY` setting. .. warning:: - **If the SECRET_KEY is not kept secret, this can lead to arbitrary remote - code execution.** + **If the SECRET_KEY is not kept secret and you are using the** + :class:`~django.contrib.sessions.serializers.PickleSerializer`, **this can + lead to arbitrary remote code execution.** An attacker in possession of the :setting:`SECRET_KEY` can not only generate falsified session data, which your site will trust, but also @@ -256,7 +257,9 @@ You can edit it multiple times. in 5 minutes. * If ``value`` is a ``datetime`` or ``timedelta`` object, the - session will expire at that specific date/time. + session will expire at that specific date/time. Note that ``datetime`` + and ``timedelta`` values are only serializable if you are using the + :class:`~django.contrib.sessions.serializers.PickleSerializer`. * If ``value`` is ``0``, the user's session cookie will expire when the user's Web browser is closed. @@ -301,6 +304,72 @@ You can edit it multiple times. Removes expired sessions from the session store. This class method is called by :djadmin:`clearsessions`. +.. _session_serialization: + +Session serialization +--------------------- + +.. versionchanged:: 1.6 + +Before version 1.6, Django defaulted to using :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaults to +using :class:`django.contrib.sessions.serializers.PickleSerializer` in +Django 1.5.x, but, for security hardening, defaults to +:class:`django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. +Even with the caveats described in :ref:`custom-serializers`, we highly +recommend sticking with JSON serialization *especially if you are using the +cookie backend*. + +Bundled Serializers +^^^^^^^^^^^^^^^^^^^ + +.. class:: serializers.JSONSerializer + + A wrapper around the JSON serializer from :mod:`django.core.signing`. Can + only serialize basic data types. See the :ref:`custom-serializers` section + for more details. + +.. class:: serializers.PickleSerializer + + Supports arbitrary Python objects, but, as described above, can lead to a + remote code execution vulnerability if :setting:`SECRET_KEY` becomes known + by an attacker. + +.. _custom-serializers: + +Write Your Own Serializer +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that unlike :class:`~django.contrib.sessions.serializers.PickleSerializer`, +the :class:`~django.contrib.sessions.serializers.JSONSerializer` cannot handle +arbitrary Python data types. As is often the case, there is a trade-off between +convenience and security. If you wish to store more advanced data types +including ``datetime`` and ``Decimal`` in JSON backed sessions, you will need +to write a custom serializer (or convert such values to a JSON serializable +object before storing them in ``request.session``). While serializing these +values is fairly straightforward +(``django.core.serializers.json.DateTimeAwareJSONEncoder`` may be helpful), +writing a decoder that can reliably get back the same thing that you put in is +more fragile. For example, you run the risk of returning a ``datetime`` that +was actually a string that just happened to be in the same format chosen for +``datetime``\s). + +Your serializer class must implement two methods, +``dumps(self, obj)`` and ``loads(self, data)``, to serialize and deserialize +the dictionary of session data, respectively. + Session object guidelines ------------------------- @@ -390,14 +459,15 @@ An API is available to manipulate session data outside of a view:: >>> from django.contrib.sessions.backends.db import SessionStore >>> import datetime >>> s = SessionStore() - >>> s['last_login'] = datetime.datetime(2005, 8, 20, 13, 35, 10) + >>> # stored as seconds since epoch since datetimes are not serializable in JSON. + >>> s['last_login'] = 1376587691 >>> s.save() >>> s.session_key '2b1189a188b44ad18c35e113ac6ceead' >>> s = SessionStore(session_key='2b1189a188b44ad18c35e113ac6ceead') >>> s['last_login'] - datetime.datetime(2005, 8, 20, 13, 35, 0) + 1376587691 In order to prevent session fixation attacks, sessions keys that don't exist are regenerated:: @@ -543,8 +613,11 @@ behavior: Technical details ================= -* The session dictionary should accept any pickleable Python object. See - the :mod:`pickle` module for more information. +* The session dictionary accepts any :mod:`json` serializable value when using + :class:`~django.contrib.sessions.serializers.JSONSerializer` or any + pickleable Python object when using + :class:`~django.contrib.sessions.serializers.PickleSerializer`. See the + :mod:`pickle` module for more information. * Session data is stored in a database table named ``django_session`` . diff --git a/tests/defer_regress/tests.py b/tests/defer_regress/tests.py index 619f65163c..ffb47a8133 100644 --- a/tests/defer_regress/tests.py +++ b/tests/defer_regress/tests.py @@ -7,6 +7,7 @@ from django.contrib.sessions.backends.db import SessionStore from django.db.models import Count from django.db.models.loading import cache from django.test import TestCase +from django.test.utils import override_settings from .models import ( ResolveThis, Item, RelatedItem, Child, Leaf, Proxy, SimpleItem, Feature, @@ -83,24 +84,6 @@ class DeferRegressionTest(TestCase): self.assertEqual(results[0].child.name, "c1") self.assertEqual(results[0].second_child.name, "c2") - # Test for #12163 - Pickling error saving session with unsaved model - # instances. - SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' - - item = Item() - item._deferred = False - s = SessionStore(SESSION_KEY) - s.clear() - s["item"] = item - s.save() - - s = SessionStore(SESSION_KEY) - s.modified = True - s.save() - - i2 = s["item"] - self.assertFalse(i2._deferred) - # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), @@ -147,6 +130,27 @@ class DeferRegressionTest(TestCase): cache.get_app("defer_regress"), include_deferred=True)) ) + @override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer') + def test_ticket_12163(self): + # Test for #12163 - Pickling error saving session with unsaved model + # instances. + SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' + + item = Item() + item._deferred = False + s = SessionStore(SESSION_KEY) + s.clear() + s["item"] = item + s.save() + + s = SessionStore(SESSION_KEY) + s.modified = True + s.save() + + i2 = s["item"] + self.assertFalse(i2._deferred) + + def test_ticket_16409(self): # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), -- cgit v1.3 From 57c82f909b212708a17edd11014be718bd02be3b Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Thu, 22 Aug 2013 12:14:56 -0300 Subject: Typos introduced in 297f5af222. --- docs/ref/settings.txt | 8 ++++---- docs/topics/i18n/translation.txt | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'docs') diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 0105b2ccf5..2f531803bc 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1291,18 +1291,18 @@ LANGUAGE_CODE Default: ``'en-us'`` A string representing the language code for this installation. This should be in -standard :term:`language ID format `. For example, U.S. English +standard :term:`language ID format `. For example, U.S. English is ``"en-us"``. See also the `list of language identifiers`_ and :doc:`/topics/i18n/index`. -:setting:`USE_I18N` must be active to this setting to have any effect. +:setting:`USE_I18N` must be active for this setting to have any effect. -it serves two purposes: +It serves two purposes: * If the locale middleware isn't in use, it decides which translation is served to all users. * If the locale middleware is active, it provides the fallback translation when - no translation exist for a given literal to the user preferred language. + no translation exist for a given literal to the user's preferred language. See :ref:`how-django-discovers-language-preference` for more details. diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 86f6637d77..120db8e5b0 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1550,15 +1550,15 @@ should be used -- installation-wide, for a particular user, or both. To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`. Django uses this language as the default translation -- the final attempt if no -better matching translation is found by one of the methods employed by the +better matching translation is found through one of the methods employed by the locale middleware (see below). -If all you want to do is run Django with your native language all you need to do +If all you want is to run Django with your native language all you need to do is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message files ` and their compiled versions (``.mo``) exist. If you want to let each individual user specify which language he or she -prefers, the you also need to use use the ``LocaleMiddleware``. +prefers, then you also need to use use the ``LocaleMiddleware``. ``LocaleMiddleware`` enables language selection based on data from the request. It customizes content for each user. -- cgit v1.3