From 051cb1f4c60ac8e7087d92ef34ed41e6684d8b9b Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 18 May 2013 12:32:47 +0200 Subject: Fixed #20411 -- Don't let invalid referers blow up CSRF same origin checks. Thanks to edevil for the report and saz for the patch. --- django/utils/http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'django/utils') diff --git a/django/utils/http.py b/django/utils/http.py index 15fac6bfca..9897df4fb0 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -226,7 +226,10 @@ def same_origin(url1, url2): Checks if two URLs are 'same-origin' """ p1, p2 = urllib_parse.urlparse(url1), urllib_parse.urlparse(url2) - return (p1.scheme, p1.hostname, p1.port) == (p2.scheme, p2.hostname, p2.port) + try: + return (p1.scheme, p1.hostname, p1.port) == (p2.scheme, p2.hostname, p2.port) + except ValueError: + return False def is_safe_url(url, host=None): """ -- cgit v1.3 From 92ebb29c5376c1811e8607e96bddc90d24e44e2a Mon Sep 17 00:00:00 2001 From: Łukasz Langa Date: Sat, 18 May 2013 14:37:04 +0200 Subject: Fixes #19919: get_language_from_request() disregards "en-us" and "en" languages when matching Accept-Language --- django/utils/translation/trans_real.py | 27 +++++++++++++---------- tests/i18n/tests.py | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 13 deletions(-) (limited to 'django/utils') diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 07353c35ee..f7b1524085 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -364,10 +364,14 @@ def get_supported_language_variant(lang_code, supported=None): if supported is None: from django.conf import settings supported = dict(settings.LANGUAGES) - if lang_code and lang_code not in supported: - lang_code = lang_code.split('-')[0] # e.g. if fr-ca is not supported fallback to fr - if lang_code and lang_code in supported and check_for_language(lang_code): - return lang_code + if lang_code: + # e.g. if fr-CA is not supported, try fr-ca; + # if that fails, fallback to fr. + variants = (lang_code, lang_code.lower(), lang_code.split('-')[0], + lang_code.lower().split('-')[0]) + for code in variants: + if code in supported and check_for_language(code): + return code raise LookupError(lang_code) def get_language_from_path(path, supported=None): @@ -438,14 +442,13 @@ def get_language_from_request(request, check_path=False): # need to check again. return _accepted[normalized] - for lang, dirname in ((accept_lang, normalized), - (accept_lang.split('-')[0], normalized.split('_')[0])): - if lang.lower() not in supported: - continue - for path in all_locale_paths(): - if os.path.exists(os.path.join(path, dirname, 'LC_MESSAGES', 'django.mo')): - _accepted[normalized] = lang - return lang + try: + accept_lang = get_supported_language_variant(accept_lang, supported) + except LookupError: + continue + else: + _accepted[normalized] = accept_lang + return accept_lang try: return get_supported_language_variant(settings.LANGUAGE_CODE, supported) diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 1022c8d2f1..29223e5add 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -30,7 +30,8 @@ from django.utils.translation import (activate, deactivate, ngettext, ngettext_lazy, ungettext, ungettext_lazy, pgettext, pgettext_lazy, - npgettext, npgettext_lazy) + npgettext, npgettext_lazy, + check_for_language) from .commands.tests import can_run_extraction_tests, can_run_compilation_tests if can_run_extraction_tests: @@ -1114,3 +1115,40 @@ class LocaleMiddlewareTests(TestCase): self.assertContains(response, "Oui/Non") response = self.client.get('/en/streaming/') self.assertContains(response, "Yes/No") + +@override_settings( + USE_I18N=True, + LANGUAGES=( + ('bg', 'Bulgarian'), + ('en-us', 'English'), + ), + MIDDLEWARE_CLASSES=( + 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.common.CommonMiddleware', + ), +) +class CountrySpecificLanguageTests(TestCase): + + urls = 'i18n.urls' + + def setUp(self): + trans_real._accepted = {} + self.rf = RequestFactory() + + def test_check_for_language(self): + self.assertTrue(check_for_language('en')) + self.assertTrue(check_for_language('en-us')) + self.assertTrue(check_for_language('en-US')) + + + def test_get_language_from_request(self): + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8,bg;q=0.6,ru;q=0.4'} + lang = get_language_from_request(r) + self.assertEqual('en-us', lang) + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'bg-bg,en-US;q=0.8,en;q=0.6,ru;q=0.4'} + lang = get_language_from_request(r) + self.assertEqual('bg', lang) -- cgit v1.3 From acd0bb39df5c9ca486e49ec55ae34538242ce071 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 18 May 2013 17:36:31 +0200 Subject: Fixed #14894 -- Ensure that activating a translation doesn't run into threading issues. Thanks to maxbublis for the report and sergeykolosov for the patch. --- django/utils/translation/trans_real.py | 2 +- tests/i18n/tests.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'django/utils') diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index f7b1524085..26be0ed729 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -140,7 +140,7 @@ def translation(language): # doesn't affect en-gb), even though they will both use the core "en" # translation. So we have to subvert Python's internal gettext caching. base_lang = lambda x: x.split('-', 1)[0] - if base_lang(lang) in [base_lang(trans) for trans in _translations]: + if base_lang(lang) in [base_lang(trans) for trans in list(_translations)]: res._info = res._info.copy() res._catalog = res._catalog.copy() diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 29223e5add..1b15720c16 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -334,6 +334,36 @@ class TranslationTests(TestCase): self.assertEqual(rendered, 'My other name is James.') +class TranslationThreadSafetyTests(TestCase): + def setUp(self): + self._old_language = get_language() + self._translations = trans_real._translations + + # here we rely on .split() being called inside the _fetch() + # in trans_real.translation() + class sideeffect_str(str): + def split(self, *args, **kwargs): + res = str.split(self, *args, **kwargs) + trans_real._translations['en-YY'] = None + return res + + trans_real._translations = {sideeffect_str('en-XX'): None} + + def tearDown(self): + trans_real._translations = self._translations + activate(self._old_language) + + def test_bug14894_translation_activate_thread_safety(self): + translation_count = len(trans_real._translations) + try: + translation.activate('pl') + except RuntimeError: + self.fail('translation.activate() is not thread-safe') + + # make sure sideeffect_str actually added a new translation + self.assertLess(translation_count, len(trans_real._translations)) + + @override_settings(USE_L10N=True) class FormattingTests(TestCase): -- cgit v1.3 From 7d77e9786a118dd95a268872dd9d36664066b96a Mon Sep 17 00:00:00 2001 From: Emil Stenström Date: Sat, 18 May 2013 13:58:45 +0200 Subject: Fixed #20246 -- Added non-breaking spaces between values an units --- AUTHORS | 1 + django/contrib/humanize/templatetags/humanize.py | 18 ++++-- django/contrib/humanize/tests.py | 26 ++++----- django/template/defaultfilters.py | 28 ++++++---- django/utils/html.py | 7 +++ django/utils/timesince.py | 7 ++- tests/defaultfilters/tests.py | 68 ++++++++++++----------- tests/template_tests/filters.py | 59 ++++++++++---------- tests/utils_tests/test_timesince.py | 71 ++++++++++++------------ 9 files changed, 155 insertions(+), 130 deletions(-) (limited to 'django/utils') diff --git a/AUTHORS b/AUTHORS index 6ecda3ff57..2e02f7c07e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -536,6 +536,7 @@ answer newbie questions, and generally made Django that much better: starrynight Vasiliy Stavenko Thomas Steinacher + Emil Stenström Johan C. Stöver Nowell Strite Thomas Stromberg diff --git a/django/contrib/humanize/templatetags/humanize.py b/django/contrib/humanize/templatetags/humanize.py index 21f4c452fa..eaee734f75 100644 --- a/django/contrib/humanize/templatetags/humanize.py +++ b/django/contrib/humanize/templatetags/humanize.py @@ -194,17 +194,20 @@ def naturaltime(value): return _('now') elif delta.seconds < 60: return ungettext( - 'a second ago', '%(count)s seconds ago', delta.seconds + # Translators: \\u00a0 is non-breaking space + 'a second ago', '%(count)s\u00a0seconds ago', delta.seconds ) % {'count': delta.seconds} elif delta.seconds // 60 < 60: count = delta.seconds // 60 return ungettext( - 'a minute ago', '%(count)s minutes ago', count + # Translators: \\u00a0 is non-breaking space + 'a minute ago', '%(count)s\u00a0minutes ago', count ) % {'count': count} else: count = delta.seconds // 60 // 60 return ungettext( - 'an hour ago', '%(count)s hours ago', count + # Translators: \\u00a0 is non-breaking space + 'an hour ago', '%(count)s\u00a0hours ago', count ) % {'count': count} else: delta = value - now @@ -216,15 +219,18 @@ def naturaltime(value): return _('now') elif delta.seconds < 60: return ungettext( - 'a second from now', '%(count)s seconds from now', delta.seconds + # Translators: \\u00a0 is non-breaking space + 'a second from now', '%(count)s\u00a0seconds from now', delta.seconds ) % {'count': delta.seconds} elif delta.seconds // 60 < 60: count = delta.seconds // 60 return ungettext( - 'a minute from now', '%(count)s minutes from now', count + # Translators: \\u00a0 is non-breaking space + 'a minute from now', '%(count)s\u00a0minutes from now', count ) % {'count': count} else: count = delta.seconds // 60 // 60 return ungettext( - 'an hour from now', '%(count)s hours from now', count + # Translators: \\u00a0 is non-breaking space + 'an hour from now', '%(count)s\u00a0hours from now', count ) % {'count': count} diff --git a/django/contrib/humanize/tests.py b/django/contrib/humanize/tests.py index 1e1c8424e6..5c39c79ac3 100644 --- a/django/contrib/humanize/tests.py +++ b/django/contrib/humanize/tests.py @@ -195,22 +195,22 @@ class HumanizeTests(TestCase): result_list = [ 'now', 'a second ago', - '30 seconds ago', + '30\xa0seconds ago', 'a minute ago', - '2 minutes ago', + '2\xa0minutes ago', 'an hour ago', - '23 hours ago', - '1 day ago', - '1 year, 4 months ago', + '23\xa0hours ago', + '1\xa0day ago', + '1\xa0year, 4\xa0months ago', 'a second from now', - '30 seconds from now', + '30\xa0seconds from now', 'a minute from now', - '2 minutes from now', + '2\xa0minutes from now', 'an hour from now', - '23 hours from now', - '1 day from now', - '2 days, 6 hours from now', - '1 year, 4 months from now', + '23\xa0hours from now', + '1\xa0day from now', + '2\xa0days, 6\xa0hours from now', + '1\xa0year, 4\xa0months from now', 'now', 'now', ] @@ -218,8 +218,8 @@ class HumanizeTests(TestCase): # date in naive arithmetic is only 2 days and 5 hours after in # aware arithmetic. result_list_with_tz_support = result_list[:] - assert result_list_with_tz_support[-4] == '2 days, 6 hours from now' - result_list_with_tz_support[-4] == '2 days, 5 hours from now' + assert result_list_with_tz_support[-4] == '2\xa0days, 6\xa0hours from now' + result_list_with_tz_support[-4] == '2\xa0days, 5\xa0hours from now' orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime try: diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index 88526e5a20..4201cfeb67 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -14,7 +14,7 @@ from django.utils import formats from django.utils.dateformat import format, time_format from django.utils.encoding import force_text, iri_to_uri from django.utils.html import (conditional_escape, escapejs, fix_ampersands, - escape, urlize as urlize_impl, linebreaks, strip_tags) + escape, urlize as urlize_impl, linebreaks, strip_tags, avoid_wrapping) from django.utils.http import urlquote from django.utils.text import Truncator, wrap, phone2numeric from django.utils.safestring import mark_safe, SafeData, mark_for_escaping @@ -810,7 +810,8 @@ def filesizeformat(bytes): try: bytes = float(bytes) except (TypeError,ValueError,UnicodeDecodeError): - return ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0} + value = ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0} + return avoid_wrapping(value) filesize_number_format = lambda value: formats.number_format(round(value, 1), 1) @@ -821,16 +822,19 @@ def filesizeformat(bytes): PB = 1<<50 if bytes < KB: - return ungettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes} - if bytes < MB: - return ugettext("%s KB") % filesize_number_format(bytes / KB) - if bytes < GB: - return ugettext("%s MB") % filesize_number_format(bytes / MB) - if bytes < TB: - return ugettext("%s GB") % filesize_number_format(bytes / GB) - if bytes < PB: - return ugettext("%s TB") % filesize_number_format(bytes / TB) - return ugettext("%s PB") % filesize_number_format(bytes / PB) + value = ungettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes} + elif bytes < MB: + value = ugettext("%s KB") % filesize_number_format(bytes / KB) + elif bytes < GB: + value = ugettext("%s MB") % filesize_number_format(bytes / MB) + elif bytes < TB: + value = ugettext("%s GB") % filesize_number_format(bytes / GB) + elif bytes < PB: + value = ugettext("%s TB") % filesize_number_format(bytes / TB) + else: + value = ugettext("%s PB") % filesize_number_format(bytes / PB) + + return avoid_wrapping(value) @register.filter(is_safe=False) def pluralize(value, arg='s'): diff --git a/django/utils/html.py b/django/utils/html.py index 8b28d97d13..edddc48e62 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -281,3 +281,10 @@ def clean_html(text): text = trailing_empty_content_re.sub('', text) return text clean_html = allow_lazy(clean_html, six.text_type) + +def avoid_wrapping(value): + """ + Avoid text wrapping in the middle of a phrase by adding non-breaking + spaces where there previously were normal spaces. + """ + return value.replace(" ", "\xa0") diff --git a/django/utils/timesince.py b/django/utils/timesince.py index d70ab2ffe1..46c387f262 100644 --- a/django/utils/timesince.py +++ b/django/utils/timesince.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime +from django.utils.html import avoid_wrapping from django.utils.timezone import is_aware, utc from django.utils.translation import ugettext, ungettext_lazy @@ -40,18 +41,18 @@ def timesince(d, now=None, reversed=False): since = delta.days * 24 * 60 * 60 + delta.seconds if since <= 0: # d is in the future compared to now, stop processing. - return ugettext('0 minutes') + return avoid_wrapping(ugettext('0 minutes')) for i, (seconds, name) in enumerate(chunks): count = since // seconds if count != 0: break - result = name % count + result = avoid_wrapping(name % count) if i + 1 < len(chunks): # Now get the second item seconds2, name2 = chunks[i + 1] count2 = (since - (seconds * count)) // seconds2 if count2 != 0: - result += ugettext(', ') + name2 % count2 + result += ugettext(', ') + avoid_wrapping(name2 % count2) return result def timeuntil(d, now=None): diff --git a/tests/defaultfilters/tests.py b/tests/defaultfilters/tests.py index 21734faf95..be16719c8e 100644 --- a/tests/defaultfilters/tests.py +++ b/tests/defaultfilters/tests.py @@ -527,24 +527,26 @@ class DefaultFiltersTests(TestCase): def test_timesince(self): # real testing is done in timesince.py, where we can provide our own 'now' + # NOTE: \xa0 avoids wrapping between value and unit self.assertEqual( timesince_filter(datetime.datetime.now() - datetime.timedelta(1)), - '1 day') + '1\xa0day') self.assertEqual( timesince_filter(datetime.datetime(2005, 12, 29), datetime.datetime(2005, 12, 30)), - '1 day') + '1\xa0day') def test_timeuntil(self): + # NOTE: \xa0 avoids wrapping between value and unit self.assertEqual( timeuntil_filter(datetime.datetime.now() + datetime.timedelta(1, 1)), - '1 day') + '1\xa0day') self.assertEqual( timeuntil_filter(datetime.datetime(2005, 12, 30), datetime.datetime(2005, 12, 29)), - '1 day') + '1\xa0day') def test_default(self): self.assertEqual(default("val", "default"), 'val') @@ -574,43 +576,45 @@ class DefaultFiltersTests(TestCase): 'get out of town') def test_filesizeformat(self): - self.assertEqual(filesizeformat(1023), '1023 bytes') - self.assertEqual(filesizeformat(1024), '1.0 KB') - self.assertEqual(filesizeformat(10*1024), '10.0 KB') - self.assertEqual(filesizeformat(1024*1024-1), '1024.0 KB') - self.assertEqual(filesizeformat(1024*1024), '1.0 MB') - self.assertEqual(filesizeformat(1024*1024*50), '50.0 MB') - self.assertEqual(filesizeformat(1024*1024*1024-1), '1024.0 MB') - self.assertEqual(filesizeformat(1024*1024*1024), '1.0 GB') - self.assertEqual(filesizeformat(1024*1024*1024*1024), '1.0 TB') - self.assertEqual(filesizeformat(1024*1024*1024*1024*1024), '1.0 PB') + # NOTE: \xa0 avoids wrapping between value and unit + self.assertEqual(filesizeformat(1023), '1023\xa0bytes') + self.assertEqual(filesizeformat(1024), '1.0\xa0KB') + self.assertEqual(filesizeformat(10*1024), '10.0\xa0KB') + self.assertEqual(filesizeformat(1024*1024-1), '1024.0\xa0KB') + self.assertEqual(filesizeformat(1024*1024), '1.0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*50), '50.0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*1024-1), '1024.0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*1024), '1.0\xa0GB') + self.assertEqual(filesizeformat(1024*1024*1024*1024), '1.0\xa0TB') + self.assertEqual(filesizeformat(1024*1024*1024*1024*1024), '1.0\xa0PB') self.assertEqual(filesizeformat(1024*1024*1024*1024*1024*2000), - '2000.0 PB') - self.assertEqual(filesizeformat(complex(1,-1)), '0 bytes') - self.assertEqual(filesizeformat(""), '0 bytes') + '2000.0\xa0PB') + self.assertEqual(filesizeformat(complex(1,-1)), '0\xa0bytes') + self.assertEqual(filesizeformat(""), '0\xa0bytes') self.assertEqual(filesizeformat("\N{GREEK SMALL LETTER ALPHA}"), - '0 bytes') + '0\xa0bytes') def test_localized_filesizeformat(self): + # NOTE: \xa0 avoids wrapping between value and unit with self.settings(USE_L10N=True): with translation.override('de', deactivate=True): - self.assertEqual(filesizeformat(1023), '1023 Bytes') - self.assertEqual(filesizeformat(1024), '1,0 KB') - self.assertEqual(filesizeformat(10*1024), '10,0 KB') - self.assertEqual(filesizeformat(1024*1024-1), '1024,0 KB') - self.assertEqual(filesizeformat(1024*1024), '1,0 MB') - self.assertEqual(filesizeformat(1024*1024*50), '50,0 MB') - self.assertEqual(filesizeformat(1024*1024*1024-1), '1024,0 MB') - self.assertEqual(filesizeformat(1024*1024*1024), '1,0 GB') - self.assertEqual(filesizeformat(1024*1024*1024*1024), '1,0 TB') + self.assertEqual(filesizeformat(1023), '1023\xa0Bytes') + self.assertEqual(filesizeformat(1024), '1,0\xa0KB') + self.assertEqual(filesizeformat(10*1024), '10,0\xa0KB') + self.assertEqual(filesizeformat(1024*1024-1), '1024,0\xa0KB') + self.assertEqual(filesizeformat(1024*1024), '1,0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*50), '50,0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*1024-1), '1024,0\xa0MB') + self.assertEqual(filesizeformat(1024*1024*1024), '1,0\xa0GB') + self.assertEqual(filesizeformat(1024*1024*1024*1024), '1,0\xa0TB') self.assertEqual(filesizeformat(1024*1024*1024*1024*1024), - '1,0 PB') + '1,0\xa0PB') self.assertEqual(filesizeformat(1024*1024*1024*1024*1024*2000), - '2000,0 PB') - self.assertEqual(filesizeformat(complex(1,-1)), '0 Bytes') - self.assertEqual(filesizeformat(""), '0 Bytes') + '2000,0\xa0PB') + self.assertEqual(filesizeformat(complex(1,-1)), '0\xa0Bytes') + self.assertEqual(filesizeformat(""), '0\xa0Bytes') self.assertEqual(filesizeformat("\N{GREEK SMALL LETTER ALPHA}"), - '0 Bytes') + '0\xa0Bytes') def test_pluralize(self): self.assertEqual(pluralize(1), '') diff --git a/tests/template_tests/filters.py b/tests/template_tests/filters.py index 7ba1681fd5..68ef15d827 100644 --- a/tests/template_tests/filters.py +++ b/tests/template_tests/filters.py @@ -35,59 +35,60 @@ def get_filter_tests(): now_tz_i = datetime.now(FixedOffset((3 * 60) + 15)) # imaginary time zone today = date.today() + # NOTE: \xa0 avoids wrapping between value and unit return { # Default compare with datetime.now() - 'filter-timesince01' : ('{{ a|timesince }}', {'a': datetime.now() + timedelta(minutes=-1, seconds = -10)}, '1 minute'), - 'filter-timesince02' : ('{{ a|timesince }}', {'a': datetime.now() - timedelta(days=1, minutes = 1)}, '1 day'), - 'filter-timesince03' : ('{{ a|timesince }}', {'a': datetime.now() - timedelta(hours=1, minutes=25, seconds = 10)}, '1 hour, 25 minutes'), + 'filter-timesince01' : ('{{ a|timesince }}', {'a': datetime.now() + timedelta(minutes=-1, seconds = -10)}, '1\xa0minute'), + 'filter-timesince02' : ('{{ a|timesince }}', {'a': datetime.now() - timedelta(days=1, minutes = 1)}, '1\xa0day'), + 'filter-timesince03' : ('{{ a|timesince }}', {'a': datetime.now() - timedelta(hours=1, minutes=25, seconds = 10)}, '1\xa0hour, 25\xa0minutes'), # Compare to a given parameter - 'filter-timesince04' : ('{{ a|timesince:b }}', {'a':now - timedelta(days=2), 'b':now - timedelta(days=1)}, '1 day'), - 'filter-timesince05' : ('{{ a|timesince:b }}', {'a':now - timedelta(days=2, minutes=1), 'b':now - timedelta(days=2)}, '1 minute'), + 'filter-timesince04' : ('{{ a|timesince:b }}', {'a':now - timedelta(days=2), 'b':now - timedelta(days=1)}, '1\xa0day'), + 'filter-timesince05' : ('{{ a|timesince:b }}', {'a':now - timedelta(days=2, minutes=1), 'b':now - timedelta(days=2)}, '1\xa0minute'), # Check that timezone is respected - 'filter-timesince06' : ('{{ a|timesince:b }}', {'a':now_tz - timedelta(hours=8), 'b':now_tz}, '8 hours'), + 'filter-timesince06' : ('{{ a|timesince:b }}', {'a':now_tz - timedelta(hours=8), 'b':now_tz}, '8\xa0hours'), # Regression for #7443 - 'filter-timesince07': ('{{ earlier|timesince }}', { 'earlier': now - timedelta(days=7) }, '1 week'), - 'filter-timesince08': ('{{ earlier|timesince:now }}', { 'now': now, 'earlier': now - timedelta(days=7) }, '1 week'), - 'filter-timesince09': ('{{ later|timesince }}', { 'later': now + timedelta(days=7) }, '0 minutes'), - 'filter-timesince10': ('{{ later|timesince:now }}', { 'now': now, 'later': now + timedelta(days=7) }, '0 minutes'), + 'filter-timesince07': ('{{ earlier|timesince }}', { 'earlier': now - timedelta(days=7) }, '1\xa0week'), + 'filter-timesince08': ('{{ earlier|timesince:now }}', { 'now': now, 'earlier': now - timedelta(days=7) }, '1\xa0week'), + 'filter-timesince09': ('{{ later|timesince }}', { 'later': now + timedelta(days=7) }, '0\xa0minutes'), + 'filter-timesince10': ('{{ later|timesince:now }}', { 'now': now, 'later': now + timedelta(days=7) }, '0\xa0minutes'), # Ensures that differing timezones are calculated correctly - 'filter-timesince11' : ('{{ a|timesince }}', {'a': now}, '0 minutes'), - 'filter-timesince12' : ('{{ a|timesince }}', {'a': now_tz}, '0 minutes'), - 'filter-timesince13' : ('{{ a|timesince }}', {'a': now_tz_i}, '0 minutes'), - 'filter-timesince14' : ('{{ a|timesince:b }}', {'a': now_tz, 'b': now_tz_i}, '0 minutes'), + 'filter-timesince11' : ('{{ a|timesince }}', {'a': now}, '0\xa0minutes'), + 'filter-timesince12' : ('{{ a|timesince }}', {'a': now_tz}, '0\xa0minutes'), + 'filter-timesince13' : ('{{ a|timesince }}', {'a': now_tz_i}, '0\xa0minutes'), + 'filter-timesince14' : ('{{ a|timesince:b }}', {'a': now_tz, 'b': now_tz_i}, '0\xa0minutes'), 'filter-timesince15' : ('{{ a|timesince:b }}', {'a': now, 'b': now_tz_i}, ''), 'filter-timesince16' : ('{{ a|timesince:b }}', {'a': now_tz_i, 'b': now}, ''), # Regression for #9065 (two date objects). - 'filter-timesince17' : ('{{ a|timesince:b }}', {'a': today, 'b': today}, '0 minutes'), - 'filter-timesince18' : ('{{ a|timesince:b }}', {'a': today, 'b': today + timedelta(hours=24)}, '1 day'), + 'filter-timesince17' : ('{{ a|timesince:b }}', {'a': today, 'b': today}, '0\xa0minutes'), + 'filter-timesince18' : ('{{ a|timesince:b }}', {'a': today, 'b': today + timedelta(hours=24)}, '1\xa0day'), # Default compare with datetime.now() - 'filter-timeuntil01' : ('{{ a|timeuntil }}', {'a':datetime.now() + timedelta(minutes=2, seconds = 10)}, '2 minutes'), - 'filter-timeuntil02' : ('{{ a|timeuntil }}', {'a':(datetime.now() + timedelta(days=1, seconds = 10))}, '1 day'), - 'filter-timeuntil03' : ('{{ a|timeuntil }}', {'a':(datetime.now() + timedelta(hours=8, minutes=10, seconds = 10))}, '8 hours, 10 minutes'), + 'filter-timeuntil01' : ('{{ a|timeuntil }}', {'a':datetime.now() + timedelta(minutes=2, seconds = 10)}, '2\xa0minutes'), + 'filter-timeuntil02' : ('{{ a|timeuntil }}', {'a':(datetime.now() + timedelta(days=1, seconds = 10))}, '1\xa0day'), + 'filter-timeuntil03' : ('{{ a|timeuntil }}', {'a':(datetime.now() + timedelta(hours=8, minutes=10, seconds = 10))}, '8\xa0hours, 10\xa0minutes'), # Compare to a given parameter - 'filter-timeuntil04' : ('{{ a|timeuntil:b }}', {'a':now - timedelta(days=1), 'b':now - timedelta(days=2)}, '1 day'), - 'filter-timeuntil05' : ('{{ a|timeuntil:b }}', {'a':now - timedelta(days=2), 'b':now - timedelta(days=2, minutes=1)}, '1 minute'), + 'filter-timeuntil04' : ('{{ a|timeuntil:b }}', {'a':now - timedelta(days=1), 'b':now - timedelta(days=2)}, '1\xa0day'), + 'filter-timeuntil05' : ('{{ a|timeuntil:b }}', {'a':now - timedelta(days=2), 'b':now - timedelta(days=2, minutes=1)}, '1\xa0minute'), # Regression for #7443 - 'filter-timeuntil06': ('{{ earlier|timeuntil }}', { 'earlier': now - timedelta(days=7) }, '0 minutes'), - 'filter-timeuntil07': ('{{ earlier|timeuntil:now }}', { 'now': now, 'earlier': now - timedelta(days=7) }, '0 minutes'), - 'filter-timeuntil08': ('{{ later|timeuntil }}', { 'later': now + timedelta(days=7, hours=1) }, '1 week'), - 'filter-timeuntil09': ('{{ later|timeuntil:now }}', { 'now': now, 'later': now + timedelta(days=7) }, '1 week'), + 'filter-timeuntil06': ('{{ earlier|timeuntil }}', { 'earlier': now - timedelta(days=7) }, '0\xa0minutes'), + 'filter-timeuntil07': ('{{ earlier|timeuntil:now }}', { 'now': now, 'earlier': now - timedelta(days=7) }, '0\xa0minutes'), + 'filter-timeuntil08': ('{{ later|timeuntil }}', { 'later': now + timedelta(days=7, hours=1) }, '1\xa0week'), + 'filter-timeuntil09': ('{{ later|timeuntil:now }}', { 'now': now, 'later': now + timedelta(days=7) }, '1\xa0week'), # Ensures that differing timezones are calculated correctly - 'filter-timeuntil10' : ('{{ a|timeuntil }}', {'a': now_tz_i}, '0 minutes'), - 'filter-timeuntil11' : ('{{ a|timeuntil:b }}', {'a': now_tz_i, 'b': now_tz}, '0 minutes'), + 'filter-timeuntil10' : ('{{ a|timeuntil }}', {'a': now_tz_i}, '0\xa0minutes'), + 'filter-timeuntil11' : ('{{ a|timeuntil:b }}', {'a': now_tz_i, 'b': now_tz}, '0\xa0minutes'), # Regression for #9065 (two date objects). - 'filter-timeuntil12' : ('{{ a|timeuntil:b }}', {'a': today, 'b': today}, '0 minutes'), - 'filter-timeuntil13' : ('{{ a|timeuntil:b }}', {'a': today, 'b': today - timedelta(hours=24)}, '1 day'), + 'filter-timeuntil12' : ('{{ a|timeuntil:b }}', {'a': today, 'b': today}, '0\xa0minutes'), + 'filter-timeuntil13' : ('{{ a|timeuntil:b }}', {'a': today, 'b': today - timedelta(hours=24)}, '1\xa0day'), 'filter-addslash01': ("{% autoescape off %}{{ a|addslashes }} {{ b|addslashes }}{% endautoescape %}", {"a": "'", "b": mark_safe("'")}, r"\' \'"), 'filter-addslash02': ("{{ a|addslashes }} {{ b|addslashes }}", {"a": "'", "b": mark_safe("'")}, r"<a>\' \'"), diff --git a/tests/utils_tests/test_timesince.py b/tests/utils_tests/test_timesince.py index 5e641a42c4..cdb95e6877 100644 --- a/tests/utils_tests/test_timesince.py +++ b/tests/utils_tests/test_timesince.py @@ -21,32 +21,33 @@ class TimesinceTests(unittest.TestCase): def test_equal_datetimes(self): """ equal datetimes. """ - self.assertEqual(timesince(self.t, self.t), '0 minutes') + # NOTE: \xa0 avoids wrapping between value and unit + self.assertEqual(timesince(self.t, self.t), '0\xa0minutes') def test_ignore_microseconds_and_seconds(self): """ Microseconds and seconds are ignored. """ self.assertEqual(timesince(self.t, self.t+self.onemicrosecond), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t+self.onesecond), - '0 minutes') + '0\xa0minutes') def test_other_units(self): """ Test other units. """ self.assertEqual(timesince(self.t, self.t+self.oneminute), - '1 minute') - self.assertEqual(timesince(self.t, self.t+self.onehour), '1 hour') - self.assertEqual(timesince(self.t, self.t+self.oneday), '1 day') - self.assertEqual(timesince(self.t, self.t+self.oneweek), '1 week') + '1\xa0minute') + self.assertEqual(timesince(self.t, self.t+self.onehour), '1\xa0hour') + self.assertEqual(timesince(self.t, self.t+self.oneday), '1\xa0day') + self.assertEqual(timesince(self.t, self.t+self.oneweek), '1\xa0week') self.assertEqual(timesince(self.t, self.t+self.onemonth), - '1 month') - self.assertEqual(timesince(self.t, self.t+self.oneyear), '1 year') + '1\xa0month') + self.assertEqual(timesince(self.t, self.t+self.oneyear), '1\xa0year') def test_multiple_units(self): """ Test multiple units. """ self.assertEqual(timesince(self.t, - self.t+2*self.oneday+6*self.onehour), '2 days, 6 hours') + self.t+2*self.oneday+6*self.onehour), '2\xa0days, 6\xa0hours') self.assertEqual(timesince(self.t, - self.t+2*self.oneweek+2*self.oneday), '2 weeks, 2 days') + self.t+2*self.oneweek+2*self.oneday), '2\xa0weeks, 2\xa0days') def test_display_first_unit(self): """ @@ -55,10 +56,10 @@ class TimesinceTests(unittest.TestCase): """ self.assertEqual(timesince(self.t, self.t+2*self.oneweek+3*self.onehour+4*self.oneminute), - '2 weeks') + '2\xa0weeks') self.assertEqual(timesince(self.t, - self.t+4*self.oneday+5*self.oneminute), '4 days') + self.t+4*self.oneday+5*self.oneminute), '4\xa0days') def test_display_second_before_first(self): """ @@ -66,30 +67,30 @@ class TimesinceTests(unittest.TestCase): get 0 minutes. """ self.assertEqual(timesince(self.t, self.t-self.onemicrosecond), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.onesecond), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.oneminute), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.onehour), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.oneday), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.oneweek), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.onemonth), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-self.oneyear), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, - self.t-2*self.oneday-6*self.onehour), '0 minutes') + self.t-2*self.oneday-6*self.onehour), '0\xa0minutes') self.assertEqual(timesince(self.t, - self.t-2*self.oneweek-2*self.oneday), '0 minutes') + self.t-2*self.oneweek-2*self.oneday), '0\xa0minutes') self.assertEqual(timesince(self.t, self.t-2*self.oneweek-3*self.onehour-4*self.oneminute), - '0 minutes') + '0\xa0minutes') self.assertEqual(timesince(self.t, - self.t-4*self.oneday-5*self.oneminute), '0 minutes') + self.t-4*self.oneday-5*self.oneminute), '0\xa0minutes') def test_different_timezones(self): """ When using two different timezones. """ @@ -97,28 +98,28 @@ class TimesinceTests(unittest.TestCase): now_tz = datetime.datetime.now(LocalTimezone(now)) now_tz_i = datetime.datetime.now(FixedOffset((3 * 60) + 15)) - self.assertEqual(timesince(now), '0 minutes') - self.assertEqual(timesince(now_tz), '0 minutes') - self.assertEqual(timeuntil(now_tz, now_tz_i), '0 minutes') + self.assertEqual(timesince(now), '0\xa0minutes') + self.assertEqual(timesince(now_tz), '0\xa0minutes') + self.assertEqual(timeuntil(now_tz, now_tz_i), '0\xa0minutes') def test_date_objects(self): """ Both timesince and timeuntil should work on date objects (#17937). """ today = datetime.date.today() - self.assertEqual(timesince(today + self.oneday), '0 minutes') - self.assertEqual(timeuntil(today - self.oneday), '0 minutes') + self.assertEqual(timesince(today + self.oneday), '0\xa0minutes') + self.assertEqual(timeuntil(today - self.oneday), '0\xa0minutes') def test_both_date_objects(self): """ Timesince should work with both date objects (#9672) """ today = datetime.date.today() - self.assertEqual(timeuntil(today + self.oneday, today), '1 day') - self.assertEqual(timeuntil(today - self.oneday, today), '0 minutes') - self.assertEqual(timeuntil(today + self.oneweek, today), '1 week') + self.assertEqual(timeuntil(today + self.oneday, today), '1\xa0day') + self.assertEqual(timeuntil(today - self.oneday, today), '0\xa0minutes') + self.assertEqual(timeuntil(today + self.oneweek, today), '1\xa0week') def test_naive_datetime_with_tzinfo_attribute(self): class naive(datetime.tzinfo): def utcoffset(self, dt): return None future = datetime.datetime(2080, 1, 1, tzinfo=naive()) - self.assertEqual(timesince(future), '0 minutes') + self.assertEqual(timesince(future), '0\xa0minutes') past = datetime.datetime(1980, 1, 1, tzinfo=naive()) - self.assertEqual(timeuntil(past), '0 minutes') + self.assertEqual(timeuntil(past), '0\xa0minutes') -- cgit v1.3 From 413735b239239bc8ff38ee893e8448daf9b2bff6 Mon Sep 17 00:00:00 2001 From: Marc Tamlyn Date: Sun, 19 May 2013 11:24:17 +0200 Subject: Fix a typo in a comment. --- django/utils/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'django/utils') diff --git a/django/utils/http.py b/django/utils/http.py index 9897df4fb0..f4911b4ec0 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -71,7 +71,7 @@ urlunquote_plus = allow_lazy(urlunquote_plus, six.text_type) def urlencode(query, doseq=0): """ A version of Python's urllib.urlencode() function that can operate on - unicode strings. The parameters are first case to UTF-8 encoded strings and + unicode strings. The parameters are first cast to UTF-8 encoded strings and then encoded as per normal. """ if isinstance(query, MultiValueDict): -- cgit v1.3 From 26e3e7ecb5e4c9af4cd5aa178f65ce1585d3ae07 Mon Sep 17 00:00:00 2001 From: Łukasz Langa Date: Sun, 19 May 2013 12:43:34 +0200 Subject: Fixed #11915: generic Accept-Language matches country-specific variants --- django/middleware/locale.py | 3 +- django/utils/translation/trans_real.py | 50 ++++++++++++++++++++++------------ tests/i18n/__init__.py | 1 + tests/i18n/tests.py | 15 +++++++++- 4 files changed, 49 insertions(+), 20 deletions(-) (limited to 'django/utils') diff --git a/django/middleware/locale.py b/django/middleware/locale.py index 9b2ef8ff32..25fbbaccdd 100644 --- a/django/middleware/locale.py +++ b/django/middleware/locale.py @@ -6,6 +6,7 @@ from django.core.urlresolvers import (is_valid_path, get_resolver, from django.http import HttpResponseRedirect from django.utils.cache import patch_vary_headers from django.utils import translation +from django.utils.datastructures import SortedDict class LocaleMiddleware(object): @@ -18,7 +19,7 @@ class LocaleMiddleware(object): """ def __init__(self): - self._supported_languages = dict(settings.LANGUAGES) + self._supported_languages = SortedDict(settings.LANGUAGES) self._is_language_prefix_patterns_used = False for url_pattern in get_resolver(None).url_patterns: if isinstance(url_pattern, LocaleRegexURLResolver): diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 26be0ed729..0aaa4ec99b 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -10,7 +10,9 @@ from threading import local import warnings from django.utils.importlib import import_module +from django.utils.datastructures import SortedDict from django.utils.encoding import force_str, force_text +from django.utils.functional import memoize from django.utils._os import upath from django.utils.safestring import mark_safe, SafeData from django.utils import six @@ -29,6 +31,7 @@ _default = None # This is a cache for normalized accept-header languages to prevent multiple # file lookups when checking the same locale on repeated requests. _accepted = {} +_checked_languages = {} # magic gettext number to separate context from message CONTEXT_SEPARATOR = "\x04" @@ -355,38 +358,54 @@ def check_for_language(lang_code): if gettext_module.find('django', path, [to_locale(lang_code)]) is not None: return True return False +check_for_language = memoize(check_for_language, _checked_languages, 1) -def get_supported_language_variant(lang_code, supported=None): +def get_supported_language_variant(lang_code, supported=None, strict=False): """ Returns the language-code that's listed in supported languages, possibly selecting a more generic variant. Raises LookupError if nothing found. + + If `strict` is False (the default), the function will look for an alternative + country-specific variant when the currently checked is not found. """ if supported is None: from django.conf import settings - supported = dict(settings.LANGUAGES) + supported = SortedDict(settings.LANGUAGES) if lang_code: - # e.g. if fr-CA is not supported, try fr-ca; - # if that fails, fallback to fr. - variants = (lang_code, lang_code.lower(), lang_code.split('-')[0], - lang_code.lower().split('-')[0]) + # if fr-CA is not supported, try fr-ca; if that fails, fallback to fr. + generic_lang_code = lang_code.split('-')[0] + variants = (lang_code, lang_code.lower(), generic_lang_code, + generic_lang_code.lower()) for code in variants: if code in supported and check_for_language(code): return code + if not strict: + # if fr-fr is not supported, try fr-ca. + for supported_code in supported: + if supported_code.startswith((generic_lang_code + '-', + generic_lang_code.lower() + '-')): + return supported_code raise LookupError(lang_code) -def get_language_from_path(path, supported=None): +def get_language_from_path(path, supported=None, strict=False): """ Returns the language-code if there is a valid language-code found in the `path`. + + If `strict` is False (the default), the function will look for an alternative + country-specific variant when the currently checked is not found. """ if supported is None: from django.conf import settings - supported = dict(settings.LANGUAGES) + supported = SortedDict(settings.LANGUAGES) regex_match = language_code_prefix_re.match(path) - if regex_match: - lang_code = regex_match.group(1) - if lang_code in supported and check_for_language(lang_code): - return lang_code + if not regex_match: + return None + lang_code = regex_match.group(1) + try: + return get_supported_language_variant(lang_code, supported, strict=strict) + except LookupError: + return None def get_language_from_request(request, check_path=False): """ @@ -400,7 +419,7 @@ def get_language_from_request(request, check_path=False): """ global _accepted from django.conf import settings - supported = dict(settings.LANGUAGES) + supported = SortedDict(settings.LANGUAGES) if check_path: lang_code = get_language_from_path(request.path_info, supported) @@ -424,11 +443,6 @@ def get_language_from_request(request, check_path=False): if accept_lang == '*': break - # We have a very restricted form for our language files (no encoding - # specifier, since they all must be UTF-8 and only one possible - # language each time. So we avoid the overhead of gettext.find() and - # work out the MO file manually. - # 'normalized' is the root name of the locale in POSIX format (which is # the format used for the directories holding the MO files). normalized = locale.locale_alias.get(to_locale(accept_lang, True)) diff --git a/tests/i18n/__init__.py b/tests/i18n/__init__.py index a3e9ce7053..c5aaa31fe3 100644 --- a/tests/i18n/__init__.py +++ b/tests/i18n/__init__.py @@ -11,6 +11,7 @@ class TransRealMixin(object): trans_real._active = local() trans_real._default = None trans_real._accepted = {} + trans_real._checked_languages = {} def tearDown(self): self.flush_caches() diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 137270f830..9f1e366c9f 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -1157,6 +1157,7 @@ class LocaleMiddlewareTests(TransRealMixin, TestCase): LANGUAGES=( ('bg', 'Bulgarian'), ('en-us', 'English'), + ('pt-br', 'Portugese (Brazil)'), ), MIDDLEWARE_CLASSES=( 'django.middleware.locale.LocaleMiddleware', @@ -1176,7 +1177,6 @@ class CountrySpecificLanguageTests(TransRealMixin, TestCase): self.assertTrue(check_for_language('en-us')) self.assertTrue(check_for_language('en-US')) - def test_get_language_from_request(self): # issue 19919 r = self.rf.get('/') @@ -1189,3 +1189,16 @@ class CountrySpecificLanguageTests(TransRealMixin, TestCase): r.META = {'HTTP_ACCEPT_LANGUAGE': 'bg-bg,en-US;q=0.8,en;q=0.6,ru;q=0.4'} lang = get_language_from_request(r) self.assertEqual('bg', lang) + + def test_specific_language_codes(self): + # issue 11915 + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt,en-US;q=0.8,en;q=0.6,ru;q=0.4'} + lang = get_language_from_request(r) + self.assertEqual('pt-br', lang) + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-pt,en-US;q=0.8,en;q=0.6,ru;q=0.4'} + lang = get_language_from_request(r) + self.assertEqual('pt-br', lang) -- cgit v1.3 From e24d486fbc0b1c42abe8b54217ff428e449c48cc Mon Sep 17 00:00:00 2001 From: Daniel Lindsley Date: Mon, 20 May 2013 22:17:56 -0700 Subject: Fixed #20212 - __reduce__ should only be defined for Py3+. --- django/utils/functional.py | 23 ++++++++++++++++++----- tests/utils_tests/test_simplelazyobject.py | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) (limited to 'django/utils') diff --git a/django/utils/functional.py b/django/utils/functional.py index 1592828c7f..bf32121e07 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -5,6 +5,11 @@ import sys from django.utils import six +try: + import copyreg +except ImportError: + import copy_reg as copyreg + # You can't trivially replace this with `functools.partial` because this binds # to classes and returns bound instances, whereas functools.partial (on @@ -323,15 +328,23 @@ class SimpleLazyObject(LazyObject): self._setup() return self._wrapped.__dict__ - # Python 3.3 will call __reduce__ when pickling; these methods are needed - # to serialize and deserialize correctly. They are not called in earlier - # versions of Python. + # Python 3.3 will call __reduce__ when pickling; this method is needed + # to serialize and deserialize correctly. @classmethod def __newobj__(cls, *args): return cls.__new__(cls, *args) - def __reduce__(self): - return (self.__newobj__, (self.__class__,), self.__getstate__()) + def __reduce_ex__(self, proto): + if proto >= 2: + # On Py3, since the default protocol is 3, pickle uses the + # ``__newobj__`` method (& more efficient opcodes) for writing. + return (self.__newobj__, (self.__class__,), self.__getstate__()) + else: + # On Py2, the default protocol is 0 (for back-compat) & the above + # code fails miserably (see regression test). Instead, we return + # exactly what's returned if there's no ``__reduce__`` method at + # all. + return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__()) # Return a meaningful representation of the lazy object for debugging # without evaluating the wrapped object. diff --git a/tests/utils_tests/test_simplelazyobject.py b/tests/utils_tests/test_simplelazyobject.py index f925e01eb6..ca7c2672eb 100644 --- a/tests/utils_tests/test_simplelazyobject.py +++ b/tests/utils_tests/test_simplelazyobject.py @@ -161,3 +161,25 @@ class TestUtilsSimpleLazyObject(TestCase): self.assertNotEqual(lazy1, lazy3) self.assertTrue(lazy1 != lazy3) self.assertFalse(lazy1 != lazy2) + + def test_pickle_py2_regression(self): + from django.contrib.auth.models import User + + # See ticket #20212 + user = User.objects.create_user('johndoe', 'john@example.com', 'pass') + x = SimpleLazyObject(lambda: user) + + # This would fail with "TypeError: can't pickle instancemethod objects", + # only on Python 2.X. + pickled = pickle.dumps(x) + + # Try the variant protocol levels. + pickled = pickle.dumps(x, 0) + pickled = pickle.dumps(x, 1) + pickled = pickle.dumps(x, 2) + + if not six.PY3: + import cPickle + + # This would fail with "TypeError: expected string or Unicode object, NoneType found". + pickled = cPickle.dumps(x) -- cgit v1.3 From fd961941cc1c9e7e1384af527792801f8f897d9f Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 21 May 2013 14:41:39 +0200 Subject: Imported copyreg from six.moves --- django/utils/functional.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'django/utils') diff --git a/django/utils/functional.py b/django/utils/functional.py index bf32121e07..93befcf59e 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -4,11 +4,7 @@ from functools import wraps import sys from django.utils import six - -try: - import copyreg -except ImportError: - import copy_reg as copyreg +from django.utils.six.moves import copyreg # You can't trivially replace this with `functools.partial` because this binds -- cgit v1.3 From dc51ec8bc214cf60ebb99732363624c23df8005f Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 22 May 2013 17:29:16 +0200 Subject: Fixed #19237 -- Used HTML parser to strip tags The regex method used until now for the strip_tags utility is fast, but subject to flaws and security issues. Consensus and good practice lead use to use a slower but safer method. --- django/utils/html.py | 28 ++++++++++++++++++++++++++-- tests/utils_tests/test_html.py | 8 ++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) (limited to 'django/utils') diff --git a/django/utils/html.py b/django/utils/html.py index edddc48e62..573235092d 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -16,6 +16,9 @@ from django.utils.functional import allow_lazy from django.utils import six from django.utils.text import normalize_newlines +from .html_parser import HTMLParser + + # Configuration for urlize() function. TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)'] WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('<', '>')] @@ -33,7 +36,6 @@ link_target_attribute_re = re.compile(r'(]*?)target=[^\s>]+') html_gunk_re = re.compile(r'(?:
|<\/i>|<\/b>|<\/em>|<\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE) hard_coded_bullets_re = re.compile(r'((?:

(?:%s).*?[a-zA-Z].*?

\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL) trailing_empty_content_re = re.compile(r'(?:

(?: |\s|
)*?

\s*)+\Z') -strip_tags_re = re.compile(r']*=(\s*"[^"]*"|\s*\'[^\']*\'|\S*)|[^>])*?>', re.IGNORECASE) def escape(text): @@ -116,9 +118,31 @@ def linebreaks(value, autoescape=False): return '\n\n'.join(paras) linebreaks = allow_lazy(linebreaks, six.text_type) + +class MLStripper(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self.reset() + self.fed = [] + def handle_data(self, d): + self.fed.append(d) + def handle_entityref(self, name): + self.fed.append('&%s;' % name) + def handle_charref(self, name): + self.fed.append('&#%s;' % name) + def get_data(self): + return ''.join(self.fed) + def strip_tags(value): """Returns the given HTML with all tags stripped.""" - return strip_tags_re.sub('', force_text(value)) + s = MLStripper() + s.feed(value) + data = s.get_data() + try: + res = s.close() + except Exception as e: + data += s.rawdata + return data strip_tags = allow_lazy(strip_tags) def remove_tags(html, tags): diff --git a/tests/utils_tests/test_html.py b/tests/utils_tests/test_html.py index 090cc32d1c..c3e9f7c878 100644 --- a/tests/utils_tests/test_html.py +++ b/tests/utils_tests/test_html.py @@ -5,6 +5,7 @@ import os from django.utils import html from django.utils._os import upath +from django.utils.encoding import force_text from django.utils.unittest import TestCase @@ -63,10 +64,12 @@ class TestUtilsHtml(TestCase): def test_strip_tags(self): f = html.strip_tags items = ( + ('

See: 'é is an apostrophe followed by e acute

', + 'See: 'é is an apostrophe followed by e acute'), ('a', 'a'), ('a', 'a'), ('e', 'e'), - ('b', 'b'), ('a

b

c', 'abc'), @@ -81,8 +84,9 @@ class TestUtilsHtml(TestCase): for filename in ('strip_tags1.html', 'strip_tags2.txt'): path = os.path.join(os.path.dirname(upath(__file__)), 'files', filename) with open(path, 'r') as fp: + content = force_text(fp.read()) start = datetime.now() - stripped = html.strip_tags(fp.read()) + stripped = html.strip_tags(content) elapsed = datetime.now() - start self.assertEqual(elapsed.seconds, 0) self.assertIn("Please try again.", stripped) -- cgit v1.3 From b664cb818d2e5896df2763299ea2c61a9af069a8 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 23 May 2013 14:00:17 +0200 Subject: Fixed #19237 (again) - Made strip_tags consistent between Python versions --- django/utils/html.py | 14 +++++++------- docs/ref/utils.txt | 10 ++++++++-- tests/utils_tests/test_html.py | 3 +++ 3 files changed, 18 insertions(+), 9 deletions(-) (limited to 'django/utils') diff --git a/django/utils/html.py b/django/utils/html.py index 573235092d..0d28c77a61 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -16,7 +16,7 @@ from django.utils.functional import allow_lazy from django.utils import six from django.utils.text import normalize_newlines -from .html_parser import HTMLParser +from .html_parser import HTMLParser, HTMLParseError # Configuration for urlize() function. @@ -136,13 +136,13 @@ class MLStripper(HTMLParser): def strip_tags(value): """Returns the given HTML with all tags stripped.""" s = MLStripper() - s.feed(value) - data = s.get_data() try: - res = s.close() - except Exception as e: - data += s.rawdata - return data + s.feed(value) + s.close() + except HTMLParseError: + return value + else: + return s.get_data() strip_tags = allow_lazy(strip_tags) def remove_tags(html, tags): diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 14ae9aa9b8..bf14af0855 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -490,7 +490,7 @@ Atom1Feed Usually you should build up HTML using Django's templates to make use of its autoescape mechanism, using the utilities in :mod:`django.utils.safestring` -where appropriate. This module provides some additional low level utilitiesfor +where appropriate. This module provides some additional low level utilities for escaping HTML. .. function:: escape(text) @@ -564,7 +564,13 @@ escaping HTML. strip_tags(value) If ``value`` is ``"Joel a slug"`` the - return value will be ``"Joel is a slug"``. + return value will be ``"Joel is a slug"``. Note that ``strip_tags`` result + may still contain unsafe HTML content, so you might use + :func:`~django.utils.html.escape` to make it a safe string. + + .. versionchanged:: 1.6 + + For improved safety, ``strip_tags`` is now parser-based. .. function:: remove_tags(value, tags) diff --git a/tests/utils_tests/test_html.py b/tests/utils_tests/test_html.py index c3e9f7c878..b973f1c64f 100644 --- a/tests/utils_tests/test_html.py +++ b/tests/utils_tests/test_html.py @@ -70,6 +70,9 @@ class TestUtilsHtml(TestCase): ('a', 'a'), ('e', 'e'), ('hi, b2!', 'b7>b2!'), ('b', 'b'), ('a

b

c', 'abc'), -- cgit v1.3 From c0439b621086ae0ae348759a5a24969be8148d6a Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 25 May 2013 14:01:52 +0200 Subject: Removed obsolete attribute of DjangoTranslation --- django/utils/translation/trans_real.py | 1 - 1 file changed, 1 deletion(-) (limited to 'django/utils') diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 0aaa4ec99b..195badfc00 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -80,7 +80,6 @@ class DjangoTranslation(gettext_module.GNUTranslations): def __init__(self, *args, **kw): gettext_module.GNUTranslations.__init__(self, *args, **kw) self.set_output_charset('utf-8') - self.django_output_charset = 'utf-8' self.__language = '??' def merge(self, other): -- cgit v1.3 From b7cf44dde88c52b3e3020bd9dae3b3d0792b9a6d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 25 May 2013 14:31:51 +0200 Subject: Fixed some minor translation-related issues --- django/contrib/admindocs/views.py | 1 + django/utils/image.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'django/utils') diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 348727eb21..4cadc8b657 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -267,6 +267,7 @@ def model_detail(request, app_label, model_name): return render_to_response('admin_doc/model_detail.html', { 'root_path': urlresolvers.reverse('admin:index'), 'name': '%s.%s' % (opts.app_label, opts.object_name), + # Translators: %s is an object type name 'summary': _("Attributes on %s objects") % opts.object_name, 'description': model.__doc__, 'fields': fields, diff --git a/django/utils/image.py b/django/utils/image.py index 54c11adfee..d251ab9d0b 100644 --- a/django/utils/image.py +++ b/django/utils/image.py @@ -124,7 +124,7 @@ def _detect_image_library(): import _imaging as PIL_imaging except ImportError as err: raise ImproperlyConfigured( - _("The '_imaging' module for the PIL could not be " + + _("The '_imaging' module for the PIL could not be " "imported: %s" % err) ) -- cgit v1.3 From 2ee447fb5f8974b432d3dd421af9a242215aea44 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Sat, 20 Apr 2013 13:38:14 +0200 Subject: Fixed #20296 -- Allowed SafeData and EscapeData to be lazy --- django/utils/safestring.py | 10 +++++-- tests/utils_tests/test_safestring.py | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 tests/utils_tests/test_safestring.py (limited to 'django/utils') diff --git a/django/utils/safestring.py b/django/utils/safestring.py index 07e0bf4cea..3774012d32 100644 --- a/django/utils/safestring.py +++ b/django/utils/safestring.py @@ -4,7 +4,7 @@ without further escaping in HTML. Marking something as a "safe string" means that the producer of the string has already turned characters that should not be interpreted by the HTML engine (e.g. '<') into the appropriate entities. """ -from django.utils.functional import curry, Promise +from django.utils.functional import curry, Promise, allow_lazy from django.utils import six class EscapeData(object): @@ -14,13 +14,13 @@ class EscapeBytes(bytes, EscapeData): """ A byte string that should be HTML-escaped when output. """ - pass + __new__ = allow_lazy(bytes.__new__, bytes) class EscapeText(six.text_type, EscapeData): """ A unicode string object that should be HTML-escaped when output. """ - pass + __new__ = allow_lazy(six.text_type.__new__, six.text_type) if six.PY3: EscapeString = EscapeText @@ -37,6 +37,8 @@ class SafeBytes(bytes, SafeData): A bytes subclass that has been specifically marked as "safe" (requires no further escaping) for HTML output purposes. """ + __new__ = allow_lazy(bytes.__new__, bytes) + def __add__(self, rhs): """ Concatenating a safe byte string with another safe byte string or safe @@ -69,6 +71,8 @@ class SafeText(six.text_type, SafeData): A unicode (Python 2) / str (Python 3) subclass that has been specifically marked as "safe" for HTML output purposes. """ + __new__ = allow_lazy(six.text_type.__new__, six.text_type) + def __add__(self, rhs): """ Concatenating a safe unicode string with another safe byte string or diff --git a/tests/utils_tests/test_safestring.py b/tests/utils_tests/test_safestring.py new file mode 100644 index 0000000000..a6f0c0a01c --- /dev/null +++ b/tests/utils_tests/test_safestring.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import, unicode_literals + + +from django.template import Template, Context +from django.test import TestCase +from django.utils.encoding import force_text, force_bytes +from django.utils.functional import lazy, Promise +from django.utils.html import escape, conditional_escape +from django.utils.safestring import mark_safe, mark_for_escaping +from django.utils import six +from django.utils import translation + +lazystr = lazy(force_text, six.text_type) +lazybytes = lazy(force_bytes, bytes) + + +class SafeStringTest(TestCase): + def assertRenderEqual(self, tpl, expected, **context): + context = Context(context) + tpl = Template(tpl) + self.assertEqual(tpl.render(context), expected) + + def test_mark_safe(self): + s = mark_safe('a&b') + + self.assertRenderEqual('{{ s }}', 'a&b', s=s) + self.assertRenderEqual('{{ s|force_escape }}', 'a&b', s=s) + + def test_mark_safe_lazy(self): + s = lazystr('a&b') + b = lazybytes(b'a&b') + + self.assertIsInstance(mark_safe(s), Promise) + self.assertIsInstance(mark_safe(b), Promise) + self.assertRenderEqual('{{ s }}', 'a&b', s=mark_safe(s)) + + def test_mark_for_escaping(self): + s = mark_for_escaping('a&b') + self.assertRenderEqual('{{ s }}', 'a&b', s=s) + self.assertRenderEqual('{{ s }}', 'a&b', s=mark_for_escaping(s)) + + def test_mark_for_escaping_lazy(self): + s = lazystr('a&b') + b = lazybytes(b'a&b') + + self.assertIsInstance(mark_for_escaping(s), Promise) + self.assertIsInstance(mark_for_escaping(b), Promise) + self.assertRenderEqual('{% autoescape off %}{{ s }}{% endautoescape %}', 'a&b', s=mark_for_escaping(s)) + + def test_regression_20296(self): + s = mark_safe(translation.ugettext_lazy("username")) + with translation.override('fr'): + self.assertRenderEqual('{{ s }}', "nom d'utilisateur", s=s) -- cgit v1.3 From d228c1192ed59ab0114d9eba82ac99df611652d2 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Wed, 15 May 2013 16:14:28 -0700 Subject: Fixed #19866 -- Added security logger and return 400 for SuspiciousOperation. SuspiciousOperations have been differentiated into subclasses, and are now logged to a 'django.security.*' logger. SuspiciousOperations that reach django.core.handlers.base.BaseHandler will now return a 400 instead of a 500. Thanks to tiwoc for the report, and Carl Meyer and Donald Stufft for review. --- django/conf/urls/__init__.py | 3 +- django/contrib/admin/exceptions.py | 6 ++++ django/contrib/admin/views/main.py | 3 +- django/contrib/auth/tests/test_views.py | 36 +++++++++++++--------- django/contrib/formtools/exceptions.py | 6 ++++ django/contrib/formtools/wizard/storage/cookie.py | 4 +-- django/contrib/sessions/backends/base.py | 14 +++++++-- django/contrib/sessions/backends/cached_db.py | 9 +++++- django/contrib/sessions/backends/db.py | 10 ++++-- django/contrib/sessions/backends/file.py | 12 ++++++-- django/contrib/sessions/exceptions.py | 11 +++++++ django/contrib/sessions/tests.py | 20 +++++++++--- django/core/exceptions.py | 34 +++++++++++++++----- django/core/files/storage.py | 4 +-- django/core/handlers/base.py | 20 ++++++++++-- django/core/urlresolvers.py | 3 ++ django/http/multipartparser.py | 4 +-- django/http/request.py | 4 +-- django/http/response.py | 4 +-- django/test/utils.py | 20 ++++++++++++ django/utils/log.py | 5 +++ django/views/defaults.py | 15 +++++++++ docs/ref/exceptions.txt | 21 +++++++++++-- docs/releases/1.6.txt | 7 +++++ docs/topics/http/views.txt | 22 +++++++++++++ docs/topics/logging.txt | 31 ++++++++++++++++++- tests/admin_views/tests.py | 34 ++++++++++---------- tests/handlers/tests.py | 9 ++++++ tests/handlers/urls.py | 1 + tests/handlers/views.py | 4 +++ tests/logging_tests/tests.py | 24 +++++++++++++-- tests/logging_tests/urls.py | 10 ++++++ tests/logging_tests/views.py | 11 +++++++ tests/test_client_regress/tests.py | 6 ++-- tests/test_client_regress/views.py | 7 +++-- tests/urlpatterns_reverse/tests.py | 4 ++- tests/urlpatterns_reverse/urls_error_handlers.py | 1 + .../urls_error_handlers_callables.py | 1 + 38 files changed, 363 insertions(+), 77 deletions(-) create mode 100644 django/contrib/admin/exceptions.py create mode 100644 django/contrib/formtools/exceptions.py create mode 100644 django/contrib/sessions/exceptions.py create mode 100644 tests/logging_tests/urls.py create mode 100644 tests/logging_tests/views.py (limited to 'django/utils') diff --git a/django/conf/urls/__init__.py b/django/conf/urls/__init__.py index 04fb1dff59..c0340c0543 100644 --- a/django/conf/urls/__init__.py +++ b/django/conf/urls/__init__.py @@ -5,8 +5,9 @@ from django.utils.importlib import import_module from django.utils import six -__all__ = ['handler403', 'handler404', 'handler500', 'include', 'patterns', 'url'] +__all__ = ['handler400', 'handler403', 'handler404', 'handler500', 'include', 'patterns', 'url'] +handler400 = 'django.views.defaults.bad_request' handler403 = 'django.views.defaults.permission_denied' handler404 = 'django.views.defaults.page_not_found' handler500 = 'django.views.defaults.server_error' diff --git a/django/contrib/admin/exceptions.py b/django/contrib/admin/exceptions.py new file mode 100644 index 0000000000..2e094c6da1 --- /dev/null +++ b/django/contrib/admin/exceptions.py @@ -0,0 +1,6 @@ +from django.core.exceptions import SuspiciousOperation + + +class DisallowedModelAdminLookup(SuspiciousOperation): + """Invalid filter was passed to admin view via URL querystring""" + pass diff --git a/django/contrib/admin/views/main.py b/django/contrib/admin/views/main.py index 21ac30b7b3..dbed21265c 100644 --- a/django/contrib/admin/views/main.py +++ b/django/contrib/admin/views/main.py @@ -14,6 +14,7 @@ from django.utils.translation import ugettext, ugettext_lazy from django.utils.http import urlencode from django.contrib.admin import FieldListFilter +from django.contrib.admin.exceptions import DisallowedModelAdminLookup from django.contrib.admin.options import IncorrectLookupParameters from django.contrib.admin.util import (quote, get_fields_from_path, lookup_needs_distinct, prepare_lookup_value) @@ -128,7 +129,7 @@ class ChangeList(six.with_metaclass(RenameChangeListMethods)): lookup_params[force_str(key)] = value if not self.model_admin.lookup_allowed(key, value): - raise SuspiciousOperation("Filtering by %s not allowed" % key) + raise DisallowedModelAdminLookup("Filtering by %s not allowed" % key) filter_specs = [] if self.list_filter: diff --git a/django/contrib/auth/tests/test_views.py b/django/contrib/auth/tests/test_views.py index fe1d7fb52f..94cad90b15 100644 --- a/django/contrib/auth/tests/test_views.py +++ b/django/contrib/auth/tests/test_views.py @@ -10,7 +10,6 @@ from django.conf import global_settings, settings from django.contrib.sites.models import Site, RequestSite from django.contrib.auth.models import User from django.core import mail -from django.core.exceptions import SuspiciousOperation from django.core.urlresolvers import reverse, NoReverseMatch from django.http import QueryDict, HttpRequest from django.utils.encoding import force_text @@ -18,7 +17,7 @@ from django.utils.html import escape from django.utils.http import urlquote from django.utils._os import upath from django.test import TestCase -from django.test.utils import override_settings +from django.test.utils import override_settings, patch_logger from django.middleware.csrf import CsrfViewMiddleware from django.contrib.sessions.middleware import SessionMiddleware @@ -155,23 +154,28 @@ class PasswordResetTest(AuthViewsTestCase): # produce a meaningful reset URL, we need to be certain that the # HTTP_HOST header isn't poisoned. This is done as a check when get_host() # is invoked, but we check here as a practical consequence. - with self.assertRaises(SuspiciousOperation): - self.client.post('/password_reset/', - {'email': 'staffmember@example.com'}, - HTTP_HOST='www.example:dr.frankenstein@evil.tld' - ) - self.assertEqual(len(mail.outbox), 0) + with patch_logger('django.security.DisallowedHost', 'error') as logger_calls: + response = self.client.post('/password_reset/', + {'email': 'staffmember@example.com'}, + HTTP_HOST='www.example:dr.frankenstein@evil.tld' + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(len(mail.outbox), 0) + self.assertEqual(len(logger_calls), 1) # Skip any 500 handler action (like sending more mail...) @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True) def test_poisoned_http_host_admin_site(self): "Poisoned HTTP_HOST headers can't be used for reset emails on admin views" - with self.assertRaises(SuspiciousOperation): - self.client.post('/admin_password_reset/', - {'email': 'staffmember@example.com'}, - HTTP_HOST='www.example:dr.frankenstein@evil.tld' - ) - self.assertEqual(len(mail.outbox), 0) + with patch_logger('django.security.DisallowedHost', 'error') as logger_calls: + response = self.client.post('/admin_password_reset/', + {'email': 'staffmember@example.com'}, + HTTP_HOST='www.example:dr.frankenstein@evil.tld' + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(len(mail.outbox), 0) + self.assertEqual(len(logger_calls), 1) + def _test_confirm_start(self): # Start by creating the email @@ -678,5 +682,7 @@ class ChangelistTests(AuthViewsTestCase): self.login() # A lookup that tries to filter on password isn't OK - with self.assertRaises(SuspiciousOperation): + with patch_logger('django.security.DisallowedModelAdminLookup', 'error') as logger_calls: response = self.client.get('/admin/auth/user/?password__startswith=sha1$') + self.assertEqual(response.status_code, 400) + self.assertEqual(len(logger_calls), 1) diff --git a/django/contrib/formtools/exceptions.py b/django/contrib/formtools/exceptions.py new file mode 100644 index 0000000000..f07ac9f745 --- /dev/null +++ b/django/contrib/formtools/exceptions.py @@ -0,0 +1,6 @@ +from django.core.exceptions import SuspiciousOperation + + +class WizardViewCookieModified(SuspiciousOperation): + """Signature of cookie modified""" + pass diff --git a/django/contrib/formtools/wizard/storage/cookie.py b/django/contrib/formtools/wizard/storage/cookie.py index e80361042f..9bf6503f18 100644 --- a/django/contrib/formtools/wizard/storage/cookie.py +++ b/django/contrib/formtools/wizard/storage/cookie.py @@ -1,8 +1,8 @@ import json -from django.core.exceptions import SuspiciousOperation from django.core.signing import BadSignature +from django.contrib.formtools.exceptions import WizardViewCookieModified from django.contrib.formtools.wizard import storage @@ -21,7 +21,7 @@ class CookieStorage(storage.BaseStorage): except KeyError: data = None except BadSignature: - raise SuspiciousOperation('WizardView cookie manipulated') + raise WizardViewCookieModified('WizardView cookie manipulated') if data is None: return None return json.loads(data, cls=json.JSONDecoder) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index f79a264500..759d7ac7ad 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import base64 from datetime import datetime, timedelta +import logging + try: from django.utils.six.moves import cPickle as pickle except ImportError: @@ -14,7 +16,9 @@ from django.utils.crypto import constant_time_compare from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone -from django.utils.encoding import force_bytes +from django.utils.encoding import force_bytes, force_text + +from django.contrib.sessions.exceptions import SuspiciousSession # session_key should not be case sensitive because some backends can store it # on case insensitive file systems. @@ -94,12 +98,16 @@ class SessionBase(object): hash, pickled = encoded_data.split(b':', 1) expected_hash = self._hash(pickled) if not constant_time_compare(hash.decode(), expected_hash): - raise SuspiciousOperation("Session data corrupted") + raise SuspiciousSession("Session data corrupted") else: return pickle.loads(pickled) - except Exception: + except Exception as e: # ValueError, SuspiciousOperation, unpickling exceptions. If any of # these happen, just return an empty dictionary (an empty session). + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) return {} def update(self, dict_): diff --git a/django/contrib/sessions/backends/cached_db.py b/django/contrib/sessions/backends/cached_db.py index 31c6fbfce3..be22c1f97a 100644 --- a/django/contrib/sessions/backends/cached_db.py +++ b/django/contrib/sessions/backends/cached_db.py @@ -2,10 +2,13 @@ Cached, database-backed sessions. """ +import logging + from django.contrib.sessions.backends.db import SessionStore as DBStore from django.core.cache import cache from django.core.exceptions import SuspiciousOperation from django.utils import timezone +from django.utils.encoding import force_text KEY_PREFIX = "django.contrib.sessions.cached_db" @@ -41,7 +44,11 @@ class SessionStore(DBStore): data = self.decode(s.session_data) cache.set(self.cache_key, data, self.get_expiry_age(expiry=s.expire_date)) - except (Session.DoesNotExist, SuspiciousOperation): + except (Session.DoesNotExist, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() data = {} return data diff --git a/django/contrib/sessions/backends/db.py b/django/contrib/sessions/backends/db.py index 30da0b7a10..206fca2700 100644 --- a/django/contrib/sessions/backends/db.py +++ b/django/contrib/sessions/backends/db.py @@ -1,8 +1,10 @@ +import logging + from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation from django.db import IntegrityError, transaction, router from django.utils import timezone - +from django.utils.encoding import force_text class SessionStore(SessionBase): """ @@ -18,7 +20,11 @@ class SessionStore(SessionBase): expire_date__gt=timezone.now() ) return self.decode(s.session_data) - except (Session.DoesNotExist, SuspiciousOperation): + except (Session.DoesNotExist, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() return {} diff --git a/django/contrib/sessions/backends/file.py b/django/contrib/sessions/backends/file.py index 3c3408e9a8..f47aa2d867 100644 --- a/django/contrib/sessions/backends/file.py +++ b/django/contrib/sessions/backends/file.py @@ -1,5 +1,6 @@ import datetime import errno +import logging import os import shutil import tempfile @@ -8,6 +9,9 @@ from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError, VALID_KEY_CHARS from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured from django.utils import timezone +from django.utils.encoding import force_text + +from django.contrib.sessions.exceptions import InvalidSessionKey class SessionStore(SessionBase): """ @@ -48,7 +52,7 @@ class SessionStore(SessionBase): # should always be md5s, so they should never contain directory # components. if not set(session_key).issubset(set(VALID_KEY_CHARS)): - raise SuspiciousOperation( + raise InvalidSessionKey( "Invalid characters in session key") return os.path.join(self.storage_path, self.file_prefix + session_key) @@ -75,7 +79,11 @@ class SessionStore(SessionBase): if file_data: try: session_data = self.decode(file_data) - except (EOFError, SuspiciousOperation): + except (EOFError, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() # Remove expired sessions. diff --git a/django/contrib/sessions/exceptions.py b/django/contrib/sessions/exceptions.py new file mode 100644 index 0000000000..4f4dc6b048 --- /dev/null +++ b/django/contrib/sessions/exceptions.py @@ -0,0 +1,11 @@ +from django.core.exceptions import SuspiciousOperation + + +class InvalidSessionKey(SuspiciousOperation): + """Invalid characters in session key""" + pass + + +class SuspiciousSession(SuspiciousOperation): + """The session may be tampered with""" + pass diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 1a7286e77e..cd8191a6a4 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -1,3 +1,4 @@ +import base64 from datetime import timedelta import os import shutil @@ -15,14 +16,16 @@ from django.contrib.sessions.models import Session from django.contrib.sessions.middleware import SessionMiddleware from django.core.cache import get_cache from django.core import management -from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation +from django.core.exceptions import ImproperlyConfigured from django.http import HttpResponse from django.test import TestCase, RequestFactory -from django.test.utils import override_settings +from django.test.utils import override_settings, patch_logger from django.utils import six from django.utils import timezone from django.utils import unittest +from django.contrib.sessions.exceptions import InvalidSessionKey + class SessionTestsMixin(object): # This does not inherit from TestCase to avoid any tests being run with this @@ -272,6 +275,15 @@ class SessionTestsMixin(object): encoded = self.session.encode(data) self.assertEqual(self.session.decode(encoded), data) + def test_decode_failure_logged_to_security(self): + bad_encode = base64.b64encode(b'flaskdj:alkdjf') + with patch_logger('django.security.SuspiciousSession', 'warning') as calls: + self.assertEqual({}, self.session.decode(bad_encode)) + # check that the failed decode is logged + self.assertEqual(len(calls), 1) + self.assertTrue('corrupted' in calls[0]) + + def test_actual_expiry(self): # Regression test for #19200 old_session_key = None @@ -411,12 +423,12 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): # This is tested directly on _key_to_file, as load() will swallow # a SuspiciousOperation in the same way as an IOError - by creating # a new session, making it unclear whether the slashes were detected. - self.assertRaises(SuspiciousOperation, + self.assertRaises(InvalidSessionKey, self.backend()._key_to_file, "a\\b\\c") def test_invalid_key_forwardslash(self): # Ensure we don't allow directory-traversal - self.assertRaises(SuspiciousOperation, + self.assertRaises(InvalidSessionKey, self.backend()._key_to_file, "a/b/c") @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.file") diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 233af40f88..2c79736e33 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -1,6 +1,7 @@ """ Global Django exception and warning classes. """ +import logging from functools import reduce @@ -9,37 +10,56 @@ class DjangoRuntimeWarning(RuntimeWarning): class ObjectDoesNotExist(Exception): - "The requested object does not exist" + """The requested object does not exist""" silent_variable_failure = True class MultipleObjectsReturned(Exception): - "The query returned multiple objects when only one was expected." + """The query returned multiple objects when only one was expected.""" pass class SuspiciousOperation(Exception): - "The user did something suspicious" + """The user did something suspicious""" + + +class SuspiciousMultipartForm(SuspiciousOperation): + """Suspect MIME request in multipart form data""" + pass + + +class SuspiciousFileOperation(SuspiciousOperation): + """A Suspicious filesystem operation was attempted""" + pass + + +class DisallowedHost(SuspiciousOperation): + """HTTP_HOST header contains invalid value""" + pass + + +class DisallowedRedirect(SuspiciousOperation): + """Redirect to scheme not in allowed list""" pass class PermissionDenied(Exception): - "The user did not have permission to do that" + """The user did not have permission to do that""" pass class ViewDoesNotExist(Exception): - "The requested view does not exist" + """The requested view does not exist""" pass class MiddlewareNotUsed(Exception): - "This middleware is not used in this server configuration" + """This middleware is not used in this server configuration""" pass class ImproperlyConfigured(Exception): - "Django is somehow improperly configured" + """Django is somehow improperly configured""" pass diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 18d15e1ab6..977b6a68a8 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -8,7 +8,7 @@ import itertools from datetime import datetime from django.conf import settings -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import SuspiciousFileOperation from django.core.files import locks, File from django.core.files.move import file_move_safe from django.utils.encoding import force_text, filepath_to_uri @@ -260,7 +260,7 @@ class FileSystemStorage(Storage): try: path = safe_join(self.location, name) except ValueError: - raise SuspiciousOperation("Attempted access to '%s' denied." % name) + raise SuspiciousFileOperation("Attempted access to '%s' denied." % name) return os.path.normpath(path) def size(self, name): diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py index 555fd98fc6..59118656c6 100644 --- a/django/core/handlers/base.py +++ b/django/core/handlers/base.py @@ -8,7 +8,7 @@ from django import http from django.conf import settings from django.core import urlresolvers from django.core import signals -from django.core.exceptions import MiddlewareNotUsed, PermissionDenied +from django.core.exceptions import MiddlewareNotUsed, PermissionDenied, SuspiciousOperation from django.db import connections, transaction from django.utils.encoding import force_text from django.utils.module_loading import import_by_path @@ -170,11 +170,27 @@ class BaseHandler(object): response = self.handle_uncaught_exception(request, resolver, sys.exc_info()) + except SuspiciousOperation as e: + # The request logger receives events for any problematic request + # The security logger receives events for all SuspiciousOperations + security_logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + security_logger.error(force_text(e)) + + try: + callback, param_dict = resolver.resolve400() + response = callback(request, **param_dict) + except: + signals.got_request_exception.send( + sender=self.__class__, request=request) + response = self.handle_uncaught_exception(request, + resolver, sys.exc_info()) + except SystemExit: # Allow sys.exit() to actually exit. See tickets #1023 and #4701 raise - except: # Handle everything else, including SuspiciousOperation, etc. + except: # Handle everything else. # Get the exception info now, in case another exception is thrown later. signals.got_request_exception.send(sender=self.__class__, request=request) response = self.handle_uncaught_exception(request, resolver, sys.exc_info()) diff --git a/django/core/urlresolvers.py b/django/core/urlresolvers.py index c3d93bb247..5f314ff490 100644 --- a/django/core/urlresolvers.py +++ b/django/core/urlresolvers.py @@ -360,6 +360,9 @@ class RegexURLResolver(LocaleRegexProvider): callback = getattr(urls, 'handler%s' % view_type) return get_callable(callback), {} + def resolve400(self): + return self._resolve_special('400') + def resolve403(self): return self._resolve_special('403') diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 26e10da1a2..eeb435fa57 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -11,7 +11,7 @@ import cgi import sys from django.conf import settings -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import SuspiciousMultipartForm from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_text from django.utils import six @@ -370,7 +370,7 @@ class LazyStream(six.Iterator): if current_number == num_bytes]) if number_equal > 40: - raise SuspiciousOperation( + raise SuspiciousMultipartForm( "The multipart parser got stuck, which shouldn't happen with" " normal uploaded files. Check for malicious upload activity;" " if there is none, report this to the Django developers." diff --git a/django/http/request.py b/django/http/request.py index 749b9f2561..e6811aa6cc 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -14,7 +14,7 @@ except ImportError: from django.conf import settings from django.core import signing -from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured +from django.core.exceptions import DisallowedHost, ImproperlyConfigured from django.core.files import uploadhandler from django.http.multipartparser import MultiPartParser from django.utils import six @@ -72,7 +72,7 @@ class HttpRequest(object): msg = "Invalid HTTP_HOST header: %r." % host if domain: msg += "You may need to add %r to ALLOWED_HOSTS." % domain - raise SuspiciousOperation(msg) + raise DisallowedHost(msg) def get_full_path(self): # RFC 3986 requires query string arguments to be in the ASCII range. diff --git a/django/http/response.py b/django/http/response.py index 671fb1c573..9aa49b1d5f 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -12,7 +12,7 @@ except ImportError: from django.conf import settings from django.core import signals from django.core import signing -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import DisallowedRedirect from django.http.cookie import SimpleCookie from django.utils import six, timezone from django.utils.encoding import force_bytes, iri_to_uri @@ -452,7 +452,7 @@ class HttpResponseRedirectBase(HttpResponse): def __init__(self, redirect_to, *args, **kwargs): parsed = urlparse(redirect_to) if parsed.scheme and parsed.scheme not in self.allowed_schemes: - raise SuspiciousOperation("Unsafe redirect to URL with protocol '%s'" % parsed.scheme) + raise DisallowedRedirect("Unsafe redirect to URL with protocol '%s'" % parsed.scheme) super(HttpResponseRedirectBase, self).__init__(*args, **kwargs) self['Location'] = iri_to_uri(redirect_to) diff --git a/django/test/utils.py b/django/test/utils.py index fb9221d25c..d178c9b29c 100644 --- a/django/test/utils.py +++ b/django/test/utils.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager +import logging import re import sys import warnings @@ -401,3 +403,21 @@ class IgnoreDeprecationWarningsMixin(object): class IgnorePendingDeprecationWarningsMixin(IgnoreDeprecationWarningsMixin): warning_class = PendingDeprecationWarning + + +@contextmanager +def patch_logger(logger_name, log_level): + """ + Context manager that takes a named logger and the logging level + and provides a simple mock-like list of messages received + """ + calls = [] + def replacement(msg): + calls.append(msg) + logger = logging.getLogger(logger_name) + orig = getattr(logger, log_level) + setattr(logger, log_level, replacement) + try: + yield calls + finally: + setattr(logger, log_level, orig) diff --git a/django/utils/log.py b/django/utils/log.py index a9b62caae1..2abf7701cb 100644 --- a/django/utils/log.py +++ b/django/utils/log.py @@ -63,6 +63,11 @@ DEFAULT_LOGGING = { 'level': 'ERROR', 'propagate': False, }, + 'django.security': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': False, + }, 'py.warnings': { 'handlers': ['console'], }, diff --git a/django/views/defaults.py b/django/views/defaults.py index 89228c50c9..c8a62fc753 100644 --- a/django/views/defaults.py +++ b/django/views/defaults.py @@ -43,6 +43,21 @@ def server_error(request, template_name='500.html'): return http.HttpResponseServerError(template.render(Context({}))) +@requires_csrf_token +def bad_request(request, template_name='400.html'): + """ + 400 error handler. + + Templates: :template:`400.html` + Context: None + """ + try: + template = loader.get_template(template_name) + except TemplateDoesNotExist: + return http.HttpResponseBadRequest('

Bad Request (400)

') + return http.HttpResponseBadRequest(template.render(Context({}))) + + # This can be called when CsrfViewMiddleware.process_view has not run, # therefore need @requires_csrf_token in case the template needs # {% csrf_token %}. diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index f9a1715180..6a5e6f49d5 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -44,9 +44,24 @@ SuspiciousOperation ------------------- .. exception:: SuspiciousOperation - The :exc:`SuspiciousOperation` exception is raised when a user has performed - an operation that should be considered suspicious from a security perspective, - such as tampering with a session cookie. + The :exc:`SuspiciousOperation` exception is raised when a user has + performed an operation that should be considered suspicious from a security + perspective, such as tampering with a session cookie. Subclasses of + SuspiciousOperation include: + + * DisallowedHost + * DisallowedModelAdminLookup + * DisallowedRedirect + * InvalidSessionKey + * SuspiciousFileOperation + * SuspiciousMultipartForm + * SuspiciousSession + * WizardViewCookieModified + + If a ``SuspiciousOperation`` exception reaches the WSGI handler level it is + logged at the ``Error`` level and results in + a :class:`~django.http.HttpResponseBadRequest`. See the :doc:`logging + documentation ` for more information. PermissionDenied ---------------- diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index 35488b4110..825c5dc28d 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -270,6 +270,13 @@ Minor features stores active language in session if it is not present there. This prevents loss of language settings after session flush, e.g. logout. +* :exc:`~django.core.exceptions.SuspiciousOperation` has been differentiated + into a number of subclasses, and each will log to a matching named logger + under the ``django.security`` logging hierarchy. Along with this change, + a ``handler400`` mechanism and default view are used whenever + a ``SuspiciousOperation`` reaches the WSGI handler to return an + ``HttpResponseBadRequest``. + Backwards incompatible changes in 1.6 ===================================== diff --git a/docs/topics/http/views.txt b/docs/topics/http/views.txt index 2ccedec2f7..5c27c9c958 100644 --- a/docs/topics/http/views.txt +++ b/docs/topics/http/views.txt @@ -231,3 +231,25 @@ same way you can for the 404 and 500 views by specifying a ``handler403`` in your URLconf:: handler403 = 'mysite.views.my_custom_permission_denied_view' + +.. _http_bad_request_view: + +The 400 (bad request) view +-------------------------- + +When a :exc:`~django.core.exceptions.SuspiciousOperation` is raised in Django, +the it may be handled by a component of Django (for example resetting the +session data). If not specifically handled, Django will consider the current +request a 'bad request' instead of a server error. + +The view ``django.views.defaults.bad_request``, is otherwise very similar to +the ``server_error`` view, but returns with the status code 400 indicating that +the error condition was the result of a client operation. + +Like the ``server_error`` view, the default ``bad_request`` should suffice for +99% of Web applications, but if you want to override the view, you can specify +``handler400`` in your URLconf, like so:: + + handler400 = 'mysite.views.my_custom_bad_request_view' + +``bad_request`` views are also only used when :setting:`DEBUG` is ``False``. diff --git a/docs/topics/logging.txt b/docs/topics/logging.txt index a31dc01cc5..a88201ad47 100644 --- a/docs/topics/logging.txt +++ b/docs/topics/logging.txt @@ -394,7 +394,7 @@ requirements of logging in Web server environment. Loggers ------- -Django provides three built-in loggers. +Django provides four built-in loggers. ``django`` ~~~~~~~~~~ @@ -434,6 +434,35 @@ For performance reasons, SQL logging is only enabled when ``settings.DEBUG`` is set to ``True``, regardless of the logging level or handlers that are installed. +``django.security.*`` +~~~~~~~~~~~~~~~~~~~~~~ + +The security loggers will receive messages on any occurrence of +:exc:`~django.core.exceptions.SuspiciousOperation`. There is a sub-logger for +each sub-type of SuspiciousOperation. The level of the log event depends on +where the exception is handled. Most occurrences are logged as a warning, while +any ``SuspiciousOperation`` that reaches the WSGI handler will be logged as an +error. For example, when an HTTP ``Host`` header is included in a request from +a client that does not match :setting:`ALLOWED_HOSTS`, Django will return a 400 +response, and an error message will be logged to the +``django.security.DisallowedHost`` logger. + +Only the parent ``django.security`` logger is configured by default, and all +child loggers will propagate to the parent logger. The ``django.security`` +logger is configured the same as the ``django.request`` logger, and any error +events will be mailed to admins. Requests resulting in a 400 response due to +a ``SuspiciousOperation`` will not be logged to the ``django.request`` logger, +but only to the ``django.security`` logger. + +To silence a particular type of SuspiciousOperation, you can override that +specific logger following this example:: + + 'loggers': { + 'django.security.DisallowedHost': { + 'handlers': ['null'], + 'propagate': False, + }, + Handlers -------- diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py index c0f6533cb2..3485ea353b 100644 --- a/tests/admin_views/tests.py +++ b/tests/admin_views/tests.py @@ -11,7 +11,6 @@ except ImportError: # Python 2 from django.conf import settings, global_settings from django.core import mail -from django.core.exceptions import SuspiciousOperation from django.core.files import temp as tempfile from django.core.urlresolvers import reverse # Register auth models with the admin. @@ -30,6 +29,7 @@ from django.db import connection from django.forms.util import ErrorList from django.template.response import TemplateResponse from django.test import TestCase +from django.test.utils import patch_logger from django.utils import formats, translation, unittest from django.utils.cache import get_max_age from django.utils.encoding import iri_to_uri, force_bytes @@ -543,20 +543,21 @@ class AdminViewBasicTest(TestCase): self.assertContains(response, '%Y-%m-%d %H:%M:%S') def test_disallowed_filtering(self): - self.assertRaises(SuspiciousOperation, - self.client.get, "/test_admin/admin/admin_views/album/?owner__email__startswith=fuzzy" - ) + with patch_logger('django.security.DisallowedModelAdminLookup', 'error') as calls: + response = self.client.get("/test_admin/admin/admin_views/album/?owner__email__startswith=fuzzy") + self.assertEqual(response.status_code, 400) + self.assertEqual(len(calls), 1) - try: - self.client.get("/test_admin/admin/admin_views/thing/?color__value__startswith=red") - self.client.get("/test_admin/admin/admin_views/thing/?color__value=red") - except SuspiciousOperation: - self.fail("Filters are allowed if explicitly included in list_filter") + # Filters are allowed if explicitly included in list_filter + response = self.client.get("/test_admin/admin/admin_views/thing/?color__value__startswith=red") + self.assertEqual(response.status_code, 200) + response = self.client.get("/test_admin/admin/admin_views/thing/?color__value=red") + self.assertEqual(response.status_code, 200) - try: - self.client.get("/test_admin/admin/admin_views/person/?age__gt=30") - except SuspiciousOperation: - self.fail("Filters should be allowed if they involve a local field without the need to whitelist them in list_filter or date_hierarchy.") + # Filters should be allowed if they involve a local field without the + # need to whitelist them in list_filter or date_hierarchy. + response = self.client.get("/test_admin/admin/admin_views/person/?age__gt=30") + self.assertEqual(response.status_code, 200) e1 = Employee.objects.create(name='Anonymous', gender=1, age=22, alive=True, code='123') e2 = Employee.objects.create(name='Visitor', gender=2, age=19, alive=True, code='124') @@ -574,10 +575,9 @@ class AdminViewBasicTest(TestCase): ForeignKey 'limit_choices_to' should be allowed, otherwise raw_id_fields can break. """ - try: - self.client.get("/test_admin/admin/admin_views/inquisition/?leader__name=Palin&leader__age=27") - except SuspiciousOperation: - self.fail("Filters should be allowed if they are defined on a ForeignKey pointing to this model") + # Filters should be allowed if they are defined on a ForeignKey pointing to this model + response = self.client.get("/test_admin/admin/admin_views/inquisition/?leader__name=Palin&leader__age=27") + self.assertEqual(response.status_code, 200) def test_hide_change_password(self): """ diff --git a/tests/handlers/tests.py b/tests/handlers/tests.py index 3680eecdd2..397b63647a 100644 --- a/tests/handlers/tests.py +++ b/tests/handlers/tests.py @@ -61,6 +61,7 @@ class TransactionsPerRequestTests(TransactionTestCase): connection.settings_dict['ATOMIC_REQUESTS'] = old_atomic_requests self.assertContains(response, 'False') + class SignalsTests(TestCase): urls = 'handlers.urls' @@ -89,3 +90,11 @@ class SignalsTests(TestCase): self.assertEqual(self.signals, ['started']) self.assertEqual(b''.join(response.streaming_content), b"streaming content") self.assertEqual(self.signals, ['started', 'finished']) + + +class HandlerSuspiciousOpsTest(TestCase): + urls = 'handlers.urls' + + def test_suspiciousop_in_view_returns_400(self): + response = self.client.get('/suspicious/') + self.assertEqual(response.status_code, 400) diff --git a/tests/handlers/urls.py b/tests/handlers/urls.py index 29858055ab..eaf764b00b 100644 --- a/tests/handlers/urls.py +++ b/tests/handlers/urls.py @@ -9,4 +9,5 @@ urlpatterns = patterns('', url(r'^streaming/$', views.streaming), url(r'^in_transaction/$', views.in_transaction), url(r'^not_in_transaction/$', views.not_in_transaction), + url(r'^suspicious/$', views.suspicious), ) diff --git a/tests/handlers/views.py b/tests/handlers/views.py index 9cc86ae6f3..1b75b27043 100644 --- a/tests/handlers/views.py +++ b/tests/handlers/views.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +from django.core.exceptions import SuspiciousOperation from django.db import connection, transaction from django.http import HttpResponse, StreamingHttpResponse @@ -15,3 +16,6 @@ def in_transaction(request): @transaction.non_atomic_requests def not_in_transaction(request): return HttpResponse(str(connection.in_atomic_block)) + +def suspicious(request): + raise SuspiciousOperation('dubious') diff --git a/tests/logging_tests/tests.py b/tests/logging_tests/tests.py index 8e5445cd42..0c2d269464 100644 --- a/tests/logging_tests/tests.py +++ b/tests/logging_tests/tests.py @@ -8,9 +8,10 @@ import warnings from django.conf import LazySettings from django.core import mail from django.test import TestCase, RequestFactory -from django.test.utils import override_settings +from django.test.utils import override_settings, patch_logger from django.utils.encoding import force_text -from django.utils.log import CallbackFilter, RequireDebugFalse, RequireDebugTrue +from django.utils.log import (CallbackFilter, RequireDebugFalse, + RequireDebugTrue) from django.utils.six import StringIO from django.utils.unittest import skipUnless @@ -354,3 +355,22 @@ class SettingsConfigureLogging(TestCase): settings.configure( LOGGING_CONFIG='logging_tests.tests.dictConfig') self.assertTrue(dictConfig.called) + + +class SecurityLoggerTest(TestCase): + + urls = 'logging_tests.urls' + + def test_suspicious_operation_creates_log_message(self): + with self.settings(DEBUG=True): + with patch_logger('django.security.SuspiciousOperation', 'error') as calls: + response = self.client.get('/suspicious/') + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0], 'dubious') + + def test_suspicious_operation_uses_sublogger(self): + with self.settings(DEBUG=True): + with patch_logger('django.security.DisallowedHost', 'error') as calls: + response = self.client.get('/suspicious_spec/') + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0], 'dubious') diff --git a/tests/logging_tests/urls.py b/tests/logging_tests/urls.py new file mode 100644 index 0000000000..c738bd565c --- /dev/null +++ b/tests/logging_tests/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + +from django.conf.urls import patterns, url + +from . import views + +urlpatterns = patterns('', + url(r'^suspicious/$', views.suspicious), + url(r'^suspicious_spec/$', views.suspicious_spec), +) diff --git a/tests/logging_tests/views.py b/tests/logging_tests/views.py new file mode 100644 index 0000000000..c685bcc005 --- /dev/null +++ b/tests/logging_tests/views.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from django.core.exceptions import SuspiciousOperation, DisallowedHost + + +def suspicious(request): + raise SuspiciousOperation('dubious') + + +def suspicious_spec(request): + raise DisallowedHost('dubious') diff --git a/tests/test_client_regress/tests.py b/tests/test_client_regress/tests.py index 2582b210c4..6a58ef6344 100644 --- a/tests/test_client_regress/tests.py +++ b/tests/test_client_regress/tests.py @@ -7,7 +7,6 @@ from __future__ import unicode_literals import os from django.conf import settings -from django.core.exceptions import SuspiciousOperation from django.core.urlresolvers import reverse from django.template import (TemplateDoesNotExist, TemplateSyntaxError, Context, Template, loader) @@ -20,6 +19,7 @@ from django.utils._os import upath from django.utils.translation import ugettext_lazy from django.http import HttpResponse +from .views import CustomTestException @override_settings( TEMPLATE_DIRS=(os.path.join(os.path.dirname(upath(__file__)), 'templates'),) @@ -619,7 +619,7 @@ class ExceptionTests(TestCase): try: response = self.client.get("/test_client_regress/staff_only/") self.fail("General users should not be able to visit this page") - except SuspiciousOperation: + except CustomTestException: pass # At this point, an exception has been raised, and should be cleared. @@ -629,7 +629,7 @@ class ExceptionTests(TestCase): self.assertTrue(login, 'Could not log in') try: self.client.get("/test_client_regress/staff_only/") - except SuspiciousOperation: + except CustomTestException: self.fail("Staff should be able to visit this page") diff --git a/tests/test_client_regress/views.py b/tests/test_client_regress/views.py index 7e86ffd8ca..71e5b526e5 100644 --- a/tests/test_client_regress/views.py +++ b/tests/test_client_regress/views.py @@ -3,12 +3,15 @@ import json from django.conf import settings from django.contrib.auth.decorators import login_required from django.http import HttpResponse, HttpResponseRedirect -from django.core.exceptions import SuspiciousOperation from django.shortcuts import render_to_response from django.core.serializers.json import DjangoJSONEncoder from django.test.client import CONTENT_TYPE_RE from django.template import RequestContext + +class CustomTestException(Exception): + pass + def no_template_view(request): "A simple view that expects a GET request, and returns a rendered template" return HttpResponse("No template used. Sample content: twice once twice. Content ends.") @@ -18,7 +21,7 @@ def staff_only_view(request): if request.user.is_staff: return HttpResponse('') else: - raise SuspiciousOperation() + raise CustomTestException() def get_view(request): "A simple login protected view" diff --git a/tests/urlpatterns_reverse/tests.py b/tests/urlpatterns_reverse/tests.py index d5d9b3a709..f54c796d30 100644 --- a/tests/urlpatterns_reverse/tests.py +++ b/tests/urlpatterns_reverse/tests.py @@ -516,7 +516,7 @@ class RequestURLconfTests(TestCase): b''.join(self.client.get('/second_test/')) class ErrorHandlerResolutionTests(TestCase): - """Tests for handler404 and handler500""" + """Tests for handler400, handler404 and handler500""" def setUp(self): from django.core.urlresolvers import RegexURLResolver @@ -528,12 +528,14 @@ class ErrorHandlerResolutionTests(TestCase): def test_named_handlers(self): from .views import empty_view handler = (empty_view, {}) + self.assertEqual(self.resolver.resolve400(), handler) self.assertEqual(self.resolver.resolve404(), handler) self.assertEqual(self.resolver.resolve500(), handler) def test_callable_handers(self): from .views import empty_view handler = (empty_view, {}) + self.assertEqual(self.callable_resolver.resolve400(), handler) self.assertEqual(self.callable_resolver.resolve404(), handler) self.assertEqual(self.callable_resolver.resolve500(), handler) diff --git a/tests/urlpatterns_reverse/urls_error_handlers.py b/tests/urlpatterns_reverse/urls_error_handlers.py index be4f42afbc..7146fdf43c 100644 --- a/tests/urlpatterns_reverse/urls_error_handlers.py +++ b/tests/urlpatterns_reverse/urls_error_handlers.py @@ -4,5 +4,6 @@ from django.conf.urls import patterns urlpatterns = patterns('') +handler400 = 'urlpatterns_reverse.views.empty_view' handler404 = 'urlpatterns_reverse.views.empty_view' handler500 = 'urlpatterns_reverse.views.empty_view' diff --git a/tests/urlpatterns_reverse/urls_error_handlers_callables.py b/tests/urlpatterns_reverse/urls_error_handlers_callables.py index fe2d3137e9..befeccaf45 100644 --- a/tests/urlpatterns_reverse/urls_error_handlers_callables.py +++ b/tests/urlpatterns_reverse/urls_error_handlers_callables.py @@ -9,5 +9,6 @@ from .views import empty_view urlpatterns = patterns('') +handler400 = empty_view handler404 = empty_view handler500 = empty_view -- cgit v1.3 From 0fa8d43e742ee8b480d938a47836b5a3720c2677 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Sun, 26 May 2013 23:47:50 -0300 Subject: Replaced `and...or...` constructs with PEP 308 conditional expressions. --- django/contrib/admin/filters.py | 2 +- django/contrib/admin/helpers.py | 4 ++-- django/contrib/admin/templatetags/log.py | 2 +- django/contrib/admindocs/views.py | 4 ++-- django/contrib/auth/forms.py | 2 +- django/contrib/gis/geos/geometry.py | 10 +++++----- django/contrib/gis/utils/layermapping.py | 2 +- django/contrib/sitemaps/__init__.py | 2 +- django/contrib/staticfiles/finders.py | 2 +- .../contrib/staticfiles/management/commands/collectstatic.py | 8 +++----- django/core/files/locks.py | 2 +- django/core/files/move.py | 6 +++--- django/core/management/commands/createcachetable.py | 4 ++-- django/core/management/commands/runserver.py | 2 +- django/db/backends/creation.py | 2 +- django/db/backends/util.py | 2 +- django/db/models/query.py | 2 +- django/db/models/sql/aggregates.py | 2 +- django/forms/forms.py | 2 +- django/middleware/common.py | 2 +- django/middleware/locale.py | 2 +- django/utils/log.py | 4 ++-- django/views/debug.py | 2 +- docs/_ext/djangodocs.py | 2 +- tests/forms_tests/tests/test_forms.py | 2 +- 25 files changed, 37 insertions(+), 39 deletions(-) (limited to 'django/utils') diff --git a/django/contrib/admin/filters.py b/django/contrib/admin/filters.py index ff66d3e3f3..4131494515 100644 --- a/django/contrib/admin/filters.py +++ b/django/contrib/admin/filters.py @@ -216,7 +216,7 @@ class RelatedFieldListFilter(FieldListFilter): } FieldListFilter.register(lambda f: ( - hasattr(f, 'rel') and bool(f.rel) or + bool(f.rel) if hasattr(f, 'rel') else isinstance(f, models.related.RelatedObject)), RelatedFieldListFilter) diff --git a/django/contrib/admin/helpers.py b/django/contrib/admin/helpers.py index adc2302587..320d3267a7 100644 --- a/django/contrib/admin/helpers.py +++ b/django/contrib/admin/helpers.py @@ -131,7 +131,7 @@ class AdminField(object): classes.append('required') if not self.is_first: classes.append('inline') - attrs = classes and {'class': ' '.join(classes)} or {} + attrs = {'class': ' '.join(classes)} if classes else {} return self.field.label_tag(contents=mark_safe(contents), attrs=attrs) def errors(self): @@ -144,7 +144,7 @@ class AdminReadonlyField(object): # {{ field.name }} must be a useful class name to identify the field. # For convenience, store other field-related data here too. if callable(field): - class_name = field.__name__ != '' and field.__name__ or '' + class_name = field.__name__ if field.__name__ != '' else '' else: class_name = field self.field = { diff --git a/django/contrib/admin/templatetags/log.py b/django/contrib/admin/templatetags/log.py index f70ee2731d..1b9e6aa7ef 100644 --- a/django/contrib/admin/templatetags/log.py +++ b/django/contrib/admin/templatetags/log.py @@ -53,4 +53,4 @@ def get_admin_log(parser, token): if tokens[4] != 'for_user': raise template.TemplateSyntaxError( "Fourth argument to 'get_admin_log' must be 'for_user'") - return AdminLogNode(limit=tokens[1], varname=tokens[3], user=(len(tokens) > 5 and tokens[5] or None)) + return AdminLogNode(limit=tokens[1], varname=tokens[3], user=(tokens[5] if len(tokens) > 5 else None)) diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 4cadc8b657..c03883def7 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -39,7 +39,7 @@ def bookmarklets(request): admin_root = urlresolvers.reverse('admin:index') return render_to_response('admin_doc/bookmarklets.html', { 'root_path': admin_root, - 'admin_url': "%s://%s%s" % (request.is_secure() and 'https' or 'http', request.get_host(), admin_root), + 'admin_url': "%s://%s%s" % ('https' if request.is_secure() else 'http', request.get_host(), admin_root), }, context_instance=RequestContext(request)) @staff_member_required @@ -287,7 +287,7 @@ def template_detail(request, template): templates.append({ 'file': template_file, 'exists': os.path.exists(template_file), - 'contents': lambda: os.path.exists(template_file) and open(template_file).read() or '', + 'contents': lambda: open(template_file).read() if os.path.exists(template_file) else '', 'site_id': settings_mod.SITE_ID, 'site': site_obj, 'order': list(settings_mod.TEMPLATE_DIRS).index(dir), diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py index 0e08d8ef31..edf2727b07 100644 --- a/django/contrib/auth/forms.py +++ b/django/contrib/auth/forms.py @@ -237,7 +237,7 @@ class PasswordResetForm(forms.Form): 'uid': int_to_base36(user.pk), 'user': user, 'token': token_generator.make_token(user), - 'protocol': use_https and 'https' or 'http', + 'protocol': 'https' if use_https else 'http', } subject = loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines diff --git a/django/contrib/gis/geos/geometry.py b/django/contrib/gis/geos/geometry.py index 01719c21d4..b088ec2dc4 100644 --- a/django/contrib/gis/geos/geometry.py +++ b/django/contrib/gis/geos/geometry.py @@ -384,7 +384,7 @@ class GEOSGeometry(GEOSBase, ListMixin): @property def wkt(self): "Returns the WKT (Well-Known Text) representation of this Geometry." - return wkt_w(self.hasz and 3 or 2).write(self).decode() + return wkt_w(3 if self.hasz else 2).write(self).decode() @property def hex(self): @@ -395,7 +395,7 @@ class GEOSGeometry(GEOSBase, ListMixin): """ # A possible faster, all-python, implementation: # str(self.wkb).encode('hex') - return wkb_w(self.hasz and 3 or 2).write_hex(self) + return wkb_w(3 if self.hasz else 2).write_hex(self) @property def hexewkb(self): @@ -407,7 +407,7 @@ class GEOSGeometry(GEOSBase, ListMixin): if self.hasz and not GEOS_PREPARE: # See: http://trac.osgeo.org/geos/ticket/216 raise GEOSException('Upgrade GEOS to 3.1 to get valid 3D HEXEWKB.') - return ewkb_w(self.hasz and 3 or 2).write_hex(self) + return ewkb_w(3 if self.hasz else 2).write_hex(self) @property def json(self): @@ -427,7 +427,7 @@ class GEOSGeometry(GEOSBase, ListMixin): as a Python buffer. SRID and Z values are not included, use the `ewkb` property instead. """ - return wkb_w(self.hasz and 3 or 2).write(self) + return wkb_w(3 if self.hasz else 2).write(self) @property def ewkb(self): @@ -439,7 +439,7 @@ class GEOSGeometry(GEOSBase, ListMixin): if self.hasz and not GEOS_PREPARE: # See: http://trac.osgeo.org/geos/ticket/216 raise GEOSException('Upgrade GEOS to 3.1 to get valid 3D EWKB.') - return ewkb_w(self.hasz and 3 or 2).write(self) + return ewkb_w(3 if self.hasz else 2).write(self) @property def kml(self): diff --git a/django/contrib/gis/utils/layermapping.py b/django/contrib/gis/utils/layermapping.py index cb56b8bce2..a502aa280d 100644 --- a/django/contrib/gis/utils/layermapping.py +++ b/django/contrib/gis/utils/layermapping.py @@ -546,7 +546,7 @@ class LayerMapping(object): # Attempting to save. m.save(using=self.using) num_saved += 1 - if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m)) + if verbose: stream.write('%s: %s\n' % ('Updated' if is_update else 'Saved', m)) except SystemExit: raise except Exception as msg: diff --git a/django/contrib/sitemaps/__init__.py b/django/contrib/sitemaps/__init__.py index 7d03ef19f5..1acae8296c 100644 --- a/django/contrib/sitemaps/__init__.py +++ b/django/contrib/sitemaps/__init__.py @@ -94,7 +94,7 @@ class Sitemap(object): 'location': loc, 'lastmod': self.__get('lastmod', item, None), 'changefreq': self.__get('changefreq', item, None), - 'priority': str(priority is not None and priority or ''), + 'priority': str(priority if priority is not None else ''), } urls.append(url_info) return urls diff --git a/django/contrib/staticfiles/finders.py b/django/contrib/staticfiles/finders.py index 8631570e79..7d266d95a0 100644 --- a/django/contrib/staticfiles/finders.py +++ b/django/contrib/staticfiles/finders.py @@ -245,7 +245,7 @@ def find(path, all=False): if matches: return matches # No match. - return all and [] or None + return [] if all else None def get_finders(): diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py index c620993f33..7c3de80e93 100644 --- a/django/contrib/staticfiles/management/commands/collectstatic.py +++ b/django/contrib/staticfiles/management/commands/collectstatic.py @@ -175,11 +175,9 @@ Type 'yes' to continue, or 'no' to cancel: """ summary = template % { 'modified_count': modified_count, 'identifier': 'static file' + ('' if modified_count == 1 else 's'), - 'action': self.symlink and 'symlinked' or 'copied', - 'destination': (destination_path and " to '%s'" - % destination_path or ''), - 'unmodified': (collected['unmodified'] and ', %s unmodified' - % unmodified_count or ''), + 'action': 'symlinked' if self.symlink else 'copied', + 'destination': (" to '%s'" % destination_path if destination_path else ''), + 'unmodified': (', %s unmodified' % unmodified_count if collected['unmodified'] else ''), 'post_processed': (collected['post_processed'] and ', %s post-processed' % post_processed_count or ''), diff --git a/django/core/files/locks.py b/django/core/files/locks.py index d1384329dc..6f0e4b9508 100644 --- a/django/core/files/locks.py +++ b/django/core/files/locks.py @@ -41,7 +41,7 @@ except (ImportError, AttributeError): def fd(f): """Get a filedescriptor from something which could be a file or an fd.""" - return hasattr(f, 'fileno') and f.fileno() or f + return f.fileno() if hasattr(f, 'fileno') else f if system_type == 'nt': def lock(file, flags): diff --git a/django/core/files/move.py b/django/core/files/move.py index 3af02634fe..4519dedf97 100644 --- a/django/core/files/move.py +++ b/django/core/files/move.py @@ -62,7 +62,7 @@ def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_ove with open(old_file_name, 'rb') as old_file: # now open the new file, not forgetting allow_overwrite fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) | - (not allow_overwrite and os.O_EXCL or 0)) + (os.O_EXCL if not allow_overwrite else 0)) try: locks.lock(fd, locks.LOCK_EX) current_chunk = None @@ -77,8 +77,8 @@ def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_ove try: os.remove(old_file_name) except OSError as e: - # Certain operating systems (Cygwin and Windows) - # fail when deleting opened files, ignore it. (For the + # Certain operating systems (Cygwin and Windows) + # fail when deleting opened files, ignore it. (For the # systems where this happens, temporary files will be auto-deleted # on close anyway.) if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13: diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py index 4d1bc0403e..d7ce3e93fd 100644 --- a/django/core/management/commands/createcachetable.py +++ b/django/core/management/commands/createcachetable.py @@ -38,7 +38,7 @@ class Command(LabelCommand): qn = connection.ops.quote_name for f in fields: field_output = [qn(f.name), f.db_type(connection=connection)] - field_output.append("%sNULL" % (not f.null and "NOT " or "")) + field_output.append("%sNULL" % ("NOT " if not f.null else "")) if f.primary_key: field_output.append("PRIMARY KEY") elif f.unique: @@ -51,7 +51,7 @@ class Command(LabelCommand): table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): - full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or '')) + full_statement.append(' %s%s' % (line, ',' if i < len(table_output)-1 else '')) full_statement.append(');') with transaction.commit_on_success_unless_managed(): curs = connection.cursor() diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py index 4fe03216d5..c4a0b78cd4 100644 --- a/django/core/management/commands/runserver.py +++ b/django/core/management/commands/runserver.py @@ -99,7 +99,7 @@ class Command(BaseCommand): "started_at": datetime.now().strftime('%B %d, %Y - %X'), "version": self.get_version(), "settings": settings.SETTINGS_MODULE, - "addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr, + "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, }) diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py index 38c284d6d3..c9e5c83ade 100644 --- a/django/db/backends/creation.py +++ b/django/db/backends/creation.py @@ -97,7 +97,7 @@ class BaseDatabaseCreation(object): style.SQL_TABLE(qn(opts.db_table)) + ' ('] for i, line in enumerate(table_output): # Combine and add commas. full_statement.append( - ' %s%s' % (line, i < len(table_output) - 1 and ',' or '')) + ' %s%s' % (line, ',' if i < len(table_output) - 1 else '')) full_statement.append(')') if opts.db_tablespace: tablespace_sql = self.connection.ops.tablespace_sql( diff --git a/django/db/backends/util.py b/django/db/backends/util.py index 084f4c200b..aa2601277a 100644 --- a/django/db/backends/util.py +++ b/django/db/backends/util.py @@ -83,7 +83,7 @@ class CursorDebugWrapper(CursorWrapper): ############################################### def typecast_date(s): - return s and datetime.date(*map(int, s.split('-'))) or None # returns None if s is null + return datetime.date(*map(int, s.split('-'))) if s else None # returns None if s is null def typecast_time(s): # does NOT store time zone information if not s: return None diff --git a/django/db/models/query.py b/django/db/models/query.py index c54380b96f..71d432f7d5 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -125,7 +125,7 @@ class QuerySet(object): else: stop = None qs.query.set_limits(start, stop) - return k.step and list(qs)[::k.step] or qs + return list(qs)[::k.step] if k.step else qs qs = self._clone() qs.query.set_limits(k, k + 1) diff --git a/django/db/models/sql/aggregates.py b/django/db/models/sql/aggregates.py index 23b79923d1..2bd2b2f76f 100644 --- a/django/db/models/sql/aggregates.py +++ b/django/db/models/sql/aggregates.py @@ -99,7 +99,7 @@ class Count(Aggregate): sql_template = '%(function)s(%(distinct)s%(field)s)' def __init__(self, col, distinct=False, **extra): - super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra) + super(Count, self).__init__(col, distinct='DISTINCT ' if distinct else '', **extra) class Max(Aggregate): sql_function = 'MAX' diff --git a/django/forms/forms.py b/django/forms/forms.py index 2c173a45dc..0c598ac775 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -134,7 +134,7 @@ class BaseForm(object): Subclasses may wish to override. """ - return self.prefix and ('%s-%s' % (self.prefix, field_name)) or field_name + return '%s-%s' % (self.prefix, field_name) if self.prefix else field_name def add_initial_prefix(self, field_name): """ diff --git a/django/middleware/common.py b/django/middleware/common.py index 1131bd698d..2c76c47756 100644 --- a/django/middleware/common.py +++ b/django/middleware/common.py @@ -85,7 +85,7 @@ class CommonMiddleware(object): return if new_url[0]: newurl = "%s://%s%s" % ( - request.is_secure() and 'https' or 'http', + 'https' if request.is_secure() else 'http', new_url[0], urlquote(new_url[1])) else: newurl = urlquote(new_url[1]) diff --git a/django/middleware/locale.py b/django/middleware/locale.py index 2e2204c859..4e0a4753ce 100644 --- a/django/middleware/locale.py +++ b/django/middleware/locale.py @@ -49,7 +49,7 @@ class LocaleMiddleware(object): if path_valid: language_url = "%s://%s/%s%s" % ( - request.is_secure() and 'https' or 'http', + 'https' if request.is_secure() else 'http', request.get_host(), language, request.get_full_path()) return HttpResponseRedirect(language_url) diff --git a/django/utils/log.py b/django/utils/log.py index 2abf7701cb..6734a7261e 100644 --- a/django/utils/log.py +++ b/django/utils/log.py @@ -92,8 +92,8 @@ class AdminEmailHandler(logging.Handler): request = record.request subject = '%s (%s IP): %s' % ( record.levelname, - (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS - and 'internal' or 'EXTERNAL'), + ('internal' if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS + else 'EXTERNAL'), record.getMessage() ) filter = get_exception_reporter_filter(request) diff --git a/django/views/debug.py b/django/views/debug.py index 209cdce8c9..4ecd936771 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -405,7 +405,7 @@ class ExceptionReporter(object): if pre_context_lineno is not None: frames.append({ 'tb': tb, - 'type': module_name.startswith('django.') and 'django' or 'user', + 'type': 'django' if module_name.startswith('django.') else 'user', 'filename': filename, 'function': function, 'lineno': lineno + 1, diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index 3ae770da20..833232a0c3 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -137,7 +137,7 @@ class DjangoHTMLTranslator(SmartyPantsHTMLTranslator): ) title = "%s%s" % ( self.version_text[node['type']] % node['version'], - len(node) and ":" or "." + ":" if len(node) else "." ) self.body.append('%s ' % title) diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py index f7eb46522f..1a3fb44a66 100644 --- a/tests/forms_tests/tests/test_forms.py +++ b/tests/forms_tests/tests/test_forms.py @@ -1398,7 +1398,7 @@ class FormsTestCase(TestCase): birthday = DateField() def add_prefix(self, field_name): - return self.prefix and '%s-prefix-%s' % (self.prefix, field_name) or field_name + return '%s-prefix-%s' % (self.prefix, field_name) if self.prefix else field_name p = Person(prefix='foo') self.assertHTMLEqual(p.as_ul(), """
    • -- cgit v1.3 From 127d9b2792131995caf87c16b73dbd61f360de2a Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 7 Nov 2012 17:23:16 +0100 Subject: Added minor splitting optimization in django.utils.ipv6._unpack_ipv4 --- django/utils/ipv6.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'django/utils') diff --git a/django/utils/ipv6.py b/django/utils/ipv6.py index 8881574eaa..eaacfb4623 100644 --- a/django/utils/ipv6.py +++ b/django/utils/ipv6.py @@ -138,8 +138,7 @@ def _unpack_ipv4(ip_str): if not ip_str.lower().startswith('0000:0000:0000:0000:0000:ffff:'): return None - hextets = ip_str.split(':') - return hextets[-1] + return ip_str.rsplit(':', 1)[1] def is_valid_ipv6_address(ip_str): """ -- cgit v1.3 From 09d0568697cf5f82737816cf8041a6c270861a7d Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 2 Jun 2013 10:31:02 -0700 Subject: Switched to using some constants the hmac module exposes. --- django/utils/crypto.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'django/utils') diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 5d0f381ffa..15db972560 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -28,10 +28,6 @@ from django.utils import six from django.utils.six.moves import xrange -_trans_5c = bytearray([(x ^ 0x5C) for x in xrange(256)]) -_trans_36 = bytearray([(x ^ 0x36) for x in xrange(256)]) - - def salted_hmac(key_salt, value, secret=None): """ Returns the HMAC-SHA1 of 'value', using a key generated from key_salt and a @@ -130,9 +126,9 @@ def _fast_hmac(key, msg, digest): if len(key) > dig1.block_size: key = digest(key).digest() key += b'\x00' * (dig1.block_size - len(key)) - dig1.update(key.translate(_trans_36)) + dig1.update(key.translate(hmac.trans_36)) dig1.update(msg) - dig2.update(key.translate(_trans_5c)) + dig2.update(key.translate(hmac.trans_5C)) dig2.update(dig1.digest()) return dig2 -- cgit v1.3 From c9d07d251fc68fe1a80c2a49e183bbef546238d9 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Mon, 3 Jun 2013 13:18:16 +0200 Subject: Fixed loaddata for Django checkouts with non ASCII chars in the name. --- django/core/management/commands/loaddata.py | 3 ++- django/utils/_os.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'django/utils') diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 45f25c21db..6856e85e45 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -16,6 +16,7 @@ from django.db import (connections, router, transaction, DEFAULT_DB_ALIAS, from django.db.models import get_app_paths from django.utils.encoding import force_text from django.utils.functional import cached_property, memoize +from django.utils._os import upath from itertools import product try: @@ -237,7 +238,7 @@ class Command(BaseCommand): dirs.append(d) dirs.extend(list(settings.FIXTURE_DIRS)) dirs.append('') - dirs = [os.path.abspath(os.path.realpath(d)) for d in dirs] + dirs = [upath(os.path.abspath(os.path.realpath(d))) for d in dirs] return dirs def parse_name(self, fixture_name): diff --git a/django/utils/_os.py b/django/utils/_os.py index 6c1cd17a83..607e02c94d 100644 --- a/django/utils/_os.py +++ b/django/utils/_os.py @@ -38,7 +38,7 @@ def upath(path): """ Always return a unicode path. """ - if not six.PY3: + if not six.PY3 and not isinstance(path, six.text_type): return path.decode(fs_encoding) return path -- cgit v1.3