From a0a0203a392f67832ba7a8a2f099e70d7db2d19e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 11 Aug 2012 15:34:51 +0200 Subject: [py3] Added python_2_unicode_compatible decorator. --- django/utils/encoding.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'django/utils/encoding.py') diff --git a/django/utils/encoding.py b/django/utils/encoding.py index eb60cfde8b..7030ab1db0 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -39,6 +39,19 @@ class StrAndUnicode(object): def __str__(self): return self.__unicode__().encode('utf-8') +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if not six.PY3: + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + def smart_text(s, encoding='utf-8', strings_only=False, errors='strict'): """ Returns a text object representing 's' -- unicode on Python 2 and str on -- cgit v1.3 From 4e68e861533846010e372ecf58e3cd0439081754 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 12 Aug 2012 14:39:48 +0200 Subject: [py3] Deprecated StrAndUnicode. This mix-in is superseded by the @python_2_unicode_compatible decorator. --- django/utils/encoding.py | 7 +++++++ docs/releases/1.5.txt | 7 +++++++ 2 files changed, 14 insertions(+) (limited to 'django/utils/encoding.py') diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 7030ab1db0..633022dd3b 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -8,6 +8,7 @@ try: from urllib.parse import quote except ImportError: # Python 2 from urllib import quote +import warnings from django.utils.functional import Promise from django.utils import six @@ -32,6 +33,12 @@ class StrAndUnicode(object): Useful as a mix-in. If you support Python 2 and 3 with a single code base, you can inherit this mix-in and just define __unicode__. """ + def __init__(self, *args, **kwargs): + warnings.warn("StrAndUnicode is deprecated. Define a __str__ method " + "and apply the @python_2_unicode_compatible decorator " + "instead.", PendingDeprecationWarning, stacklevel=2) + super(StrAndUnicode, self).__init__(*args, **kwargs) + if six.PY3: def __str__(self): return self.__unicode__() diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index f789ebde40..4aeb61af94 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -273,3 +273,10 @@ our own copy of ``simplejson``. You can safely change any use of The :func:`~django.utils.itercompat.product` function has been deprecated. Use the built-in :func:`itertools.product` instead. + +``django.utils.encoding.StrAndUnicode`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :class:`~django.utils.encoding.StrAndUnicode` mix-in has been deprecated. +Define a ``__str__`` method and apply the +:func:`~django.utils.encoding.python_2_unicode_compatible` decorator instead. -- cgit v1.3 From 7d48e077b573afb4184301be6565a88e8a9b6e28 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 14 Aug 2012 17:24:31 +0200 Subject: [py3] Fixed staticfiles tests --- django/contrib/staticfiles/storage.py | 4 ++-- django/utils/encoding.py | 2 +- tests/regressiontests/staticfiles_tests/tests.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'django/utils/encoding.py') diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py index 3b431edf94..1c841e9889 100644 --- a/django/contrib/staticfiles/storage.py +++ b/django/contrib/staticfiles/storage.py @@ -51,8 +51,8 @@ class CachedFilesMixin(object): default_template = """url("%s")""" patterns = ( ("*.css", ( - br"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", - (br"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""), + r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", + (r"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""), )), ) diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 633022dd3b..7b80f13496 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -225,7 +225,7 @@ def filepath_to_uri(path): return path # I know about `os.sep` and `os.altsep` but I want to leave # some flexibility for hardcoding separators. - return quote(smart_bytes(path).replace("\\", "/"), safe=b"/~!*()'") + return quote(smart_bytes(path.replace("\\", "/")), safe=b"/~!*()'") # The encoding of the default system locale but falls back to the # given fallback encoding if the encoding is unsupported by python or could diff --git a/tests/regressiontests/staticfiles_tests/tests.py b/tests/regressiontests/staticfiles_tests/tests.py index d68f211535..515b7c3a53 100644 --- a/tests/regressiontests/staticfiles_tests/tests.py +++ b/tests/regressiontests/staticfiles_tests/tests.py @@ -527,11 +527,11 @@ class TestCollectionCachedStorage(BaseCollectionTestCase, """ Handle cache key creation correctly, see #17861. """ - name = b"/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + chr(22) + chr(180) + name = "/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/long filename/ with spaces Here and ?#%#$/other/stuff/some crazy/" + "\x16" + "\xb4" cache_key = storage.staticfiles_storage.cache_key(name) cache_validator = BaseCache({}) cache_validator.validate_key(cache_key) - self.assertEqual(cache_key, 'staticfiles:e95bbc36387084582df2a70750d7b351') + self.assertEqual(cache_key, 'staticfiles:821ea71ef36f95b3922a77f7364670e7') # we set DEBUG to False here since the template tag wouldn't work otherwise @@ -569,8 +569,8 @@ class TestCollectionSimpleCachedStorage(BaseCollectionTestCase, self.assertEqual(relpath, "cached/styles.deploy12345.css") with storage.staticfiles_storage.open(relpath) as relfile: content = relfile.read() - self.assertNotIn("cached/other.css", content) - self.assertIn("other.deploy12345.css", content) + self.assertNotIn(b"cached/other.css", content) + self.assertIn(b"other.deploy12345.css", content) if sys.platform != 'win32': -- cgit v1.3 From 547b181046539f548839e42544521503fbe4d821 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 18 Aug 2012 16:04:06 +0200 Subject: [py3] Ported django.utils.safestring. Backwards compatibility aliases were created under Python 2. --- django/db/backends/mysql/base.py | 8 +-- django/db/backends/postgresql_psycopg2/base.py | 6 +-- django/db/backends/sqlite3/base.py | 4 +- django/utils/encoding.py | 4 +- django/utils/safestring.py | 74 +++++++++++++++----------- docs/howto/custom-template-tags.txt | 6 +-- docs/ref/utils.txt | 22 ++++++-- tests/regressiontests/i18n/tests.py | 12 ++--- 8 files changed, 82 insertions(+), 54 deletions(-) (limited to 'django/utils/encoding.py') diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 2222f89cf0..cec3b04f7e 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -38,7 +38,7 @@ from django.db.backends.mysql.creation import DatabaseCreation from django.db.backends.mysql.introspection import DatabaseIntrospection from django.db.backends.mysql.validation import DatabaseValidation from django.utils.functional import cached_property -from django.utils.safestring import SafeString, SafeUnicode +from django.utils.safestring import SafeBytes, SafeText from django.utils import six from django.utils import timezone @@ -75,7 +75,7 @@ def adapt_datetime_with_timezone_support(value, conv): # MySQLdb-1.2.1 returns TIME columns as timedelta -- they are more like # timedelta in terms of actual behavior as they are signed and include days -- # and Django expects time, so we still need to override that. We also need to -# add special handling for SafeUnicode and SafeString as MySQLdb's type +# add special handling for SafeText and SafeBytes as MySQLdb's type # checking is too tight to catch those (see Django ticket #6052). # Finally, MySQLdb always returns naive datetime objects. However, when # timezone support is active, Django expects timezone-aware datetime objects. @@ -402,8 +402,8 @@ class DatabaseWrapper(BaseDatabaseWrapper): kwargs['client_flag'] = CLIENT.FOUND_ROWS kwargs.update(settings_dict['OPTIONS']) self.connection = Database.connect(**kwargs) - self.connection.encoders[SafeUnicode] = self.connection.encoders[six.text_type] - self.connection.encoders[SafeString] = self.connection.encoders[bytes] + self.connection.encoders[SafeText] = self.connection.encoders[six.text_type] + self.connection.encoders[SafeBytes] = self.connection.encoders[bytes] connection_created.send(sender=self.__class__, connection=self) cursor = self.connection.cursor() if new_connection: diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index d553594ec1..f6f534da8c 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -14,7 +14,7 @@ from django.db.backends.postgresql_psycopg2.creation import DatabaseCreation from django.db.backends.postgresql_psycopg2.version import get_version from django.db.backends.postgresql_psycopg2.introspection import DatabaseIntrospection from django.utils.log import getLogger -from django.utils.safestring import SafeUnicode, SafeString +from django.utils.safestring import SafeText, SafeBytes from django.utils import six from django.utils.timezone import utc @@ -29,8 +29,8 @@ DatabaseError = Database.DatabaseError IntegrityError = Database.IntegrityError psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) -psycopg2.extensions.register_adapter(SafeString, psycopg2.extensions.QuotedString) -psycopg2.extensions.register_adapter(SafeUnicode, psycopg2.extensions.QuotedString) +psycopg2.extensions.register_adapter(SafeBytes, psycopg2.extensions.QuotedString) +psycopg2.extensions.register_adapter(SafeText, psycopg2.extensions.QuotedString) logger = getLogger('django.db.backends') diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 775ca20e45..31a16b6a2b 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -20,7 +20,7 @@ from django.db.backends.sqlite3.creation import DatabaseCreation from django.db.backends.sqlite3.introspection import DatabaseIntrospection from django.utils.dateparse import parse_date, parse_datetime, parse_time from django.utils.functional import cached_property -from django.utils.safestring import SafeString +from django.utils.safestring import SafeBytes from django.utils import six from django.utils import timezone @@ -80,7 +80,7 @@ if Database.version_info >= (2, 4, 1): # slow-down, this adapter is only registered for sqlite3 versions # needing it (Python 2.6 and up). Database.register_adapter(str, lambda s: s.decode('utf-8')) - Database.register_adapter(SafeString, lambda s: s.decode('utf-8')) + Database.register_adapter(SafeBytes, lambda s: s.decode('utf-8')) class DatabaseFeatures(BaseDatabaseFeatures): # SQLite cannot handle us only partially reading from a cursor's result set diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 7b80f13496..3ac2c508b6 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -119,8 +119,8 @@ def force_text(s, encoding='utf-8', strings_only=False, errors='strict'): errors) for arg in s]) else: # Note: We use .decode() here, instead of six.text_type(s, encoding, - # errors), so that if s is a SafeString, it ends up being a - # SafeUnicode at the end. + # errors), so that if s is a SafeBytes, it ends up being a + # SafeText at the end. s = s.decode(encoding, errors) except UnicodeDecodeError as e: if not isinstance(s, Exception): diff --git a/django/utils/safestring.py b/django/utils/safestring.py index bfaefd07ee..07e0bf4cea 100644 --- a/django/utils/safestring.py +++ b/django/utils/safestring.py @@ -10,36 +10,43 @@ from django.utils import six class EscapeData(object): pass -class EscapeString(bytes, EscapeData): +class EscapeBytes(bytes, EscapeData): """ - A string that should be HTML-escaped when output. + A byte string that should be HTML-escaped when output. """ pass -class EscapeUnicode(six.text_type, EscapeData): +class EscapeText(six.text_type, EscapeData): """ - A unicode object that should be HTML-escaped when output. + A unicode string object that should be HTML-escaped when output. """ pass +if six.PY3: + EscapeString = EscapeText +else: + EscapeString = EscapeBytes + # backwards compatibility for Python 2 + EscapeUnicode = EscapeText + class SafeData(object): pass -class SafeString(bytes, SafeData): +class SafeBytes(bytes, SafeData): """ - A string subclass that has been specifically marked as "safe" (requires no + A bytes subclass that has been specifically marked as "safe" (requires no further escaping) for HTML output purposes. """ def __add__(self, rhs): """ - Concatenating a safe string with another safe string or safe unicode - object is safe. Otherwise, the result is no longer safe. + Concatenating a safe byte string with another safe byte string or safe + unicode string is safe. Otherwise, the result is no longer safe. """ - t = super(SafeString, self).__add__(rhs) - if isinstance(rhs, SafeUnicode): - return SafeUnicode(t) - elif isinstance(rhs, SafeString): - return SafeString(t) + t = super(SafeBytes, self).__add__(rhs) + if isinstance(rhs, SafeText): + return SafeText(t) + elif isinstance(rhs, SafeBytes): + return SafeBytes(t) return t def _proxy_method(self, *args, **kwargs): @@ -51,25 +58,25 @@ class SafeString(bytes, SafeData): method = kwargs.pop('method') data = method(self, *args, **kwargs) if isinstance(data, bytes): - return SafeString(data) + return SafeBytes(data) else: - return SafeUnicode(data) + return SafeText(data) decode = curry(_proxy_method, method=bytes.decode) -class SafeUnicode(six.text_type, SafeData): +class SafeText(six.text_type, SafeData): """ - A unicode subclass that has been specifically marked as "safe" for HTML - output purposes. + A unicode (Python 2) / str (Python 3) subclass that has been specifically + marked as "safe" for HTML output purposes. """ def __add__(self, rhs): """ - Concatenating a safe unicode object with another safe string or safe - unicode object is safe. Otherwise, the result is no longer safe. + Concatenating a safe unicode string with another safe byte string or + safe unicode string is safe. Otherwise, the result is no longer safe. """ - t = super(SafeUnicode, self).__add__(rhs) + t = super(SafeText, self).__add__(rhs) if isinstance(rhs, SafeData): - return SafeUnicode(t) + return SafeText(t) return t def _proxy_method(self, *args, **kwargs): @@ -81,12 +88,19 @@ class SafeUnicode(six.text_type, SafeData): method = kwargs.pop('method') data = method(self, *args, **kwargs) if isinstance(data, bytes): - return SafeString(data) + return SafeBytes(data) else: - return SafeUnicode(data) + return SafeText(data) encode = curry(_proxy_method, method=six.text_type.encode) +if six.PY3: + SafeString = SafeText +else: + SafeString = SafeBytes + # backwards compatibility for Python 2 + SafeUnicode = SafeText + def mark_safe(s): """ Explicitly mark a string as safe for (HTML) output purposes. The returned @@ -97,10 +111,10 @@ def mark_safe(s): if isinstance(s, SafeData): return s if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes): - return SafeString(s) + return SafeBytes(s) if isinstance(s, (six.text_type, Promise)): - return SafeUnicode(s) - return SafeString(bytes(s)) + return SafeText(s) + return SafeString(str(s)) def mark_for_escaping(s): """ @@ -113,8 +127,8 @@ def mark_for_escaping(s): if isinstance(s, (SafeData, EscapeData)): return s if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes): - return EscapeString(s) + return EscapeBytes(s) if isinstance(s, (six.text_type, Promise)): - return EscapeUnicode(s) - return EscapeString(bytes(s)) + return EscapeText(s) + return EscapeBytes(bytes(s)) diff --git a/docs/howto/custom-template-tags.txt b/docs/howto/custom-template-tags.txt index 054c831fad..5b27af82d6 100644 --- a/docs/howto/custom-template-tags.txt +++ b/docs/howto/custom-template-tags.txt @@ -189,7 +189,7 @@ passed around inside the template code: They're commonly used for output that contains raw HTML that is intended to be interpreted as-is on the client side. - Internally, these strings are of type ``SafeString`` or ``SafeUnicode``. + Internally, these strings are of type ``SafeBytes`` or ``SafeText``. They share a common base class of ``SafeData``, so you can test for them using code like: @@ -204,8 +204,8 @@ passed around inside the template code: not. These strings are only escaped once, however, even if auto-escaping applies. - Internally, these strings are of type ``EscapeString`` or - ``EscapeUnicode``. Generally you don't have to worry about these; they + Internally, these strings are of type ``EscapeBytes`` or + ``EscapeText``. Generally you don't have to worry about these; they exist for the implementation of the :tfilter:`escape` filter. Template filter code falls into one of two situations: diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 97643c29e9..20775fcc81 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -560,15 +560,29 @@ string" means that the producer of the string has already turned characters that should not be interpreted by the HTML engine (e.g. '<') into the appropriate entities. +.. class:: SafeBytes + + .. versionadded:: 1.5 + + A :class:`bytes` subclass that has been specifically marked as "safe" + (requires no further escaping) for HTML output purposes. + .. class:: SafeString - A string subclass that has been specifically marked as "safe" (requires no - further escaping) for HTML output purposes. + A :class:`str` subclass that has been specifically marked as "safe" + (requires no further escaping) for HTML output purposes. This is + :class:`SafeBytes` on Python 2 and :class:`SafeText` on Python 3. + +.. class:: SafeText + + .. versionadded:: 1.5 + + A :class:`str` (in Python 3) or :class:`unicode` (in Python 2) subclass + that has been specifically marked as "safe" for HTML output purposes. .. class:: SafeUnicode - A unicode subclass that has been specifically marked as "safe" for HTML - output purposes. + Historical name of :class:`SafeText`. Only available under Python 2. .. function:: mark_safe(s) diff --git a/tests/regressiontests/i18n/tests.py b/tests/regressiontests/i18n/tests.py index 1a8ff25ff3..c13c1c6f86 100644 --- a/tests/regressiontests/i18n/tests.py +++ b/tests/regressiontests/i18n/tests.py @@ -18,7 +18,7 @@ from django.utils.formats import (get_format, date_format, time_format, number_format) from django.utils.importlib import import_module from django.utils.numberformat import format as nformat -from django.utils.safestring import mark_safe, SafeString, SafeUnicode +from django.utils.safestring import mark_safe, SafeBytes, SafeString, SafeText from django.utils import six from django.utils.six import PY3 from django.utils.translation import (ugettext, ugettext_lazy, activate, @@ -235,9 +235,9 @@ class TranslationTests(TestCase): s = mark_safe(str('Password')) self.assertEqual(SafeString, type(s)) with translation.override('de', deactivate=True): - self.assertEqual(SafeUnicode, type(ugettext(s))) - self.assertEqual('aPassword', SafeString('a') + s) - self.assertEqual('Passworda', s + SafeString('a')) + self.assertEqual(SafeText, type(ugettext(s))) + self.assertEqual('aPassword', SafeText('a') + s) + self.assertEqual('Passworda', s + SafeText('a')) self.assertEqual('Passworda', s + mark_safe('a')) self.assertEqual('aPassword', mark_safe('a') + s) self.assertEqual('as', mark_safe('a') + mark_safe('s')) @@ -897,9 +897,9 @@ class TestModels(TestCase): def test_safestr(self): c = Company(cents_paid=12, products_delivered=1) - c.name = SafeUnicode('Iñtërnâtiônàlizætiøn1') + c.name = SafeText('Iñtërnâtiônàlizætiøn1') c.save() - c.name = SafeString('Iñtërnâtiônàlizætiøn1'.encode('utf-8')) + c.name = SafeBytes('Iñtërnâtiônàlizætiøn1'.encode('utf-8')) c.save() -- cgit v1.3 From a120fac65a17137bc8ac710477478474e3f9973e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 18 Aug 2012 16:36:55 +0200 Subject: Introduced force_bytes and force_str. This is consistent with the smart_* series of functions and it's going to be used by the next commit. --- django/utils/encoding.py | 19 +++++++++++++++++++ docs/ref/utils.txt | 17 ++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) (limited to 'django/utils/encoding.py') diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 3ac2c508b6..7027b82a61 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -139,6 +139,19 @@ def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): """ Returns a bytestring version of 's', encoded as specified in 'encoding'. + If strings_only is True, don't convert (some) non-string-like objects. + """ + if isinstance(s, Promise): + # The input is the result of a gettext_lazy() call. + return s + return force_bytes(s, encoding, strings_only, errors) + + +def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): + """ + Similar to smart_bytes, except that lazy instances are resolved to + strings, rather than kept as lazy objects. + If strings_only is True, don't convert (some) non-string-like objects. """ if isinstance(s, bytes): @@ -169,8 +182,10 @@ def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): if six.PY3: smart_str = smart_text + force_str = force_text else: smart_str = smart_bytes + force_str = force_bytes # backwards compatibility for Python 2 smart_unicode = smart_text force_unicode = force_text @@ -181,6 +196,10 @@ Apply smart_text in Python 3 and smart_bytes in Python 2. This is suitable for writing to sys.stdout (for instance). """ +force_str.__doc__ = """\ +Apply force_text in Python 3 and force_bytes in Python 2. +""" + def iri_to_uri(iri): """ Convert an Internationalized Resource Identifier (IRI) portion to a URI diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 775d70738b..de19578cac 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -240,14 +240,29 @@ The functions defined in this module share the following properties: If ``strings_only`` is ``True``, don't convert (some) non-string-like objects. +.. function:: force_bytes(s, encoding='utf-8', strings_only=False, errors='strict') + + .. versionadded:: 1.5 + + Similar to ``smart_bytes``, except that lazy instances are resolved to + bytestrings, rather than kept as lazy objects. + + If ``strings_only`` is ``True``, don't convert (some) non-string-like + objects. + .. function:: smart_str(s, encoding='utf-8', strings_only=False, errors='strict') Alias of :func:`smart_bytes` on Python 2 and :func:`smart_text` on Python - 3. This function always returns a :class:`str`. + 3. This function returns a :class:`str` or a lazy string. For instance, this is suitable for writing to :attr:`sys.stdout` on Python 2 and 3. +.. function:: force_str(s, encoding='utf-8', strings_only=False, errors='strict') + + Alias of :func:`force_bytes` on Python 2 and :func:`force_text` on Python + 3. This function always returns a :class:`str`. + .. function:: iri_to_uri(iri) Convert an Internationalized Resource Identifier (IRI) portion to a URI -- cgit v1.3 From ebc773ada3e4f40cf5084268387b873d7fe22e8b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 28 Aug 2012 20:59:56 +0200 Subject: Replaced many smart_bytes by force_bytes In all those occurrences, we didn't care about preserving the lazy status of the strings, but we really wanted to obtain a real bytestring. --- django/contrib/admin/util.py | 4 ++-- django/contrib/admin/views/main.py | 4 ++-- django/contrib/admindocs/utils.py | 4 ++-- django/contrib/auth/hashers.py | 8 ++++---- django/contrib/gis/sitemaps/views.py | 3 +-- django/contrib/markup/templatetags/markup.py | 6 +++--- django/contrib/sessions/backends/base.py | 4 ++-- django/contrib/staticfiles/storage.py | 6 +++--- django/core/cache/backends/db.py | 4 ++-- django/core/cache/backends/filebased.py | 4 ++-- django/core/files/base.py | 2 +- django/core/serializers/json.py | 1 - django/core/serializers/pyyaml.py | 1 - django/db/backends/oracle/base.py | 10 +++++----- django/db/backends/util.py | 4 ++-- django/http/__init__.py | 10 +++++----- django/templatetags/cache.py | 6 +++--- django/test/client.py | 10 +++++----- django/test/testcases.py | 2 +- django/utils/cache.py | 6 +++--- django/utils/crypto.py | 8 ++++---- django/utils/encoding.py | 6 +++--- django/utils/html.py | 4 ++-- django/views/debug.py | 4 ++-- docs/ref/models/instances.txt | 4 ++-- docs/ref/settings.txt | 2 +- docs/releases/1.5.txt | 2 +- docs/topics/cache.txt | 2 +- tests/regressiontests/admin_util/tests.py | 2 +- tests/regressiontests/admin_views/tests.py | 8 ++++---- tests/regressiontests/file_uploads/tests.py | 6 +++--- tests/regressiontests/file_uploads/views.py | 4 ++-- 32 files changed, 74 insertions(+), 77 deletions(-) (limited to 'django/utils/encoding.py') diff --git a/django/contrib/admin/util.py b/django/contrib/admin/util.py index ff90e1d007..889f692ac3 100644 --- a/django/contrib/admin/util.py +++ b/django/contrib/admin/util.py @@ -12,7 +12,7 @@ from django.utils import formats from django.utils.html import format_html from django.utils.text import capfirst from django.utils import timezone -from django.utils.encoding import force_text, smart_text, smart_bytes +from django.utils.encoding import force_str, force_text, smart_text from django.utils import six from django.utils.translation import ungettext from django.core.urlresolvers import reverse @@ -277,7 +277,7 @@ def label_for_field(name, model, model_admin=None, return_attr=False): label = force_text(model._meta.verbose_name) attr = six.text_type elif name == "__str__": - label = smart_bytes(model._meta.verbose_name) + label = force_str(model._meta.verbose_name) attr = bytes else: if callable(name): diff --git a/django/contrib/admin/views/main.py b/django/contrib/admin/views/main.py index 30074f1648..74ef095b4b 100644 --- a/django/contrib/admin/views/main.py +++ b/django/contrib/admin/views/main.py @@ -6,7 +6,7 @@ from django.core.paginator import InvalidPage from django.db import models from django.db.models.fields import FieldDoesNotExist from django.utils.datastructures import SortedDict -from django.utils.encoding import force_text, smart_bytes +from django.utils.encoding import force_str, force_text from django.utils.translation import ugettext, ugettext_lazy from django.utils.http import urlencode @@ -94,7 +94,7 @@ class ChangeList(object): # 'key' will be used as a keyword argument later, so Python # requires it to be a string. del lookup_params[key] - lookup_params[smart_bytes(key)] = value + lookup_params[force_str(key)] = value if not self.model_admin.lookup_allowed(key, value): raise SuspiciousOperation("Filtering by %s not allowed" % key) diff --git a/django/contrib/admindocs/utils.py b/django/contrib/admindocs/utils.py index 0e10eb4fa3..9be0093dfc 100644 --- a/django/contrib/admindocs/utils.py +++ b/django/contrib/admindocs/utils.py @@ -6,7 +6,7 @@ from email.errors import HeaderParseError from django.utils.safestring import mark_safe from django.core.urlresolvers import reverse -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes try: import docutils.core import docutils.nodes @@ -66,7 +66,7 @@ def parse_rst(text, default_reference_context, thing_being_parsed=None): "link_base" : reverse('django-admindocs-docroot').rstrip('/') } if thing_being_parsed: - thing_being_parsed = smart_bytes("<%s>" % thing_being_parsed) + thing_being_parsed = force_bytes("<%s>" % thing_being_parsed) parts = docutils.core.publish_parts(text, source_path=thing_being_parsed, destination_path=None, writer_name='html', settings_overrides=overrides) diff --git a/django/contrib/auth/hashers.py b/django/contrib/auth/hashers.py index cdaf75636f..bd0c6778c9 100644 --- a/django/contrib/auth/hashers.py +++ b/django/contrib/auth/hashers.py @@ -8,7 +8,7 @@ from django.conf import settings from django.test.signals import setting_changed from django.utils import importlib from django.utils.datastructures import SortedDict -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.core.exceptions import ImproperlyConfigured from django.utils.crypto import ( pbkdf2, constant_time_compare, get_random_string) @@ -299,7 +299,7 @@ class SHA1PasswordHasher(BasePasswordHasher): def encode(self, password, salt): assert password assert salt and '$' not in salt - hash = hashlib.sha1(smart_bytes(salt + password)).hexdigest() + hash = hashlib.sha1(force_bytes(salt + password)).hexdigest() return "%s$%s$%s" % (self.algorithm, salt, hash) def verify(self, password, encoded): @@ -327,7 +327,7 @@ class MD5PasswordHasher(BasePasswordHasher): def encode(self, password, salt): assert password assert salt and '$' not in salt - hash = hashlib.md5(smart_bytes(salt + password)).hexdigest() + hash = hashlib.md5(force_bytes(salt + password)).hexdigest() return "%s$%s$%s" % (self.algorithm, salt, hash) def verify(self, password, encoded): @@ -361,7 +361,7 @@ class UnsaltedMD5PasswordHasher(BasePasswordHasher): return '' def encode(self, password, salt): - return hashlib.md5(smart_bytes(password)).hexdigest() + return hashlib.md5(force_bytes(password)).hexdigest() def verify(self, password, encoded): encoded_2 = self.encode(password, '') diff --git a/django/contrib/gis/sitemaps/views.py b/django/contrib/gis/sitemaps/views.py index 8bcdba1b44..36e48f7d20 100644 --- a/django/contrib/gis/sitemaps/views.py +++ b/django/contrib/gis/sitemaps/views.py @@ -8,7 +8,6 @@ from django.core.paginator import EmptyPage, PageNotAnInteger from django.contrib.gis.db.models.fields import GeometryField from django.db import connections, DEFAULT_DB_ALIAS from django.db.models import get_model -from django.utils.encoding import smart_bytes from django.utils import six from django.utils.translation import ugettext as _ @@ -61,7 +60,7 @@ def sitemap(request, sitemaps, section=None): raise Http404(_("Page %s empty") % page) except PageNotAnInteger: raise Http404(_("No page '%s'") % page) - xml = smart_bytes(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})) + xml = loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls}) return HttpResponse(xml, content_type='application/xml') def kml(request, label, model, field_name=None, compress=False, using=DEFAULT_DB_ALIAS): diff --git a/django/contrib/markup/templatetags/markup.py b/django/contrib/markup/templatetags/markup.py index af9c842f42..18b7475ca7 100644 --- a/django/contrib/markup/templatetags/markup.py +++ b/django/contrib/markup/templatetags/markup.py @@ -13,7 +13,7 @@ markup syntaxes to HTML; currently there is support for: from django import template from django.conf import settings -from django.utils.encoding import smart_bytes, force_text +from django.utils.encoding import force_bytes, force_text from django.utils.safestring import mark_safe register = template.Library() @@ -27,7 +27,7 @@ def textile(value): raise template.TemplateSyntaxError("Error in 'textile' filter: The Python textile library isn't installed.") return force_text(value) else: - return mark_safe(force_text(textile.textile(smart_bytes(value), encoding='utf-8', output='utf-8'))) + return mark_safe(force_text(textile.textile(force_bytes(value), encoding='utf-8', output='utf-8'))) @register.filter(is_safe=True) def markdown(value, arg=''): @@ -80,5 +80,5 @@ def restructuredtext(value): return force_text(value) else: docutils_settings = getattr(settings, "RESTRUCTUREDTEXT_FILTER_SETTINGS", {}) - parts = publish_parts(source=smart_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings) + parts = publish_parts(source=force_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings) return mark_safe(force_text(parts["fragment"])) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 2fb7991b49..c8393f23c6 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -14,7 +14,7 @@ from django.utils.crypto import constant_time_compare from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes class CreateError(Exception): """ @@ -84,7 +84,7 @@ class SessionBase(object): return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii') def decode(self, session_data): - encoded_data = base64.b64decode(smart_bytes(session_data)) + encoded_data = base64.b64decode(force_bytes(session_data)) try: # could produce ValueError if there is no ':' hash, pickled = encoded_data.split(b':', 1) diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py index 4be7540c6e..9691b7849d 100644 --- a/django/contrib/staticfiles/storage.py +++ b/django/contrib/staticfiles/storage.py @@ -16,7 +16,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files.base import ContentFile from django.core.files.storage import FileSystemStorage, get_storage_class from django.utils.datastructures import SortedDict -from django.utils.encoding import force_text, smart_bytes +from django.utils.encoding import force_bytes, force_text from django.utils.functional import LazyObject from django.utils.importlib import import_module @@ -118,7 +118,7 @@ class CachedFilesMixin(object): return urlunsplit(unparsed_name) def cache_key(self, name): - return 'staticfiles:%s' % hashlib.md5(smart_bytes(name)).hexdigest() + return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest() def url(self, name, force=False): """ @@ -254,7 +254,7 @@ class CachedFilesMixin(object): if hashed_file_exists: self.delete(hashed_name) # then save the processed result - content_file = ContentFile(smart_bytes(content)) + content_file = ContentFile(force_bytes(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_text(saved_name.replace('\\', '/')) processed = True diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py index 52db4d1b1d..348b03f733 100644 --- a/django/core/cache/backends/db.py +++ b/django/core/cache/backends/db.py @@ -12,7 +12,7 @@ from django.conf import settings from django.core.cache.backends.base import BaseCache from django.db import connections, router, transaction, DatabaseError from django.utils import timezone -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes class Options(object): @@ -73,7 +73,7 @@ class DatabaseCache(BaseDatabaseCache): transaction.commit_unless_managed(using=db) return default value = connections[db].ops.process_clob(row[1]) - return pickle.loads(base64.b64decode(smart_bytes(value))) + return pickle.loads(base64.b64decode(force_bytes(value))) def set(self, key, value, timeout=None, version=None): key = self.make_key(key, version=version) diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index c54e8d280f..96194d458f 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -10,7 +10,7 @@ except ImportError: import pickle from django.core.cache.backends.base import BaseCache -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes class FileBasedCache(BaseCache): def __init__(self, dir, params): @@ -137,7 +137,7 @@ class FileBasedCache(BaseCache): Thus, a cache key of "foo" gets turnned into a file named ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``. """ - path = hashlib.md5(smart_bytes(key)).hexdigest() + path = hashlib.md5(force_bytes(key)).hexdigest() path = os.path.join(path[:2], path[2:4], path[4:]) return os.path.join(self._dir, path) diff --git a/django/core/files/base.py b/django/core/files/base.py index 4a422be90d..4f40606c53 100644 --- a/django/core/files/base.py +++ b/django/core/files/base.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import os from io import BytesIO, UnsupportedOperation -from django.utils.encoding import smart_bytes, smart_text +from django.utils.encoding import smart_text from django.core.files.utils import FileProxyMixin from django.utils.encoding import python_2_unicode_compatible diff --git a/django/core/serializers/json.py b/django/core/serializers/json.py index 4ba0d7fd79..ed65f2922c 100644 --- a/django/core/serializers/json.py +++ b/django/core/serializers/json.py @@ -12,7 +12,6 @@ import json from django.core.serializers.base import DeserializationError from django.core.serializers.python import Serializer as PythonSerializer from django.core.serializers.python import Deserializer as PythonDeserializer -from django.utils.encoding import smart_bytes from django.utils import six from django.utils.timezone import is_aware diff --git a/django/core/serializers/pyyaml.py b/django/core/serializers/pyyaml.py index 9be1ea4492..4c11626bad 100644 --- a/django/core/serializers/pyyaml.py +++ b/django/core/serializers/pyyaml.py @@ -12,7 +12,6 @@ from django.db import models from django.core.serializers.base import DeserializationError from django.core.serializers.python import Serializer as PythonSerializer from django.core.serializers.python import Deserializer as PythonDeserializer -from django.utils.encoding import smart_bytes from django.utils import six diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 5913250206..6bf2e815a7 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -51,7 +51,7 @@ from django.db.backends.signals import connection_created from django.db.backends.oracle.client import DatabaseClient from django.db.backends.oracle.creation import DatabaseCreation from django.db.backends.oracle.introspection import DatabaseIntrospection -from django.utils.encoding import smart_bytes, force_text +from django.utils.encoding import force_bytes, force_text from django.utils import six from django.utils import timezone @@ -64,7 +64,7 @@ IntegrityError = Database.IntegrityError if int(Database.version.split('.', 1)[0]) >= 5 and not hasattr(Database, 'UNICODE'): convert_unicode = force_text else: - convert_unicode = smart_bytes + convert_unicode = force_bytes class DatabaseFeatures(BaseDatabaseFeatures): @@ -602,9 +602,9 @@ class OracleParam(object): elif param is False: param = "0" if hasattr(param, 'bind_parameter'): - self.smart_bytes = param.bind_parameter(cursor) + self.force_bytes = param.bind_parameter(cursor) else: - self.smart_bytes = convert_unicode(param, cursor.charset, + self.force_bytes = convert_unicode(param, cursor.charset, strings_only) if hasattr(param, 'input_size'): # If parameter has `input_size` attribute, use that. @@ -683,7 +683,7 @@ class FormatStylePlaceholderCursor(object): self.setinputsizes(*sizes) def _param_generator(self, params): - return [p.smart_bytes for p in params] + return [p.force_bytes for p in params] def execute(self, query, params=None): if params is None: diff --git a/django/db/backends/util.py b/django/db/backends/util.py index 9d70248ebf..75d4d07a66 100644 --- a/django/db/backends/util.py +++ b/django/db/backends/util.py @@ -6,7 +6,7 @@ import hashlib from time import time from django.conf import settings -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.utils.log import getLogger from django.utils.timezone import utc @@ -138,7 +138,7 @@ def truncate_name(name, length=None, hash_len=4): if length is None or len(name) <= length: return name - hsh = hashlib.md5(smart_bytes(name)).hexdigest()[:hash_len] + hsh = hashlib.md5(force_bytes(name)).hexdigest()[:hash_len] return '%s%s' % (name[:length-hash_len], hsh) def format_number(value, max_digits, decimal_places): diff --git a/django/http/__init__.py b/django/http/__init__.py index c6f104695d..b458fe6c2e 100644 --- a/django/http/__init__.py +++ b/django/http/__init__.py @@ -85,7 +85,7 @@ from django.core.files import uploadhandler from django.http.multipartparser import MultiPartParser from django.http.utils import * from django.utils.datastructures import MultiValueDict, ImmutableList -from django.utils.encoding import smart_bytes, smart_str, iri_to_uri, force_text +from django.utils.encoding import force_bytes, force_text, smart_str, iri_to_uri from django.utils.http import cookie_date from django.utils import six from django.utils import timezone @@ -489,13 +489,13 @@ class QueryDict(MultiValueDict): """ output = [] if safe: - safe = smart_bytes(safe, self.encoding) + safe = force_bytes(safe, self.encoding) encode = lambda k, v: '%s=%s' % ((quote(k, safe), quote(v, safe))) else: encode = lambda k, v: urlencode({k: v}) for k, list_ in self.lists(): - k = smart_bytes(k, self.encoding) - output.extend([encode(k, smart_bytes(v, self.encoding)) + k = force_bytes(k, self.encoding) + output.extend([encode(k, force_bytes(v, self.encoding)) for v in list_]) return '&'.join(output) @@ -680,7 +680,7 @@ class HttpResponse(object): # force conversion to bytes in case chunk is a subclass return bytes(value) return b''.join(make_bytes(e) for e in self._container) - return b''.join(smart_bytes(e, self._charset) for e in self._container) + return b''.join(force_bytes(e, self._charset) for e in self._container) @content.setter def content(self, value): diff --git a/django/templatetags/cache.py b/django/templatetags/cache.py index e431f99d0d..36db4807c7 100644 --- a/django/templatetags/cache.py +++ b/django/templatetags/cache.py @@ -4,7 +4,7 @@ import hashlib from django.template import Library, Node, TemplateSyntaxError, Variable, VariableDoesNotExist from django.template import resolve_variable from django.core.cache import cache -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.utils.http import urlquote register = Library() @@ -26,8 +26,8 @@ class CacheNode(Node): except (ValueError, TypeError): raise TemplateSyntaxError('"cache" tag got a non-integer timeout value: %r' % expire_time) # Build a key for this fragment and all vary-on's. - key = smart_bytes(':'.join([urlquote(resolve_variable(var, context)) for var in self.vary_on])) - args = hashlib.md5(key) + key = ':'.join([urlquote(resolve_variable(var, context)) for var in self.vary_on]) + args = hashlib.md5(force_bytes(key)) cache_key = 'template.cache.%s.%s' % (self.fragment_name, args.hexdigest()) value = cache.get(cache_key) if value is None: diff --git a/django/test/client.py b/django/test/client.py index 2b61c51ce1..8fd765ec9a 100644 --- a/django/test/client.py +++ b/django/test/client.py @@ -21,7 +21,7 @@ from django.http import SimpleCookie, HttpRequest, QueryDict from django.template import TemplateDoesNotExist from django.test import signals from django.utils.functional import curry -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.utils.http import urlencode from django.utils.importlib import import_module from django.utils.itercompat import is_iterable @@ -110,7 +110,7 @@ def encode_multipart(boundary, data): as an application/octet-stream; otherwise, str(value) will be sent. """ lines = [] - to_bytes = lambda s: smart_bytes(s, settings.DEFAULT_CHARSET) + to_bytes = lambda s: force_bytes(s, settings.DEFAULT_CHARSET) # Not by any means perfect, but good enough for our purposes. is_file = lambda thing: hasattr(thing, "read") and callable(thing.read) @@ -147,7 +147,7 @@ def encode_multipart(boundary, data): return b'\r\n'.join(lines) def encode_file(boundary, key, file): - to_bytes = lambda s: smart_bytes(s, settings.DEFAULT_CHARSET) + to_bytes = lambda s: force_bytes(s, settings.DEFAULT_CHARSET) content_type = mimetypes.guess_type(file.name)[0] if content_type is None: content_type = 'application/octet-stream' @@ -222,7 +222,7 @@ class RequestFactory(object): charset = match.group(1) else: charset = settings.DEFAULT_CHARSET - return smart_bytes(data, encoding=charset) + return force_bytes(data, encoding=charset) def _get_path(self, parsed): # If there are parameters, add them @@ -293,7 +293,7 @@ class RequestFactory(object): def generic(self, method, path, data='', content_type='application/octet-stream', **extra): parsed = urlparse(path) - data = smart_bytes(data, settings.DEFAULT_CHARSET) + data = force_bytes(data, settings.DEFAULT_CHARSET) r = { 'PATH_INFO': self._get_path(parsed), 'QUERY_STRING': parsed[4], diff --git a/django/test/testcases.py b/django/test/testcases.py index e0f2655ee8..f12c431d3a 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -41,7 +41,7 @@ from django.test.utils import (get_warnings_state, restore_warnings_state, override_settings) from django.test.utils import ContextList from django.utils import unittest as ut2 -from django.utils.encoding import smart_bytes, force_text +from django.utils.encoding import force_text from django.utils import six from django.utils.unittest.util import safe_repr from django.views.static import serve diff --git a/django/utils/cache.py b/django/utils/cache.py index 3e99833aa6..91c4796988 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -24,7 +24,7 @@ import time from django.conf import settings from django.core.cache import get_cache -from django.utils.encoding import iri_to_uri, force_text, smart_bytes +from django.utils.encoding import iri_to_uri, force_bytes, force_text from django.utils.http import http_date from django.utils.timezone import get_current_timezone_name from django.utils.translation import get_language @@ -181,14 +181,14 @@ def _generate_cache_key(request, method, headerlist, key_prefix): value = request.META.get(header, None) if value is not None: ctx.update(value) - path = hashlib.md5(smart_bytes(iri_to_uri(request.get_full_path()))) + path = hashlib.md5(force_bytes(iri_to_uri(request.get_full_path()))) cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % ( key_prefix, method, path.hexdigest(), ctx.hexdigest()) return _i18n_cache_key_suffix(request, cache_key) def _generate_cache_header_key(key_prefix, request): """Returns a cache key for the header cache.""" - path = hashlib.md5(smart_bytes(iri_to_uri(request.get_full_path()))) + path = hashlib.md5(force_bytes(iri_to_uri(request.get_full_path()))) cache_key = 'views.decorators.cache.cache_header.%s.%s' % ( key_prefix, path.hexdigest()) return _i18n_cache_key_suffix(request, cache_key) diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 1fdcc3082e..57bc60dc4f 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -23,7 +23,7 @@ except NotImplementedError: using_sysrandom = False from django.conf import settings -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.utils import six from django.utils.six.moves import xrange @@ -51,7 +51,7 @@ def salted_hmac(key_salt, value, secret=None): # line is redundant and could be replaced by key = key_salt + secret, since # the hmac module does the same thing for keys longer than the block size. # However, we need to ensure that we *always* do this. - return hmac.new(key, msg=smart_bytes(value), digestmod=hashlib.sha1) + return hmac.new(key, msg=force_bytes(value), digestmod=hashlib.sha1) def get_random_string(length=12, @@ -147,8 +147,8 @@ def pbkdf2(password, salt, iterations, dklen=0, digest=None): assert iterations > 0 if not digest: digest = hashlib.sha256 - password = smart_bytes(password) - salt = smart_bytes(salt) + password = force_bytes(password) + salt = force_bytes(salt) hlen = digest().digest_size if not dklen: dklen = hlen diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 7027b82a61..998cf4f3f7 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -174,7 +174,7 @@ def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): # An Exception subclass containing non-ASCII data that doesn't # know how to print itself properly. We shouldn't raise a # further exception. - return ' '.join([smart_bytes(arg, encoding, strings_only, + return ' '.join([force_bytes(arg, encoding, strings_only, errors) for arg in s]) return six.text_type(s).encode(encoding, errors) else: @@ -225,7 +225,7 @@ def iri_to_uri(iri): # converted. if iri is None: return iri - return quote(smart_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~") + return quote(force_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~") def filepath_to_uri(path): """Convert an file system path to a URI portion that is suitable for @@ -244,7 +244,7 @@ def filepath_to_uri(path): return path # I know about `os.sep` and `os.altsep` but I want to leave # some flexibility for hardcoding separators. - return quote(smart_bytes(path.replace("\\", "/")), safe=b"/~!*()'") + return quote(force_bytes(path.replace("\\", "/")), safe=b"/~!*()'") # The encoding of the default system locale but falls back to the # given fallback encoding if the encoding is unsupported by python or could diff --git a/django/utils/html.py b/django/utils/html.py index 0ee789ebb5..2b669cc8ec 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -11,7 +11,7 @@ except ImportError: # Python 2 from urlparse import urlsplit, urlunsplit from django.utils.safestring import SafeData, mark_safe -from django.utils.encoding import smart_bytes, force_text +from django.utils.encoding import force_bytes, force_text from django.utils.functional import allow_lazy from django.utils import six from django.utils.text import normalize_newlines @@ -164,7 +164,7 @@ def smart_urlquote(url): # contains a % not followed by two hexadecimal digits. See #9655. if '%' not in url or unquoted_percents_re.search(url): # See http://bugs.python.org/issue2637 - url = quote(smart_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~') + url = quote(force_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~') return force_text(url) diff --git a/django/views/debug.py b/django/views/debug.py index b275ef9e73..ed99d8dfe6 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -14,7 +14,7 @@ from django.template import Template, Context, TemplateDoesNotExist from django.template.defaultfilters import force_escape, pprint from django.utils.html import escape from django.utils.importlib import import_module -from django.utils.encoding import smart_text, smart_bytes +from django.utils.encoding import force_bytes, smart_text from django.utils import six HIDDEN_SETTINGS = re.compile('API|TOKEN|KEY|SECRET|PASS|PROFANITIES_LIST|SIGNATURE') @@ -440,7 +440,7 @@ def technical_404_response(request, exception): 'root_urlconf': settings.ROOT_URLCONF, 'request_path': request.path_info[1:], # Trim leading slash 'urlpatterns': tried, - 'reason': smart_bytes(exception, errors='replace'), + 'reason': force_bytes(exception, errors='replace'), 'request': request, 'settings': get_safe_settings(), }) diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index b7ae3890cf..b01abc1936 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -459,9 +459,9 @@ using ``__str__()`` like this:: last_name = models.CharField(max_length=50) def __str__(self): - # Note use of django.utils.encoding.smart_bytes() here because + # Note use of django.utils.encoding.force_bytes() here because # first_name and last_name will be unicode strings. - return smart_bytes('%s %s' % (self.first_name, self.last_name)) + return force_bytes('%s %s' % (self.first_name, self.last_name)) ``get_absolute_url`` -------------------- diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 32ba5f0ced..4729a2b6f1 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -183,7 +183,7 @@ compose a prefix, version and key into a final cache key. The default implementation is equivalent to the function:: def make_key(key, key_prefix, version): - return ':'.join([key_prefix, str(version), smart_bytes(key)]) + return ':'.join([key_prefix, str(version), key]) You may use any key function you want, as long as it has the same argument signature. diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index 6042569cea..5728d8559a 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -227,7 +227,7 @@ If you have written a :ref:`custom password hasher `, your ``encode()``, ``verify()`` or ``safe_summary()`` methods should accept Unicode parameters (``password``, ``salt`` or ``encoded``). If any of the hashing methods need byte strings, you can use the -:func:`~django.utils.encoding.smart_bytes` utility to encode the strings. +:func:`~django.utils.encoding.force_bytes` utility to encode the strings. Validation of previous_page_number and next_page_number ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/cache.txt b/docs/topics/cache.txt index 219b6c7795..f13238e342 100644 --- a/docs/topics/cache.txt +++ b/docs/topics/cache.txt @@ -864,7 +864,7 @@ key version to provide a final cache key. By default, the three parts are joined using colons to produce a final string:: def make_key(key, key_prefix, version): - return ':'.join([key_prefix, str(version), smart_bytes(key)]) + return ':'.join([key_prefix, str(version), key]) If you want to combine the parts in different ways, or apply other processing to the final key (e.g., taking a hash digest of the key diff --git a/tests/regressiontests/admin_util/tests.py b/tests/regressiontests/admin_util/tests.py index 6b6dad4336..d04740ce95 100644 --- a/tests/regressiontests/admin_util/tests.py +++ b/tests/regressiontests/admin_util/tests.py @@ -171,7 +171,7 @@ class UtilTests(unittest.TestCase): ) self.assertEqual( label_for_field("__str__", Article), - b"article" + str("article") ) self.assertRaises( diff --git a/tests/regressiontests/admin_views/tests.py b/tests/regressiontests/admin_views/tests.py index 679e3a0346..cf7d4855fb 100644 --- a/tests/regressiontests/admin_views/tests.py +++ b/tests/regressiontests/admin_views/tests.py @@ -30,7 +30,7 @@ from django.template.response import TemplateResponse from django.test import TestCase from django.utils import formats, translation, unittest from django.utils.cache import get_max_age -from django.utils.encoding import iri_to_uri, smart_bytes +from django.utils.encoding import iri_to_uri, force_bytes from django.utils.html import escape from django.utils.http import urlencode from django.utils import six @@ -84,7 +84,7 @@ class AdminViewBasicTest(TestCase): content. """ self.assertEqual(response.status_code, 200) - self.assertTrue(response.content.index(smart_bytes(text1)) < response.content.index(smart_bytes(text2)), + self.assertTrue(response.content.index(force_bytes(text1)) < response.content.index(force_bytes(text2)), failing_msg ) @@ -1378,9 +1378,9 @@ class AdminViewStringPrimaryKeyTest(TestCase): logentry.content_type = None logentry.save() - counted_presence_before = response.content.count(smart_bytes(should_contain)) + counted_presence_before = response.content.count(force_bytes(should_contain)) response = self.client.get('/test_admin/admin/') - counted_presence_after = response.content.count(smart_bytes(should_contain)) + counted_presence_after = response.content.count(force_bytes(should_contain)) self.assertEqual(counted_presence_before - 1, counted_presence_after) diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py index 19af992a36..a545ed649e 100644 --- a/tests/regressiontests/file_uploads/tests.py +++ b/tests/regressiontests/file_uploads/tests.py @@ -12,7 +12,7 @@ from django.core.files import temp as tempfile from django.core.files.uploadedfile import SimpleUploadedFile from django.http.multipartparser import MultiPartParser from django.test import TestCase, client -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from django.utils.six import StringIO from django.utils import unittest @@ -54,7 +54,7 @@ class FileUploadTests(TestCase): post_data[key + '_hash'] = hashlib.sha1(post_data[key].read()).hexdigest() post_data[key].seek(0) except AttributeError: - post_data[key + '_hash'] = hashlib.sha1(smart_bytes(post_data[key])).hexdigest() + post_data[key + '_hash'] = hashlib.sha1(force_bytes(post_data[key])).hexdigest() response = self.client.post('/file_uploads/verify/', post_data) @@ -68,7 +68,7 @@ class FileUploadTests(TestCase): 'Content-Type: application/octet-stream', 'Content-Transfer-Encoding: base64', '', - base64.b64encode(smart_bytes(test_string)).decode('ascii'), + base64.b64encode(force_bytes(test_string)).decode('ascii'), '--' + client.BOUNDARY + '--', '', ]).encode('utf-8') diff --git a/tests/regressiontests/file_uploads/views.py b/tests/regressiontests/file_uploads/views.py index 95b2124fd9..fcf32cecea 100644 --- a/tests/regressiontests/file_uploads/views.py +++ b/tests/regressiontests/file_uploads/views.py @@ -7,7 +7,7 @@ import os from django.core.files.uploadedfile import UploadedFile from django.http import HttpResponse, HttpResponseServerError from django.utils import six -from django.utils.encoding import smart_bytes +from django.utils.encoding import force_bytes from .models import FileModel, UPLOAD_TO from .tests import UNICODE_FILENAME @@ -46,7 +46,7 @@ def file_upload_view_verify(request): if isinstance(value, UploadedFile): new_hash = hashlib.sha1(value.read()).hexdigest() else: - new_hash = hashlib.sha1(smart_bytes(value)).hexdigest() + new_hash = hashlib.sha1(force_bytes(value)).hexdigest() if new_hash != submitted_hash: return HttpResponseServerError() -- cgit v1.3 From cc9b767fc60a8b0455a67f00ca23077b55e709bb Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 4 Sep 2012 09:24:39 +0200 Subject: Fixed #18902 -- Made force_bytes properly handle exception input Thanks Aymeric Augustin for the report and the initial patch. --- django/utils/encoding.py | 2 +- tests/regressiontests/utils/encoding.py | 17 +++++++++++++++++ tests/regressiontests/utils/tests.py | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/regressiontests/utils/encoding.py (limited to 'django/utils/encoding.py') diff --git a/django/utils/encoding.py b/django/utils/encoding.py index 998cf4f3f7..3b284f3ed0 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -174,7 +174,7 @@ def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): # An Exception subclass containing non-ASCII data that doesn't # know how to print itself properly. We shouldn't raise a # further exception. - return ' '.join([force_bytes(arg, encoding, strings_only, + return b' '.join([force_bytes(arg, encoding, strings_only, errors) for arg in s]) return six.text_type(s).encode(encoding, errors) else: diff --git a/tests/regressiontests/utils/encoding.py b/tests/regressiontests/utils/encoding.py new file mode 100644 index 0000000000..d191845518 --- /dev/null +++ b/tests/regressiontests/utils/encoding.py @@ -0,0 +1,17 @@ +# -*- encoding: utf-8 -*- +from __future__ import unicode_literals + +from django.utils import unittest +from django.utils.encoding import force_bytes + + +class TestEncodingUtils(unittest.TestCase): + def test_force_bytes_exception(self): + """ + Test that force_bytes knows how to convert to bytes an exception + containing non-ASCII characters in its args. + """ + error_msg = "This is an exception, voilà" + exc = ValueError(error_msg) + result = force_bytes(exc) + self.assertEqual(result, error_msg.encode('utf-8')) diff --git a/tests/regressiontests/utils/tests.py b/tests/regressiontests/utils/tests.py index 08f5f760cc..f4fa75b177 100644 --- a/tests/regressiontests/utils/tests.py +++ b/tests/regressiontests/utils/tests.py @@ -13,6 +13,7 @@ from .dateformat import DateFormatTests from .dateparse import DateParseTests from .datetime_safe import DatetimeTests from .decorators import DecoratorFromMiddlewareTests +from .encoding import TestEncodingUtils from .feedgenerator import FeedgeneratorTest from .functional import FunctionalTestCase from .html import TestUtilsHtml -- cgit v1.3