Merge branch 'master' into schema-alteration

20 files changed, 370 insertions, 200 deletions

diff --git a/django/utils/2to3_fixes/__init__.py b/django/utils/2to3_fixes/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/django/utils/2to3_fixes/__init__.py
diff --git a/django/utils/2to3_fixes/fix_unicode.py b/django/utils/2to3_fixes/fix_unicode.py
new file mode 100644
index 0000000000..613734ca86
--- /dev/null
+++ b/django/utils/2to3_fixes/fix_unicode.py
@@ -0,0 +1,36 @@
+"""Fixer for __unicode__ methods.
+
+Uses the django.utils.encoding.python_2_unicode_compatible decorator.
+"""
+
+from __future__ import unicode_literals
+
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import find_indentation, Name, syms, touch_import
+from lib2to3.pgen2 import token
+from lib2to3.pytree import Leaf, Node
+
+
+class FixUnicode(fixer_base.BaseFix):
+
+    BM_compatible = True
+    PATTERN = """
+    classdef< 'class' any+ ':'
+              suite< any*
+                     funcdef< 'def' unifunc='__unicode__'
+                              parameters< '(' NAME ')' > any+ >
+                     any* > >
+    """
+
+    def transform(self, node, results):
+        unifunc = results["unifunc"]
+        strfunc = Name("__str__", prefix=unifunc.prefix)
+        unifunc.replace(strfunc)
+
+        klass = node.clone()
+        klass.prefix = '\n' + find_indentation(node)
+        decorator = Node(syms.decorator, [Leaf(token.AT, "@"), Name('python_2_unicode_compatible')])
+        decorated = Node(syms.decorated, [decorator, klass], prefix=node.prefix)
+        node.replace(decorated)
+
+        touch_import('django.utils.encoding', 'python_2_unicode_compatible', decorated)
diff --git a/django/utils/archive.py b/django/utils/archive.py
index 829b55dd28..0faf1fa781 100644
--- a/django/utils/archive.py
+++ b/django/utils/archive.py
@@ -46,7 +46,8 @@ def extract(path, to_path=''):
     Unpack the tar or zip file at the specified path to the directory
     specified by to_path.
     """
-    Archive(path).extract(to_path)
+    with Archive(path) as archive:
+        archive.extract(to_path)
 
 
 class Archive(object):
@@ -77,12 +78,21 @@ class Archive(object):
                 "Path not a recognized archive format: %s" % filename)
         return cls
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
     def extract(self, to_path=''):
         self._archive.extract(to_path)
 
     def list(self):
         self._archive.list()
 
+    def close(self):
+        self._archive.close()
+
 
 class BaseArchive(object):
     """
@@ -161,6 +171,9 @@ class TarArchive(BaseArchive):
                     if extracted:
                         extracted.close()
 
+    def close(self):
+        self._archive.close()
+
 
 class ZipArchive(BaseArchive):
 
@@ -189,6 +202,9 @@ class ZipArchive(BaseArchive):
                 with open(filename, 'wb') as outfile:
                     outfile.write(data)
 
+    def close(self):
+        self._archive.close()
+
 extension_map = {
     '.tar': TarArchive,
     '.tar.bz2': TarArchive,
diff --git a/django/utils/autoreload.py b/django/utils/autoreload.py
index b6c055383c..2daafedd85 100644
--- a/django/utils/autoreload.py
+++ b/django/utils/autoreload.py
@@ -31,7 +31,7 @@
 import os, sys, time, signal
 
 try:
-    import thread
+    from django.utils.six.moves import _thread as thread
 except ImportError:
     from django.utils.six.moves import _dummy_thread as thread
 
@@ -54,7 +54,8 @@ _win = (sys.platform == "win32")
 
 def code_changed():
     global _mtimes, _win
-    for filename in filter(lambda v: v, map(lambda m: getattr(m, "__file__", None), sys.modules.values())):
+    filenames = [getattr(m, "__file__", None) for m in sys.modules.values()]
+    for filename in filter(None, filenames):
         if filename.endswith(".pyc") or filename.endswith(".pyo"):
             filename = filename[:-1]
         if filename.endswith("$py.class"):
diff --git a/django/utils/cache.py b/django/utils/cache.py
index 42b0de4ce6..91c4796988 100644
--- a/django/utils/cache.py
+++ b/django/utils/cache.py
@@ -16,6 +16,7 @@ cache keys to prevent delivery of wrong content.
 An example: i18n middleware would need to distinguish caches by the
 "Accept-language" header.
 """
+from __future__ import unicode_literals
 
 import hashlib
 import re
@@ -23,7 +24,7 @@ import time
 
 from django.conf import settings
 from django.core.cache import get_cache
-from django.utils.encoding import iri_to_uri, force_text
+from django.utils.encoding import iri_to_uri, force_bytes, force_text
 from django.utils.http import http_date
 from django.utils.timezone import get_current_timezone_name
 from django.utils.translation import get_language
@@ -65,7 +66,7 @@ def patch_cache_control(response, **kwargs):
     # max-age, use the minimum of the two ages. In practice this happens when
     # a decorator and a piece of middleware both operate on a given view.
     if 'max-age' in cc and 'max_age' in kwargs:
-        kwargs['max_age'] = min(cc['max-age'], kwargs['max_age'])
+        kwargs['max_age'] = min(int(cc['max-age']), kwargs['max_age'])
 
     # Allow overriding private caching and vice versa
     if 'private' in cc and 'public' in kwargs:
@@ -170,7 +171,7 @@ def _i18n_cache_key_suffix(request, cache_key):
         # User-defined tzinfo classes may return absolutely anything.
         # Hence this paranoid conversion to create a valid cache key.
         tz_name = force_text(get_current_timezone_name(), errors='ignore')
-        cache_key += '.%s' % tz_name.encode('ascii', 'ignore').replace(' ', '_')
+        cache_key += '.%s' % tz_name.encode('ascii', 'ignore').decode('ascii').replace(' ', '_')
     return cache_key
 
 def _generate_cache_key(request, method, headerlist, key_prefix):
@@ -180,14 +181,14 @@ def _generate_cache_key(request, method, headerlist, key_prefix):
         value = request.META.get(header, None)
         if value is not None:
             ctx.update(value)
-    path = hashlib.md5(iri_to_uri(request.get_full_path()))
+    path = hashlib.md5(force_bytes(iri_to_uri(request.get_full_path())))
     cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % (
         key_prefix, method, path.hexdigest(), ctx.hexdigest())
     return _i18n_cache_key_suffix(request, cache_key)
 
 def _generate_cache_header_key(key_prefix, request):
     """Returns a cache key for the header cache."""
-    path = hashlib.md5(iri_to_uri(request.get_full_path()))
+    path = hashlib.md5(force_bytes(iri_to_uri(request.get_full_path())))
     cache_key = 'views.decorators.cache.cache_header.%s.%s' % (
         key_prefix, path.hexdigest())
     return _i18n_cache_key_suffix(request, cache_key)
diff --git a/django/utils/crypto.py b/django/utils/crypto.py
index 70a07e7fde..57bc60dc4f 100644
--- a/django/utils/crypto.py
+++ b/django/utils/crypto.py
@@ -23,7 +23,8 @@ except NotImplementedError:
     using_sysrandom = False
 
 from django.conf import settings
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
+from django.utils import six
 from django.utils.six.moves import xrange
 
 
@@ -50,7 +51,7 @@ def salted_hmac(key_salt, value, secret=None):
     # line is redundant and could be replaced by key = key_salt + secret, since
     # the hmac module does the same thing for keys longer than the block size.
     # However, we need to ensure that we *always* do this.
-    return hmac.new(key, msg=smart_bytes(value), digestmod=hashlib.sha1)
+    return hmac.new(key, msg=force_bytes(value), digestmod=hashlib.sha1)
 
 
 def get_random_string(length=12,
@@ -88,8 +89,12 @@ def constant_time_compare(val1, val2):
     if len(val1) != len(val2):
         return False
     result = 0
-    for x, y in zip(val1, val2):
-        result |= ord(x) ^ ord(y)
+    if six.PY3 and isinstance(val1, bytes) and isinstance(val2, bytes):
+        for x, y in zip(val1, val2):
+            result |= x ^ y
+    else:
+        for x, y in zip(val1, val2):
+            result |= ord(x) ^ ord(y)
     return result == 0
 
 
@@ -142,8 +147,8 @@ def pbkdf2(password, salt, iterations, dklen=0, digest=None):
     assert iterations > 0
     if not digest:
         digest = hashlib.sha256
-    password = smart_bytes(password)
-    salt = smart_bytes(salt)
+    password = force_bytes(password)
+    salt = force_bytes(salt)
     hlen = digest().digest_size
     if not dklen:
         dklen = hlen
diff --git a/django/utils/dateparse.py b/django/utils/dateparse.py
index 032eb493b6..b4bd559a66 100644
--- a/django/utils/dateparse.py
+++ b/django/utils/dateparse.py
@@ -15,16 +15,16 @@ date_re = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})$'
 )
 
-datetime_re = re.compile(
-    r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'[T ](?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
+time_re = re.compile(
+    r'(?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
     r'(?::(?P<second>\d{1,2})(?:\.(?P<microsecond>\d{1,6})\d{0,6})?)?'
-    r'(?P<tzinfo>Z|[+-]\d{1,2}:\d{1,2})?$'
 )
 
-time_re = re.compile(
-    r'(?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
+datetime_re = re.compile(
+    r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
+    r'[T ](?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
     r'(?::(?P<second>\d{1,2})(?:\.(?P<microsecond>\d{1,6})\d{0,6})?)?'
+    r'(?P<tzinfo>Z|[+-]\d{2}:?\d{2})?$'
 )
 
 def parse_date(value):
@@ -43,8 +43,6 @@ def parse_time(value):
 
     This function doesn't support time zone offsets.
 
-    Sub-microsecond precision is accepted, but ignored.
-
     Raises ValueError if the input is well formatted but not a valid time.
     Returns None if the input isn't well formatted, in particular if it
     contains an offset.
@@ -63,8 +61,6 @@ def parse_datetime(value):
     This function supports time zone offsets. When the input contains one,
     the output uses an instance of FixedOffset as tzinfo.
 
-    Sub-microsecond precision is accepted, but ignored.
-
     Raises ValueError if the input is well formatted but not a valid datetime.
     Returns None if the input isn't well formatted.
     """
@@ -77,7 +73,7 @@ def parse_datetime(value):
         if tzinfo == 'Z':
             tzinfo = utc
         elif tzinfo is not None:
-            offset = 60 * int(tzinfo[1:3]) + int(tzinfo[4:6])
+            offset = 60 * int(tzinfo[1:3]) + int(tzinfo[-2:])
             if tzinfo[0] == '-':
                 offset = -offset
             tzinfo = FixedOffset(offset)
diff --git a/django/utils/encoding.py b/django/utils/encoding.py
index eb60cfde8b..3b284f3ed0 100644
--- a/django/utils/encoding.py
+++ b/django/utils/encoding.py
@@ -8,6 +8,7 @@ try:
     from urllib.parse import quote
 except ImportError:     # Python 2
     from urllib import quote
+import warnings
 
 from django.utils.functional import Promise
 from django.utils import six
@@ -32,6 +33,12 @@ class StrAndUnicode(object):
     Useful as a mix-in. If you support Python 2 and 3 with a single code base,
     you can inherit this mix-in and just define __unicode__.
     """
+    def __init__(self, *args, **kwargs):
+        warnings.warn("StrAndUnicode is deprecated. Define a __str__ method "
+                      "and apply the @python_2_unicode_compatible decorator "
+                      "instead.", PendingDeprecationWarning, stacklevel=2)
+        super(StrAndUnicode, self).__init__(*args, **kwargs)
+
     if six.PY3:
         def __str__(self):
             return self.__unicode__()
@@ -39,6 +46,19 @@ class StrAndUnicode(object):
         def __str__(self):
             return self.__unicode__().encode('utf-8')
 
+def python_2_unicode_compatible(klass):
+    """
+    A decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if not six.PY3:
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
 def smart_text(s, encoding='utf-8', strings_only=False, errors='strict'):
     """
     Returns a text object representing 's' -- unicode on Python 2 and str on
@@ -99,8 +119,8 @@ def force_text(s, encoding='utf-8', strings_only=False, errors='strict'):
                             errors) for arg in s])
         else:
             # Note: We use .decode() here, instead of six.text_type(s, encoding,
-            # errors), so that if s is a SafeString, it ends up being a
-            # SafeUnicode at the end.
+            # errors), so that if s is a SafeBytes, it ends up being a
+            # SafeText at the end.
             s = s.decode(encoding, errors)
     except UnicodeDecodeError as e:
         if not isinstance(s, Exception):
@@ -121,6 +141,19 @@ def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
 
     If strings_only is True, don't convert (some) non-string-like objects.
     """
+    if isinstance(s, Promise):
+        # The input is the result of a gettext_lazy() call.
+        return s
+    return force_bytes(s, encoding, strings_only, errors)
+
+
+def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
+    """
+    Similar to smart_bytes, except that lazy instances are resolved to
+    strings, rather than kept as lazy objects.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
+    """
     if isinstance(s, bytes):
         if encoding == 'utf-8':
             return s
@@ -141,7 +174,7 @@ def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
                 # An Exception subclass containing non-ASCII data that doesn't
                 # know how to print itself properly. We shouldn't raise a
                 # further exception.
-                return ' '.join([smart_bytes(arg, encoding, strings_only,
+                return b' '.join([force_bytes(arg, encoding, strings_only,
                         errors) for arg in s])
             return six.text_type(s).encode(encoding, errors)
     else:
@@ -149,8 +182,10 @@ def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
 
 if six.PY3:
     smart_str = smart_text
+    force_str = force_text
 else:
     smart_str = smart_bytes
+    force_str = force_bytes
     # backwards compatibility for Python 2
     smart_unicode = smart_text
     force_unicode = force_text
@@ -161,6 +196,10 @@ Apply smart_text in Python 3 and smart_bytes in Python 2.
 This is suitable for writing to sys.stdout (for instance).
 """
 
+force_str.__doc__ = """\
+Apply force_text in Python 3 and force_bytes in Python 2.
+"""
+
 def iri_to_uri(iri):
     """
     Convert an Internationalized Resource Identifier (IRI) portion to a URI
@@ -186,7 +225,7 @@ def iri_to_uri(iri):
     # converted.
     if iri is None:
         return iri
-    return quote(smart_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~")
+    return quote(force_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~")
 
 def filepath_to_uri(path):
     """Convert an file system path to a URI portion that is suitable for
@@ -205,7 +244,7 @@ def filepath_to_uri(path):
         return path
     # I know about `os.sep` and `os.altsep` but I want to leave
     # some flexibility for hardcoding separators.
-    return quote(smart_bytes(path).replace("\\", "/"), safe=b"/~!*()'")
+    return quote(force_bytes(path.replace("\\", "/")), safe=b"/~!*()'")
 
 # The encoding of the default system locale but falls back to the
 # given fallback encoding if the encoding is unsupported by python or could
diff --git a/django/utils/formats.py b/django/utils/formats.py
index 2e54d792fa..555982eede 100644
--- a/django/utils/formats.py
+++ b/django/utils/formats.py
@@ -4,7 +4,7 @@ import datetime
 from django.conf import settings
 from django.utils import dateformat, numberformat, datetime_safe
 from django.utils.importlib import import_module
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
 from django.utils.functional import lazy
 from django.utils.safestring import mark_safe
 from django.utils import six
@@ -66,7 +66,7 @@ def get_format(format_type, lang=None, use_l10n=None):
     If use_l10n is provided and is not None, that will force the value to
     be localized (or not), overriding the value of settings.USE_L10N.
     """
-    format_type = smart_str(format_type)
+    format_type = force_str(format_type)
     if use_l10n or (use_l10n is None and settings.USE_L10N):
         if lang is None:
             lang = get_language()
@@ -160,14 +160,14 @@ def localize_input(value, default=None):
         return number_format(value)
     elif isinstance(value, datetime.datetime):
         value = datetime_safe.new_datetime(value)
-        format = smart_str(default or get_format('DATETIME_INPUT_FORMATS')[0])
+        format = force_str(default or get_format('DATETIME_INPUT_FORMATS')[0])
         return value.strftime(format)
     elif isinstance(value, datetime.date):
         value = datetime_safe.new_date(value)
-        format = smart_str(default or get_format('DATE_INPUT_FORMATS')[0])
+        format = force_str(default or get_format('DATE_INPUT_FORMATS')[0])
         return value.strftime(format)
     elif isinstance(value, datetime.time):
-        format = smart_str(default or get_format('TIME_INPUT_FORMATS')[0])
+        format = force_str(default or get_format('TIME_INPUT_FORMATS')[0])
         return value.strftime(format)
     return value
 
diff --git a/django/utils/functional.py b/django/utils/functional.py
index 085ec40b63..085a8fce59 100644
--- a/django/utils/functional.py
+++ b/django/utils/functional.py
@@ -238,7 +238,6 @@ class LazyObject(object):
         raise NotImplementedError
 
     # introspection support:
-    __members__ = property(lambda self: self.__dir__())
     __dir__ = new_method_proxy(dir)
 
 
diff --git a/django/utils/html.py b/django/utils/html.py
index 7dd53a1353..2b669cc8ec 100644
--- a/django/utils/html.py
+++ b/django/utils/html.py
@@ -11,7 +11,7 @@ except ImportError:     # Python 2
     from urlparse import urlsplit, urlunsplit
 
 from django.utils.safestring import SafeData, mark_safe
-from django.utils.encoding import smart_bytes, force_text
+from django.utils.encoding import force_bytes, force_text
 from django.utils.functional import allow_lazy
 from django.utils import six
 from django.utils.text import normalize_newlines
@@ -123,6 +123,17 @@ def strip_tags(value):
     return re.sub(r'<[^>]*?>', '', force_text(value))
 strip_tags = allow_lazy(strip_tags)
 
+def remove_tags(html, tags):
+    """Returns the given HTML with given tags removed."""
+    tags = [re.escape(tag) for tag in tags.split()]
+    tags_re = '(%s)' % '|'.join(tags)
+    starttag_re = re.compile(r'<%s(/?>|(\s+[^>]*>))' % tags_re, re.U)
+    endtag_re = re.compile('</%s>' % tags_re)
+    html = starttag_re.sub('', html)
+    html = endtag_re.sub('', html)
+    return html
+remove_tags = allow_lazy(remove_tags, six.text_type)
+
 def strip_spaces_between_tags(value):
     """Returns the given HTML with spaces between tags removed."""
     return re.sub(r'>\s+<', '><', force_text(value))
@@ -143,7 +154,7 @@ def smart_urlquote(url):
     # Handle IDN before quoting.
     scheme, netloc, path, query, fragment = urlsplit(url)
     try:
-        netloc = netloc.encode('idna') # IDN -> ACE
+        netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
     except UnicodeError: # invalid domain part
         pass
     else:
@@ -153,7 +164,7 @@ def smart_urlquote(url):
     # contains a % not followed by two hexadecimal digits. See #9655.
     if '%' not in url or unquoted_percents_re.search(url):
         # See http://bugs.python.org/issue2637
-        url = quote(smart_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~')
+        url = quote(force_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~')
 
     return force_text(url)
 
@@ -206,7 +217,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
             elif not ':' in middle and simple_email_re.match(middle):
                 local, domain = middle.rsplit('@', 1)
                 try:
-                    domain = domain.encode('idna')
+                    domain = domain.encode('idna').decode('ascii')
                 except UnicodeError:
                     continue
                 url = 'mailto:%s@%s' % (local, domain)
diff --git a/django/utils/html_parser.py b/django/utils/html_parser.py
index ee56c01aec..d7311f253b 100644
--- a/django/utils/html_parser.py
+++ b/django/utils/html_parser.py
@@ -1,102 +1,114 @@
 from django.utils.six.moves import html_parser as _html_parser
 import re
+import sys
 
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
+current_version = sys.version_info
+
+use_workaround = (
+    (current_version < (2, 6, 8)) or
+    (current_version >= (2, 7) and current_version < (2, 7, 3)) or
+    (current_version >= (3, 0) and current_version < (3, 2, 3))
+)
 
 HTMLParseError = _html_parser.HTMLParseError
 
-class HTMLParser(_html_parser.HTMLParser):
-    """
-    Patched version of stdlib's HTMLParser with patch from:
-    http://bugs.python.org/issue670664
-    """
-    def __init__(self):
-        _html_parser.HTMLParser.__init__(self)
-        self.cdata_tag = None
+if not use_workaround:
+    HTMLParser = _html_parser.HTMLParser
+else:
+    tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
 
-    def set_cdata_mode(self, tag):
-        try:
-            self.interesting = _html_parser.interesting_cdata
-        except AttributeError:
-            self.interesting = re.compile(r'</\s*%s\s*>' % tag.lower(), re.I)
-        self.cdata_tag = tag.lower()
+    class HTMLParser(_html_parser.HTMLParser):
+        """
+        Patched version of stdlib's HTMLParser with patch from:
+        http://bugs.python.org/issue670664
+        """
+        def __init__(self):
+            _html_parser.HTMLParser.__init__(self)
+            self.cdata_tag = None
 
-    def clear_cdata_mode(self):
-        self.interesting = _html_parser.interesting_normal
-        self.cdata_tag = None
+        def set_cdata_mode(self, tag):
+            try:
+                self.interesting = _html_parser.interesting_cdata
+            except AttributeError:
+                self.interesting = re.compile(r'</\s*%s\s*>' % tag.lower(), re.I)
+            self.cdata_tag = tag.lower()
 
-    # Internal -- handle starttag, return end or -1 if not terminated
-    def parse_starttag(self, i):
-        self.__starttag_text = None
-        endpos = self.check_for_whole_start_tag(i)
-        if endpos < 0:
-            return endpos
-        rawdata = self.rawdata
-        self.__starttag_text = rawdata[i:endpos]
+        def clear_cdata_mode(self):
+            self.interesting = _html_parser.interesting_normal
+            self.cdata_tag = None
 
-        # Now parse the data between i+1 and j into a tag and attrs
-        attrs = []
-        match = tagfind.match(rawdata, i + 1)
-        assert match, 'unexpected call to parse_starttag()'
-        k = match.end()
-        self.lasttag = tag = match.group(1).lower()
+        # Internal -- handle starttag, return end or -1 if not terminated
+        def parse_starttag(self, i):
+            self.__starttag_text = None
+            endpos = self.check_for_whole_start_tag(i)
+            if endpos < 0:
+                return endpos
+            rawdata = self.rawdata
+            self.__starttag_text = rawdata[i:endpos]
 
-        while k < endpos:
-            m = _html_parser.attrfind.match(rawdata, k)
-            if not m:
-                break
-            attrname, rest, attrvalue = m.group(1, 2, 3)
-            if not rest:
-                attrvalue = None
-            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
-                 attrvalue[:1] == '"' == attrvalue[-1:]:
-                attrvalue = attrvalue[1:-1]
-            if attrvalue:
-                attrvalue = self.unescape(attrvalue)
-            attrs.append((attrname.lower(), attrvalue))
-            k = m.end()
+            # Now parse the data between i+1 and j into a tag and attrs
+            attrs = []
+            match = tagfind.match(rawdata, i + 1)
+            assert match, 'unexpected call to parse_starttag()'
+            k = match.end()
+            self.lasttag = tag = match.group(1).lower()
 
-        end = rawdata[k:endpos].strip()
-        if end not in (">", "/>"):
-            lineno, offset = self.getpos()
-            if "\n" in self.__starttag_text:
-                lineno = lineno + self.__starttag_text.count("\n")
-                offset = len(self.__starttag_text) \
-                         - self.__starttag_text.rfind("\n")
+            while k < endpos:
+                m = _html_parser.attrfind.match(rawdata, k)
+                if not m:
+                    break
+                attrname, rest, attrvalue = m.group(1, 2, 3)
+                if not rest:
+                    attrvalue = None
+                elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
+                     attrvalue[:1] == '"' == attrvalue[-1:]:
+                    attrvalue = attrvalue[1:-1]
+                if attrvalue:
+                    attrvalue = self.unescape(attrvalue)
+                attrs.append((attrname.lower(), attrvalue))
+                k = m.end()
+
+            end = rawdata[k:endpos].strip()
+            if end not in (">", "/>"):
+                lineno, offset = self.getpos()
+                if "\n" in self.__starttag_text:
+                    lineno = lineno + self.__starttag_text.count("\n")
+                    offset = len(self.__starttag_text) \
+                             - self.__starttag_text.rfind("\n")
+                else:
+                    offset = offset + len(self.__starttag_text)
+                self.error("junk characters in start tag: %r"
+                           % (rawdata[k:endpos][:20],))
+            if end.endswith('/>'):
+                # XHTML-style empty tag: <span attr="value" />
+                self.handle_startendtag(tag, attrs)
             else:
-                offset = offset + len(self.__starttag_text)
-            self.error("junk characters in start tag: %r"
-                       % (rawdata[k:endpos][:20],))
-        if end.endswith('/>'):
-            # XHTML-style empty tag: <span attr="value" />
-            self.handle_startendtag(tag, attrs)
-        else:
-            self.handle_starttag(tag, attrs)
-            if tag in self.CDATA_CONTENT_ELEMENTS:
-                self.set_cdata_mode(tag) # <--------------------------- Changed
-        return endpos
+                self.handle_starttag(tag, attrs)
+                if tag in self.CDATA_CONTENT_ELEMENTS:
+                    self.set_cdata_mode(tag) # <--------------------------- Changed
+            return endpos
 
-    # Internal -- parse endtag, return end or -1 if incomplete
-    def parse_endtag(self, i):
-        rawdata = self.rawdata
-        assert rawdata[i:i + 2] == "</", "unexpected call to parse_endtag"
-        match = _html_parser.endendtag.search(rawdata, i + 1) # >
-        if not match:
-            return -1
-        j = match.end()
-        match = _html_parser.endtagfind.match(rawdata, i) # </ + tag + >
-        if not match:
-            if self.cdata_tag is not None: # *** add ***
-                self.handle_data(rawdata[i:j]) # *** add ***
-                return j # *** add ***
-            self.error("bad end tag: %r" % (rawdata[i:j],))
-        # --- changed start ---------------------------------------------------
-        tag = match.group(1).strip()
-        if self.cdata_tag is not None:
-            if tag.lower() != self.cdata_tag:
-                self.handle_data(rawdata[i:j])
-                return j
-        # --- changed end -----------------------------------------------------
-        self.handle_endtag(tag.lower())
-        self.clear_cdata_mode()
-        return j
+        # Internal -- parse endtag, return end or -1 if incomplete
+        def parse_endtag(self, i):
+            rawdata = self.rawdata
+            assert rawdata[i:i + 2] == "</", "unexpected call to parse_endtag"
+            match = _html_parser.endendtag.search(rawdata, i + 1) # >
+            if not match:
+                return -1
+            j = match.end()
+            match = _html_parser.endtagfind.match(rawdata, i) # </ + tag + >
+            if not match:
+                if self.cdata_tag is not None: # *** add ***
+                    self.handle_data(rawdata[i:j]) # *** add ***
+                    return j # *** add ***
+                self.error("bad end tag: %r" % (rawdata[i:j],))
+            # --- changed start ---------------------------------------------------
+            tag = match.group(1).strip()
+            if self.cdata_tag is not None:
+                if tag.lower() != self.cdata_tag:
+                    self.handle_data(rawdata[i:j])
+                    return j
+            # --- changed end -----------------------------------------------------
+            self.handle_endtag(tag.lower())
+            self.clear_cdata_mode()
+            return j
diff --git a/django/utils/http.py b/django/utils/http.py
index 22e81a33d7..d3c70f1209 100644
--- a/django/utils/http.py
+++ b/django/utils/http.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import calendar
 import datetime
 import re
@@ -13,7 +15,7 @@ except ImportError:     # Python 2
 from email.utils import formatdate
 
 from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import force_text, smart_str
+from django.utils.encoding import force_str, force_text
 from django.utils.functional import allow_lazy
 from django.utils import six
 
@@ -37,7 +39,7 @@ def urlquote(url, safe='/'):
     can safely be used as part of an argument to a subsequent iri_to_uri() call
     without double-quoting occurring.
     """
-    return force_text(urllib_parse.quote(smart_str(url), smart_str(safe)))
+    return force_text(urllib_parse.quote(force_str(url), force_str(safe)))
 urlquote = allow_lazy(urlquote, six.text_type)
 
 def urlquote_plus(url, safe=''):
@@ -47,7 +49,7 @@ def urlquote_plus(url, safe=''):
     returned string can safely be used as part of an argument to a subsequent
     iri_to_uri() call without double-quoting occurring.
     """
-    return force_text(urllib_parse.quote_plus(smart_str(url), smart_str(safe)))
+    return force_text(urllib_parse.quote_plus(force_str(url), force_str(safe)))
 urlquote_plus = allow_lazy(urlquote_plus, six.text_type)
 
 def urlunquote(quoted_url):
@@ -55,7 +57,7 @@ def urlunquote(quoted_url):
     A wrapper for Python's urllib.unquote() function that can operate on
     the result of django.utils.http.urlquote().
     """
-    return force_text(urllib_parse.unquote(smart_str(quoted_url)))
+    return force_text(urllib_parse.unquote(force_str(quoted_url)))
 urlunquote = allow_lazy(urlunquote, six.text_type)
 
 def urlunquote_plus(quoted_url):
@@ -63,7 +65,7 @@ def urlunquote_plus(quoted_url):
     A wrapper for Python's urllib.unquote_plus() function that can operate on
     the result of django.utils.http.urlquote_plus().
     """
-    return force_text(urllib_parse.unquote_plus(smart_str(quoted_url)))
+    return force_text(urllib_parse.unquote_plus(force_str(quoted_url)))
 urlunquote_plus = allow_lazy(urlunquote_plus, six.text_type)
 
 def urlencode(query, doseq=0):
@@ -77,8 +79,8 @@ def urlencode(query, doseq=0):
     elif hasattr(query, 'items'):
         query = query.items()
     return urllib_parse.urlencode(
-        [(smart_str(k),
-         [smart_str(i) for i in v] if isinstance(v, (list,tuple)) else smart_str(v))
+        [(force_str(k),
+         [force_str(i) for i in v] if isinstance(v, (list,tuple)) else force_str(v))
             for k, v in query],
         doseq)
 
@@ -211,7 +213,7 @@ def parse_etags(etag_str):
     if not etags:
         # etag_str has wrong format, treat it as an opaque string then
         return [etag_str]
-    etags = [e.decode('string_escape') for e in etags]
+    etags = [e.encode('ascii').decode('unicode_escape') for e in etags]
     return etags
 
 def quote_etag(etag):
diff --git a/django/utils/ipv6.py b/django/utils/ipv6.py
index 2ccae8cdd0..7624bb9c76 100644
--- a/django/utils/ipv6.py
+++ b/django/utils/ipv6.py
@@ -263,6 +263,6 @@ def _is_shorthand_ip(ip_str):
     """
     if ip_str.count('::') == 1:
         return True
-    if filter(lambda x: len(x) < 4, ip_str.split(':')):
+    if any(len(x) < 4 for x in ip_str.split(':')):
         return True
     return False
diff --git a/django/utils/safestring.py b/django/utils/safestring.py
index bfaefd07ee..07e0bf4cea 100644
--- a/django/utils/safestring.py
+++ b/django/utils/safestring.py
@@ -10,36 +10,43 @@ from django.utils import six
 class EscapeData(object):
     pass
 
-class EscapeString(bytes, EscapeData):
+class EscapeBytes(bytes, EscapeData):
     """
-    A string that should be HTML-escaped when output.
+    A byte string that should be HTML-escaped when output.
     """
     pass
 
-class EscapeUnicode(six.text_type, EscapeData):
+class EscapeText(six.text_type, EscapeData):
     """
-    A unicode object that should be HTML-escaped when output.
+    A unicode string object that should be HTML-escaped when output.
     """
     pass
 
+if six.PY3:
+    EscapeString = EscapeText
+else:
+    EscapeString = EscapeBytes
+    # backwards compatibility for Python 2
+    EscapeUnicode = EscapeText
+
 class SafeData(object):
     pass
 
-class SafeString(bytes, SafeData):
+class SafeBytes(bytes, SafeData):
     """
-    A string subclass that has been specifically marked as "safe" (requires no
+    A bytes subclass that has been specifically marked as "safe" (requires no
     further escaping) for HTML output purposes.
     """
     def __add__(self, rhs):
         """
-        Concatenating a safe string with another safe string or safe unicode
-        object is safe. Otherwise, the result is no longer safe.
+        Concatenating a safe byte string with another safe byte string or safe
+        unicode string is safe. Otherwise, the result is no longer safe.
         """
-        t = super(SafeString, self).__add__(rhs)
-        if isinstance(rhs, SafeUnicode):
-            return SafeUnicode(t)
-        elif isinstance(rhs, SafeString):
-            return SafeString(t)
+        t = super(SafeBytes, self).__add__(rhs)
+        if isinstance(rhs, SafeText):
+            return SafeText(t)
+        elif isinstance(rhs, SafeBytes):
+            return SafeBytes(t)
         return t
 
     def _proxy_method(self, *args, **kwargs):
@@ -51,25 +58,25 @@ class SafeString(bytes, SafeData):
         method = kwargs.pop('method')
         data = method(self, *args, **kwargs)
         if isinstance(data, bytes):
-            return SafeString(data)
+            return SafeBytes(data)
         else:
-            return SafeUnicode(data)
+            return SafeText(data)
 
     decode = curry(_proxy_method, method=bytes.decode)
 
-class SafeUnicode(six.text_type, SafeData):
+class SafeText(six.text_type, SafeData):
     """
-    A unicode subclass that has been specifically marked as "safe" for HTML
-    output purposes.
+    A unicode (Python 2) / str (Python 3) subclass that has been specifically
+    marked as "safe" for HTML output purposes.
     """
     def __add__(self, rhs):
         """
-        Concatenating a safe unicode object with another safe string or safe
-        unicode object is safe. Otherwise, the result is no longer safe.
+        Concatenating a safe unicode string with another safe byte string or
+        safe unicode string is safe. Otherwise, the result is no longer safe.
         """
-        t = super(SafeUnicode, self).__add__(rhs)
+        t = super(SafeText, self).__add__(rhs)
         if isinstance(rhs, SafeData):
-            return SafeUnicode(t)
+            return SafeText(t)
         return t
 
     def _proxy_method(self, *args, **kwargs):
@@ -81,12 +88,19 @@ class SafeUnicode(six.text_type, SafeData):
         method = kwargs.pop('method')
         data = method(self, *args, **kwargs)
         if isinstance(data, bytes):
-            return SafeString(data)
+            return SafeBytes(data)
         else:
-            return SafeUnicode(data)
+            return SafeText(data)
 
     encode = curry(_proxy_method, method=six.text_type.encode)
 
+if six.PY3:
+    SafeString = SafeText
+else:
+    SafeString = SafeBytes
+    # backwards compatibility for Python 2
+    SafeUnicode = SafeText
+
 def mark_safe(s):
     """
     Explicitly mark a string as safe for (HTML) output purposes. The returned
@@ -97,10 +111,10 @@ def mark_safe(s):
     if isinstance(s, SafeData):
         return s
     if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes):
-        return SafeString(s)
+        return SafeBytes(s)
     if isinstance(s, (six.text_type, Promise)):
-        return SafeUnicode(s)
-    return SafeString(bytes(s))
+        return SafeText(s)
+    return SafeString(str(s))
 
 def mark_for_escaping(s):
     """
@@ -113,8 +127,8 @@ def mark_for_escaping(s):
     if isinstance(s, (SafeData, EscapeData)):
         return s
     if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes):
-        return EscapeString(s)
+        return EscapeBytes(s)
     if isinstance(s, (six.text_type, Promise)):
-        return EscapeUnicode(s)
-    return EscapeString(bytes(s))
+        return EscapeText(s)
+    return EscapeBytes(bytes(s))
 
diff --git a/django/utils/six.py b/django/utils/six.py
index ceb5d70e58..e4ce939844 100644
--- a/django/utils/six.py
+++ b/django/utils/six.py
@@ -5,7 +5,7 @@ import sys
 import types
 
 __author__ = "Benjamin Peterson <benjamin@python.org>"
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 
 
 # True if we are running on Python 3.
@@ -26,19 +26,23 @@ else:
     text_type = unicode
     binary_type = str
 
-    # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
-    class X(object):
-        def __len__(self):
-            return 1 << 31
-    try:
-        len(X())
-    except OverflowError:
-        # 32-bit
+    if sys.platform == "java":
+        # Jython always uses 32 bits.
         MAXSIZE = int((1 << 31) - 1)
     else:
-        # 64-bit
-        MAXSIZE = int((1 << 63) - 1)
-    del X
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+            del X
 
 
 def _add_doc(func, doc):
@@ -201,12 +205,19 @@ else:
     _iteritems = "iteritems"
 
 
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
 if PY3:
     def get_unbound_function(unbound):
         return unbound
 
-
-    advance_iterator = next
+    Iterator = object
 
     def callable(obj):
         return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
@@ -214,9 +225,10 @@ else:
     def get_unbound_function(unbound):
         return unbound.im_func
 
+    class Iterator(object):
 
-    def advance_iterator(it):
-        return it.next()
+        def next(self):
+            return type(self).__next__(self)
 
     callable = callable
 _add_doc(get_unbound_function,
@@ -231,15 +243,15 @@ get_function_defaults = operator.attrgetter(_func_defaults)
 
 def iterkeys(d):
     """Return an iterator over the keys of a dictionary."""
-    return getattr(d, _iterkeys)()
+    return iter(getattr(d, _iterkeys)())
 
 def itervalues(d):
     """Return an iterator over the values of a dictionary."""
-    return getattr(d, _itervalues)()
+    return iter(getattr(d, _itervalues)())
 
 def iteritems(d):
     """Return an iterator over the (key, value) pairs of a dictionary."""
-    return getattr(d, _iteritems)()
+    return iter(getattr(d, _iteritems)())
 
 
 if PY3:
@@ -365,4 +377,6 @@ def iterlists(d):
     """Return an iterator over the values of a MultiValueDict."""
     return getattr(d, _iterlists)()
 
+
 add_move(MovedModule("_dummy_thread", "dummy_thread"))
+add_move(MovedModule("_thread", "thread"))
diff --git a/django/utils/text.py b/django/utils/text.py
index 0838d79c65..c19708458b 100644
--- a/django/utils/text.py
+++ b/django/utils/text.py
@@ -4,13 +4,19 @@ import re
 import unicodedata
 import warnings
 from gzip import GzipFile
-from django.utils.six.moves import html_entities
 from io import BytesIO
 
 from django.utils.encoding import force_text
 from django.utils.functional import allow_lazy, SimpleLazyObject
 from django.utils import six
+from django.utils.six.moves import html_entities
 from django.utils.translation import ugettext_lazy, ugettext as _, pgettext
+from django.utils.safestring import mark_safe
+
+if not six.PY3:
+    # Import force_unicode even though this module doesn't use it, because some
+    # people rely on it being here.
+    from django.utils.encoding import force_unicode
 
 # Capitalizes the first letter of a string.
 capfirst = lambda x: x and force_text(x)[0].upper() + force_text(x)[1:]
@@ -287,7 +293,7 @@ ustring_re = re.compile("([\u0080-\uffff])")
 def javascript_quote(s, quote_double_quotes=False):
 
     def fix(match):
-        return b"\u%04x" % ord(match.group(1))
+        return "\\u%04x" % ord(match.group(1))
 
     if type(s) == bytes:
         s = s.decode('utf-8')
@@ -378,3 +384,14 @@ def unescape_string_literal(s):
     quote = s[0]
     return s[1:-1].replace(r'\%s' % quote, quote).replace(r'\\', '\\')
 unescape_string_literal = allow_lazy(unescape_string_literal)
+
+def slugify(value):
+    """
+    Converts to lowercase, removes non-word characters (alphanumerics and
+    underscores) and converts spaces to hyphens. Also strips leading and
+    trailing whitespace.
+    """
+    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
+    value = re.sub('[^\w\s-]', '', value).strip().lower()
+    return mark_safe(re.sub('[-\s]+', '-', value))
+slugify = allow_lazy(slugify, six.text_type)
diff --git a/django/utils/translation/__init__.py b/django/utils/translation/__init__.py
index febde404a5..f3cc6348f6 100644
--- a/django/utils/translation/__init__.py
+++ b/django/utils/translation/__init__.py
@@ -79,10 +79,10 @@ def pgettext(context, message):
 def npgettext(context, singular, plural, number):
     return _trans.npgettext(context, singular, plural, number)
 
-ngettext_lazy = lazy(ngettext, bytes)
-gettext_lazy = lazy(gettext, bytes)
-ungettext_lazy = lazy(ungettext, six.text_type)
+gettext_lazy = lazy(gettext, str)
+ngettext_lazy = lazy(ngettext, str)
 ugettext_lazy = lazy(ugettext, six.text_type)
+ungettext_lazy = lazy(ungettext, six.text_type)
 pgettext_lazy = lazy(pgettext, six.text_type)
 npgettext_lazy = lazy(npgettext, six.text_type)
 
diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py
index 9e6eadcd48..9fd33a7ea8 100644
--- a/django/utils/translation/trans_real.py
+++ b/django/utils/translation/trans_real.py
@@ -9,7 +9,7 @@ import gettext as gettext_module
 from threading import local
 
 from django.utils.importlib import import_module
-from django.utils.encoding import smart_str, smart_text
+from django.utils.encoding import force_str, force_text
 from django.utils.safestring import mark_safe, SafeData
 from django.utils import six
 from django.utils.six import StringIO
@@ -259,6 +259,11 @@ def do_translate(message, translation_function):
     return result
 
 def gettext(message):
+    """
+    Returns a string of the translation of the message.
+
+    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
+    """
     return do_translate(message, 'gettext')
 
 if six.PY3:
@@ -296,8 +301,10 @@ def do_ntranslate(singular, plural, number, translation_function):
 
 def ngettext(singular, plural, number):
     """
-    Returns a UTF-8 bytestring of the translation of either the singular or
-    plural, based on the number.
+    Returns a string of the translation of either the singular or plural,
+    based on the number.
+
+    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
     """
     return do_ntranslate(singular, plural, number, 'ngettext')
 
@@ -447,7 +454,7 @@ def templatize(src, origin=None):
     from django.conf import settings
     from django.template import (Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK,
             TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK)
-    src = smart_text(src, settings.FILE_CHARSET)
+    src = force_text(src, settings.FILE_CHARSET)
     out = StringIO()
     message_context = None
     intrans = False
@@ -462,7 +469,7 @@ def templatize(src, origin=None):
                 content = ''.join(comment)
                 translators_comment_start = None
                 for lineno, line in enumerate(content.splitlines(True)):
-                    if line.lstrip().startswith(smart_text(TRANSLATOR_COMMENT_MARK)):
+                    if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK):
                         translators_comment_start = lineno
                 for lineno, line in enumerate(content.splitlines(True)):
                     if translators_comment_start is not None and lineno >= translators_comment_start:
@@ -577,7 +584,7 @@ def templatize(src, origin=None):
                 out.write(' # %s' % t.contents)
             else:
                 out.write(blankout(t.contents, 'X'))
-    return smart_str(out.getvalue())
+    return force_str(out.getvalue())
 
 def parse_accept_lang_header(lang_string):
     """
diff --git a/django/utils/tzinfo.py b/django/utils/tzinfo.py
index 208b7e7191..654c99778e 100644
--- a/django/utils/tzinfo.py
+++ b/django/utils/tzinfo.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import time
 from datetime import timedelta, tzinfo
 
-from django.utils.encoding import smart_text, smart_str, DEFAULT_LOCALE_ENCODING
+from django.utils.encoding import force_str, force_text, DEFAULT_LOCALE_ENCODING
 
 # Python's doc say: "A tzinfo subclass must have an __init__() method that can
 # be called with no arguments". FixedOffset and LocalTimezone don't honor this
@@ -53,7 +53,7 @@ class LocalTimezone(tzinfo):
         self._tzname = self.tzname(dt)
 
     def __repr__(self):
-        return smart_str(self._tzname)
+        return force_str(self._tzname)
 
     def __getinitargs__(self):
         return self.__dt,
@@ -72,7 +72,7 @@ class LocalTimezone(tzinfo):
 
     def tzname(self, dt):
         try:
-            return smart_text(time.tzname[self._isdst(dt)],
+            return force_text(time.tzname[self._isdst(dt)],
                                  DEFAULT_LOCALE_ENCODING)
         except UnicodeDecodeError:
             return None