Merge branch 'master' into schema-alteration

26 files changed, 116 insertions, 101 deletions

diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py
index f7573b2e31..d527e44d8b 100644
--- a/django/core/cache/backends/base.py
+++ b/django/core/cache/backends/base.py
@@ -3,7 +3,7 @@
 import warnings
 
 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_bytes
 from django.utils.importlib import import_module
 
 class InvalidCacheBackendError(ImproperlyConfigured):
@@ -23,7 +23,7 @@ def default_key_func(key, key_prefix, version):
     the `key_prefix'. KEY_FUNCTION can be used to specify an alternate
     function with custom key making behavior.
     """
-    return ':'.join([key_prefix, str(version), smart_str(key)])
+    return ':'.join([key_prefix, str(version), smart_bytes(key)])
 
 def get_key_func(key_func):
     """
@@ -62,7 +62,7 @@ class BaseCache(object):
         except (ValueError, TypeError):
             self._cull_frequency = 3
 
-        self.key_prefix = smart_str(params.get('KEY_PREFIX', ''))
+        self.key_prefix = smart_bytes(params.get('KEY_PREFIX', ''))
         self.version = params.get('VERSION', 1)
         self.key_func = get_key_func(params.get('KEY_FUNCTION', None))
 
diff --git a/django/core/context_processors.py b/django/core/context_processors.py
index 325f64d224..a503270cf4 100644
--- a/django/core/context_processors.py
+++ b/django/core/context_processors.py
@@ -9,7 +9,7 @@ RequestContext.
 
 from django.conf import settings
 from django.middleware.csrf import get_token
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_bytes
 from django.utils.functional import lazy
 
 def csrf(request):
@@ -25,7 +25,7 @@ def csrf(request):
             # instead of returning an empty dict.
             return b'NOTPROVIDED'
         else:
-            return smart_str(token)
+            return smart_bytes(token)
     _get_val = lazy(_get_val, str)
 
     return {'csrf_token': _get_val() }
diff --git a/django/core/exceptions.py b/django/core/exceptions.py
index e3d1dc9c7e..f0f14cffda 100644
--- a/django/core/exceptions.py
+++ b/django/core/exceptions.py
@@ -43,7 +43,7 @@ class ValidationError(Exception):
     """An error while validating data."""
     def __init__(self, message, code=None, params=None):
         import operator
-        from django.utils.encoding import force_unicode
+        from django.utils.encoding import force_text
         """
         ValidationError can be passed any object that can be printed (usually
         a string), a list of objects or a dictionary.
@@ -54,11 +54,11 @@ class ValidationError(Exception):
             message = reduce(operator.add, message.values())
 
         if isinstance(message, list):
-            self.messages = [force_unicode(msg) for msg in message]
+            self.messages = [force_text(msg) for msg in message]
         else:
             self.code = code
             self.params = params
-            message = force_unicode(message)
+            message = force_text(message)
             self.messages = [message]
 
     def __str__(self):
diff --git a/django/core/files/base.py b/django/core/files/base.py
index 04853fad0c..d0b25250a5 100644
--- a/django/core/files/base.py
+++ b/django/core/files/base.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import os
 from io import BytesIO
 
-from django.utils.encoding import smart_str, smart_unicode
+from django.utils.encoding import smart_bytes, smart_text
 from django.core.files.utils import FileProxyMixin
 
 class File(FileProxyMixin):
@@ -18,16 +18,17 @@ class File(FileProxyMixin):
             self.mode = file.mode
 
     def __str__(self):
-        return smart_str(self.name or '')
+        return smart_bytes(self.name or '')
 
     def __unicode__(self):
-        return smart_unicode(self.name or '')
+        return smart_text(self.name or '')
 
     def __repr__(self):
         return "<%s: %s>" % (self.__class__.__name__, self or "None")
 
-    def __nonzero__(self):
+    def __bool__(self):
         return bool(self.name)
+    __nonzero__ = __bool__ # Python 2
 
     def __len__(self):
         return self.size
@@ -135,8 +136,9 @@ class ContentFile(File):
     def __str__(self):
         return 'Raw content'
 
-    def __nonzero__(self):
+    def __bool__(self):
         return True
+    __nonzero__ = __bool__ # Python 2
 
     def open(self, mode=None):
         self.seek(0)
diff --git a/django/core/files/images.py b/django/core/files/images.py
index 228a7118c5..7d7eac65db 100644
--- a/django/core/files/images.py
+++ b/django/core/files/images.py
@@ -47,13 +47,18 @@ def get_image_dimensions(file_or_path, close=False):
         file = open(file_or_path, 'rb')
         close = True
     try:
+        # Most of the time PIL only needs a small chunk to parse the image and
+        # get the dimensions, but with some TIFF files PIL needs to parse the
+        # whole file.
+        chunk_size = 1024
         while 1:
-            data = file.read(1024)
+            data = file.read(chunk_size)
             if not data:
                 break
             p.feed(data)
             if p.image:
                 return p.image.size
+            chunk_size = chunk_size*2
         return None
     finally:
         if close:
diff --git a/django/core/files/storage.py b/django/core/files/storage.py
index 5179980513..7542dcda46 100644
--- a/django/core/files/storage.py
+++ b/django/core/files/storage.py
@@ -11,7 +11,7 @@ from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
 from django.core.files import locks, File
 from django.core.files.move import file_move_safe
-from django.utils.encoding import force_unicode, filepath_to_uri
+from django.utils.encoding import force_text, filepath_to_uri
 from django.utils.functional import LazyObject
 from django.utils.importlib import import_module
 from django.utils.text import get_valid_filename
@@ -48,7 +48,7 @@ class Storage(object):
         name = self._save(name, content)
 
         # Store filenames with forward slashes, even on Windows
-        return force_unicode(name.replace('\\', '/'))
+        return force_text(name.replace('\\', '/'))
 
     # These methods are part of the public API, with default implementations.
 
diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py
index 97d53482e4..3a6c632975 100644
--- a/django/core/files/uploadedfile.py
+++ b/django/core/files/uploadedfile.py
@@ -8,7 +8,7 @@ from io import BytesIO
 from django.conf import settings
 from django.core.files.base import File
 from django.core.files import temp as tempfile
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_bytes
 
 __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile',
            'SimpleUploadedFile')
@@ -30,7 +30,7 @@ class UploadedFile(File):
         self.charset = charset
 
     def __repr__(self):
-        return smart_str("<%s: %s (%s)>" % (
+        return smart_bytes("<%s: %s (%s)>" % (
             self.__class__.__name__, self.name, self.content_type))
 
     def _get_name(self):
diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py
index 7fd7d19c4a..5a6825f0a7 100644
--- a/django/core/handlers/base.py
+++ b/django/core/handlers/base.py
@@ -4,7 +4,7 @@ import sys
 
 from django import http
 from django.core import signals
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
 from django.utils.importlib import import_module
 from django.utils.log import getLogger
 from django.utils import six
@@ -250,7 +250,7 @@ def get_script_name(environ):
     """
     from django.conf import settings
     if settings.FORCE_SCRIPT_NAME is not None:
-        return force_unicode(settings.FORCE_SCRIPT_NAME)
+        return force_text(settings.FORCE_SCRIPT_NAME)
 
     # If Apache's mod_rewrite had a whack at the URL, Apache set either
     # SCRIPT_URL or REDIRECT_URL to the full resource URL before applying any
@@ -261,5 +261,5 @@ def get_script_name(environ):
     if not script_url:
         script_url = environ.get('REDIRECT_URL', '')
     if script_url:
-        return force_unicode(script_url[:-len(environ.get('PATH_INFO', ''))])
-    return force_unicode(environ.get('SCRIPT_NAME', ''))
+        return force_text(script_url[:-len(environ.get('PATH_INFO', ''))])
+    return force_text(environ.get('SCRIPT_NAME', ''))
diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py
index 617068a21c..70b23f8515 100644
--- a/django/core/handlers/wsgi.py
+++ b/django/core/handlers/wsgi.py
@@ -9,7 +9,7 @@ from django.core import signals
 from django.core.handlers import base
 from django.core.urlresolvers import set_script_prefix
 from django.utils import datastructures
-from django.utils.encoding import force_unicode, smart_str, iri_to_uri
+from django.utils.encoding import force_text, smart_bytes, iri_to_uri
 from django.utils.log import getLogger
 
 logger = getLogger('django.request')
@@ -127,7 +127,7 @@ class LimitedStream(object):
 class WSGIRequest(http.HttpRequest):
     def __init__(self, environ):
         script_name = base.get_script_name(environ)
-        path_info = force_unicode(environ.get('PATH_INFO', '/'))
+        path_info = force_text(environ.get('PATH_INFO', '/'))
         if not path_info or path_info == script_name:
             # Sometimes PATH_INFO exists, but is empty (e.g. accessing
             # the SCRIPT_NAME URL without a trailing slash). We really need to
@@ -245,6 +245,6 @@ class WSGIHandler(base.BaseHandler):
         status = '%s %s' % (response.status_code, status_text)
         response_headers = [(str(k), str(v)) for k, v in response.items()]
         for c in response.cookies.values():
-            response_headers.append((b'Set-Cookie', str(c.output(header=''))))
-        start_response(smart_str(status), response_headers)
+            response_headers.append((str('Set-Cookie'), str(c.output(header=''))))
+        start_response(smart_bytes(status), response_headers)
         return response
diff --git a/django/core/mail/message.py b/django/core/mail/message.py
index 629ad464f9..b332ffba04 100644
--- a/django/core/mail/message.py
+++ b/django/core/mail/message.py
@@ -11,11 +11,10 @@ from email.mime.multipart import MIMEMultipart
 from email.mime.base import MIMEBase
 from email.header import Header
 from email.utils import formatdate, getaddresses, formataddr, parseaddr
-from io import BytesIO
 
 from django.conf import settings
 from django.core.mail.utils import DNS_NAME
-from django.utils.encoding import smart_str, force_unicode
+from django.utils.encoding import force_text
 from django.utils import six
 
 
@@ -79,38 +78,38 @@ ADDRESS_HEADERS = set([
 def forbid_multi_line_headers(name, val, encoding):
     """Forbids multi-line headers, to prevent header injection."""
     encoding = encoding or settings.DEFAULT_CHARSET
-    val = force_unicode(val)
+    val = force_text(val)
     if '\n' in val or '\r' in val:
         raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name))
     try:
-        val = val.encode('ascii')
+        val.encode('ascii')
     except UnicodeEncodeError:
         if name.lower() in ADDRESS_HEADERS:
             val = ', '.join(sanitize_address(addr, encoding)
                 for addr in getaddresses((val,)))
         else:
-            val = str(Header(val, encoding))
+            val = Header(val, encoding).encode()
     else:
         if name.lower() == 'subject':
-            val = Header(val)
-    return smart_str(name), val
+            val = Header(val).encode()
+    return str(name), val
 
 
 def sanitize_address(addr, encoding):
     if isinstance(addr, six.string_types):
-        addr = parseaddr(force_unicode(addr))
+        addr = parseaddr(force_text(addr))
     nm, addr = addr
-    nm = str(Header(nm, encoding))
+    nm = Header(nm, encoding).encode()
     try:
-        addr = addr.encode('ascii')
+        addr.encode('ascii')
     except UnicodeEncodeError:  # IDN
         if '@' in addr:
             localpart, domain = addr.split('@', 1)
             localpart = str(Header(localpart, encoding))
-            domain = domain.encode('idna')
+            domain = domain.encode('idna').decode('ascii')
             addr = '@'.join([localpart, domain])
         else:
-            addr = str(Header(addr, encoding))
+            addr = Header(addr, encoding).encode()
     return formataddr((nm, addr))
 
 
@@ -132,7 +131,7 @@ class SafeMIMEText(MIMEText):
         This overrides the default as_string() implementation to not mangle
         lines that begin with 'From '. See bug #13433 for details.
         """
-        fp = BytesIO()
+        fp = six.StringIO()
         g = Generator(fp, mangle_from_ = False)
         g.flatten(self, unixfrom=unixfrom)
         return fp.getvalue()
@@ -156,7 +155,7 @@ class SafeMIMEMultipart(MIMEMultipart):
         This overrides the default as_string() implementation to not mangle
         lines that begin with 'From '. See bug #13433 for details.
         """
-        fp = BytesIO()
+        fp = six.StringIO()
         g = Generator(fp, mangle_from_ = False)
         g.flatten(self, unixfrom=unixfrom)
         return fp.getvalue()
@@ -210,8 +209,7 @@ class EmailMessage(object):
 
     def message(self):
         encoding = self.encoding or settings.DEFAULT_CHARSET
-        msg = SafeMIMEText(smart_str(self.body, encoding),
-                           self.content_subtype, encoding)
+        msg = SafeMIMEText(self.body, self.content_subtype, encoding)
         msg = self._create_message(msg)
         msg['Subject'] = self.subject
         msg['From'] = self.extra_headers.get('From', self.from_email)
@@ -293,7 +291,7 @@ class EmailMessage(object):
         basetype, subtype = mimetype.split('/', 1)
         if basetype == 'text':
             encoding = self.encoding or settings.DEFAULT_CHARSET
-            attachment = SafeMIMEText(smart_str(content, encoding), subtype, encoding)
+            attachment = SafeMIMEText(content, subtype, encoding)
         else:
             # Encode non-text attachments with base64.
             attachment = MIMEBase(basetype, subtype)
@@ -313,9 +311,11 @@ class EmailMessage(object):
         attachment = self._create_mime_attachment(content, mimetype)
         if filename:
             try:
-                filename = filename.encode('ascii')
+                filename.encode('ascii')
             except UnicodeEncodeError:
-                filename = ('utf-8', '', filename.encode('utf-8'))
+                if not six.PY3:
+                    filename = filename.encode('utf-8')
+                filename = ('utf-8', '', filename)
             attachment.add_header('Content-Disposition', 'attachment',
                                   filename=filename)
         return attachment
diff --git a/django/core/management/__init__.py b/django/core/management/__init__.py
index 68048e5672..268cd63c38 100644
--- a/django/core/management/__init__.py
+++ b/django/core/management/__init__.py
@@ -8,6 +8,7 @@ import warnings
 from django.core.management.base import BaseCommand, CommandError, handle_default_options
 from django.core.management.color import color_style
 from django.utils.importlib import import_module
+from django.utils import six
 
 # For backwards compatibility: get_version() used to be in this module.
 from django import get_version
@@ -228,7 +229,7 @@ class ManagementUtility(object):
                 "Available subcommands:",
             ]
             commands_dict = collections.defaultdict(lambda: [])
-            for name, app in get_commands().iteritems():
+            for name, app in six.iteritems(get_commands()):
                 if app == 'django.core':
                     app = 'django'
                 else:
@@ -294,7 +295,7 @@ class ManagementUtility(object):
         except IndexError:
             curr = ''
 
-        subcommands = get_commands().keys() + ['help']
+        subcommands = list(get_commands()) + ['help']
         options = [('--help', None)]
 
         # subcommand
diff --git a/django/core/management/base.py b/django/core/management/base.py
index a204f6f0bc..5e630d5207 100644
--- a/django/core/management/base.py
+++ b/django/core/management/base.py
@@ -6,7 +6,6 @@ be executed through ``django-admin.py`` or ``manage.py``).
 import os
 import sys
 
-from io import BytesIO
 from optparse import make_option, OptionParser
 import traceback
 
@@ -14,6 +13,7 @@ import django
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management.color import color_style
 from django.utils.encoding import smart_str
+from django.utils.six import StringIO
 
 
 class CommandError(Exception):
@@ -273,7 +273,7 @@ class BaseCommand(object):
 
         """
         from django.core.management.validation import get_validation_errors
-        s = BytesIO()
+        s = StringIO()
         num_errors = get_validation_errors(s, app)
         if num_errors:
             s.seek(0)
diff --git a/django/core/management/commands/compilemessages.py b/django/core/management/commands/compilemessages.py
index fdc3535cf6..b7392b9173 100644
--- a/django/core/management/commands/compilemessages.py
+++ b/django/core/management/commands/compilemessages.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import codecs
 import os
 import sys
@@ -7,7 +9,7 @@ from django.core.management.base import BaseCommand, CommandError
 def has_bom(fn):
     with open(fn, 'rb') as f:
         sample = f.read(4)
-    return sample[:3] == '\xef\xbb\xbf' or \
+    return sample[:3] == b'\xef\xbb\xbf' or \
             sample.startswith(codecs.BOM_UTF16_LE) or \
             sample.startswith(codecs.BOM_UTF16_BE)
 
diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py
index fd6dbbbd2c..411042ee76 100644
--- a/django/core/management/commands/createcachetable.py
+++ b/django/core/management/commands/createcachetable.py
@@ -4,7 +4,7 @@ from django.core.cache.backends.db import BaseDatabaseCache
 from django.core.management.base import LabelCommand, CommandError
 from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS
 from django.db.utils import DatabaseError
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
 
 
 class Command(LabelCommand):
@@ -60,7 +60,7 @@ class Command(LabelCommand):
             transaction.rollback_unless_managed(using=db)
             raise CommandError(
                 "Cache table '%s' could not be created.\nThe error was: %s." %
-                    (tablename, force_unicode(e)))
+                    (tablename, force_text(e)))
         for statement in index_output:
             curs.execute(statement)
         transaction.commit_unless_managed(using=db)
diff --git a/django/core/management/commands/diffsettings.py b/django/core/management/commands/diffsettings.py
index 98b53b405d..aa7395e5ee 100644
--- a/django/core/management/commands/diffsettings.py
+++ b/django/core/management/commands/diffsettings.py
@@ -22,9 +22,7 @@ class Command(NoArgsCommand):
         default_settings = module_to_dict(global_settings)
 
         output = []
-        keys = user_settings.keys()
-        keys.sort()
-        for key in keys:
+        for key in sorted(user_settings.keys()):
             if key not in default_settings:
                 output.append("%s = %s  ###" % (key, user_settings[key]))
             elif user_settings[key] != default_settings[key]:
diff --git a/django/core/management/commands/flush.py b/django/core/management/commands/flush.py
index ac7b7a3599..b8b78434ce 100644
--- a/django/core/management/commands/flush.py
+++ b/django/core/management/commands/flush.py
@@ -7,6 +7,7 @@ from django.core.management.base import NoArgsCommand, CommandError
 from django.core.management.color import no_style
 from django.core.management.sql import sql_flush, emit_post_sync_signal
 from django.utils.importlib import import_module
+from django.utils.six.moves import input
 
 
 class Command(NoArgsCommand):
@@ -45,7 +46,7 @@ class Command(NoArgsCommand):
         sql_list = sql_flush(self.style, connection, only_django=True, reset_sequences=reset_sequences)
 
         if interactive:
-            confirm = raw_input("""You have requested a flush of the database.
+            confirm = input("""You have requested a flush of the database.
 This will IRREVERSIBLY DESTROY all data currently in the %r database,
 and return each table to the state it was in after syncdb.
 Are you sure you want to do this?
diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py
index 34f8041d33..1896e53cee 100644
--- a/django/core/management/commands/loaddata.py
+++ b/django/core/management/commands/loaddata.py
@@ -14,7 +14,7 @@ from django.core.management.color import no_style
 from django.db import (connections, router, transaction, DEFAULT_DB_ALIAS,
       IntegrityError, DatabaseError)
 from django.db.models import get_apps
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
 from itertools import product
 
 try:
@@ -189,7 +189,7 @@ class Command(BaseCommand):
                                                         'app_label': obj.object._meta.app_label,
                                                         'object_name': obj.object._meta.object_name,
                                                         'pk': obj.object.pk,
-                                                        'error_msg': force_unicode(e)
+                                                        'error_msg': force_text(e)
                                                     },)
                                                 raise
 
diff --git a/django/core/serializers/__init__.py b/django/core/serializers/__init__.py
index 09bcb651d5..cf7e66190f 100644
--- a/django/core/serializers/__init__.py
+++ b/django/core/serializers/__init__.py
@@ -18,6 +18,7 @@ To add your own serializers, use the SERIALIZATION_MODULES setting::
 
 from django.conf import settings
 from django.utils import importlib
+from django.utils import six
 from django.core.serializers.base import SerializerDoesNotExist
 
 # Built-in serializers
@@ -75,12 +76,12 @@ def get_serializer(format):
 def get_serializer_formats():
     if not _serializers:
         _load_serializers()
-    return _serializers.keys()
+    return list(_serializers)
 
 def get_public_serializer_formats():
     if not _serializers:
         _load_serializers()
-    return [k for k, v in _serializers.iteritems() if not v.Serializer.internal_use_only]
+    return [k for k, v in six.iteritems(_serializers) if not v.Serializer.internal_use_only]
 
 def get_deserializer(format):
     if not _serializers:
diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py
index 19886f7d53..bdb43db9f3 100644
--- a/django/core/serializers/base.py
+++ b/django/core/serializers/base.py
@@ -5,7 +5,7 @@ Module for abstract serializer/unserializer base classes.
 from io import BytesIO
 
 from django.db import models
-from django.utils.encoding import smart_unicode
+from django.utils.encoding import smart_text
 from django.utils import six
 
 class SerializerDoesNotExist(KeyError):
@@ -136,10 +136,12 @@ class Deserializer(object):
     def __iter__(self):
         return self
 
-    def next(self):
+    def __next__(self):
         """Iteration iterface -- return the next item in the stream"""
         raise NotImplementedError
 
+    next = __next__             # Python 2 compatibility
+
 class DeserializedObject(object):
     """
     A deserialized model.
diff --git a/django/core/serializers/json.py b/django/core/serializers/json.py
index 8b56d0e7b8..3bac24d33a 100644
--- a/django/core/serializers/json.py
+++ b/django/core/serializers/json.py
@@ -12,7 +12,7 @@ import json
 from django.core.serializers.base import DeserializationError
 from django.core.serializers.python import Serializer as PythonSerializer
 from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_bytes
 from django.utils import six
 from django.utils.timezone import is_aware
 
diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py
index 333161c929..a1fff6f9bb 100644
--- a/django/core/serializers/python.py
+++ b/django/core/serializers/python.py
@@ -8,7 +8,8 @@ from __future__ import unicode_literals
 from django.conf import settings
 from django.core.serializers import base
 from django.db import models, DEFAULT_DB_ALIAS
-from django.utils.encoding import smart_unicode, is_protected_type
+from django.utils.encoding import smart_text, is_protected_type
+from django.utils import six
 
 class Serializer(base.Serializer):
     """
@@ -33,8 +34,8 @@ class Serializer(base.Serializer):
 
     def get_dump_object(self, obj):
         return {
-            "pk": smart_unicode(obj._get_pk_val(), strings_only=True),
-            "model": smart_unicode(obj._meta),
+            "pk": smart_text(obj._get_pk_val(), strings_only=True),
+            "model": smart_text(obj._meta),
             "fields": self._current
         }
 
@@ -64,7 +65,7 @@ class Serializer(base.Serializer):
             if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
                 m2m_value = lambda value: value.natural_key()
             else:
-                m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
+                m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True)
             self._current[field.name] = [m2m_value(related)
                                for related in getattr(obj, field.name).iterator()]
 
@@ -87,9 +88,9 @@ def Deserializer(object_list, **options):
         m2m_data = {}
 
         # Handle each field
-        for (field_name, field_value) in d["fields"].iteritems():
+        for (field_name, field_value) in six.iteritems(d["fields"]):
             if isinstance(field_value, str):
-                field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
+                field_value = smart_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
 
             field = Model._meta.get_field(field_name)
 
@@ -97,19 +98,19 @@ def Deserializer(object_list, **options):
             if field.rel and isinstance(field.rel, models.ManyToManyRel):
                 if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
                     def m2m_convert(value):
-                        if hasattr(value, '__iter__'):
+                        if hasattr(value, '__iter__') and not isinstance(value, six.text_type):
                             return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
                         else:
-                            return smart_unicode(field.rel.to._meta.pk.to_python(value))
+                            return smart_text(field.rel.to._meta.pk.to_python(value))
                 else:
-                    m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
+                    m2m_convert = lambda v: smart_text(field.rel.to._meta.pk.to_python(v))
                 m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
 
             # Handle FK fields
             elif field.rel and isinstance(field.rel, models.ManyToOneRel):
                 if field_value is not None:
                     if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
-                        if hasattr(field_value, '__iter__'):
+                        if hasattr(field_value, '__iter__') and not isinstance(field_value, six.text_type):
                             obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value)
                             value = getattr(obj, field.rel.field_name)
                             # If this is a natural foreign key to an object that
diff --git a/django/core/serializers/pyyaml.py b/django/core/serializers/pyyaml.py
index ac0e6cf82d..9be1ea4492 100644
--- a/django/core/serializers/pyyaml.py
+++ b/django/core/serializers/pyyaml.py
@@ -12,7 +12,7 @@ from django.db import models
 from django.core.serializers.base import DeserializationError
 from django.core.serializers.python import Serializer as PythonSerializer
 from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_str
+from django.utils.encoding import smart_bytes
 from django.utils import six
 
 
diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py
index 9d9c023b64..666587dc77 100644
--- a/django/core/serializers/xml_serializer.py
+++ b/django/core/serializers/xml_serializer.py
@@ -8,7 +8,7 @@ from django.conf import settings
 from django.core.serializers import base
 from django.db import models, DEFAULT_DB_ALIAS
 from django.utils.xmlutils import SimplerXMLGenerator
-from django.utils.encoding import smart_unicode
+from django.utils.encoding import smart_text
 from xml.dom import pulldom
 
 class Serializer(base.Serializer):
@@ -46,11 +46,11 @@ class Serializer(base.Serializer):
         self.indent(1)
         obj_pk = obj._get_pk_val()
         if obj_pk is None:
-            attrs = {"model": smart_unicode(obj._meta),}
+            attrs = {"model": smart_text(obj._meta),}
         else:
             attrs = {
-                "pk": smart_unicode(obj._get_pk_val()),
-                "model": smart_unicode(obj._meta),
+                "pk": smart_text(obj._get_pk_val()),
+                "model": smart_text(obj._meta),
             }
 
         self.xml.startElement("object", attrs)
@@ -96,10 +96,10 @@ class Serializer(base.Serializer):
                 # Iterable natural keys are rolled out as subelements
                 for key_value in related:
                     self.xml.startElement("natural", {})
-                    self.xml.characters(smart_unicode(key_value))
+                    self.xml.characters(smart_text(key_value))
                     self.xml.endElement("natural")
             else:
-                self.xml.characters(smart_unicode(related_att))
+                self.xml.characters(smart_text(related_att))
         else:
             self.xml.addQuickElement("None")
         self.xml.endElement("field")
@@ -120,13 +120,13 @@ class Serializer(base.Serializer):
                     self.xml.startElement("object", {})
                     for key_value in natural:
                         self.xml.startElement("natural", {})
-                        self.xml.characters(smart_unicode(key_value))
+                        self.xml.characters(smart_text(key_value))
                         self.xml.endElement("natural")
                     self.xml.endElement("object")
             else:
                 def handle_m2m(value):
                     self.xml.addQuickElement("object", attrs={
-                        'pk' : smart_unicode(value._get_pk_val())
+                        'pk' : smart_text(value._get_pk_val())
                     })
             for relobj in getattr(obj, field.name).iterator():
                 handle_m2m(relobj)
@@ -141,7 +141,7 @@ class Serializer(base.Serializer):
         self.xml.startElement("field", {
             "name" : field.name,
             "rel"  : field.rel.__class__.__name__,
-            "to"   : smart_unicode(field.rel.to._meta),
+            "to"   : smart_text(field.rel.to._meta),
         })
 
 class Deserializer(base.Deserializer):
@@ -154,13 +154,15 @@ class Deserializer(base.Deserializer):
         self.event_stream = pulldom.parse(self.stream)
         self.db = options.pop('using', DEFAULT_DB_ALIAS)
 
-    def next(self):
+    def __next__(self):
         for event, node in self.event_stream:
             if event == "START_ELEMENT" and node.nodeName == "object":
                 self.event_stream.expandNode(node)
                 return self._handle_object(node)
         raise StopIteration
 
+    next = __next__             # Python 2 compatibility
+
     def _handle_object(self, node):
         """
         Convert an <object> node to a DeserializedObject.
diff --git a/django/core/signing.py b/django/core/signing.py
index cd9759e536..9ab8c5b8b0 100644
--- a/django/core/signing.py
+++ b/django/core/signing.py
@@ -41,7 +41,7 @@ from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils import baseconv
 from django.utils.crypto import constant_time_compare, salted_hmac
-from django.utils.encoding import force_unicode, smart_str
+from django.utils.encoding import force_text, smart_bytes
 from django.utils.importlib import import_module
 
 
@@ -135,7 +135,7 @@ def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, ma
     """
     Reverse of dumps(), raises BadSignature if signature fails
     """
-    base64d = smart_str(
+    base64d = smart_bytes(
         TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
     decompress = False
     if base64d[0] == '.':
@@ -159,16 +159,16 @@ class Signer(object):
         return base64_hmac(self.salt + 'signer', value, self.key)
 
     def sign(self, value):
-        value = smart_str(value)
+        value = smart_bytes(value)
         return '%s%s%s' % (value, self.sep, self.signature(value))
 
     def unsign(self, signed_value):
-        signed_value = smart_str(signed_value)
+        signed_value = smart_bytes(signed_value)
         if not self.sep in signed_value:
             raise BadSignature('No "%s" found in value' % self.sep)
         value, sig = signed_value.rsplit(self.sep, 1)
         if constant_time_compare(sig, self.signature(value)):
-            return force_unicode(value)
+            return force_text(value)
         raise BadSignature('Signature "%s" does not match' % sig)
 
 
@@ -178,7 +178,7 @@ class TimestampSigner(Signer):
         return baseconv.base62.encode(int(time.time()))
 
     def sign(self, value):
-        value = smart_str('%s%s%s' % (value, self.sep, self.timestamp()))
+        value = smart_bytes('%s%s%s' % (value, self.sep, self.timestamp()))
         return '%s%s%s' % (value, self.sep, self.signature(value))
 
     def unsign(self, value, max_age=None):
diff --git a/django/core/urlresolvers.py b/django/core/urlresolvers.py
index 7397cf3b3d..2fe744e8eb 100644
--- a/django/core/urlresolvers.py
+++ b/django/core/urlresolvers.py
@@ -14,7 +14,7 @@ from threading import local
 from django.http import Http404
 from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
 from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import iri_to_uri, force_unicode, smart_str
+from django.utils.encoding import iri_to_uri, force_text, smart_bytes
 from django.utils.functional import memoize, lazy
 from django.utils.importlib import import_module
 from django.utils.module_loading import module_has_submodule
@@ -163,7 +163,7 @@ class LocaleRegexProvider(object):
             if isinstance(self._regex, six.string_types):
                 regex = self._regex
             else:
-                regex = force_unicode(self._regex)
+                regex = force_text(self._regex)
             try:
                 compiled_regex = re.compile(regex, re.UNICODE)
             except re.error as e:
@@ -190,7 +190,7 @@ class RegexURLPattern(LocaleRegexProvider):
         self.name = name
 
     def __repr__(self):
-        return smart_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
+        return smart_bytes('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
 
     def add_prefix(self, prefix):
         """
@@ -240,7 +240,7 @@ class RegexURLResolver(LocaleRegexProvider):
         self._app_dict = {}
 
     def __repr__(self):
-        return smart_str('<%s %s (%s:%s) %s>' % (self.__class__.__name__, self.urlconf_name, self.app_name, self.namespace, self.regex.pattern))
+        return smart_bytes('<%s %s (%s:%s) %s>' % (self.__class__.__name__, self.urlconf_name, self.app_name, self.namespace, self.regex.pattern))
 
     def _populate(self):
         lookups = MultiValueDict()
@@ -373,10 +373,10 @@ class RegexURLResolver(LocaleRegexProvider):
                 if args:
                     if len(args) != len(params) + len(prefix_args):
                         continue
-                    unicode_args = [force_unicode(val) for val in args]
+                    unicode_args = [force_text(val) for val in args]
                     candidate =  (prefix_norm + result) % dict(zip(prefix_args + params, unicode_args))
                 else:
-                    if set(kwargs.keys() + defaults.keys()) != set(params + defaults.keys() + prefix_args):
+                    if set(kwargs.keys()) | set(defaults.keys()) != set(params) | set(defaults.keys()) | set(prefix_args):
                         continue
                     matches = True
                     for k, v in defaults.items():
@@ -385,7 +385,7 @@ class RegexURLResolver(LocaleRegexProvider):
                             break
                     if not matches:
                         continue
-                    unicode_kwargs = dict([(k, force_unicode(v)) for (k, v) in kwargs.items()])
+                    unicode_kwargs = dict([(k, force_text(v)) for (k, v) in kwargs.items()])
                     candidate = (prefix_norm + result) % unicode_kwargs
                 if re.search('^%s%s' % (_prefix, pattern), candidate, re.UNICODE):
                     return candidate
diff --git a/django/core/validators.py b/django/core/validators.py
index 03ff8be3bc..456fa3cec5 100644
--- a/django/core/validators.py
+++ b/django/core/validators.py
@@ -8,7 +8,7 @@ except ImportError:     # Python 2
 
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_unicode
+from django.utils.encoding import smart_text
 from django.utils.ipv6 import is_valid_ipv6_address
 from django.utils import six
 
@@ -36,7 +36,7 @@ class RegexValidator(object):
         """
         Validates that the input matches the regular expression.
         """
-        if not self.regex.search(smart_unicode(value)):
+        if not self.regex.search(smart_text(value)):
             raise ValidationError(self.message, code=self.code)
 
 class URLValidator(RegexValidator):
@@ -54,7 +54,7 @@ class URLValidator(RegexValidator):
         except ValidationError as e:
             # Trivial case failed. Try for possible IDN domain
             if value:
-                value = smart_unicode(value)
+                value = smart_text(value)
                 scheme, netloc, path, query, fragment = urlsplit(value)
                 try:
                     netloc = netloc.encode('idna') # IDN -> ACE
@@ -138,7 +138,7 @@ def ip_address_validators(protocol, unpack_ipv4):
         return ip_address_validator_map[protocol.lower()]
     except KeyError:
         raise ValueError("The protocol '%s' is unknown. Supported: %s"
-                         % (protocol, ip_address_validator_map.keys()))
+                         % (protocol, list(ip_address_validator_map)))
 
 comma_separated_int_list_re = re.compile('^[\d,]+$')
 validate_comma_separated_integer_list = RegexValidator(comma_separated_int_list_re, _('Enter only digits separated by commas.'), 'invalid')