Merge remote-tracking branch 'core/master' into schema-alteration

Conflicts:
	django/db/models/fields/related.py

23 files changed, 336 insertions, 277 deletions

diff --git a/django/db/__init__.py b/django/db/__init__.py
index 08c901ab7b..2421ddeab8 100644
--- a/django/db/__init__.py
+++ b/django/db/__init__.py
@@ -1,24 +1,19 @@
 import warnings
 
-from django.conf import settings
 from django.core import signals
-from django.core.exceptions import ImproperlyConfigured
 from django.db.utils import (DEFAULT_DB_ALIAS,
     DataError, OperationalError, IntegrityError, InternalError,
     ProgrammingError, NotSupportedError, DatabaseError,
     InterfaceError, Error,
     load_backend, ConnectionHandler, ConnectionRouter)
+from django.utils.functional import cached_property
 
 __all__ = ('backend', 'connection', 'connections', 'router', 'DatabaseError',
     'IntegrityError', 'DEFAULT_DB_ALIAS')
 
+connections = ConnectionHandler()
 
-if settings.DATABASES and DEFAULT_DB_ALIAS not in settings.DATABASES:
-    raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS)
-
-connections = ConnectionHandler(settings.DATABASES)
-
-router = ConnectionRouter(settings.DATABASE_ROUTERS)
+router = ConnectionRouter()
 
 # `connection`, `DatabaseError` and `IntegrityError` are convenient aliases
 # for backend bits.
@@ -45,7 +40,28 @@ class DefaultConnectionProxy(object):
         return delattr(connections[DEFAULT_DB_ALIAS], name)
 
 connection = DefaultConnectionProxy()
-backend = load_backend(connection.settings_dict['ENGINE'])
+
+class DefaultBackendProxy(object):
+    """
+    Temporary proxy class used during deprecation period of the `backend` module
+    variable.
+    """
+    @cached_property
+    def _backend(self):
+        warnings.warn("Accessing django.db.backend is deprecated.",
+            PendingDeprecationWarning, stacklevel=2)
+        return load_backend(connections[DEFAULT_DB_ALIAS].settings_dict['ENGINE'])
+
+    def __getattr__(self, item):
+        return getattr(self._backend, item)
+
+    def __setattr__(self, name, value):
+        return setattr(self._backend, name, value)
+
+    def __delattr__(self, name):
+        return delattr(self._backend, name)
+
+backend = DefaultBackendProxy()
 
 def close_connection(**kwargs):
     warnings.warn(
diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py
index 0b9e75cbbc..1b8e6ae447 100644
--- a/django/db/backends/__init__.py
+++ b/django/db/backends/__init__.py
@@ -390,9 +390,10 @@ class BaseDatabaseWrapper(object):
     def disable_constraint_checking(self):
         """
         Backends can implement as needed to temporarily disable foreign key
-        constraint checking.
+        constraint checking. Should return True if the constraints were 
+        disabled and will need to be reenabled.
         """
-        pass
+        return False
 
     def enable_constraint_checking(self):
         """
@@ -784,12 +785,12 @@ class BaseDatabaseOperations(object):
         """
         return cursor.fetchone()[0]
 
-    def field_cast_sql(self, db_type):
+    def field_cast_sql(self, db_type, internal_type):
         """
-        Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary
-        to cast it before using it in a WHERE statement. Note that the
-        resulting string should contain a '%s' placeholder for the column being
-        searched against.
+        Given a column type (e.g. 'BLOB', 'VARCHAR'), and an internal type
+        (e.g. 'GenericIPAddressField'), returns the SQL necessary to cast it
+        before using it in a WHERE statement. Note that the resulting string
+        should contain a '%s' placeholder for the column being searched against.
         """
         return '%s'
 
diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py
index 21cea1fef8..98830407fb 100644
--- a/django/db/backends/creation.py
+++ b/django/db/backends/creation.py
@@ -99,7 +99,7 @@ class BaseDatabaseCreation(object):
                           style.SQL_TABLE(qn(opts.db_table)) + ' (']
         for i, line in enumerate(table_output):  # Combine and add commas.
             full_statement.append(
-                '    %s%s' % (line, i < len(table_output) - 1 and ',' or ''))
+                '    %s%s' % (line, ',' if i < len(table_output) - 1 else ''))
         full_statement.append(')')
         if opts.db_tablespace:
             tablespace_sql = self.connection.ops.tablespace_sql(
diff --git a/django/db/backends/mysql/compiler.py b/django/db/backends/mysql/compiler.py
index 50a085212b..4e033e3d93 100644
--- a/django/db/backends/mysql/compiler.py
+++ b/django/db/backends/mysql/compiler.py
@@ -17,8 +17,7 @@ class SQLCompiler(compiler.SQLCompiler):
             values.append(value)
         return row[:index_extra_select] + tuple(values)
 
-    def as_subquery_condition(self, alias, columns):
-        qn = self.quote_name_unless_alias
+    def as_subquery_condition(self, alias, columns, qn):
         qn2 = self.connection.ops.quote_name
         sql, params = self.as_sql()
         return '(%s) IN (%s)' % (', '.join(['%s.%s' % (qn(alias), qn2(column)) for column in columns]), sql), params
diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py
index 66860d3d01..798c735d7b 100644
--- a/django/db/backends/oracle/base.py
+++ b/django/db/backends/oracle/base.py
@@ -44,6 +44,11 @@ except ImportError as e:
     from django.core.exceptions import ImproperlyConfigured
     raise ImproperlyConfigured("Error loading cx_Oracle module: %s" % e)
 
+try:
+    import pytz
+except ImportError:
+    pytz = None
+
 from django.db import utils
 from django.db.backends import *
 from django.db.backends.oracle.client import DatabaseClient
@@ -78,6 +83,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_subqueries_in_group_by = False
     supports_transactions = True
     supports_timezones = False
+    has_zoneinfo_database = pytz is not None
     supports_bitwise_or = False
     can_defer_constraint_checks = True
     ignores_nulls_in_unique_constraints = False
@@ -243,9 +249,6 @@ WHEN (new.%(col_name)s IS NULL)
                 value = value.date()
         return value
 
-    def datetime_cast_sql(self):
-        return "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS.FF')"
-
     def deferrable_sql(self):
         return " DEFERRABLE INITIALLY DEFERRED"
 
@@ -255,7 +258,7 @@ WHEN (new.%(col_name)s IS NULL)
     def fetch_returned_insert_id(self, cursor):
         return int(cursor._insert_id_var.getvalue())
 
-    def field_cast_sql(self, db_type):
+    def field_cast_sql(self, db_type, internal_type):
         if db_type and db_type.endswith('LOB'):
             return "DBMS_LOB.SUBSTR(%s)"
         else:
@@ -434,6 +437,17 @@ WHEN (new.%(col_name)s IS NULL)
         second = '%s-12-31'
         return [first % value, second % value]
 
+    def year_lookup_bounds_for_datetime_field(self, value):
+        # The default implementation uses datetime objects for the bounds.
+        # This must be overridden here, to use a formatted date (string) as
+        # 'second' instead -- cx_Oracle chops the fraction-of-second part
+        # off of datetime objects, leaving almost an entire second out of
+        # the year under the default implementation.
+        bounds = super(DatabaseOperations, self).year_lookup_bounds_for_datetime_field(value)
+        if settings.USE_TZ:
+            bounds = [b.astimezone(timezone.utc).replace(tzinfo=None) for b in bounds]
+        return [b.isoformat(b' ') for b in bounds]
+
     def combine_expression(self, connector, sub_expressions):
         "Oracle requires special cases for %% and & operators in query expressions"
         if connector == '%%':
diff --git a/django/db/backends/oracle/creation.py b/django/db/backends/oracle/creation.py
index c8a436c8cd..7f1192ed8a 100644
--- a/django/db/backends/oracle/creation.py
+++ b/django/db/backends/oracle/creation.py
@@ -1,5 +1,7 @@
 import sys
 import time
+
+from django.conf import settings
 from django.db.backends.creation import BaseDatabaseCreation
 from django.utils.six.moves import input
 
@@ -112,7 +114,6 @@ class DatabaseCreation(BaseDatabaseCreation):
                     print("Tests cancelled.")
                     sys.exit(1)
 
-        from django.db import settings
         real_settings = settings.DATABASES[self.connection.alias]
         real_settings['SAVED_USER'] = self.connection.settings_dict['SAVED_USER'] = self.connection.settings_dict['USER']
         real_settings['SAVED_PASSWORD'] = self.connection.settings_dict['SAVED_PASSWORD'] = self.connection.settings_dict['PASSWORD']
diff --git a/django/db/backends/oracle/introspection.py b/django/db/backends/oracle/introspection.py
index ff56dca5c2..361308a62c 100644
--- a/django/db/backends/oracle/introspection.py
+++ b/django/db/backends/oracle/introspection.py
@@ -1,4 +1,5 @@
 from django.db.backends import BaseDatabaseIntrospection, FieldInfo
+from django.utils.encoding import force_text
 import cx_Oracle
 import re
 
@@ -48,7 +49,9 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
         cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name))
         description = []
         for desc in cursor.description:
-            description.append(FieldInfo(*((desc[0].lower(),) + desc[1:])))
+            name = force_text(desc[0]) # cx_Oracle always returns a 'str' on both Python 2 and 3
+            name = name % {} # cx_Oracle, for some reason, doubles percent signs.
+            description.append(FieldInfo(*(name.lower(),) + desc[1:]))
         return description
 
     def table_name_converter(self, name):
@@ -87,6 +90,18 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
             relations[row[0]] = (row[2], row[1].lower())
         return relations
 
+    def get_key_columns(self, cursor, table_name):
+        cursor.execute("""
+            SELECT ccol.column_name, rcol.table_name AS referenced_table, rcol.column_name AS referenced_column
+            FROM user_constraints c
+            JOIN user_cons_columns ccol
+              ON ccol.constraint_name = c.constraint_name 
+            JOIN user_cons_columns rcol
+              ON rcol.constraint_name = c.r_constraint_name 
+            WHERE c.table_name = %s AND c.constraint_type = 'R'""" , [table_name.upper()])
+        return [tuple(cell.lower() for cell in row) 
+                for row in cursor.fetchall()]
+
     def get_indexes(self, cursor, table_name):
         sql = """
     SELECT LOWER(uic1.column_name) AS column_name,
diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py
index b17a0c17bb..f06eec5a1d 100644
--- a/django/db/backends/postgresql_psycopg2/operations.py
+++ b/django/db/backends/postgresql_psycopg2/operations.py
@@ -78,8 +78,8 @@ class DatabaseOperations(BaseDatabaseOperations):
 
         return lookup
 
-    def field_cast_sql(self, db_type):
-        if db_type == 'inet':
+    def field_cast_sql(self, db_type, internal_type):
+        if internal_type == "GenericIPAddressField" or internal_type == "IPAddressField":
             return 'HOST(%s)'
         return '%s'
 
diff --git a/django/db/backends/util.py b/django/db/backends/util.py
index 084f4c200b..aa2601277a 100644
--- a/django/db/backends/util.py
+++ b/django/db/backends/util.py
@@ -83,7 +83,7 @@ class CursorDebugWrapper(CursorWrapper):
 ###############################################
 
 def typecast_date(s):
-    return s and datetime.date(*map(int, s.split('-'))) or None # returns None if s is null
+    return datetime.date(*map(int, s.split('-'))) if s else None # returns None if s is null
 
 def typecast_time(s): # does NOT store time zone information
     if not s: return None
diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py
index 5f17229753..3eac2167d4 100644
--- a/django/db/models/__init__.py
+++ b/django/db/models/__init__.py
@@ -1,7 +1,7 @@
 from functools import wraps
 
 from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
-from django.db.models.loading import get_apps, get_app, get_models, get_model, register_models
+from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models
 from django.db.models.query import Q
 from django.db.models.expressions import F
 from django.db.models.manager import Manager
diff --git a/django/db/models/base.py b/django/db/models/base.py
index 556249fa54..5f1c21c255 100644
--- a/django/db/models/base.py
+++ b/django/db/models/base.py
@@ -632,15 +632,7 @@ class Model(six.with_metaclass(ModelBase)):
             base_qs = cls._base_manager.using(using)
             values = [(f, None, (getattr(self, f.attname) if raw else f.pre_save(self, False)))
                       for f in non_pks]
-            if not values:
-                # We can end up here when saving a model in inheritance chain where
-                # update_fields doesn't target any field in current model. In that
-                # case we just say the update succeeded. Another case ending up here
-                # is a model with just PK - in that case check that the PK still
-                # exists.
-                updated = update_fields is not None or base_qs.filter(pk=pk_val).exists()
-            else:
-                updated = self._do_update(base_qs, using, pk_val, values)
+            updated = self._do_update(base_qs, using, pk_val, values, update_fields)
             if force_update and not updated:
                 raise DatabaseError("Forced update did not affect any rows.")
             if update_fields and not updated:
@@ -664,13 +656,21 @@ class Model(six.with_metaclass(ModelBase)):
                 setattr(self, meta.pk.attname, result)
         return updated
 
-    def _do_update(self, base_qs, using, pk_val, values):
+    def _do_update(self, base_qs, using, pk_val, values, update_fields):
         """
         This method will try to update the model. If the model was updated (in
         the sense that an update query was done and a matching row was found
         from the DB) the method will return True.
         """
-        return base_qs.filter(pk=pk_val)._update(values) > 0
+        if not values:
+            # We can end up here when saving a model in inheritance chain where
+            # update_fields doesn't target any field in current model. In that
+            # case we just say the update succeeded. Another case ending up here
+            # is a model with just PK - in that case check that the PK still
+            # exists.
+            return update_fields is not None or base_qs.filter(pk=pk_val).exists()
+        else:
+            return base_qs.filter(pk=pk_val)._update(values) > 0
 
     def _do_insert(self, manager, using, fields, update_pk, raw):
         """
diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
index 86a0711d7c..691eeffb08 100644
--- a/django/db/models/fields/__init__.py
+++ b/django/db/models/fields/__init__.py
@@ -243,6 +243,8 @@ class Field(object):
         obj = copy.copy(self)
         if self.rel:
             obj.rel = copy.copy(self.rel)
+            if hasattr(self.rel, 'field') and self.rel.field is self:
+                obj.rel.field = obj
         memodict[id(self)] = obj
         return obj
 
diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py
index 5ef713e5e6..754a97633b 100644
--- a/django/db/models/fields/related.py
+++ b/django/db/models/fields/related.py
@@ -11,7 +11,7 @@ from django.db.models.deletion import CASCADE
 from django.utils.encoding import smart_text
 from django.utils import six
 from django.utils.deprecation import RenameMethodsBase
-from django.utils.translation import ugettext_lazy as _, string_concat
+from django.utils.translation import ugettext_lazy as _
 from django.utils.functional import curry, cached_property
 from django.core import exceptions
 from django import forms
@@ -199,7 +199,9 @@ class SingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjectDescri
                     setattr(rel_obj, self.related.field.get_cache_name(), instance)
             setattr(instance, self.cache_name, rel_obj)
         if rel_obj is None:
-            raise self.related.model.DoesNotExist
+            raise self.related.model.DoesNotExist("%s has no %s." % (
+                                                  instance.__class__.__name__,
+                                                  self.related.get_accessor_name()))
         else:
             return rel_obj
 
@@ -224,8 +226,7 @@ class SingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjectDescri
                 value._state.db = router.db_for_write(value.__class__, instance=instance)
             elif value._state.db is not None and instance._state.db is not None:
                 if not router.allow_relation(value, instance):
-                    raise ValueError('Cannot assign "%r": instance is on database "%s", value is on database "%s"' %
-                                        (value, instance._state.db, value._state.db))
+                    raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value)
 
         related_pk = tuple([getattr(instance, field.attname) for field in self.related.field.foreign_related_fields])
         if None in related_pk:
@@ -302,7 +303,8 @@ class ReverseSingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjec
                     setattr(rel_obj, self.field.related.get_cache_name(), instance)
             setattr(instance, self.cache_name, rel_obj)
         if rel_obj is None and not self.field.null:
-            raise self.field.rel.to.DoesNotExist
+            raise self.field.rel.to.DoesNotExist(
+                "%s has no %s." % (self.field.model.__name__, self.field.name))
         else:
             return rel_obj
 
@@ -323,8 +325,7 @@ class ReverseSingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjec
                 value._state.db = router.db_for_write(value.__class__, instance=instance)
             elif value._state.db is not None and instance._state.db is not None:
                 if not router.allow_relation(value, instance):
-                    raise ValueError('Cannot assign "%r": instance is on database "%s", value is on database "%s"' %
-                                        (value, instance._state.db, value._state.db))
+                    raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value)
 
         # If we're setting the value of a OneToOneField to None, we need to clear
         # out the cache on any old related object. Otherwise, deleting the
@@ -1379,9 +1380,6 @@ class ManyToManyField(RelatedField):
 
         super(ManyToManyField, self).__init__(**kwargs)
 
-        msg = _('Hold down "Control", or "Command" on a Mac, to select more than one.')
-        self.help_text = string_concat(self.help_text, ' ', msg)
-
     def deconstruct(self):
         name, path, args, kwargs = super(ManyToManyField, self).deconstruct()
         # Handle the simpler arguments
diff --git a/django/db/models/loading.py b/django/db/models/loading.py
index 075cae4c61..535df7ce80 100644
--- a/django/db/models/loading.py
+++ b/django/db/models/loading.py
@@ -152,7 +152,9 @@ class BaseAppCache(object):
         return self.loaded
 
     def get_apps(self):
-        "Returns a list of all installed modules that contain models."
+        """
+        Returns a list of all installed modules that contain models.
+        """
         self._populate()
 
         # Ensure the returned list is always in the same order (with new apps
@@ -162,6 +164,23 @@ class BaseAppCache(object):
         apps.sort()
         return [elt[1] for elt in apps]
 
+    def get_app_paths(self):
+        """
+        Returns a list of paths to all installed apps.
+
+        Useful for discovering files at conventional locations inside apps
+        (static files, templates, etc.)
+        """
+        self._populate()
+
+        app_paths = []
+        for app in self.get_apps():
+            if hasattr(app, '__path__'):        # models/__init__.py package
+                app_paths.extend([upath(path) for path in app.__path__])
+            else:                               # models.py module
+                app_paths.append(upath(app.__file__))
+        return app_paths
+
     def get_app(self, app_label, emptyOK=False):
         """
         Returns the module containing the models for the given app_label. If
@@ -302,6 +321,7 @@ cache = AppCache()
 # These methods were always module level, so are kept that way for backwards
 # compatibility.
 get_apps = cache.get_apps
+get_app_paths = cache.get_app_paths
 get_app = cache.get_app
 get_app_errors = cache.get_app_errors
 get_models = cache.get_models
diff --git a/django/db/models/manager.py b/django/db/models/manager.py
index 43a8264f11..a1aa79f809 100644
--- a/django/db/models/manager.py
+++ b/django/db/models/manager.py
@@ -186,6 +186,12 @@ class Manager(six.with_metaclass(RenameManagerMethods)):
     def latest(self, *args, **kwargs):
         return self.get_queryset().latest(*args, **kwargs)
 
+    def first(self):
+        return self.get_queryset().first()
+
+    def last(self):
+        return self.get_queryset().last()
+
     def order_by(self, *args, **kwargs):
         return self.get_queryset().order_by(*args, **kwargs)
 
diff --git a/django/db/models/query.py b/django/db/models/query.py
index d3763d3934..b0ce25f5b5 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -9,7 +9,7 @@ import warnings
 
 from django.conf import settings
 from django.core import exceptions
-from django.db import connections, router, transaction, IntegrityError
+from django.db import connections, router, transaction, DatabaseError
 from django.db.models.constants import LOOKUP_SEP
 from django.db.models.fields import AutoField
 from django.db.models.query_utils import (Q, select_related_descend,
@@ -20,11 +20,6 @@ from django.utils.functional import partition
 from django.utils import six
 from django.utils import timezone
 
-# Used to control how many objects are worked with at once in some cases (e.g.
-# when deleting objects).
-CHUNK_SIZE = 100
-ITER_CHUNK_SIZE = CHUNK_SIZE
-
 # The maximum number of items to display in a QuerySet.__repr__
 REPR_OUTPUT_SIZE = 20
 
@@ -41,7 +36,6 @@ class QuerySet(object):
         self._db = using
         self.query = query or sql.Query(self.model)
         self._result_cache = None
-        self._iter = None
         self._sticky_filter = False
         self._for_write = False
         self._prefetch_related_lookups = []
@@ -57,8 +51,8 @@ class QuerySet(object):
         Deep copy of a QuerySet doesn't populate the cache
         """
         obj = self.__class__()
-        for k,v in self.__dict__.items():
-            if k in ('_iter','_result_cache'):
+        for k, v in self.__dict__.items():
+            if k == '_result_cache':
                 obj.__dict__[k] = None
             else:
                 obj.__dict__[k] = copy.deepcopy(v, memo)
@@ -69,10 +63,8 @@ class QuerySet(object):
         Allows the QuerySet to be pickled.
         """
         # Force the cache to be fully populated.
-        len(self)
-
+        self._fetch_all()
         obj_dict = self.__dict__.copy()
-        obj_dict['_iter'] = None
         return obj_dict
 
     def __repr__(self):
@@ -82,95 +74,31 @@ class QuerySet(object):
         return repr(data)
 
     def __len__(self):
-        # Since __len__ is called quite frequently (for example, as part of
-        # list(qs), we make some effort here to be as efficient as possible
-        # whilst not messing up any existing iterators against the QuerySet.
-        if self._result_cache is None:
-            if self._iter:
-                self._result_cache = list(self._iter)
-            else:
-                self._result_cache = list(self.iterator())
-        elif self._iter:
-            self._result_cache.extend(self._iter)
-        if self._prefetch_related_lookups and not self._prefetch_done:
-            self._prefetch_related_objects()
+        self._fetch_all()
         return len(self._result_cache)
 
     def __iter__(self):
-        if self._prefetch_related_lookups and not self._prefetch_done:
-            # We need all the results in order to be able to do the prefetch
-            # in one go. To minimize code duplication, we use the __len__
-            # code path which also forces this, and also does the prefetch
-            len(self)
-
-        if self._result_cache is None:
-            self._iter = self.iterator()
-            self._result_cache = []
-        if self._iter:
-            return self._result_iter()
-        # Python's list iterator is better than our version when we're just
-        # iterating over the cache.
+        """
+        The queryset iterator protocol uses three nested iterators in the
+        default case:
+            1. sql.compiler:execute_sql()
+               - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
+                 using cursor.fetchmany(). This part is responsible for
+                 doing some column masking, and returning the rows in chunks.
+            2. sql/compiler.results_iter()
+               - Returns one row at time. At this point the rows are still just
+                 tuples. In some cases the return values are converted to
+                 Python values at this location (see resolve_columns(),
+                 resolve_aggregate()).
+            3. self.iterator()
+               - Responsible for turning the rows into model objects.
+        """
+        self._fetch_all()
         return iter(self._result_cache)
 
-    def _result_iter(self):
-        pos = 0
-        while 1:
-            upper = len(self._result_cache)
-            while pos < upper:
-                yield self._result_cache[pos]
-                pos = pos + 1
-            if not self._iter:
-                raise StopIteration
-            if len(self._result_cache) <= pos:
-                self._fill_cache()
-
-    def __bool__(self):
-        if self._prefetch_related_lookups and not self._prefetch_done:
-            # We need all the results in order to be able to do the prefetch
-            # in one go. To minimize code duplication, we use the __len__
-            # code path which also forces this, and also does the prefetch
-            len(self)
-
-        if self._result_cache is not None:
-            return bool(self._result_cache)
-        try:
-            next(iter(self))
-        except StopIteration:
-            return False
-        return True
-
-    def __nonzero__(self):      # Python 2 compatibility
-        return type(self).__bool__(self)
-
-    def __contains__(self, val):
-        # The 'in' operator works without this method, due to __iter__. This
-        # implementation exists only to shortcut the creation of Model
-        # instances, by bailing out early if we find a matching element.
-        pos = 0
-        if self._result_cache is not None:
-            if val in self._result_cache:
-                return True
-            elif self._iter is None:
-                # iterator is exhausted, so we have our answer
-                return False
-            # remember not to check these again:
-            pos = len(self._result_cache)
-        else:
-            # We need to start filling the result cache out. The following
-            # ensures that self._iter is not None and self._result_cache is not
-            # None
-            it = iter(self)
-
-        # Carry on, one result at a time.
-        while True:
-            if len(self._result_cache) <= pos:
-                self._fill_cache(num=1)
-            if self._iter is None:
-                # we ran out of items
-                return False
-            if self._result_cache[pos] == val:
-                return True
-            pos += 1
+    def __nonzero__(self):
+        self._fetch_all()
+        return bool(self._result_cache)
 
     def __getitem__(self, k):
         """
@@ -184,19 +112,6 @@ class QuerySet(object):
                 "Negative indexing is not supported."
 
         if self._result_cache is not None:
-            if self._iter is not None:
-                # The result cache has only been partially populated, so we may
-                # need to fill it out a bit more.
-                if isinstance(k, slice):
-                    if k.stop is not None:
-                        # Some people insist on passing in strings here.
-                        bound = int(k.stop)
-                    else:
-                        bound = None
-                else:
-                    bound = k + 1
-                if len(self._result_cache) < bound:
-                    self._fill_cache(bound - len(self._result_cache))
             return self._result_cache[k]
 
         if isinstance(k, slice):
@@ -210,7 +125,7 @@ class QuerySet(object):
             else:
                 stop = None
             qs.query.set_limits(start, stop)
-            return k.step and list(qs)[::k.step] or qs
+            return list(qs)[::k.step] if k.step else qs
 
         qs = self._clone()
         qs.query.set_limits(k, k + 1)
@@ -370,7 +285,7 @@ class QuerySet(object):
         If the QuerySet is already fully cached this simply returns the length
         of the cached results set to avoid multiple SELECT COUNT(*) calls.
         """
-        if self._result_cache is not None and not self._iter:
+        if self._result_cache is not None:
             return len(self._result_cache)
 
         return self.query.get_count(using=self.db)
@@ -388,13 +303,11 @@ class QuerySet(object):
             return clone._result_cache[0]
         if not num:
             raise self.model.DoesNotExist(
-                "%s matching query does not exist. "
-                "Lookup parameters were %s" %
-                (self.model._meta.object_name, kwargs))
+                "%s matching query does not exist." %
+                self.model._meta.object_name)
         raise self.model.MultipleObjectsReturned(
-            "get() returned more than one %s -- it returned %s! "
-            "Lookup parameters were %s" %
-            (self.model._meta.object_name, num, kwargs))
+            "get() returned more than one %s -- it returned %s!" %
+            (self.model._meta.object_name, num))
 
     def create(self, **kwargs):
         """
@@ -450,8 +363,6 @@ class QuerySet(object):
         Returns a tuple of (object, created), where created is a boolean
         specifying whether an object was created.
         """
-        assert kwargs, \
-                'get_or_create() must be passed at least one keyword argument'
         defaults = kwargs.pop('defaults', {})
         lookup = kwargs.copy()
         for f in self.model._meta.fields:
@@ -469,13 +380,13 @@ class QuerySet(object):
                 obj.save(force_insert=True, using=self.db)
                 transaction.savepoint_commit(sid, using=self.db)
                 return obj, True
-            except IntegrityError:
+            except DatabaseError:
                 transaction.savepoint_rollback(sid, using=self.db)
                 exc_info = sys.exc_info()
                 try:
                     return self.get(**lookup), False
                 except self.model.DoesNotExist:
-                    # Re-raise the IntegrityError with its original traceback.
+                    # Re-raise the DatabaseError with its original traceback.
                     six.reraise(*exc_info)
 
     def _earliest_or_latest(self, field_name=None, direction="-"):
@@ -500,6 +411,26 @@ class QuerySet(object):
     def latest(self, field_name=None):
         return self._earliest_or_latest(field_name=field_name, direction="-")
 
+    def first(self):
+        """
+        Returns the first object of a query, returns None if no match is found.
+        """
+        qs = self if self.ordered else self.order_by('pk')
+        try:
+            return qs[0]
+        except IndexError:
+            return None
+
+    def last(self):
+        """
+        Returns the last object of a query, returns None if no match is found.
+        """
+        qs = self.reverse() if self.ordered else self.order_by('-pk')
+        try:
+            return qs[0]
+        except IndexError:
+            return None
+
     def in_bulk(self, id_list):
         """
         Returns a dictionary mapping each of the given IDs to the object with
@@ -714,6 +645,8 @@ class QuerySet(object):
 
         If fields are specified, they must be ForeignKey fields and only those
         related objects are included in the selection.
+
+        If select_related(None) is called, the list is cleared.
         """
         if 'depth' in kwargs:
             warnings.warn('The "depth" keyword argument has been deprecated.\n'
@@ -723,7 +656,9 @@ class QuerySet(object):
             raise TypeError('Unexpected keyword arguments to select_related: %s'
                     % (list(kwargs),))
         obj = self._clone()
-        if fields:
+        if fields == (None,):
+            obj.query.select_related = False
+        elif fields:
             if depth:
                 raise TypeError('Cannot pass both "depth" and fields to select_related()')
             obj.query.add_select_related(fields)
@@ -915,17 +850,11 @@ class QuerySet(object):
             c._setup_query()
         return c
 
-    def _fill_cache(self, num=None):
-        """
-        Fills the result cache with 'num' more entries (or until the results
-        iterator is exhausted).
-        """
-        if self._iter:
-            try:
-                for i in range(num or ITER_CHUNK_SIZE):
-                    self._result_cache.append(next(self._iter))
-            except StopIteration:
-                self._iter = None
+    def _fetch_all(self):
+        if self._result_cache is None:
+            self._result_cache = list(self.iterator())
+        if self._prefetch_related_lookups and not self._prefetch_done:
+            self._prefetch_related_objects()
 
     def _next_is_sticky(self):
         """
@@ -1618,8 +1547,18 @@ def prefetch_related_objects(result_cache, related_lookups):
             if len(obj_list) == 0:
                 break
 
+            current_lookup = LOOKUP_SEP.join(attrs[0:level+1])
+            if current_lookup in done_queries:
+                # Skip any prefetching, and any object preparation
+                obj_list = done_queries[current_lookup]
+                continue
+
+            # Prepare objects:
             good_objects = True
             for obj in obj_list:
+                # Since prefetching can re-use instances, it is possible to have
+                # the same instance multiple times in obj_list, so obj might
+                # already be prepared.
                 if not hasattr(obj, '_prefetched_objects_cache'):
                     try:
                         obj._prefetched_objects_cache = {}
@@ -1630,9 +1569,6 @@ def prefetch_related_objects(result_cache, related_lookups):
                         # now.
                         good_objects = False
                         break
-                else:
-                    # We already did this list
-                    break
             if not good_objects:
                 break
 
@@ -1657,23 +1593,18 @@ def prefetch_related_objects(result_cache, related_lookups):
                                  "prefetch_related()." % lookup)
 
             if prefetcher is not None and not is_fetched:
-                # Check we didn't do this already
-                current_lookup = LOOKUP_SEP.join(attrs[0:level+1])
-                if current_lookup in done_queries:
-                    obj_list = done_queries[current_lookup]
-                else:
-                    obj_list, additional_prl = prefetch_one_level(obj_list, prefetcher, attr)
-                    # We need to ensure we don't keep adding lookups from the
-                    # same relationships to stop infinite recursion. So, if we
-                    # are already on an automatically added lookup, don't add
-                    # the new lookups from relationships we've seen already.
-                    if not (lookup in auto_lookups and
-                            descriptor in followed_descriptors):
-                        for f in additional_prl:
-                            new_prl = LOOKUP_SEP.join([current_lookup, f])
-                            auto_lookups.append(new_prl)
-                        done_queries[current_lookup] = obj_list
-                    followed_descriptors.add(descriptor)
+                obj_list, additional_prl = prefetch_one_level(obj_list, prefetcher, attr)
+                # We need to ensure we don't keep adding lookups from the
+                # same relationships to stop infinite recursion. So, if we
+                # are already on an automatically added lookup, don't add
+                # the new lookups from relationships we've seen already.
+                if not (lookup in auto_lookups and
+                        descriptor in followed_descriptors):
+                    for f in additional_prl:
+                        new_prl = LOOKUP_SEP.join([current_lookup, f])
+                        auto_lookups.append(new_prl)
+                    done_queries[current_lookup] = obj_list
+                followed_descriptors.add(descriptor)
             else:
                 # Either a singly related object that has already been fetched
                 # (e.g. via select_related), or hopefully some other property
diff --git a/django/db/models/signals.py b/django/db/models/signals.py
index 09f93d0f77..3e321893c1 100644
--- a/django/db/models/signals.py
+++ b/django/db/models/signals.py
@@ -12,6 +12,7 @@ post_save = Signal(providing_args=["instance", "raw", "created", "using", "updat
 pre_delete = Signal(providing_args=["instance", "using"], use_caching=True)
 post_delete = Signal(providing_args=["instance", "using"], use_caching=True)
 
+pre_syncdb = Signal(providing_args=["app", "create_models", "verbosity", "interactive", "db"])
 post_syncdb = Signal(providing_args=["class", "app", "created_models", "verbosity", "interactive", "db"], use_caching=True)
 
 m2m_changed = Signal(providing_args=["action", "instance", "reverse", "model", "pk_set", "using"], use_caching=True)
diff --git a/django/db/models/sql/aggregates.py b/django/db/models/sql/aggregates.py
index 23b79923d1..2bd2b2f76f 100644
--- a/django/db/models/sql/aggregates.py
+++ b/django/db/models/sql/aggregates.py
@@ -99,7 +99,7 @@ class Count(Aggregate):
     sql_template = '%(function)s(%(distinct)s%(field)s)'
 
     def __init__(self, col, distinct=False, **extra):
-        super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra)
+        super(Count, self).__init__(col, distinct='DISTINCT ' if distinct else '', **extra)
 
 class Max(Aggregate):
     sql_function = 'MAX'
diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index bbe310c8c3..0bfd1b38d3 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -729,7 +729,8 @@ class SQLCompiler(object):
                     row = self.resolve_columns(row, fields)
 
                 if has_aggregate_select:
-                    aggregate_start = len(self.query.extra_select) + len(self.query.select)
+                    loaded_fields = self.query.get_loaded_field_names().get(self.query.model, set()) or self.query.select
+                    aggregate_start = len(self.query.extra_select) + len(loaded_fields)
                     aggregate_end = aggregate_start + len(self.query.aggregate_select)
                     row = tuple(row[:aggregate_start]) + tuple([
                         self.query.resolve_aggregate(value, aggregate, self.connection)
@@ -786,8 +787,7 @@ class SQLCompiler(object):
             return list(result)
         return result
 
-    def as_subquery_condition(self, alias, columns):
-        qn = self.quote_name_unless_alias
+    def as_subquery_condition(self, alias, columns, qn):
         qn2 = self.connection.ops.quote_name
         if len(columns) == 1:
             sql, params = self.as_sql()
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 0a4152587d..154b6bd204 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -1422,7 +1422,9 @@ class Query(object):
         query.clear_ordering(True)
         # Try to have as simple as possible subquery -> trim leading joins from
         # the subquery.
-        trimmed_joins = query.trim_start(names_with_path)
+        trimmed_prefix, contains_louter = query.trim_start(names_with_path)
+        query.remove_inherited_models()
+
         # Add extra check to make sure the selected field will not be null
         # since we are adding a IN <subquery> clause. This prevents the
         # database from tripping over IN (...,NULL,...) selects and returning
@@ -1431,38 +1433,20 @@ class Query(object):
             alias, col = query.select[0].col
             query.where.add((Constraint(alias, col, query.select[0].field), 'isnull', False), AND)
 
-        # Still make sure that the trimmed parts in the inner query and
-        # trimmed prefix are in sync. So, use the trimmed_joins to make sure
-        # as many path elements are in the prefix as there were trimmed joins.
-        # In addition, convert the path elements back to names so that
-        # add_filter() can handle them.
-        trimmed_prefix = []
-        paths_in_prefix = trimmed_joins
-        for name, path in names_with_path:
-            if paths_in_prefix - len(path) < 0:
-                break
-            trimmed_prefix.append(name)
-            paths_in_prefix -= len(path)
-        join_field = path[paths_in_prefix].join_field
-        # TODO: This should be made properly multicolumn
-        # join aware. It is likely better to not use build_filter
-        # at all, instead construct joins up to the correct point,
-        # then construct the needed equality constraint manually,
-        # or maybe using SubqueryConstraint would work, too.
-        # The foreign_related_fields attribute is right here, we
-        # don't ever split joins for direct case.
-        trimmed_prefix.append(
-            join_field.field.foreign_related_fields[0].name)
-        trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
         condition = self.build_filter(
             ('%s__in' % trimmed_prefix, query),
             current_negated=True, branch_negated=True, can_reuse=can_reuse)
-        # Intentionally leave the other alias as blank, if the condition
-        # refers it, things will break here.
-        extra_restriction = join_field.get_extra_restriction(
-            self.where_class, None, [t for t in query.tables if query.alias_refcount[t]][0])
-        if extra_restriction:
-            query.where.add(extra_restriction, 'AND')
+        if contains_louter:
+            or_null_condition = self.build_filter(
+                ('%s__isnull' % trimmed_prefix, True),
+                current_negated=True, branch_negated=True, can_reuse=can_reuse)
+            condition.add(or_null_condition, OR)
+            # Note that the end result will be:
+            # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL.
+            # This might look crazy but due to how IN works, this seems to be
+            # correct. If the IS NOT NULL check is removed then outercol NOT
+            # IN will return UNKNOWN. If the IS NULL check is removed, then if
+            # outercol IS NULL we will not match the row.
         return condition
 
     def set_empty(self):
@@ -1821,35 +1805,58 @@ class Query(object):
     def trim_start(self, names_with_path):
         """
         Trims joins from the start of the join path. The candidates for trim
-        are the PathInfos in names_with_path structure. Outer joins are not
-        eligible for removal. Also sets the select column so the start
-        matches the join.
+        are the PathInfos in names_with_path structure that are m2m joins.
+
+        Also sets the select column so the start matches the join.
+
+        This method is meant to be used for generating the subquery joins &
+        cols in split_exclude().
 
-        This method is mostly useful for generating the subquery joins & col
-        in "WHERE somecol IN (subquery)". This construct is needed by
-        split_exclude().
+        Returns a lookup usable for doing outerq.filter(lookup=self). Returns
+        also if the joins in the prefix contain a LEFT OUTER join.
         _"""
         all_paths = []
         for _, paths in names_with_path:
             all_paths.extend(paths)
-        direct_join = True
+        contains_louter = False
         for pos, path in enumerate(all_paths):
+            if path.m2m:
+                break
             if self.alias_map[self.tables[pos + 1]].join_type == self.LOUTER:
-                direct_join = False
-                pos -= 1
+                contains_louter = True
+            self.unref_alias(self.tables[pos])
+        # The path.join_field is a Rel, lets get the other side's field
+        join_field = path.join_field.field
+        # Build the filter prefix.
+        trimmed_prefix = []
+        paths_in_prefix = pos
+        for name, path in names_with_path:
+            if paths_in_prefix - len(path) < 0:
                 break
+            trimmed_prefix.append(name)
+            paths_in_prefix -= len(path)
+        trimmed_prefix.append(
+            join_field.foreign_related_fields[0].name)
+        trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
+        # Lets still see if we can trim the first join from the inner query
+        # (that is, self). We can't do this for LEFT JOINs because we would
+        # miss those rows that have nothing on the outer side.
+        if self.alias_map[self.tables[pos + 1]].join_type != self.LOUTER:
+            select_fields = [r[0] for r in join_field.related_fields]
+            select_alias = self.tables[pos + 1]
             self.unref_alias(self.tables[pos])
-        if path.direct:
-            direct_join = not direct_join
-        join_side = 0 if direct_join else 1
-        select_alias = self.tables[pos + 1]
-        join_field = path.join_field
-        if hasattr(join_field, 'field'):
-            join_field = join_field.field
-        select_fields = [r[join_side] for r in join_field.related_fields]
+            extra_restriction = join_field.get_extra_restriction(
+                self.where_class, None, self.tables[pos + 1])
+            if extra_restriction:
+                self.where.add(extra_restriction, AND)
+        else:
+            # TODO: It might be possible to trim more joins from the start of the
+            # inner query if it happens to have a longer join chain containing the
+            # values in select_fields. Lets punt this one for now.
+            select_fields = [r[1] for r in join_field.related_fields]
+            select_alias = self.tables[pos]
         self.select = [SelectInfo((select_alias, f.column), f) for f in select_fields]
-        self.remove_inherited_models()
-        return pos
+        return trimmed_prefix, contains_louter
 
     def is_nullable(self, field):
         """
diff --git a/django/db/models/sql/where.py b/django/db/models/sql/where.py
index 029226383d..2a342d417a 100644
--- a/django/db/models/sql/where.py
+++ b/django/db/models/sql/where.py
@@ -174,6 +174,8 @@ class WhereNode(tree.Node):
         it.
         """
         lvalue, lookup_type, value_annotation, params_or_value = child
+        field_internal_type = lvalue.field.get_internal_type() if lvalue.field else None
+        
         if isinstance(lvalue, Constraint):
             try:
                 lvalue, params = lvalue.process(lookup_type, params_or_value, connection)
@@ -187,7 +189,7 @@ class WhereNode(tree.Node):
 
         if isinstance(lvalue, tuple):
             # A direct database column lookup.
-            field_sql, field_params = self.sql_for_columns(lvalue, qn, connection), []
+            field_sql, field_params = self.sql_for_columns(lvalue, qn, connection, field_internal_type), []
         else:
             # A smart object with an as_sql() method.
             field_sql, field_params = lvalue.as_sql(qn, connection)
@@ -257,7 +259,7 @@ class WhereNode(tree.Node):
 
         raise TypeError('Invalid lookup_type: %r' % lookup_type)
 
-    def sql_for_columns(self, data, qn, connection):
+    def sql_for_columns(self, data, qn, connection, internal_type=None):
         """
         Returns the SQL fragment used for the left-hand side of a column
         constraint (for example, the "T1.foo" portion in the clause
@@ -268,7 +270,7 @@ class WhereNode(tree.Node):
             lhs = '%s.%s' % (qn(table_alias), qn(name))
         else:
             lhs = qn(name)
-        return connection.ops.field_cast_sql(db_type) % lhs
+        return connection.ops.field_cast_sql(db_type, internal_type) % lhs
 
     def relabel_aliases(self, change_map):
         """
@@ -397,13 +399,21 @@ class SubqueryConstraint(object):
         if hasattr(query, 'values'):
             if query._db and connection.alias != query._db:
                 raise ValueError("Can't do subqueries with queries on different DBs.")
-            query = query.values(*self.targets).query
+            # Do not override already existing values.
+            if not hasattr(query, 'field_names'):
+                query = query.values(*self.targets)
+            else:
+                query = query._clone()
+            query = query.query
             query.clear_ordering(True)
 
         query_compiler = query.get_compiler(connection=connection)
-        return query_compiler.as_subquery_condition(self.alias, self.columns)
+        return query_compiler.as_subquery_condition(self.alias, self.columns, qn)
+
+    def relabel_aliases(self, change_map):
+        self.alias = change_map.get(self.alias, self.alias)
 
-    def relabeled_clone(self, relabels):
+    def clone(self):
         return self.__class__(
-            relabels.get(self.alias, self.alias),
-            self.columns, self.query_object)
+            self.alias, self.columns, self.targets,
+            self.query_object)
diff --git a/django/db/transaction.py b/django/db/transaction.py
index 48e7f900dd..f770f2efa7 100644
--- a/django/db/transaction.py
+++ b/django/db/transaction.py
@@ -333,6 +333,23 @@ def atomic(using=None, savepoint=True):
         return Atomic(using, savepoint)
 
 
+def _non_atomic_requests(view, using):
+    try:
+        view._non_atomic_requests.add(using)
+    except AttributeError:
+        view._non_atomic_requests = set([using])
+    return view
+
+
+def non_atomic_requests(using=None):
+    if callable(using):
+        return _non_atomic_requests(using, DEFAULT_DB_ALIAS)
+    else:
+        if using is None:
+            using = DEFAULT_DB_ALIAS
+        return lambda view: _non_atomic_requests(view, using)
+
+
 ############################################
 # Deprecated decorators / context managers #
 ############################################
diff --git a/django/db/utils.py b/django/db/utils.py
index e84060f9b3..bd7e10d24c 100644
--- a/django/db/utils.py
+++ b/django/db/utils.py
@@ -6,6 +6,7 @@ import warnings
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from django.utils.functional import cached_property
 from django.utils.importlib import import_module
 from django.utils.module_loading import import_by_path
 from django.utils._os import upath
@@ -90,8 +91,7 @@ class DatabaseErrorWrapper(object):
                 except AttributeError:
                     args = (exc_value,)
                 dj_exc_value = dj_exc_type(*args)
-                if six.PY3:
-                    dj_exc_value.__cause__ = exc_value
+                dj_exc_value.__cause__ = exc_value
                 # Only set the 'errors_occurred' flag for errors that may make
                 # the connection unusable.
                 if dj_exc_type not in (DataError, IntegrityError):
@@ -138,16 +138,27 @@ class ConnectionDoesNotExist(Exception):
 
 
 class ConnectionHandler(object):
-    def __init__(self, databases):
-        if not databases:
-            self.databases = {
+    def __init__(self, databases=None):
+        """
+        databases is an optional dictionary of database definitions (structured
+        like settings.DATABASES).
+        """
+        self._databases = databases
+        self._connections = local()
+
+    @cached_property
+    def databases(self):
+        if self._databases is None:
+            self._databases = settings.DATABASES
+        if self._databases == {}:
+            self._databases = {
                 DEFAULT_DB_ALIAS: {
                     'ENGINE': 'django.db.backends.dummy',
                 },
             }
-        else:
-            self.databases = databases
-        self._connections = local()
+        if DEFAULT_DB_ALIAS not in self._databases:
+            raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS)
+        return self._databases
 
     def ensure_defaults(self, alias):
         """
@@ -202,14 +213,24 @@ class ConnectionHandler(object):
 
 
 class ConnectionRouter(object):
-    def __init__(self, routers):
-        self.routers = []
-        for r in routers:
+    def __init__(self, routers=None):
+        """
+        If routers is not specified, will default to settings.DATABASE_ROUTERS.
+        """
+        self._routers = routers
+
+    @cached_property
+    def routers(self):
+        if self._routers is None:
+            self._routers = settings.DATABASE_ROUTERS
+        routers = []
+        for r in self._routers:
             if isinstance(r, six.string_types):
                 router = import_by_path(r)()
             else:
                 router = r
-            self.routers.append(router)
+            routers.append(router)
+        return routers
 
     def _router_func(action):
         def _route_db(self, model, **hints):