Fixed #20625 -- Chainable Manager/QuerySet methods.

Additionally this patch solves the orthogonal problem that specialized
`QuerySet` like `ValuesQuerySet` didn't inherit from the current `QuerySet`
type. This wasn't an issue until now because we didn't officially support
custom `QuerySet` but it became necessary with the introduction of this new
feature.

Thanks aaugustin, akaariai, carljm, charettes, mjtamlyn, shaib and timgraham
for the reviews.

3 files changed, 105 insertions, 118 deletions

diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py
index 33151e068d..2ee525faf1 100644
--- a/django/db/models/__init__.py
+++ b/django/db/models/__init__.py
@@ -2,7 +2,7 @@ from functools import wraps
 
 from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
 from django.db.models.loading import get_apps, get_app_path, get_app_paths, get_app, get_models, get_model, register_models, UnavailableApp
-from django.db.models.query import Q
+from django.db.models.query import Q, QuerySet
 from django.db.models.expressions import F
 from django.db.models.manager import Manager
 from django.db.models.base import Model
diff --git a/django/db/models/manager.py b/django/db/models/manager.py
index b369aedb64..f57944ebbc 100644
--- a/django/db/models/manager.py
+++ b/django/db/models/manager.py
@@ -1,4 +1,6 @@
 import copy
+import inspect
+
 from django.db import router
 from django.db.models.query import QuerySet, insert_query, RawQuerySet
 from django.db.models import signals
@@ -56,17 +58,51 @@ class RenameManagerMethods(RenameMethodsBase):
     )
 
 
-class Manager(six.with_metaclass(RenameManagerMethods)):
+class BaseManager(six.with_metaclass(RenameManagerMethods)):
     # Tracks each time a Manager instance is created. Used to retain order.
     creation_counter = 0
 
     def __init__(self):
-        super(Manager, self).__init__()
+        super(BaseManager, self).__init__()
         self._set_creation_counter()
         self.model = None
         self._inherited = False
         self._db = None
 
+    @classmethod
+    def _get_queryset_methods(cls, queryset_class):
+        def create_method(name, method):
+            def manager_method(self, *args, **kwargs):
+                return getattr(self.get_queryset(), name)(*args, **kwargs)
+            manager_method.__name__ = method.__name__
+            manager_method.__doc__ = method.__doc__
+            return manager_method
+
+        new_methods = {}
+        # Refs http://bugs.python.org/issue1785.
+        predicate = inspect.isfunction if six.PY3 else inspect.ismethod
+        for name, method in inspect.getmembers(queryset_class, predicate=predicate):
+            # Only copy missing methods.
+            if hasattr(cls, name):
+                continue
+            # Only copy public methods or methods with the attribute `queryset_only=False`.
+            queryset_only = getattr(method, 'queryset_only', None)
+            if queryset_only or (queryset_only is None and name.startswith('_')):
+                continue
+            # Copy the method onto the manager.
+            new_methods[name] = create_method(name, method)
+        return new_methods
+
+    @classmethod
+    def from_queryset(cls, queryset_class, class_name=None):
+        if class_name is None:
+            class_name = '%sFrom%s' % (cls.__name__, queryset_class.__name__)
+        class_dict = {
+            '_queryset_class': queryset_class,
+        }
+        class_dict.update(cls._get_queryset_methods(queryset_class))
+        return type(class_name, (cls,), class_dict)
+
     def contribute_to_class(self, model, name):
         # TODO: Use weakref because of possible memory leak / circular reference.
         self.model = model
@@ -92,8 +128,8 @@ class Manager(six.with_metaclass(RenameManagerMethods)):
         Sets the creation counter value for this instance and increments the
         class-level copy.
         """
-        self.creation_counter = Manager.creation_counter
-        Manager.creation_counter += 1
+        self.creation_counter = BaseManager.creation_counter
+        BaseManager.creation_counter += 1
 
     def _copy_to_model(self, model):
         """
@@ -117,130 +153,30 @@ class Manager(six.with_metaclass(RenameManagerMethods)):
     def db(self):
         return self._db or router.db_for_read(self.model)
 
-    #######################
-    # PROXIES TO QUERYSET #
-    #######################
-
     def get_queryset(self):
-        """Returns a new QuerySet object.  Subclasses can override this method
-        to easily customize the behavior of the Manager.
         """
-        return QuerySet(self.model, using=self._db)
-
-    def none(self):
-        return self.get_queryset().none()
+        Returns a new QuerySet object.  Subclasses can override this method to
+        easily customize the behavior of the Manager.
+        """
+        return self._queryset_class(self.model, using=self._db)
 
     def all(self):
+        # We can't proxy this method through the `QuerySet` like we do for the
+        # rest of the `QuerySet` methods. This is because `QuerySet.all()`
+        # works by creating a "copy" of the current queryset and in making said
+        # copy, all the cached `prefetch_related` lookups are lost. See the
+        # implementation of `RelatedManager.get_queryset()` for a better
+        # understanding of how this comes into play.
         return self.get_queryset()
 
-    def count(self):
-        return self.get_queryset().count()
-
-    def dates(self, *args, **kwargs):
-        return self.get_queryset().dates(*args, **kwargs)
-
-    def datetimes(self, *args, **kwargs):
-        return self.get_queryset().datetimes(*args, **kwargs)
-
-    def distinct(self, *args, **kwargs):
-        return self.get_queryset().distinct(*args, **kwargs)
-
-    def extra(self, *args, **kwargs):
-        return self.get_queryset().extra(*args, **kwargs)
-
-    def get(self, *args, **kwargs):
-        return self.get_queryset().get(*args, **kwargs)
-
-    def get_or_create(self, **kwargs):
-        return self.get_queryset().get_or_create(**kwargs)
-
-    def update_or_create(self, **kwargs):
-        return self.get_queryset().update_or_create(**kwargs)
-
-    def create(self, **kwargs):
-        return self.get_queryset().create(**kwargs)
-
-    def bulk_create(self, *args, **kwargs):
-        return self.get_queryset().bulk_create(*args, **kwargs)
-
-    def filter(self, *args, **kwargs):
-        return self.get_queryset().filter(*args, **kwargs)
-
-    def aggregate(self, *args, **kwargs):
-        return self.get_queryset().aggregate(*args, **kwargs)
-
-    def annotate(self, *args, **kwargs):
-        return self.get_queryset().annotate(*args, **kwargs)
-
-    def complex_filter(self, *args, **kwargs):
-        return self.get_queryset().complex_filter(*args, **kwargs)
-
-    def exclude(self, *args, **kwargs):
-        return self.get_queryset().exclude(*args, **kwargs)
-
-    def in_bulk(self, *args, **kwargs):
-        return self.get_queryset().in_bulk(*args, **kwargs)
-
-    def iterator(self, *args, **kwargs):
-        return self.get_queryset().iterator(*args, **kwargs)
-
-    def earliest(self, *args, **kwargs):
-        return self.get_queryset().earliest(*args, **kwargs)
-
-    def latest(self, *args, **kwargs):
-        return self.get_queryset().latest(*args, **kwargs)
-
-    def first(self):
-        return self.get_queryset().first()
-
-    def last(self):
-        return self.get_queryset().last()
-
-    def order_by(self, *args, **kwargs):
-        return self.get_queryset().order_by(*args, **kwargs)
-
-    def select_for_update(self, *args, **kwargs):
-        return self.get_queryset().select_for_update(*args, **kwargs)
-
-    def select_related(self, *args, **kwargs):
-        return self.get_queryset().select_related(*args, **kwargs)
-
-    def prefetch_related(self, *args, **kwargs):
-        return self.get_queryset().prefetch_related(*args, **kwargs)
-
-    def values(self, *args, **kwargs):
-        return self.get_queryset().values(*args, **kwargs)
-
-    def values_list(self, *args, **kwargs):
-        return self.get_queryset().values_list(*args, **kwargs)
-
-    def update(self, *args, **kwargs):
-        return self.get_queryset().update(*args, **kwargs)
-
-    def reverse(self, *args, **kwargs):
-        return self.get_queryset().reverse(*args, **kwargs)
-
-    def defer(self, *args, **kwargs):
-        return self.get_queryset().defer(*args, **kwargs)
-
-    def only(self, *args, **kwargs):
-        return self.get_queryset().only(*args, **kwargs)
-
-    def using(self, *args, **kwargs):
-        return self.get_queryset().using(*args, **kwargs)
-
-    def exists(self, *args, **kwargs):
-        return self.get_queryset().exists(*args, **kwargs)
-
     def _insert(self, objs, fields, **kwargs):
         return insert_query(self.model, objs, fields, **kwargs)
 
-    def _update(self, values, **kwargs):
-        return self.get_queryset()._update(values, **kwargs)
-
     def raw(self, raw_query, params=None, *args, **kwargs):
         return RawQuerySet(raw_query=raw_query, model=self.model, params=params, using=self._db, *args, **kwargs)
 
+Manager = BaseManager.from_queryset(QuerySet, class_name='Manager')
+
 
 class ManagerDescriptor(object):
     # This class ensures managers aren't accessible via model instances.
diff --git a/django/db/models/query.py b/django/db/models/query.py
index 087c10de8e..406838f907 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -10,7 +10,7 @@ from django.conf import settings
 from django.core import exceptions
 from django.db import connections, router, transaction, DatabaseError, IntegrityError
 from django.db.models.constants import LOOKUP_SEP
-from django.db.models.fields import AutoField
+from django.db.models.fields import AutoField, Empty
 from django.db.models.query_utils import (Q, select_related_descend,
     deferred_class_factory, InvalidQuery)
 from django.db.models.deletion import Collector
@@ -30,10 +30,23 @@ REPR_OUTPUT_SIZE = 20
 EmptyResultSet = sql.EmptyResultSet
 
 
+def _pickle_queryset(class_bases, class_dict):
+    """
+    Used by `__reduce__` to create the initial version of the `QuerySet` class
+    onto which the output of `__getstate__` will be applied.
+
+    See `__reduce__` for more details.
+    """
+    new = Empty()
+    new.__class__ = type(class_bases[0].__name__, class_bases, class_dict)
+    return new
+
+
 class QuerySet(object):
     """
     Represents a lazy database lookup for a set of objects.
     """
+
     def __init__(self, model=None, query=None, using=None):
         self.model = model
         self._db = using
@@ -45,6 +58,13 @@ class QuerySet(object):
         self._prefetch_done = False
         self._known_related_objects = {}        # {rel_field, {pk: rel_obj}}
 
+    def as_manager(cls):
+        # Address the circular dependency between `Queryset` and `Manager`.
+        from django.db.models.manager import Manager
+        return Manager.from_queryset(cls)()
+    as_manager.queryset_only = True
+    as_manager = classmethod(as_manager)
+
     ########################
     # PYTHON MAGIC METHODS #
     ########################
@@ -70,6 +90,26 @@ class QuerySet(object):
         obj_dict = self.__dict__.copy()
         return obj_dict
 
+    def __reduce__(self):
+        """
+        Used by pickle to deal with the types that we create dynamically when
+        specialized queryset such as `ValuesQuerySet` are used in conjunction
+        with querysets that are *subclasses* of `QuerySet`.
+
+        See `_clone` implementation for more details.
+        """
+        if hasattr(self, '_specialized_queryset_class'):
+            class_bases = (
+                self._specialized_queryset_class,
+                self._base_queryset_class,
+            )
+            class_dict = {
+                '_specialized_queryset_class': self._specialized_queryset_class,
+                '_base_queryset_class': self._base_queryset_class,
+            }
+            return _pickle_queryset, (class_bases, class_dict), self.__getstate__()
+        return super(QuerySet, self).__reduce__()
+
     def __repr__(self):
         data = list(self[:REPR_OUTPUT_SIZE + 1])
         if len(data) > REPR_OUTPUT_SIZE:
@@ -528,6 +568,7 @@ class QuerySet(object):
         # Clear the result cache, in case this QuerySet gets reused.
         self._result_cache = None
     delete.alters_data = True
+    delete.queryset_only = True
 
     def _raw_delete(self, using):
         """
@@ -567,6 +608,7 @@ class QuerySet(object):
         self._result_cache = None
         return query.get_compiler(self.db).execute_sql(None)
     _update.alters_data = True
+    _update.queryset_only = False
 
     def exists(self):
         if self._result_cache is None:
@@ -886,6 +928,15 @@ class QuerySet(object):
     def _clone(self, klass=None, setup=False, **kwargs):
         if klass is None:
             klass = self.__class__
+        elif not issubclass(self.__class__, klass):
+            base_queryset_class = getattr(self, '_base_queryset_class', self.__class__)
+            class_bases = (klass, base_queryset_class)
+            class_dict = {
+                '_base_queryset_class': base_queryset_class,
+                '_specialized_queryset_class': klass,
+            }
+            klass = type(klass.__name__, class_bases, class_dict)
+
         query = self.query.clone()
         if self._sticky_filter:
             query.filter_is_sticky = True