[1.8.x] Fixed #24225, #24264, #24282 -- Rewrote model reloading in migration project state

Instead of naively reloading only directly related models (FK, O2O, M2M
relationship) the project state needs to reload their relations as well
as the model changes as well. Furthermore inheriting models (and super
models) need to be reloaded in order to keep inherited fields in sync.

To prevent endless recursive calls an iterative approach is taken.

Backport of b29f3b51204d53c1c8745966476543d068c173a2 from master

1 files changed, 110 insertions, 40 deletions

diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py
index 2e2fbaa0f8..fdce3c77c3 100644
--- a/django/db/migrations/state.py
+++ b/django/db/migrations/state.py
@@ -8,7 +8,9 @@ from django.apps.registry import Apps, apps as global_apps
 from django.conf import settings
 from django.db import models
 from django.db.models.fields.proxy import OrderWrt
-from django.db.models.fields.related import do_pending_lookups
+from django.db.models.fields.related import (
+    RECURSIVE_RELATIONSHIP_CONSTANT, do_pending_lookups,
+)
 from django.db.models.options import DEFAULT_NAMES, normalize_together
 from django.utils import six
 from django.utils.encoding import force_text, smart_text
@@ -21,6 +23,39 @@ class InvalidBasesError(ValueError):
     pass
 
 
+def _get_app_label_and_model_name(model, app_label=''):
+    if isinstance(model, six.string_types):
+        split = model.split('.', 1)
+        return (tuple(split) if len(split) == 2 else (app_label, split[0]))
+    else:
+        return model._meta.app_label, model._meta.model_name
+
+
+def get_related_models_recursive(model):
+    """
+    Returns all models that have a direct or indirect relationship
+    to the given model.
+    """
+    def _related_models(m):
+        return [
+            f.related_model for f in m._meta.get_fields(include_parents=True, include_hidden=True)
+            if f.is_relation and not isinstance(f.related_model, six.string_types)
+        ] + [
+            subclass for subclass in m.__subclasses__()
+            if issubclass(subclass, models.Model)
+        ]
+
+    seen = set()
+    queue = _related_models(model)
+    for rel_mod in queue:
+        rel_app_label, rel_model_name = rel_mod._meta.app_label, rel_mod._meta.model_name
+        if (rel_app_label, rel_model_name) in seen:
+            continue
+        seen.add((rel_app_label, rel_model_name))
+        queue.extend(_related_models(rel_mod))
+    return seen - {(model._meta.app_label, model._meta.model_name)}
+
+
 class ProjectState(object):
     """
     Represents the entire project's overall state.
@@ -46,27 +81,57 @@ class ProjectState(object):
 
     def reload_model(self, app_label, model_name):
         if 'apps' in self.__dict__:  # hasattr would cache the property
-            # Get relations before reloading the models, as _meta.apps may change
             try:
-                related_old = {
-                    f.related_model for f in
-                    self.apps.get_model(app_label, model_name)._meta.related_objects
-                }
+                old_model = self.apps.get_model(app_label, model_name)
             except LookupError:
-                related_old = set()
-            self._reload_one_model(app_label, model_name)
-            # Reload models if there are relations
-            model = self.apps.get_model(app_label, model_name)
-            related_m2m = {f.related_model for f in model._meta.many_to_many}
-            for rel_model in related_old.union(related_m2m):
-                self._reload_one_model(rel_model._meta.app_label, rel_model._meta.model_name)
-            if related_m2m:
-                # Re-render this model after related models have been reloaded
-                self._reload_one_model(app_label, model_name)
+                related_models = set()
+            else:
+                # Get all relations to and from the old model before reloading,
+                # as _meta.apps may change
+                related_models = get_related_models_recursive(old_model)
+
+            # Get all outgoing references from the model to be rendered
+            model_state = self.models[(app_label, model_name)]
+            for name, field in model_state.fields:
+                if field.is_relation:
+                    if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT:
+                        continue
+                    rel_app_label, rel_model_name = _get_app_label_and_model_name(field.rel.to, app_label)
+                    related_models.add((rel_app_label, rel_model_name.lower()))
 
-    def _reload_one_model(self, app_label, model_name):
-        self.apps.unregister_model(app_label, model_name)
-        self.models[app_label, model_name].render(self.apps)
+            # Unregister all related models
+            for rel_app_label, rel_model_name in related_models:
+                self.apps.unregister_model(rel_app_label, rel_model_name)
+
+            # Unregister the current model
+            self.apps.unregister_model(app_label, model_name)
+
+            # Gather all models states of those models that will be rerendered.
+            # This includes:
+            # 1. The current model
+            try:
+                model_state = self.models[app_label, model_name]
+            except KeyError:
+                states_to_be_rendered = []
+            else:
+                states_to_be_rendered = [model_state]
+
+            # 2. All related models of unmigrated apps
+            for model_state in self.apps.real_models:
+                if (model_state.app_label, model_state.name_lower) in related_models:
+                    states_to_be_rendered.append(model_state)
+
+            # 3. All related models of migrated apps
+            for rel_app_label, rel_model_name in related_models:
+                try:
+                    model_state = self.models[rel_app_label, rel_model_name]
+                except KeyError:
+                    pass
+                else:
+                    states_to_be_rendered.append(model_state)
+
+            # Render all models
+            self.apps.render_multiple(states_to_be_rendered)
 
     def clone(self):
         "Returns an exact copy of this ProjectState"
@@ -136,21 +201,42 @@ class StateApps(Apps):
         # are some variables that refer to the Apps object.
         # FKs/M2Ms from real apps are also not included as they just
         # mess things up with partial states (due to lack of dependencies)
-        real_models = []
+        self.real_models = []
         for app_label in real_apps:
             app = global_apps.get_app_config(app_label)
             for model in app.get_models():
-                real_models.append(ModelState.from_model(model, exclude_rels=True))
+                self.real_models.append(ModelState.from_model(model, exclude_rels=True))
         # Populate the app registry with a stub for each application.
         app_labels = {model_state.app_label for model_state in models.values()}
         app_configs = [AppConfigStub(label) for label in sorted(real_apps + list(app_labels))]
         super(StateApps, self).__init__(app_configs)
 
+        self.render_multiple(list(models.values()) + self.real_models)
+
+        # If there are some lookups left, see if we can first resolve them
+        # ourselves - sometimes fields are added after class_prepared is sent
+        for lookup_model, operations in self._pending_lookups.items():
+            try:
+                model = self.get_model(lookup_model[0], lookup_model[1])
+            except LookupError:
+                app_label = "%s.%s" % (lookup_model[0], lookup_model[1])
+                if app_label == settings.AUTH_USER_MODEL and ignore_swappable:
+                    continue
+                # Raise an error with a best-effort helpful message
+                # (only for the first issue). Error message should look like:
+                # "ValueError: Lookup failed for model referenced by
+                # field migrations.Book.author: migrations.Author"
+                msg = "Lookup failed for model referenced by field {field}: {model[0]}.{model[1]}"
+                raise ValueError(msg.format(field=operations[0][1], model=lookup_model))
+            else:
+                do_pending_lookups(model)
+
+    def render_multiple(self, model_states):
         # We keep trying to render the models in a loop, ignoring invalid
         # base errors, until the size of the unrendered models doesn't
         # decrease by at least one, meaning there's a base dependency loop/
         # missing base.
-        unrendered_models = list(models.values()) + real_models
+        unrendered_models = model_states
         while unrendered_models:
             new_unrendered_models = []
             for model in unrendered_models:
@@ -167,24 +253,6 @@ class StateApps(Apps):
                 )
             unrendered_models = new_unrendered_models
 
-        # If there are some lookups left, see if we can first resolve them
-        # ourselves - sometimes fields are added after class_prepared is sent
-        for lookup_model, operations in self._pending_lookups.items():
-            try:
-                model = self.get_model(lookup_model[0], lookup_model[1])
-            except LookupError:
-                app_label = "%s.%s" % (lookup_model[0], lookup_model[1])
-                if app_label == settings.AUTH_USER_MODEL and ignore_swappable:
-                    continue
-                # Raise an error with a best-effort helpful message
-                # (only for the first issue). Error message should look like:
-                # "ValueError: Lookup failed for model referenced by
-                # field migrations.Book.author: migrations.Author"
-                msg = "Lookup failed for model referenced by field {field}: {model[0]}.{model[1]}"
-                raise ValueError(msg.format(field=operations[0][1], model=lookup_model))
-            else:
-                do_pending_lookups(model)
-
     def clone(self):
         """
         Return a clone of this registry, mainly used by the migration framework.
@@ -192,6 +260,8 @@ class StateApps(Apps):
         clone = StateApps([], {})
         clone.all_models = copy.deepcopy(self.all_models)
         clone.app_configs = copy.deepcopy(self.app_configs)
+        # No need to actually clone them, they'll never change
+        clone.real_models = self.real_models
         return clone
 
     def register_model(self, app_label, model):