Fixed #26291 -- Allowed loaddata to handle forward references in natural_key fixtures.

4 files changed, 97 insertions, 15 deletions

diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py
index 0b6a1e0898..aae156f911 100644
--- a/django/core/management/commands/loaddata.py
+++ b/django/core/management/commands/loaddata.py
@@ -109,8 +109,11 @@ class Command(BaseCommand):
             return
 
         with connection.constraint_checks_disabled():
+            self.objs_with_deferred_fields = []
             for fixture_label in fixture_labels:
                 self.load_label(fixture_label)
+            for obj in self.objs_with_deferred_fields:
+                obj.save_deferred_fields(using=self.using)
 
         # Since we disabled constraint checks, we must manually check for
         # any invalid keys that might have been added
@@ -163,6 +166,7 @@ class Command(BaseCommand):
 
                 objects = serializers.deserialize(
                     ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore,
+                    handle_forward_references=True,
                 )
 
                 for obj in objects:
@@ -189,6 +193,8 @@ class Command(BaseCommand):
                                 'error_msg': e,
                             },)
                             raise
+                    if obj.deferred_fields:
+                        self.objs_with_deferred_fields.append(obj)
                 if objects and show_progress:
                     self.stdout.write('')  # add a newline after progress indicator
                 self.loaded_object_count += loaded_objects_in_fixture
diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py
index 4ff0c753f8..0d7946dc3f 100644
--- a/django/core/serializers/base.py
+++ b/django/core/serializers/base.py
@@ -3,8 +3,11 @@ Module for abstract serializer/unserializer base classes.
 """
 from io import StringIO
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 
+DEFER_FIELD = object()
+
 
 class SerializerDoesNotExist(KeyError):
     """The requested serializer was not found."""
@@ -201,9 +204,10 @@ class DeserializedObject:
     (and not touch the many-to-many stuff.)
     """
 
-    def __init__(self, obj, m2m_data=None):
+    def __init__(self, obj, m2m_data=None, deferred_fields=None):
         self.object = obj
         self.m2m_data = m2m_data
+        self.deferred_fields = deferred_fields
 
     def __repr__(self):
         return "<%s: %s(pk=%s)>" % (
@@ -225,6 +229,25 @@ class DeserializedObject:
         # the m2m data twice.
         self.m2m_data = None
 
+    def save_deferred_fields(self, using=None):
+        self.m2m_data = {}
+        for field, field_value in self.deferred_fields.items():
+            opts = self.object._meta
+            label = opts.app_label + '.' + opts.model_name
+            if isinstance(field.remote_field, models.ManyToManyRel):
+                try:
+                    values = deserialize_m2m_values(field, field_value, using, handle_forward_references=False)
+                except M2MDeserializationError as e:
+                    raise DeserializationError.WithData(e.original_exc, label, self.object.pk, e.pk)
+                self.m2m_data[field.name] = values
+            elif isinstance(field.remote_field, models.ManyToOneRel):
+                try:
+                    value = deserialize_fk_value(field, field_value, using, handle_forward_references=False)
+                except Exception as e:
+                    raise DeserializationError.WithData(e, label, self.object.pk, field_value)
+                setattr(self.object, field.attname, value)
+        self.save()
+
 
 def build_instance(Model, data, db):
     """
@@ -244,7 +267,7 @@ def build_instance(Model, data, db):
     return obj
 
 
-def deserialize_m2m_values(field, field_value, using):
+def deserialize_m2m_values(field, field_value, using, handle_forward_references):
     model = field.remote_field.model
     if hasattr(model._default_manager, 'get_by_natural_key'):
         def m2m_convert(value):
@@ -262,10 +285,13 @@ def deserialize_m2m_values(field, field_value, using):
             values.append(m2m_convert(pk))
         return values
     except Exception as e:
-        raise M2MDeserializationError(e, pk)
+        if isinstance(e, ObjectDoesNotExist) and handle_forward_references:
+            return DEFER_FIELD
+        else:
+            raise M2MDeserializationError(e, pk)
 
 
-def deserialize_fk_value(field, field_value, using):
+def deserialize_fk_value(field, field_value, using, handle_forward_references):
     if field_value is None:
         return None
     model = field.remote_field.model
@@ -273,7 +299,13 @@ def deserialize_fk_value(field, field_value, using):
     field_name = field.remote_field.field_name
     if (hasattr(default_manager, 'get_by_natural_key') and
             hasattr(field_value, '__iter__') and not isinstance(field_value, str)):
-        obj = default_manager.db_manager(using).get_by_natural_key(*field_value)
+        try:
+            obj = default_manager.db_manager(using).get_by_natural_key(*field_value)
+        except ObjectDoesNotExist:
+            if handle_forward_references:
+                return DEFER_FIELD
+            else:
+                raise
         value = getattr(obj, field_name)
         # If this is a natural foreign key to an object that has a FK/O2O as
         # the foreign key, use the FK value.
diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py
index 922e2c6a8f..08739c98fc 100644
--- a/django/core/serializers/python.py
+++ b/django/core/serializers/python.py
@@ -83,6 +83,7 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False
     It's expected that you pass the Python objects themselves (instead of a
     stream or a string) to the constructor
     """
+    handle_forward_references = options.pop('handle_forward_references', False)
     field_names_cache = {}  # Model: <list of field_names>
 
     for d in object_list:
@@ -101,6 +102,7 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False
             except Exception as e:
                 raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), None)
         m2m_data = {}
+        deferred_fields = {}
 
         if Model not in field_names_cache:
             field_names_cache[Model] = {f.name for f in Model._meta.get_fields()}
@@ -118,17 +120,23 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False
             # Handle M2M relations
             if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel):
                 try:
-                    values = base.deserialize_m2m_values(field, field_value, using)
+                    values = base.deserialize_m2m_values(field, field_value, using, handle_forward_references)
                 except base.M2MDeserializationError as e:
                     raise base.DeserializationError.WithData(e.original_exc, d['model'], d.get('pk'), e.pk)
-                m2m_data[field.name] = values
+                if values == base.DEFER_FIELD:
+                    deferred_fields[field] = field_value
+                else:
+                    m2m_data[field.name] = values
             # Handle FK fields
             elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel):
                 try:
-                    value = base.deserialize_fk_value(field, field_value, using)
+                    value = base.deserialize_fk_value(field, field_value, using, handle_forward_references)
                 except Exception as e:
                     raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value)
-                data[field.attname] = value
+                if value == base.DEFER_FIELD:
+                    deferred_fields[field] = field_value
+                else:
+                    data[field.attname] = value
             # Handle all other fields
             else:
                 try:
@@ -137,7 +145,7 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False
                     raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value)
 
         obj = base.build_instance(Model, data, using)
-        yield base.DeserializedObject(obj, m2m_data)
+        yield base.DeserializedObject(obj, m2m_data, deferred_fields)
 
 
 def _get_model(model_identifier):
diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py
index 076be1fe1f..47c6b88e93 100644
--- a/django/core/serializers/xml_serializer.py
+++ b/django/core/serializers/xml_serializer.py
@@ -8,6 +8,7 @@ from xml.sax.expatreader import ExpatParser as _ExpatParser
 
 from django.apps import apps
 from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.serializers import base
 from django.db import DEFAULT_DB_ALIAS, models
 from django.utils.xmlutils import (
@@ -151,6 +152,7 @@ class Deserializer(base.Deserializer):
 
     def __init__(self, stream_or_string, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False, **options):
         super().__init__(stream_or_string, **options)
+        self.handle_forward_references = options.pop('handle_forward_references', False)
         self.event_stream = pulldom.parse(self.stream, self._make_parser())
         self.db = using
         self.ignore = ignorenonexistent
@@ -181,6 +183,7 @@ class Deserializer(base.Deserializer):
         # Also start building a dict of m2m data (this is saved as
         # {m2m_accessor_attribute : [list_of_related_objects]})
         m2m_data = {}
+        deferred_fields = {}
 
         field_names = {f.name for f in Model._meta.get_fields()}
         # Deserialize each field.
@@ -200,9 +203,26 @@ class Deserializer(base.Deserializer):
 
             # As is usually the case, relation fields get the special treatment.
             if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel):
-                m2m_data[field.name] = self._handle_m2m_field_node(field_node, field)
+                value = self._handle_m2m_field_node(field_node, field)
+                if value == base.DEFER_FIELD:
+                    deferred_fields[field] = [
+                        [
+                            getInnerText(nat_node).strip()
+                            for nat_node in obj_node.getElementsByTagName('natural')
+                        ]
+                        for obj_node in field_node.getElementsByTagName('object')
+                    ]
+                else:
+                    m2m_data[field.name] = value
             elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel):
-                data[field.attname] = self._handle_fk_field_node(field_node, field)
+                value = self._handle_fk_field_node(field_node, field)
+                if value == base.DEFER_FIELD:
+                    deferred_fields[field] = [
+                        getInnerText(k).strip()
+                        for k in field_node.getElementsByTagName('natural')
+                    ]
+                else:
+                    data[field.attname] = value
             else:
                 if field_node.getElementsByTagName('None'):
                     value = None
@@ -213,7 +233,7 @@ class Deserializer(base.Deserializer):
         obj = base.build_instance(Model, data, self.db)
 
         # Return a DeserializedObject so that the m2m data has a place to live.
-        return base.DeserializedObject(obj, m2m_data)
+        return base.DeserializedObject(obj, m2m_data, deferred_fields)
 
     def _handle_fk_field_node(self, node, field):
         """
@@ -229,7 +249,13 @@ class Deserializer(base.Deserializer):
                 if keys:
                     # If there are 'natural' subelements, it must be a natural key
                     field_value = [getInnerText(k).strip() for k in keys]
-                    obj = model._default_manager.db_manager(self.db).get_by_natural_key(*field_value)
+                    try:
+                        obj = model._default_manager.db_manager(self.db).get_by_natural_key(*field_value)
+                    except ObjectDoesNotExist:
+                        if self.handle_forward_references:
+                            return base.DEFER_FIELD
+                        else:
+                            raise
                     obj_pk = getattr(obj, field.remote_field.field_name)
                     # If this is a natural foreign key to an object that
                     # has a FK/O2O as the foreign key, use the FK value
@@ -264,7 +290,17 @@ class Deserializer(base.Deserializer):
         else:
             def m2m_convert(n):
                 return model._meta.pk.to_python(n.getAttribute('pk'))
-        return [m2m_convert(c) for c in node.getElementsByTagName("object")]
+        values = []
+        try:
+            for c in node.getElementsByTagName('object'):
+                values.append(m2m_convert(c))
+        except Exception as e:
+            if isinstance(e, ObjectDoesNotExist) and self.handle_forward_references:
+                return base.DEFER_FIELD
+            else:
+                raise base.M2MDeserializationError(e, c)
+        else:
+            return values
 
     def _get_model_from_node(self, node, attr):
         """