Fixed #10109 -- Removed the use of raw SQL in many-to-many fields by introducing an autogenerated through model.

This is the first part of Alex Gaynor's GSoC project to add Multi-db support to Django.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11710 bcc190cf-cafb-0310-a4f2-bffc1f526a37

5 files changed, 101 insertions, 90 deletions

diff --git a/django/core/management/commands/syncdb.py b/django/core/management/commands/syncdb.py
index fe51d45bb3..165006efd1 100644
--- a/django/core/management/commands/syncdb.py
+++ b/django/core/management/commands/syncdb.py
@@ -57,12 +57,15 @@ class Command(NoArgsCommand):
         # Create the tables for each model
         for app in models.get_apps():
             app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
+            model_list = models.get_models(app, include_auto_created=True)
             for model in model_list:
                 # Create the model's database table, if it doesn't already exist.
                 if verbosity >= 2:
                     print "Processing %s.%s model" % (app_name, model._meta.object_name)
-                if connection.introspection.table_name_converter(model._meta.db_table) in tables:
+                opts = model._meta
+                if (connection.introspection.table_name_converter(opts.db_table) in tables or
+                    (opts.auto_created and
+                    connection.introspection.table_name_converter(opts.auto_created._meta.db_table in tables))):
                     continue
                 sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
                 seen_models.add(model)
@@ -78,19 +81,6 @@ class Command(NoArgsCommand):
                     cursor.execute(statement)
                 tables.append(connection.introspection.table_name_converter(model._meta.db_table))
 
-        # Create the m2m tables. This must be done after all tables have been created
-        # to ensure that all referred tables will exist.
-        for app in models.get_apps():
-            app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
-            for model in model_list:
-                if model in created_models:
-                    sql = connection.creation.sql_for_many_to_many(model, self.style)
-                    if sql:
-                        if verbosity >= 2:
-                            print "Creating many-to-many tables for %s.%s model" % (app_name, model._meta.object_name)
-                        for statement in sql:
-                            cursor.execute(statement)
 
         transaction.commit_unless_managed()
 
diff --git a/django/core/management/sql.py b/django/core/management/sql.py
index 14fd3f8214..caf40d088d 100644
--- a/django/core/management/sql.py
+++ b/django/core/management/sql.py
@@ -23,7 +23,7 @@ def sql_create(app, style):
     # We trim models from the current app so that the sqlreset command does not
     # generate invalid SQL (leaving models out of known_models is harmless, so
     # we can be conservative).
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     final_output = []
     tables = connection.introspection.table_names()
     known_models = set([model for model in connection.introspection.installed_models(tables) if model not in app_models])
@@ -40,10 +40,6 @@ def sql_create(app, style):
         # Keep track of the fact that we've created the table for this model.
         known_models.add(model)
 
-    # Create the many-to-many join tables.
-    for model in app_models:
-        final_output.extend(connection.creation.sql_for_many_to_many(model, style))
-
     # Handle references to tables that are from other apps
     # but don't exist physically.
     not_installed_models = set(pending_references.keys())
@@ -82,7 +78,7 @@ def sql_delete(app, style):
     to_delete = set()
 
     references_to_delete = {}
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     for model in app_models:
         if cursor and connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             # The table exists, so it needs to be dropped
@@ -97,13 +93,6 @@ def sql_delete(app, style):
         if connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             output.extend(connection.creation.sql_destroy_model(model, references_to_delete, style))
 
-    # Output DROP TABLE statements for many-to-many tables.
-    for model in app_models:
-        opts = model._meta
-        for f in opts.local_many_to_many:
-            if cursor and connection.introspection.table_name_converter(f.m2m_db_table()) in table_names:
-                output.extend(connection.creation.sql_destroy_many_to_many(model, f, style))
-
     # Close database connection explicitly, in case this output is being piped
     # directly into a database client, to avoid locking issues.
     if cursor:
diff --git a/django/core/management/validation.py b/django/core/management/validation.py
index b971558ac7..97164d75c3 100644
--- a/django/core/management/validation.py
+++ b/django/core/management/validation.py
@@ -79,27 +79,28 @@ def get_validation_errors(outfile, app=None):
                 rel_opts = f.rel.to._meta
                 rel_name = RelatedObject(f.rel.to, cls, f).get_accessor_name()
                 rel_query_name = f.related_query_name()
-                for r in rel_opts.fields:
-                    if r.name == rel_name:
-                        e.add(opts, "Accessor for field '%s' clashes with field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
-                    if r.name == rel_query_name:
-                        e.add(opts, "Reverse query name for field '%s' clashes with field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
-                for r in rel_opts.local_many_to_many:
-                    if r.name == rel_name:
-                        e.add(opts, "Accessor for field '%s' clashes with m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
-                    if r.name == rel_query_name:
-                        e.add(opts, "Reverse query name for field '%s' clashes with m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
-                for r in rel_opts.get_all_related_many_to_many_objects():
-                    if r.get_accessor_name() == rel_name:
-                        e.add(opts, "Accessor for field '%s' clashes with related m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
-                    if r.get_accessor_name() == rel_query_name:
-                        e.add(opts, "Reverse query name for field '%s' clashes with related m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
-                for r in rel_opts.get_all_related_objects():
-                    if r.field is not f:
+                if not f.rel.is_hidden():
+                    for r in rel_opts.fields:
+                        if r.name == rel_name:
+                            e.add(opts, "Accessor for field '%s' clashes with field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
+                        if r.name == rel_query_name:
+                            e.add(opts, "Reverse query name for field '%s' clashes with field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
+                    for r in rel_opts.local_many_to_many:
+                        if r.name == rel_name:
+                            e.add(opts, "Accessor for field '%s' clashes with m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
+                        if r.name == rel_query_name:
+                            e.add(opts, "Reverse query name for field '%s' clashes with m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
+                    for r in rel_opts.get_all_related_many_to_many_objects():
                         if r.get_accessor_name() == rel_name:
-                            e.add(opts, "Accessor for field '%s' clashes with related field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
+                            e.add(opts, "Accessor for field '%s' clashes with related m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
                         if r.get_accessor_name() == rel_query_name:
-                            e.add(opts, "Reverse query name for field '%s' clashes with related field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
+                            e.add(opts, "Reverse query name for field '%s' clashes with related m2m field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
+                    for r in rel_opts.get_all_related_objects():
+                        if r.field is not f:
+                            if r.get_accessor_name() == rel_name:
+                                e.add(opts, "Accessor for field '%s' clashes with related field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
+                            if r.get_accessor_name() == rel_query_name:
+                                e.add(opts, "Reverse query name for field '%s' clashes with related field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.get_accessor_name(), f.name))
 
         seen_intermediary_signatures = []
         for i, f in enumerate(opts.local_many_to_many):
@@ -117,48 +118,80 @@ def get_validation_errors(outfile, app=None):
             if f.unique:
                 e.add(opts, "ManyToManyFields cannot be unique.  Remove the unique argument on '%s'." % f.name)
 
-            if getattr(f.rel, 'through', None) is not None:
-                if hasattr(f.rel, 'through_model'):
-                    from_model, to_model = cls, f.rel.to
-                    if from_model == to_model and f.rel.symmetrical:
-                        e.add(opts, "Many-to-many fields with intermediate tables cannot be symmetrical.")
-                    seen_from, seen_to, seen_self = False, False, 0
-                    for inter_field in f.rel.through_model._meta.fields:
-                        rel_to = getattr(inter_field.rel, 'to', None)
-                        if from_model == to_model: # relation to self
-                            if rel_to == from_model:
-                                seen_self += 1
-                            if seen_self > 2:
-                                e.add(opts, "Intermediary model %s has more than two foreign keys to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, from_model._meta.object_name))
-                        else:
-                            if rel_to == from_model:
-                                if seen_from:
-                                    e.add(opts, "Intermediary model %s has more than one foreign key to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, from_model._meta.object_name))
-                                else:
-                                    seen_from = True
-                            elif rel_to == to_model:
-                                if seen_to:
-                                    e.add(opts, "Intermediary model %s has more than one foreign key to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, rel_to._meta.object_name))
-                                else:
-                                    seen_to = True
-                    if f.rel.through_model not in models.get_models():
-                        e.add(opts, "'%s' specifies an m2m relation through model %s, which has not been installed." % (f.name, f.rel.through))
-                    signature = (f.rel.to, cls, f.rel.through_model)
-                    if signature in seen_intermediary_signatures:
-                        e.add(opts, "The model %s has two manually-defined m2m relations through the model %s, which is not permitted. Please consider using an extra field on your intermediary model instead." % (cls._meta.object_name, f.rel.through_model._meta.object_name))
+            if f.rel.through is not None and not isinstance(f.rel.through, basestring):
+                from_model, to_model = cls, f.rel.to
+                if from_model == to_model and f.rel.symmetrical and not f.rel.through._meta.auto_created:
+                    e.add(opts, "Many-to-many fields with intermediate tables cannot be symmetrical.")
+                seen_from, seen_to, seen_self = False, False, 0
+                for inter_field in f.rel.through._meta.fields:
+                    rel_to = getattr(inter_field.rel, 'to', None)
+                    if from_model == to_model: # relation to self
+                        if rel_to == from_model:
+                            seen_self += 1
+                        if seen_self > 2:
+                            e.add(opts, "Intermediary model %s has more than "
+                                "two foreign keys to %s, which is ambiguous "
+                                "and is not permitted." % (
+                                    f.rel.through._meta.object_name,
+                                    from_model._meta.object_name
+                                )
+                            )
                     else:
-                        seen_intermediary_signatures.append(signature)
-                    seen_related_fk, seen_this_fk = False, False
-                    for field in f.rel.through_model._meta.fields:
-                        if field.rel:
-                            if not seen_related_fk and field.rel.to == f.rel.to:
-                                seen_related_fk = True
-                            elif field.rel.to == cls:
-                                seen_this_fk = True
-                    if not seen_related_fk or not seen_this_fk:
-                        e.add(opts, "'%s' has a manually-defined m2m relation through model %s, which does not have foreign keys to %s and %s" % (f.name, f.rel.through, f.rel.to._meta.object_name, cls._meta.object_name))
+                        if rel_to == from_model:
+                            if seen_from:
+                                e.add(opts, "Intermediary model %s has more "
+                                    "than one foreign key to %s, which is "
+                                    "ambiguous and is not permitted." % (
+                                        f.rel.through._meta.object_name,
+                                         from_model._meta.object_name
+                                     )
+                                 )
+                            else:
+                                seen_from = True
+                        elif rel_to == to_model:
+                            if seen_to:
+                                e.add(opts, "Intermediary model %s has more "
+                                    "than one foreign key to %s, which is "
+                                    "ambiguous and is not permitted." % (
+                                        f.rel.through._meta.object_name,
+                                        rel_to._meta.object_name
+                                    )
+                                )
+                            else:
+                                seen_to = True
+                if f.rel.through not in models.get_models(include_auto_created=True):
+                    e.add(opts, "'%s' specifies an m2m relation through model "
+                        "%s, which has not been installed." % (f.name, f.rel.through)
+                    )
+                signature = (f.rel.to, cls, f.rel.through)
+                if signature in seen_intermediary_signatures:
+                    e.add(opts, "The model %s has two manually-defined m2m "
+                        "relations through the model %s, which is not "
+                        "permitted. Please consider using an extra field on "
+                        "your intermediary model instead." % (
+                            cls._meta.object_name,
+                            f.rel.through._meta.object_name
+                        )
+                    )
                 else:
-                    e.add(opts, "'%s' specifies an m2m relation through model %s, which has not been installed" % (f.name, f.rel.through))
+                    seen_intermediary_signatures.append(signature)
+                seen_related_fk, seen_this_fk = False, False
+                for field in f.rel.through._meta.fields:
+                    if field.rel:
+                        if not seen_related_fk and field.rel.to == f.rel.to:
+                            seen_related_fk = True
+                        elif field.rel.to == cls:
+                            seen_this_fk = True
+                if not seen_related_fk or not seen_this_fk:
+                    e.add(opts, "'%s' has a manually-defined m2m relation "
+                        "through model %s, which does not have foreign keys "
+                        "to %s and %s" % (f.name, f.rel.through._meta.object_name,
+                            f.rel.to._meta.object_name, cls._meta.object_name)
+                    )
+            elif isinstance(f.rel.through, basestring):
+                e.add(opts, "'%s' specifies an m2m relation through model %s, "
+                    "which has not been installed" % (f.name, f.rel.through)
+                )
 
             rel_opts = f.rel.to._meta
             rel_name = RelatedObject(f.rel.to, cls, f).get_accessor_name()
diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py
index b672e4efc3..7b77804009 100644
--- a/django/core/serializers/python.py
+++ b/django/core/serializers/python.py
@@ -56,7 +56,7 @@ class Serializer(base.Serializer):
         self._current[field.name] = smart_unicode(related, strings_only=True)
 
     def handle_m2m_field(self, obj, field):
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
                                for related in getattr(obj, field.name).iterator()]
 
diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py
index 2d74fe28f3..4cde0b039d 100644
--- a/django/core/serializers/xml_serializer.py
+++ b/django/core/serializers/xml_serializer.py
@@ -98,7 +98,7 @@ class Serializer(base.Serializer):
         serialized as references to the object's PK (i.e. the related *data*
         is not dumped, just the relation).
         """
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
             for relobj in getattr(obj, field.name).iterator():
                 self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
@@ -233,4 +233,3 @@ def getInnerText(node):
         else:
            pass
     return u"".join(inner_text)
-