Fixed #26808 -- Added Meta.indexes for class-based indexes.

* Added the index name to its deconstruction.
* Added indexes to sqlite3.schema._remake_table() so that indexes
  aren't dropped when _remake_table() is called.

Thanks timgraham & MarkusH for review and advice.

4 files changed, 131 insertions, 8 deletions

diff --git a/tests/migrations/test_autodetector.py b/tests/migrations/test_autodetector.py
index 7265330057..e1469760ee 100644
--- a/tests/migrations/test_autodetector.py
+++ b/tests/migrations/test_autodetector.py
@@ -370,6 +370,20 @@ class AutodetectorTests(TestCase):
         ("authors", models.ManyToManyField("testapp.Author", through="otherapp.Attribution")),
         ("title", models.CharField(max_length=200)),
     ])
+    book_indexes = ModelState("otherapp", "Book", [
+        ("id", models.AutoField(primary_key=True)),
+        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
+        ("title", models.CharField(max_length=200)),
+    ], {
+        "indexes": [models.Index(fields=["author", "title"], name="book_title_author_idx")],
+    })
+    book_unordered_indexes = ModelState("otherapp", "Book", [
+        ("id", models.AutoField(primary_key=True)),
+        ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
+        ("title", models.CharField(max_length=200)),
+    ], {
+        "indexes": [models.Index(fields=["title", "author"], name="book_author_title_idx")],
+    })
     book_foo_together = ModelState("otherapp", "Book", [
         ("id", models.AutoField(primary_key=True)),
         ("author", models.ForeignKey("testapp.Author", models.CASCADE)),
@@ -432,7 +446,10 @@ class AutodetectorTests(TestCase):
         ("id", models.AutoField(primary_key=True)),
         ("knight", models.ForeignKey("eggs.Knight", models.CASCADE)),
         ("parent", models.ForeignKey("eggs.Rabbit", models.CASCADE)),
-    ], {"unique_together": {("parent", "knight")}})
+    ], {
+        "unique_together": {("parent", "knight")},
+        "indexes": [models.Index(fields=["parent", "knight"], name='rabbit_circular_fk_index')],
+    })
 
     def repr_changes(self, changes, include_dependencies=False):
         output = ""
@@ -978,16 +995,18 @@ class AutodetectorTests(TestCase):
         self.assertOperationAttributes(changes, "testapp", 0, 2, name="publisher")
         self.assertMigrationDependencies(changes, 'testapp', 0, [])
 
-    def test_same_app_circular_fk_dependency_and_unique_together(self):
+    def test_same_app_circular_fk_dependency_with_unique_together_and_indexes(self):
         """
         #22275 - Tests that a migration with circular FK dependency does not try
-        to create unique together constraint before creating all required fields
-        first.
+        to create unique together constraint and indexes before creating all
+        required fields first.
         """
         changes = self.get_changes([], [self.knight, self.rabbit])
         # Right number/type of migrations?
         self.assertNumberMigrations(changes, 'eggs', 1)
-        self.assertOperationTypes(changes, 'eggs', 0, ["CreateModel", "CreateModel", "AlterUniqueTogether"])
+        self.assertOperationTypes(
+            changes, 'eggs', 0, ["CreateModel", "CreateModel", "AddIndex", "AlterUniqueTogether"]
+        )
         self.assertNotIn("unique_together", changes['eggs'][0].operations[0].options)
         self.assertNotIn("unique_together", changes['eggs'][0].operations[1].options)
         self.assertMigrationDependencies(changes, 'eggs', 0, [])
@@ -1135,6 +1154,51 @@ class AutodetectorTests(TestCase):
         for t in tests:
             test(*t)
 
+    def test_create_model_with_indexes(self):
+        """Test creation of new model with indexes already defined."""
+        author = ModelState('otherapp', 'Author', [
+            ('id', models.AutoField(primary_key=True)),
+            ('name', models.CharField(max_length=200)),
+        ], {'indexes': [models.Index(fields=['name'], name='create_model_with_indexes_idx')]})
+        changes = self.get_changes([], [author])
+        added_index = models.Index(fields=['name'], name='create_model_with_indexes_idx')
+        # Right number of migrations?
+        self.assertEqual(len(changes['otherapp']), 1)
+        # Right number of actions?
+        migration = changes['otherapp'][0]
+        self.assertEqual(len(migration.operations), 2)
+        # Right actions order?
+        self.assertOperationTypes(changes, 'otherapp', 0, ['CreateModel', 'AddIndex'])
+        self.assertOperationAttributes(changes, 'otherapp', 0, 0, name='Author')
+        self.assertOperationAttributes(changes, 'otherapp', 0, 1, model_name='author', index=added_index)
+
+    def test_add_indexes(self):
+        """Test change detection of new indexes."""
+        changes = self.get_changes([self.author_empty, self.book], [self.author_empty, self.book_indexes])
+        self.assertNumberMigrations(changes, 'otherapp', 1)
+        self.assertOperationTypes(changes, 'otherapp', 0, ['AddIndex'])
+        added_index = models.Index(fields=['author', 'title'], name='book_title_author_idx')
+        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', index=added_index)
+
+    def test_remove_indexes(self):
+        """Test change detection of removed indexes."""
+        changes = self.get_changes([self.author_empty, self.book_indexes], [self.author_empty, self.book])
+        # Right number/type of migrations?
+        self.assertNumberMigrations(changes, 'otherapp', 1)
+        self.assertOperationTypes(changes, 'otherapp', 0, ['RemoveIndex'])
+        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', name='book_title_author_idx')
+
+    def test_order_fields_indexes(self):
+        """Test change detection of reordering of fields in indexes."""
+        changes = self.get_changes(
+            [self.author_empty, self.book_indexes], [self.author_empty, self.book_unordered_indexes]
+        )
+        self.assertNumberMigrations(changes, 'otherapp', 1)
+        self.assertOperationTypes(changes, 'otherapp', 0, ['RemoveIndex', 'AddIndex'])
+        self.assertOperationAttributes(changes, 'otherapp', 0, 0, model_name='book', name='book_title_author_idx')
+        added_index = models.Index(fields=['title', 'author'], name='book_author_title_idx')
+        self.assertOperationAttributes(changes, 'otherapp', 0, 1, model_name='book', index=added_index)
+
     def test_add_foo_together(self):
         """Tests index/unique_together detection."""
         changes = self.get_changes([self.author_empty, self.book], [self.author_empty, self.book_foo_together])
diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py
index 86e58cc3e6..fdb956ae26 100644
--- a/tests/migrations/test_operations.py
+++ b/tests/migrations/test_operations.py
@@ -51,7 +51,7 @@ class OperationTestBase(MigrationTestBase):
         return project_state, new_state
 
     def set_up_test_model(
-            self, app_label, second_model=False, third_model=False, multicol_index=False,
+            self, app_label, second_model=False, third_model=False, index=False, multicol_index=False,
             related_model=False, mti_model=False, proxy_model=False, manager_model=False,
             unique_together=False, options=False, db_table=None, index_together=False):
         """
@@ -96,6 +96,11 @@ class OperationTestBase(MigrationTestBase):
             ],
             options=model_options,
         )]
+        if index:
+            operations.append(migrations.AddIndex(
+                "Pony",
+                models.Index(fields=["pink"], name="pony_pink_idx")
+            ))
         if multicol_index:
             operations.append(migrations.AddIndex(
                 "Pony",
@@ -1447,6 +1452,43 @@ class OperationTests(OperationTestBase):
         self.assertEqual(definition[1], [])
         self.assertEqual(definition[2], {'model_name': "Pony", 'name': "pony_test_idx"})
 
+        # Also test a field dropped with index - sqlite remake issue
+        operations = [
+            migrations.RemoveIndex("Pony", "pony_test_idx"),
+            migrations.RemoveField("Pony", "pink"),
+        ]
+        self.assertColumnExists("test_rmin_pony", "pink")
+        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
+        # Test database alteration
+        new_state = project_state.clone()
+        self.apply_operations('test_rmin', new_state, operations=operations)
+        self.assertColumnNotExists("test_rmin_pony", "pink")
+        self.assertIndexNotExists("test_rmin_pony", ["pink", "weight"])
+        # And test reversal
+        self.unapply_operations("test_rmin", project_state, operations=operations)
+        self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
+
+    def test_alter_field_with_index(self):
+        """
+        Test AlterField operation with an index to ensure indexes created via
+        Meta.indexes don't get dropped with sqlite3 remake.
+        """
+        project_state = self.set_up_test_model("test_alflin", index=True)
+        operation = migrations.AlterField("Pony", "pink", models.IntegerField(null=True))
+        new_state = project_state.clone()
+        operation.state_forwards("test_alflin", new_state)
+        # Test the database alteration
+        self.assertColumnNotNull("test_alflin_pony", "pink")
+        with connection.schema_editor() as editor:
+            operation.database_forwards("test_alflin", editor, project_state, new_state)
+        # Ensure that index hasn't been dropped
+        self.assertIndexExists("test_alflin_pony", ["pink"])
+        # And test reversal
+        with connection.schema_editor() as editor:
+            operation.database_backwards("test_alflin", editor, new_state, project_state)
+        # Ensure the index is still there
+        self.assertIndexExists("test_alflin_pony", ["pink"])
+
     def test_alter_index_together(self):
         """
         Tests the AlterIndexTogether operation.
diff --git a/tests/migrations/test_state.py b/tests/migrations/test_state.py
index e955f1c63f..9428ea060d 100644
--- a/tests/migrations/test_state.py
+++ b/tests/migrations/test_state.py
@@ -65,6 +65,7 @@ class StateTests(SimpleTestCase):
                 apps = new_apps
                 verbose_name = "tome"
                 db_table = "test_tome"
+                indexes = [models.Index(fields=['title'])]
 
         class Food(models.Model):
 
@@ -116,6 +117,8 @@ class StateTests(SimpleTestCase):
         food_no_managers_state = project_state.models['migrations', 'foodnomanagers']
         food_no_default_manager_state = project_state.models['migrations', 'foodnodefaultmanager']
         food_order_manager_state = project_state.models['migrations', 'foodorderedmanagers']
+        book_index = models.Index(fields=['title'])
+        book_index.set_name_with_model(Book)
 
         self.assertEqual(author_state.app_label, "migrations")
         self.assertEqual(author_state.name, "Author")
@@ -135,7 +138,10 @@ class StateTests(SimpleTestCase):
         self.assertEqual(book_state.fields[1][1].max_length, 1000)
         self.assertIs(book_state.fields[2][1].null, False)
         self.assertEqual(book_state.fields[3][1].__class__.__name__, "ManyToManyField")
-        self.assertEqual(book_state.options, {"verbose_name": "tome", "db_table": "test_tome", "indexes": []})
+        self.assertEqual(
+            book_state.options,
+            {"verbose_name": "tome", "db_table": "test_tome", "indexes": [book_index]},
+        )
         self.assertEqual(book_state.bases, (models.Model, ))
 
         self.assertEqual(author_proxy_state.app_label, "migrations")
@@ -947,6 +953,13 @@ class ModelStateTests(SimpleTestCase):
         ):
             ModelState('app', 'Model', [('field', field)])
 
+    def test_sanity_index_name(self):
+        field = models.IntegerField()
+        options = {'indexes': [models.Index(fields=['field'])]}
+        msg = "Indexes passed to ModelState require a name attribute. <Index: fields='field'> doesn't have one."
+        with self.assertRaisesMessage(ValueError, msg):
+            ModelState('app', 'Model', [('field', field)], options=options)
+
     def test_fields_immutability(self):
         """
         Tests that rendering a model state doesn't alter its internal fields.
diff --git a/tests/model_indexes/tests.py b/tests/model_indexes/tests.py
index e979a00de7..b3f5d04dc1 100644
--- a/tests/model_indexes/tests.py
+++ b/tests/model_indexes/tests.py
@@ -16,6 +16,9 @@ class IndexesTests(TestCase):
         index = models.Index(fields=['title'])
         same_index = models.Index(fields=['title'])
         another_index = models.Index(fields=['title', 'author'])
+        index.model = Book
+        same_index.model = Book
+        another_index.model = Book
         self.assertEqual(index, same_index)
         self.assertNotEqual(index, another_index)
 
@@ -56,7 +59,8 @@ class IndexesTests(TestCase):
 
     def test_deconstruction(self):
         index = models.Index(fields=['title'])
+        index.set_name_with_model(Book)
         path, args, kwargs = index.deconstruct()
         self.assertEqual(path, 'django.db.models.Index')
         self.assertEqual(args, ())
-        self.assertEqual(kwargs, {'fields': ['title']})
+        self.assertEqual(kwargs, {'fields': ['title'], 'name': 'model_index_title_196f42_idx'})