Fixed #19462 -- Made assertQuerysetEqual detect undefined ordering

If there are more than one values to compare against and the qs isn't
ordered then assertQuerysetEqual will raise a ValueError.

7 files changed, 45 insertions, 26 deletions

diff --git a/tests/modeltests/expressions/tests.py b/tests/modeltests/expressions/tests.py
index ca47521ccd..a351496442 100644
--- a/tests/modeltests/expressions/tests.py
+++ b/tests/modeltests/expressions/tests.py
@@ -158,6 +158,7 @@ class ExpressionsTests(TestCase):
                 "Max Mustermann",
             ],
             lambda c: six.text_type(c.point_of_contact),
+            ordered=False
         )
 
         c = Company.objects.all()[0]
@@ -170,7 +171,8 @@ class ExpressionsTests(TestCase):
                 "Foobar Ltd.",
                 "Test GmbH",
             ],
-            lambda c: c.name
+            lambda c: c.name,
+            ordered=False
         )
 
         Company.objects.exclude(
diff --git a/tests/modeltests/field_subclassing/tests.py b/tests/modeltests/field_subclassing/tests.py
index 48755123f2..0ec317dea5 100644
--- a/tests/modeltests/field_subclassing/tests.py
+++ b/tests/modeltests/field_subclassing/tests.py
@@ -77,7 +77,8 @@ class CustomField(TestCase):
                 "12",
                 "23",
             ],
-            lambda m: str(m.data)
+            lambda m: str(m.data),
+            ordered=False
         )
 
     def test_field_subclassing(self):
diff --git a/tests/modeltests/fixtures/tests.py b/tests/modeltests/fixtures/tests.py
index b667c8c6d4..415ed6dcf2 100644
--- a/tests/modeltests/fixtures/tests.py
+++ b/tests/modeltests/fixtures/tests.py
@@ -96,8 +96,8 @@ class FixtureLoadingTests(TestCase):
         management.call_command('loaddata', 'fixture6.json', verbosity=0, commit=False)
         self.assertQuerysetEqual(Tag.objects.all(), [
             '<Tag: <Article: Copyright is fine the way it is> tagged "copyright">',
-            '<Tag: <Article: Copyright is fine the way it is> tagged "law">'
-        ])
+            '<Tag: <Article: Copyright is fine the way it is> tagged "law">',
+        ], ordered=False)
 
         # Load fixture 7, XML file with dynamic ContentType fields. Testing ManyToOne.
         management.call_command('loaddata', 'fixture7.xml', verbosity=0, commit=False)
@@ -105,8 +105,8 @@ class FixtureLoadingTests(TestCase):
             '<Tag: <Article: Copyright is fine the way it is> tagged "copyright">',
             '<Tag: <Article: Copyright is fine the way it is> tagged "legal">',
             '<Tag: <Article: Django conquers world!> tagged "django">',
-            '<Tag: <Article: Django conquers world!> tagged "world domination">'
-        ])
+            '<Tag: <Article: Django conquers world!> tagged "world domination">',
+        ], ordered=False)
 
         # Load fixture 8, JSON file with dynamic Permission fields. Testing ManyToMany.
         management.call_command('loaddata', 'fixture8.json', verbosity=0, commit=False)
@@ -114,7 +114,7 @@ class FixtureLoadingTests(TestCase):
             '<Visa: Django Reinhardt Can add user, Can change user, Can delete user>',
             '<Visa: Stephane Grappelli Can add user>',
             '<Visa: Prince >'
-        ])
+        ], ordered=False)
 
         # Load fixture 9, XML file with dynamic Permission fields. Testing ManyToMany.
         management.call_command('loaddata', 'fixture9.xml', verbosity=0, commit=False)
@@ -122,7 +122,7 @@ class FixtureLoadingTests(TestCase):
             '<Visa: Django Reinhardt Can add user, Can change user, Can delete user>',
             '<Visa: Stephane Grappelli Can add user, Can delete user>',
             '<Visa: Artist formerly known as "Prince" Can change user>'
-        ])
+        ], ordered=False)
 
         self.assertQuerysetEqual(Book.objects.all(), [
             '<Book: Achieving self-awareness of Python programs>',
@@ -280,7 +280,7 @@ class FixtureLoadingTests(TestCase):
         self.assertQuerysetEqual(Tag.objects.all(), [
             '<Tag: <Article: Time to reform copyright> tagged "copyright">',
             '<Tag: <Article: Time to reform copyright> tagged "law">'
-        ])
+        ], ordered=False)
 
         # Dump the current contents of the database as a JSON fixture
         self._dumpdata_assert(['fixtures'], '[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", "pub_date": "2006-06-16T12:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16T13:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "law", "tagged_id": 3}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Prince"}}, {"pk": 10, "model": "fixtures.book", "fields": {"name": "Achieving self-awareness of Python programs", "authors": []}}]', natural_keys=True)
diff --git a/tests/modeltests/generic_relations/tests.py b/tests/modeltests/generic_relations/tests.py
index 14871e4e09..73b0a483a2 100644
--- a/tests/modeltests/generic_relations/tests.py
+++ b/tests/modeltests/generic_relations/tests.py
@@ -169,8 +169,8 @@ class GenericRelationsTests(TestCase):
         # Filtering works
         self.assertQuerysetEqual(tiger.comparisons.filter(comparative="cooler"), [
             "<Comparison: tiger is cooler than cheetah>",
-            "<Comparison: tiger is cooler than bear>"
-        ])
+            "<Comparison: tiger is cooler than bear>",
+        ], ordered=False)
 
         # Filtering and deleting works
         subjective = ["cooler"]
@@ -178,7 +178,7 @@ class GenericRelationsTests(TestCase):
         self.assertQuerysetEqual(Comparison.objects.all(), [
             "<Comparison: cheetah is faster than tiger>",
             "<Comparison: tiger is stronger than cheetah>"
-        ])
+        ], ordered=False)
 
         # If we delete cheetah, Comparisons with cheetah as 'first_obj' will be
         # deleted since Animal has an explicit GenericRelation to Comparison
diff --git a/tests/modeltests/m2m_recursive/tests.py b/tests/modeltests/m2m_recursive/tests.py
index 5742836929..a3f2c670d6 100644
--- a/tests/modeltests/m2m_recursive/tests.py
+++ b/tests/modeltests/m2m_recursive/tests.py
@@ -28,7 +28,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David"
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is friends with Bill?
         self.assertQuerysetEqual(
@@ -43,7 +44,8 @@ class RecursiveM2MTests(TestCase):
                 "Anne",
                 "David"
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is friends with David?
         self.assertQuerysetEqual(
@@ -51,7 +53,8 @@ class RecursiveM2MTests(TestCase):
                 "Anne",
                 "Chuck",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Bill is already friends with Anne - add Anne again, but in the
         # reverse direction
@@ -64,7 +67,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is friends with Bill?
         self.assertQuerysetEqual(
@@ -81,7 +85,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is friends with Bill?
         self.assertQuerysetEqual(
@@ -125,7 +130,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is stalking Anne?
         self.assertQuerysetEqual(
@@ -172,7 +178,8 @@ class RecursiveM2MTests(TestCase):
                 "Anne",
                 "Chuck",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Bill is already being stalked by Anne - add Anne again, but in the
         # reverse direction
@@ -184,7 +191,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is stalking Anne?
         self.assertQuerysetEqual(
@@ -215,7 +223,8 @@ class RecursiveM2MTests(TestCase):
                 "Chuck",
                 "David",
             ],
-            attrgetter("name")
+            attrgetter("name"),
+            ordered=False
         )
         # Who is stalking Anne?
         self.assertQuerysetEqual(
diff --git a/tests/modeltests/many_to_one/tests.py b/tests/modeltests/many_to_one/tests.py
index 4fb19dbc69..44ae689dd4 100644
--- a/tests/modeltests/many_to_one/tests.py
+++ b/tests/modeltests/many_to_one/tests.py
@@ -267,7 +267,9 @@ class ManyToOneTests(TestCase):
             ["<Reporter: John Smith>"])
         self.assertQuerysetEqual(
             Reporter.objects.filter(article__headline__startswith='T'),
-            ["<Reporter: John Smith>", "<Reporter: John Smith>"])
+            ["<Reporter: John Smith>", "<Reporter: John Smith>"],
+            ordered=False
+        )
         self.assertQuerysetEqual(
             Reporter.objects.filter(article__headline__startswith='T').distinct(),
             ["<Reporter: John Smith>"])
@@ -285,7 +287,9 @@ class ManyToOneTests(TestCase):
                 "<Reporter: John Smith>",
                 "<Reporter: John Smith>",
                 "<Reporter: John Smith>",
-            ])
+            ],
+            ordered=False
+        )
         self.assertQuerysetEqual(
             Reporter.objects.filter(article__reporter__first_name__startswith='John').distinct(),
             ["<Reporter: John Smith>"])
diff --git a/tests/modeltests/model_forms/tests.py b/tests/modeltests/model_forms/tests.py
index 47d72abdc2..9699b155c0 100644
--- a/tests/modeltests/model_forms/tests.py
+++ b/tests/modeltests/model_forms/tests.py
@@ -1044,9 +1044,12 @@ class OldFormForXTests(TestCase):
         self.assertQuerysetEqual(f.clean([c1.id]), ["Entertainment"])
         self.assertQuerysetEqual(f.clean([c2.id]), ["It's a test"])
         self.assertQuerysetEqual(f.clean([str(c1.id)]), ["Entertainment"])
-        self.assertQuerysetEqual(f.clean([str(c1.id), str(c2.id)]), ["Entertainment", "It's a test"])
-        self.assertQuerysetEqual(f.clean([c1.id, str(c2.id)]), ["Entertainment", "It's a test"])
-        self.assertQuerysetEqual(f.clean((c1.id, str(c2.id))), ["Entertainment", "It's a test"])
+        self.assertQuerysetEqual(f.clean([str(c1.id), str(c2.id)]), ["Entertainment", "It's a test"],
+                                 ordered=False)
+        self.assertQuerysetEqual(f.clean([c1.id, str(c2.id)]), ["Entertainment", "It's a test"],
+                                 ordered=False)
+        self.assertQuerysetEqual(f.clean((c1.id, str(c2.id))), ["Entertainment", "It's a test"],
+                                 ordered=False)
         with self.assertRaises(ValidationError):
             f.clean(['100'])
         with self.assertRaises(ValidationError):