Fixed #16501 -- Added an allow_unicode parameter to SlugField.

Thanks Flavio Curella and Berker Peksag for the initial patch.

8 files changed, 72 insertions, 16 deletions

diff --git a/tests/admin_views/admin.py b/tests/admin_views/admin.py
index 1def206df7..367ac3df62 100644
--- a/tests/admin_views/admin.py
+++ b/tests/admin_views/admin.py
@@ -673,12 +673,15 @@ class MainPrepopulatedAdmin(admin.ModelAdmin):
     inlines = [RelatedPrepopulatedInline1, RelatedPrepopulatedInline2]
     fieldsets = (
         (None, {
-            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2',),)
+            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2', 'slug3'))
         }),
     )
     formfield_overrides = {models.CharField: {'strip': False}}
-    prepopulated_fields = {'slug1': ['name', 'pubdate'],
-                           'slug2': ['status', 'name']}
+    prepopulated_fields = {
+        'slug1': ['name', 'pubdate'],
+        'slug2': ['status', 'name'],
+        'slug3': ['name'],
+    }
 
 
 class UnorderedObjectAdmin(admin.ModelAdmin):
diff --git a/tests/admin_views/models.py b/tests/admin_views/models.py
index 0be371fa7c..6c009f85d7 100644
--- a/tests/admin_views/models.py
+++ b/tests/admin_views/models.py
@@ -716,6 +716,7 @@ class MainPrepopulated(models.Model):
                  ('option two', 'Option Two')))
     slug1 = models.SlugField(blank=True)
     slug2 = models.SlugField(blank=True)
+    slug3 = models.SlugField(blank=True, allow_unicode=True)
 
 
 class RelatedPrepopulated(models.Model):
diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py
index fedae380b4..55ccee7fe1 100644
--- a/tests/admin_views/tests.py
+++ b/tests/admin_views/tests.py
@@ -4412,11 +4412,13 @@ class SeleniumAdminViewsFirefoxTests(AdminSeleniumWebDriverTestCase):
         # Main form ----------------------------------------------------------
         self.selenium.find_element_by_css_selector('#id_pubdate').send_keys('2012-02-18')
         self.get_select_option('#id_status', 'option two').click()
-        self.selenium.find_element_by_css_selector('#id_name').send_keys(' this is the mAin nÀMë and it\'s awεšome')
+        self.selenium.find_element_by_css_selector('#id_name').send_keys(' this is the mAin nÀMë and it\'s awεšomeııı')
         slug1 = self.selenium.find_element_by_css_selector('#id_slug1').get_attribute('value')
         slug2 = self.selenium.find_element_by_css_selector('#id_slug2').get_attribute('value')
-        self.assertEqual(slug1, 'main-name-and-its-awesome-2012-02-18')
-        self.assertEqual(slug2, 'option-two-main-name-and-its-awesome')
+        slug3 = self.selenium.find_element_by_css_selector('#id_slug3').get_attribute('value')
+        self.assertEqual(slug1, 'main-name-and-its-awesomeiii-2012-02-18')
+        self.assertEqual(slug2, 'option-two-main-name-and-its-awesomeiii')
+        self.assertEqual(slug3, 'main-n\xe0m\xeb-and-its-aw\u03b5\u0161ome\u0131\u0131\u0131')
 
         # Stacked inlines ----------------------------------------------------
         # Initial inline
@@ -4463,11 +4465,11 @@ class SeleniumAdminViewsFirefoxTests(AdminSeleniumWebDriverTestCase):
         self.wait_page_loaded()
         self.assertEqual(MainPrepopulated.objects.all().count(), 1)
         MainPrepopulated.objects.get(
-            name=' this is the mAin nÀMë and it\'s awεšome',
+            name=' this is the mAin nÀMë and it\'s awεšomeııı',
             pubdate='2012-02-18',
             status='option two',
-            slug1='main-name-and-its-awesome-2012-02-18',
-            slug2='option-two-main-name-and-its-awesome',
+            slug1='main-name-and-its-awesomeiii-2012-02-18',
+            slug2='option-two-main-name-and-its-awesomeiii',
         )
         self.assertEqual(RelatedPrepopulated.objects.all().count(), 4)
         RelatedPrepopulated.objects.get(
diff --git a/tests/forms_tests/tests/test_fields.py b/tests/forms_tests/tests/test_fields.py
index 1644bda4e2..6234f5c455 100644
--- a/tests/forms_tests/tests/test_fields.py
+++ b/tests/forms_tests/tests/test_fields.py
@@ -1556,6 +1556,16 @@ class FieldsTests(SimpleTestCase):
         f = SlugField()
         self.assertEqual(f.clean('    aa-bb-cc    '), 'aa-bb-cc')
 
+    def test_slugfield_unicode_normalization(self):
+        f = SlugField(allow_unicode=True)
+        self.assertEqual(f.clean('a'), 'a')
+        self.assertEqual(f.clean('1'), '1')
+        self.assertEqual(f.clean('a1'), 'a1')
+        self.assertEqual(f.clean('你好'), '你好')
+        self.assertEqual(f.clean('  你-好  '), '你-好')
+        self.assertEqual(f.clean('ıçğüş'), 'ıçğüş')
+        self.assertEqual(f.clean('foo-ıç-bar'), 'foo-ıç-bar')
+
     # UUIDField ###################################################################
 
     def test_uuidfield_1(self):
diff --git a/tests/model_fields/models.py b/tests/model_fields/models.py
index 350e98a80b..931468c9bf 100644
--- a/tests/model_fields/models.py
+++ b/tests/model_fields/models.py
@@ -86,6 +86,10 @@ class BigS(models.Model):
     s = models.SlugField(max_length=255)
 
 
+class UnicodeSlugField(models.Model):
+    s = models.SlugField(max_length=255, allow_unicode=True)
+
+
 class SmallIntegerModel(models.Model):
     value = models.SmallIntegerField()
 
diff --git a/tests/model_fields/tests.py b/tests/model_fields/tests.py
index 353c15f491..8bbead1a36 100644
--- a/tests/model_fields/tests.py
+++ b/tests/model_fields/tests.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import datetime
@@ -20,6 +21,7 @@ from django.db.models.fields import (
 from django.db.models.fields.files import FileField, ImageField
 from django.test.utils import requires_tz_support
 from django.utils import six, timezone
+from django.utils.encoding import force_str
 from django.utils.functional import lazy
 
 from .models import (
@@ -27,7 +29,8 @@ from .models import (
     Document, FksToBooleans, FkToChar, FloatModel, Foo, GenericIPAddress,
     IntegerModel, NullBooleanModel, PositiveIntegerModel,
     PositiveSmallIntegerModel, Post, PrimaryKeyCharModel, RenamedField,
-    SmallIntegerModel, VerboseNameField, Whiz, WhizIter, WhizIterEmpty,
+    SmallIntegerModel, UnicodeSlugField, VerboseNameField, Whiz, WhizIter,
+    WhizIterEmpty,
 )
 
 
@@ -113,7 +116,6 @@ class BasicFieldTests(test.TestCase):
         self.assertIsInstance(field.formfield(choices_form_class=klass), klass)
 
     def test_field_str(self):
-        from django.utils.encoding import force_str
         f = Foo._meta.get_field('a')
         self.assertEqual(force_str(f), "model_fields.Foo.a")
 
@@ -515,6 +517,14 @@ class SlugFieldTests(test.TestCase):
         bs = BigS.objects.get(pk=bs.pk)
         self.assertEqual(bs.s, 'slug' * 50)
 
+    def test_slugfield_unicode_max_length(self):
+        """
+        SlugField with allow_unicode=True should honor max_length.
+        """
+        bs = UnicodeSlugField.objects.create(s='你好你好' * 50)
+        bs = UnicodeSlugField.objects.get(pk=bs.pk)
+        self.assertEqual(bs.s, '你好你好' * 50)
+
 
 class ValidationTest(test.SimpleTestCase):
     def test_charfield_raises_error_on_empty_string(self):
diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py
index d8675fa8c5..e6f5c0f378 100644
--- a/tests/utils_tests/test_text.py
+++ b/tests/utils_tests/test_text.py
@@ -172,11 +172,16 @@ class TestUtilsText(SimpleTestCase):
 
     def test_slugify(self):
         items = (
-            ('Hello, World!', 'hello-world'),
-            ('spam & eggs', 'spam-eggs'),
+            # given - expected - unicode?
+            ('Hello, World!', 'hello-world', False),
+            ('spam & eggs', 'spam-eggs', False),
+            ('spam & ıçüş', 'spam-ıçüş', True),
+            ('foo ıç bar', 'foo-ıç-bar', True),
+            ('    foo ıç bar', 'foo-ıç-bar', True),
+            ('你好', '你好', True),
         )
-        for value, output in items:
-            self.assertEqual(text.slugify(value), output)
+        for value, output, is_unicode in items:
+            self.assertEqual(text.slugify(value, allow_unicode=is_unicode), output)
 
     def test_unescape_entities(self):
         items = [
diff --git a/tests/validators/tests.py b/tests/validators/tests.py
index 977bc66bb1..2acafbfcef 100644
--- a/tests/validators/tests.py
+++ b/tests/validators/tests.py
@@ -14,7 +14,7 @@ from django.core.validators import (
     MinLengthValidator, MinValueValidator, RegexValidator, URLValidator,
     int_list_validator, validate_comma_separated_integer_list, validate_email,
     validate_integer, validate_ipv4_address, validate_ipv6_address,
-    validate_ipv46_address, validate_slug,
+    validate_ipv46_address, validate_slug, validate_unicode_slug,
 )
 from django.test import SimpleTestCase
 from django.test.utils import str_prefix
@@ -89,15 +89,36 @@ TEST_DATA = [
     (validate_slug, 'longer-slug-still-ok', None),
     (validate_slug, '--------', None),
     (validate_slug, 'nohyphensoranything', None),
+    (validate_slug, 'a', None),
+    (validate_slug, '1', None),
+    (validate_slug, 'a1', None),
 
     (validate_slug, '', ValidationError),
     (validate_slug, ' text ', ValidationError),
     (validate_slug, ' ', ValidationError),
     (validate_slug, 'some@mail.com', ValidationError),
     (validate_slug, '你好', ValidationError),
+    (validate_slug, '你 好', ValidationError),
     (validate_slug, '\n', ValidationError),
     (validate_slug, 'trailing-newline\n', ValidationError),
 
+    (validate_unicode_slug, 'slug-ok', None),
+    (validate_unicode_slug, 'longer-slug-still-ok', None),
+    (validate_unicode_slug, '--------', None),
+    (validate_unicode_slug, 'nohyphensoranything', None),
+    (validate_unicode_slug, 'a', None),
+    (validate_unicode_slug, '1', None),
+    (validate_unicode_slug, 'a1', None),
+    (validate_unicode_slug, '你好', None),
+
+    (validate_unicode_slug, '', ValidationError),
+    (validate_unicode_slug, ' text ', ValidationError),
+    (validate_unicode_slug, ' ', ValidationError),
+    (validate_unicode_slug, 'some@mail.com', ValidationError),
+    (validate_unicode_slug, '\n', ValidationError),
+    (validate_unicode_slug, '你 好', ValidationError),
+    (validate_unicode_slug, 'trailing-newline\n', ValidationError),
+
     (validate_ipv4_address, '1.1.1.1', None),
     (validate_ipv4_address, '255.0.0.0', None),
     (validate_ipv4_address, '0.0.0.0', None),