newforms: Fixed #3008 -- Widgets now support strings containing utf-8 characters. Thanks for reporting, Nebojša Đorđević

git-svn-id: http://code.djangoproject.com/svn/django/trunk@4076 bcc190cf-cafb-0310-a4f2-bffc1f526a37

4 files changed, 51 insertions, 33 deletions

diff --git a/django/newforms/fields.py b/django/newforms/fields.py
index b9e2ed35c7..179555fc77 100644
--- a/django/newforms/fields.py
+++ b/django/newforms/fields.py
@@ -2,7 +2,7 @@
 Field classes
 """
 
-from util import ValidationError, DEFAULT_ENCODING
+from util import ValidationError, DEFAULT_ENCODING, smart_unicode
 from widgets import TextInput, CheckboxInput, Select, SelectMultiple
 import datetime
 import re
@@ -55,10 +55,7 @@ class CharField(Field):
         "Validates max_length and min_length. Returns a Unicode object."
         Field.clean(self, value)
         if value in EMPTY_VALUES: value = u''
-        if not isinstance(value, basestring):
-            value = unicode(str(value), DEFAULT_ENCODING)
-        elif not isinstance(value, unicode):
-            value = unicode(value, DEFAULT_ENCODING)
+        value = smart_unicode(value)
         if self.max_length is not None and len(value) > self.max_length:
             raise ValidationError(u'Ensure this value has at most %d characters.' % self.max_length)
         if self.min_length is not None and len(value) < self.min_length:
@@ -165,10 +162,7 @@ class RegexField(Field):
         """
         Field.clean(self, value)
         if value in EMPTY_VALUES: value = u''
-        if not isinstance(value, basestring):
-            value = unicode(str(value), DEFAULT_ENCODING)
-        elif not isinstance(value, unicode):
-            value = unicode(value, DEFAULT_ENCODING)
+        value = smart_unicode(value)
         if not self.regex.search(value):
             raise ValidationError(self.error_message)
         return value
@@ -244,10 +238,7 @@ class ChoiceField(Field):
         """
         value = Field.clean(self, value)
         if value in EMPTY_VALUES: value = u''
-        if not isinstance(value, basestring):
-            value = unicode(str(value), DEFAULT_ENCODING)
-        elif not isinstance(value, unicode):
-            value = unicode(value, DEFAULT_ENCODING)
+        value = smart_unicode(value)
         valid_values = set([str(k) for k, v in self.choices])
         if value not in valid_values:
             raise ValidationError(u'Select a valid choice. %s is not one of the available choices.' % value)
@@ -267,11 +258,8 @@ class MultipleChoiceField(ChoiceField):
             raise ValidationError(u'This field is required.')
         new_value = []
         for val in value:
-            if not isinstance(val, basestring):
-                value = unicode(str(val), DEFAULT_ENCODING)
-            elif not isinstance(val, unicode):
-                value = unicode(val, DEFAULT_ENCODING)
-            new_value.append(value)
+            val = smart_unicode(val)
+            new_value.append(val)
         # Validate that each value in the value list is in self.choices.
         valid_values = set([k for k, v in self.choices])
         for val in new_value:
diff --git a/django/newforms/util.py b/django/newforms/util.py
index 3887010c85..6b101fc152 100644
--- a/django/newforms/util.py
+++ b/django/newforms/util.py
@@ -2,6 +2,8 @@
 DEFAULT_ENCODING = 'utf-8' # TODO: First look at django.conf.settings, then fall back to this.
 
 def smart_unicode(s):
+    if not isinstance(s, basestring):
+        s = unicode(str(s))
     if not isinstance(s, unicode):
         s = unicode(s, DEFAULT_ENCODING)
     return s
diff --git a/django/newforms/widgets.py b/django/newforms/widgets.py
index c9b3d02913..318c76e55d 100644
--- a/django/newforms/widgets.py
+++ b/django/newforms/widgets.py
@@ -8,6 +8,7 @@ __all__ = (
     'Select', 'SelectMultiple', 'RadioSelect',
 )
 
+from util import smart_unicode
 from django.utils.html import escape
 from itertools import chain
 
@@ -18,7 +19,7 @@ except NameError:
 
 # Converts a dictionary to a single string with key="value", XML-style with
 # a leading space. Assumes keys do not need to be XML-escaped.
-flatatt = lambda attrs: ''.join([' %s="%s"' % (k, escape(v)) for k, v in attrs.items()])
+flatatt = lambda attrs: u''.join([u' %s="%s"' % (k, escape(v)) for k, v in attrs.items()])
 
 class Widget(object):
     requires_data_list = False # Determines whether render()'s 'value' argument should be a list.
@@ -43,7 +44,7 @@ class Input(Widget):
     def render(self, name, value, attrs=None):
         if value is None: value = ''
         final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
-        if value != '': final_attrs['value'] = value # Only add the 'value' attribute if a value is non-empty.
+        if value != '': final_attrs['value'] = smart_unicode(value) # Only add the 'value' attribute if a value is non-empty.
         return u'<input%s />' % flatatt(final_attrs)
 
 class TextInput(Input):
@@ -61,6 +62,7 @@ class FileInput(Input):
 class Textarea(Widget):
     def render(self, name, value, attrs=None):
         if value is None: value = ''
+        value = smart_unicode(value)
         final_attrs = self.build_attrs(attrs, name=name)
         return u'<textarea%s>%s</textarea>' % (flatatt(final_attrs), escape(value))
 
@@ -80,10 +82,11 @@ class Select(Widget):
         if value is None: value = ''
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<select%s>' % flatatt(final_attrs)]
-        str_value = str(value) # Normalize to string.
+        str_value = smart_unicode(value) # Normalize to string.
         for option_value, option_label in chain(self.choices, choices):
-            selected_html = (str(option_value) == str_value) and ' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(option_label)))
+            option_value = smart_unicode(option_value)
+            selected_html = (option_value == str_value) and u' selected="selected"' or ''
+            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(smart_unicode(option_label))))
         output.append(u'</select>')
         return u'\n'.join(output)
 
@@ -98,10 +101,11 @@ class SelectMultiple(Widget):
         if value is None: value = []
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<select multiple="multiple"%s>' % flatatt(final_attrs)]
-        str_values = set([str(v) for v in value]) # Normalize to strings.
+        str_values = set([smart_unicode(v) for v in value]) # Normalize to strings.
         for option_value, option_label in chain(self.choices, choices):
-            selected_html = (str(option_value) in str_values) and ' selected="selected"' or ''
-            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(option_label)))
+            option_value = smart_unicode(option_value)
+            selected_html = (option_value in str_values) and ' selected="selected"' or ''
+            output.append(u'<option value="%s"%s>%s</option>' % (escape(option_value), selected_html, escape(smart_unicode(option_label))))
         output.append(u'</select>')
         return u'\n'.join(output)
 
@@ -116,7 +120,7 @@ class RadioInput(object):
         return u'<label>%s %s</label>' % (self.tag(), self.choice_label)
 
     def is_checked(self):
-        return self.value == str(self.choice_value)
+        return self.value == smart_unicode(self.choice_value)
 
     def tag(self):
         final_attrs = dict(self.attrs, type='radio', name=self.name, value=self.choice_value)
@@ -142,7 +146,7 @@ class RadioSelect(Select):
     def render(self, name, value, attrs=None, choices=()):
         "Returns a RadioFieldRenderer instance rather than a Unicode string."
         if value is None: value = ''
-        str_value = str(value) # Normalize to string.
+        str_value = smart_unicode(value) # Normalize to string.
         return RadioFieldRenderer(name, str_value, attrs, list(chain(self.choices, choices)))
 
 class CheckboxSelectMultiple(Widget):
diff --git a/tests/regressiontests/forms/tests.py b/tests/regressiontests/forms/tests.py
index b02ccb8ee8..c4dd7074a5 100644
--- a/tests/regressiontests/forms/tests.py
+++ b/tests/regressiontests/forms/tests.py
@@ -1,4 +1,5 @@
-"""
+# -*- coding: utf-8 -*-
+r"""
 >>> from django.newforms import *
 >>> import datetime
 >>> import re
@@ -17,6 +18,11 @@ u'<input type="text" name="email" value="some &quot;quoted&quot; &amp; ampersand
 >>> w.render('email', 'test@example.com', attrs={'class': 'fun'})
 u'<input type="text" name="email" value="test@example.com" class="fun" />'
 
+# Note that doctest in Python 2.4 (and maybe 2.5?) doesn't support non-ascii
+# characters in output, so we're displaying the repr() here.
+>>> w.render('email', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'})
+u'<input type="text" name="email" value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" class="fun" />'
+
 You can also pass 'attrs' to the constructor:
 >>> w = TextInput(attrs={'class': 'fun'})
 >>> w.render('email', '')
@@ -55,6 +61,9 @@ u'<input type="password" class="fun" value="foo@example.com" name="email" />'
 >>> w.render('email', '', attrs={'class': 'special'})
 u'<input type="password" class="special" name="email" />'
 
+>>> w.render('email', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'})
+u'<input type="password" class="fun" value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" name="email" />'
+
 # HiddenInput Widget ############################################################
 
 >>> w = HiddenInput()
@@ -81,6 +90,14 @@ u'<input type="hidden" class="fun" value="foo@example.com" name="email" />'
 >>> w.render('email', '', attrs={'class': 'special'})
 u'<input type="hidden" class="special" name="email" />'
 
+>>> w.render('email', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'})
+u'<input type="hidden" class="fun" value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" name="email" />'
+
+'attrs' passed to render() get precedence over those passed to the constructor:
+>>> w = HiddenInput(attrs={'class': 'pretty'})
+>>> w.render('email', '', attrs={'class': 'special'})
+u'<input type="hidden" class="special" name="email" />'
+
 # FileInput Widget ############################################################
 
 >>> w = FileInput()
@@ -102,10 +119,8 @@ u'<input type="file" class="fun" name="email" />'
 >>> w.render('email', 'foo@example.com')
 u'<input type="file" class="fun" value="foo@example.com" name="email" />'
 
-'attrs' passed to render() get precedence over those passed to the constructor:
->>> w = HiddenInput(attrs={'class': 'pretty'})
->>> w.render('email', '', attrs={'class': 'special'})
-u'<input type="hidden" class="special" name="email" />'
+>>> w.render('email', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'})
+u'<input type="file" class="fun" value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" name="email" />'
 
 # Textarea Widget #############################################################
 
@@ -133,6 +148,9 @@ u'<textarea class="pretty" name="msg">example</textarea>'
 >>> w.render('msg', '', attrs={'class': 'special'})
 u'<textarea class="special" name="msg"></textarea>'
 
+>>> w.render('msg', 'ŠĐĆŽćžšđ', attrs={'class': 'fun'})
+u'<textarea class="fun" name="msg">\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111</textarea>'
+
 # CheckboxInput Widget ########################################################
 
 >>> w = CheckboxInput()
@@ -236,6 +254,9 @@ If 'choices' is passed to both the constructor and render(), then they'll both b
 <option value="5">5</option>
 </select>
 
+>>> w.render('email', 'ŠĐĆŽćžšđ', choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')])
+u'<select name="email">\n<option value="1">1</option>\n<option value="2">2</option>\n<option value="3">3</option>\n<option value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" selected="selected">\u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111</option>\n<option value="\u0107\u017e\u0161\u0111">abc\u0107\u017e\u0161\u0111</option>\n</select>'
+
 # SelectMultiple Widget #######################################################
 
 >>> w = SelectMultiple()
@@ -340,6 +361,9 @@ If 'choices' is passed to both the constructor and render(), then they'll both b
 <option value="5">5</option>
 </select>
 
+>>> w.render('nums', ['ŠĐĆŽćžšđ'], choices=[('ŠĐĆŽćžšđ', 'ŠĐabcĆŽćžšđ'), ('ćžšđ', 'abcćžšđ')])
+u'<select multiple="multiple" name="nums">\n<option value="1">1</option>\n<option value="2">2</option>\n<option value="3">3</option>\n<option value="\u0160\u0110\u0106\u017d\u0107\u017e\u0161\u0111" selected="selected">\u0160\u0110abc\u0106\u017d\u0107\u017e\u0161\u0111</option>\n<option value="\u0107\u017e\u0161\u0111">abc\u0107\u017e\u0161\u0111</option>\n</select>'
+
 # RadioSelect Widget ##########################################################
 
 >>> w = RadioSelect()