diff options
| author | Andrew Godwin <andrew@aeracode.org> | 2012-08-10 12:40:37 +0100 |
|---|---|---|
| committer | Andrew Godwin <andrew@aeracode.org> | 2012-08-10 12:40:37 +0100 |
| commit | 184cf9ab798d5b25d855649ddb2ca580949778df (patch) | |
| tree | 8512633ec04a6979b0953e32e73c9a43d09e5805 /django | |
| parent | c4b2a3262cc79383d6562cfc7e9af20135c8e0bf (diff) | |
| parent | 7275576235ae2e87f3de7b0facb3f9b0a2368f28 (diff) | |
Merge branch 'master' into schema-alteration
Diffstat (limited to 'django')
174 files changed, 1532 insertions, 1325 deletions
diff --git a/django/conf/__init__.py b/django/conf/__init__.py index 77454b3fb9..e1c3fd1808 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -158,7 +158,7 @@ class UserSettingsHolder(BaseSettings): return getattr(self.default_settings, name) def __dir__(self): - return self.__dict__.keys() + dir(self.default_settings) + return list(self.__dict__) + dir(self.default_settings) # For Python < 2.6: __members__ = property(lambda self: self.__dir__()) diff --git a/django/conf/locale/fi/formats.py b/django/conf/locale/fi/formats.py index 9a658eed40..e76144a9e4 100644 --- a/django/conf/locale/fi/formats.py +++ b/django/conf/locale/fi/formats.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals # see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date DATE_FORMAT = 'j. E Y' TIME_FORMAT = 'G.i.s' -# DATETIME_FORMAT = +DATETIME_FORMAT = r'j. E Y \k\e\l\l\o G.i.s' YEAR_MONTH_FORMAT = 'F Y' MONTH_DAY_FORMAT = 'j. F' SHORT_DATE_FORMAT = 'j.n.Y' diff --git a/django/conf/locale/gl/formats.py b/django/conf/locale/gl/formats.py index 1ff9f34521..ba7f6c52a0 100644 --- a/django/conf/locale/gl/formats.py +++ b/django/conf/locale/gl/formats.py @@ -1,17 +1,18 @@ # -*- encoding: utf-8 -*- # This file is distributed under the same license as the Django package. # +from __future__ import unicode_literals # The *_FORMAT strings use the Django date format syntax, # see http://docs.djangoproject.com/en/dev/ref/templates/builtins/#date -DATE_FORMAT = 'd F Y' +DATE_FORMAT = r'j \d\e F \d\e Y' TIME_FORMAT = 'H:i:s' -# DATETIME_FORMAT = -YEAR_MONTH_FORMAT = 'F Y' -MONTH_DAY_FORMAT = 'j F' -SHORT_DATE_FORMAT = 'j M, Y' -# SHORT_DATETIME_FORMAT = -# FIRST_DAY_OF_WEEK = +DATETIME_FORMAT = r'j \d\e F \d\e Y \á\s H:i' +YEAR_MONTH_FORMAT = r'F \d\e Y' +MONTH_DAY_FORMAT = r'j \d\e F' +SHORT_DATE_FORMAT = 'd-m-Y' +SHORT_DATETIME_FORMAT = 'd-m-Y, H:i' +FIRST_DAY_OF_WEEK = 1 # Monday # The *_INPUT_FORMATS strings use the Python strftime format syntax, # see http://docs.python.org/library/datetime.html#strftime-strptime-behavior diff --git a/django/conf/urls/i18n.py b/django/conf/urls/i18n.py index 6e56af8271..426c2b2d30 100644 --- a/django/conf/urls/i18n.py +++ b/django/conf/urls/i18n.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.conf.urls import patterns +from django.conf.urls import patterns, url from django.core.urlresolvers import LocaleRegexURLResolver def i18n_patterns(prefix, *args): @@ -16,5 +16,5 @@ def i18n_patterns(prefix, *args): urlpatterns = patterns('', - (r'^setlang/$', 'django.views.i18n.set_language'), + url(r'^setlang/$', 'django.views.i18n.set_language', name='set_language'), ) diff --git a/django/contrib/admin/actions.py b/django/contrib/admin/actions.py index 5b56402428..201101736e 100644 --- a/django/contrib/admin/actions.py +++ b/django/contrib/admin/actions.py @@ -7,7 +7,7 @@ from django.contrib.admin import helpers from django.contrib.admin.util import get_deleted_objects, model_ngettext from django.db import router from django.template.response import TemplateResponse -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy, ugettext as _ def delete_selected(modeladmin, request, queryset): @@ -42,7 +42,7 @@ def delete_selected(modeladmin, request, queryset): n = queryset.count() if n: for obj in queryset: - obj_display = force_unicode(obj) + obj_display = force_text(obj) modeladmin.log_deletion(request, obj, obj_display) queryset.delete() modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % { @@ -52,9 +52,9 @@ def delete_selected(modeladmin, request, queryset): return None if len(queryset) == 1: - objects_name = force_unicode(opts.verbose_name) + objects_name = force_text(opts.verbose_name) else: - objects_name = force_unicode(opts.verbose_name_plural) + objects_name = force_text(opts.verbose_name_plural) if perms_needed or protected: title = _("Cannot delete %(name)s") % {"name": objects_name} diff --git a/django/contrib/admin/filters.py b/django/contrib/admin/filters.py index 538bf54df9..cecae216c0 100644 --- a/django/contrib/admin/filters.py +++ b/django/contrib/admin/filters.py @@ -9,7 +9,7 @@ import datetime from django.db import models from django.core.exceptions import ImproperlyConfigured -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ from django.utils import timezone @@ -195,7 +195,7 @@ class RelatedFieldListFilter(FieldListFilter): } for pk_val, val in self.lookup_choices: yield { - 'selected': self.lookup_val == smart_unicode(pk_val), + 'selected': self.lookup_val == smart_text(pk_val), 'query_string': cl.get_query_string({ self.lookup_kwarg: pk_val, }, [self.lookup_kwarg_isnull]), @@ -272,7 +272,7 @@ class ChoicesFieldListFilter(FieldListFilter): } for lookup, title in self.field.flatchoices: yield { - 'selected': smart_unicode(lookup) == self.lookup_val, + 'selected': smart_text(lookup) == self.lookup_val, 'query_string': cl.get_query_string({ self.lookup_kwarg: lookup}), 'display': title, @@ -381,7 +381,7 @@ class AllValuesFieldListFilter(FieldListFilter): if val is None: include_none = True continue - val = smart_unicode(val) + val = smart_text(val) yield { 'selected': self.lookup_val == val, 'query_string': cl.get_query_string({ diff --git a/django/contrib/admin/helpers.py b/django/contrib/admin/helpers.py index 1bc843cdf9..90370bd978 100644 --- a/django/contrib/admin/helpers.py +++ b/django/contrib/admin/helpers.py @@ -9,7 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.db.models.fields.related import ManyToManyRel from django.forms.util import flatatt from django.template.defaultfilters import capfirst -from django.utils.encoding import force_unicode, smart_unicode +from django.utils.encoding import force_text, smart_text from django.utils.html import conditional_escape, format_html from django.utils.safestring import mark_safe from django.utils import six @@ -94,7 +94,7 @@ class Fieldset(object): class Fieldline(object): def __init__(self, form, field, readonly_fields=None, model_admin=None): self.form = form # A django.forms.Form instance - if not hasattr(field, "__iter__"): + if not hasattr(field, "__iter__") or isinstance(field, six.text_type): self.fields = [field] else: self.fields = field @@ -122,7 +122,7 @@ class AdminField(object): def label_tag(self): classes = [] - contents = conditional_escape(force_unicode(self.field.label)) + contents = conditional_escape(force_text(self.field.label)) if self.is_checkbox: classes.append('vCheckboxLabel') else: @@ -166,7 +166,7 @@ class AdminReadonlyField(object): label = self.field['label'] return format_html('<label{0}>{1}:</label>', flatatt(attrs), - capfirst(force_unicode(label))) + capfirst(force_text(label))) def contents(self): from django.contrib.admin.templatetags.admin_list import _boolean_icon @@ -182,7 +182,7 @@ class AdminReadonlyField(object): if boolean: result_repr = _boolean_icon(value) else: - result_repr = smart_unicode(value) + result_repr = smart_text(value) if getattr(attr, "allow_tags", False): result_repr = mark_safe(result_repr) else: @@ -325,11 +325,11 @@ class AdminErrorList(forms.util.ErrorList): """ def __init__(self, form, inline_formsets): if form.is_bound: - self.extend(form.errors.values()) + self.extend(list(six.itervalues(form.errors))) for inline_formset in inline_formsets: self.extend(inline_formset.non_form_errors()) for errors_in_inline_form in inline_formset.errors: - self.extend(errors_in_inline_form.values()) + self.extend(list(six.itervalues(errors_in_inline_form))) def normalize_fieldsets(fieldsets): """ diff --git a/django/contrib/admin/models.py b/django/contrib/admin/models.py index 58bbbabfdf..e31c6d84ed 100644 --- a/django/contrib/admin/models.py +++ b/django/contrib/admin/models.py @@ -5,7 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.auth.models import User from django.contrib.admin.util import quote from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text ADDITION = 1 CHANGE = 2 @@ -13,7 +13,7 @@ DELETION = 3 class LogEntryManager(models.Manager): def log_action(self, user_id, content_type_id, object_id, object_repr, action_flag, change_message=''): - e = self.model(None, None, user_id, content_type_id, smart_unicode(object_id), object_repr[:200], action_flag, change_message) + e = self.model(None, None, user_id, content_type_id, smart_text(object_id), object_repr[:200], action_flag, change_message) e.save() class LogEntry(models.Model): @@ -34,7 +34,7 @@ class LogEntry(models.Model): ordering = ('-action_time',) def __repr__(self): - return smart_unicode(self.action_time) + return smart_text(self.action_time) def __unicode__(self): if self.action_flag == ADDITION: diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index c13a6bc5cc..081d00121b 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -28,7 +28,7 @@ from django.utils import six from django.utils.text import capfirst, get_text_list from django.utils.translation import ugettext as _ from django.utils.translation import ungettext -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text HORIZONTAL, VERTICAL = 1, 2 # returns the <ul> class for a given radio_admin field @@ -425,7 +425,7 @@ class ModelAdmin(BaseModelAdmin): if self.declared_fieldsets: return self.declared_fieldsets form = self.get_form(request, obj) - fields = form.base_fields.keys() + list(self.get_readonly_fields(request, obj)) + fields = list(form.base_fields) + list(self.get_readonly_fields(request, obj)) return [(None, {'fields': fields})] def get_form(self, request, obj=None, **kwargs): @@ -520,7 +520,7 @@ class ModelAdmin(BaseModelAdmin): user_id = request.user.pk, content_type_id = ContentType.objects.get_for_model(object).pk, object_id = object.pk, - object_repr = force_unicode(object), + object_repr = force_text(object), action_flag = ADDITION ) @@ -535,7 +535,7 @@ class ModelAdmin(BaseModelAdmin): user_id = request.user.pk, content_type_id = ContentType.objects.get_for_model(object).pk, object_id = object.pk, - object_repr = force_unicode(object), + object_repr = force_text(object), action_flag = CHANGE, change_message = message ) @@ -560,7 +560,7 @@ class ModelAdmin(BaseModelAdmin): """ A list_display column containing a checkbox widget. """ - return helpers.checkbox.render(helpers.ACTION_CHECKBOX_NAME, force_unicode(obj.pk)) + return helpers.checkbox.render(helpers.ACTION_CHECKBOX_NAME, force_text(obj.pk)) action_checkbox.short_description = mark_safe('<input type="checkbox" id="action-toggle" />') action_checkbox.allow_tags = True @@ -608,7 +608,7 @@ class ModelAdmin(BaseModelAdmin): tuple (name, description). """ choices = [] + default_choices - for func, name, description in self.get_actions(request).itervalues(): + for func, name, description in six.itervalues(self.get_actions(request)): choice = (name, description % model_format_dict(self.opts)) choices.append(choice) return choices @@ -674,17 +674,17 @@ class ModelAdmin(BaseModelAdmin): for formset in formsets: for added_object in formset.new_objects: change_message.append(_('Added %(name)s "%(object)s".') - % {'name': force_unicode(added_object._meta.verbose_name), - 'object': force_unicode(added_object)}) + % {'name': force_text(added_object._meta.verbose_name), + 'object': force_text(added_object)}) for changed_object, changed_fields in formset.changed_objects: change_message.append(_('Changed %(list)s for %(name)s "%(object)s".') % {'list': get_text_list(changed_fields, _('and')), - 'name': force_unicode(changed_object._meta.verbose_name), - 'object': force_unicode(changed_object)}) + 'name': force_text(changed_object._meta.verbose_name), + 'object': force_text(changed_object)}) for deleted_object in formset.deleted_objects: change_message.append(_('Deleted %(name)s "%(object)s".') - % {'name': force_unicode(deleted_object._meta.verbose_name), - 'object': force_unicode(deleted_object)}) + % {'name': force_text(deleted_object._meta.verbose_name), + 'object': force_text(deleted_object)}) change_message = ' '.join(change_message) return change_message or _('No fields changed.') @@ -769,7 +769,7 @@ class ModelAdmin(BaseModelAdmin): opts = obj._meta pk_value = obj._get_pk_val() - msg = _('The %(name)s "%(obj)s" was added successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': force_unicode(obj)} + msg = _('The %(name)s "%(obj)s" was added successfully.') % {'name': force_text(opts.verbose_name), 'obj': force_text(obj)} # Here, we distinguish between different save types by checking for # the presence of keys in request.POST. if "_continue" in request.POST: @@ -782,10 +782,10 @@ class ModelAdmin(BaseModelAdmin): return HttpResponse( '<!DOCTYPE html><html><head><title></title></head><body>' '<script type="text/javascript">opener.dismissAddAnotherPopup(window, "%s", "%s");</script></body></html>' % \ - # escape() calls force_unicode. + # escape() calls force_text. (escape(pk_value), escapejs(obj))) elif "_addanother" in request.POST: - self.message_user(request, msg + ' ' + (_("You may add another %s below.") % force_unicode(opts.verbose_name))) + self.message_user(request, msg + ' ' + (_("You may add another %s below.") % force_text(opts.verbose_name))) return HttpResponseRedirect(request.path) else: self.message_user(request, msg) @@ -819,7 +819,7 @@ class ModelAdmin(BaseModelAdmin): pk_value = obj._get_pk_val() - msg = _('The %(name)s "%(obj)s" was changed successfully.') % {'name': force_unicode(verbose_name), 'obj': force_unicode(obj)} + msg = _('The %(name)s "%(obj)s" was changed successfully.') % {'name': force_text(verbose_name), 'obj': force_text(obj)} if "_continue" in request.POST: self.message_user(request, msg + ' ' + _("You may edit it again below.")) if "_popup" in request.REQUEST: @@ -827,14 +827,14 @@ class ModelAdmin(BaseModelAdmin): else: return HttpResponseRedirect(request.path) elif "_saveasnew" in request.POST: - msg = _('The %(name)s "%(obj)s" was added successfully. You may edit it again below.') % {'name': force_unicode(verbose_name), 'obj': obj} + msg = _('The %(name)s "%(obj)s" was added successfully. You may edit it again below.') % {'name': force_text(verbose_name), 'obj': obj} self.message_user(request, msg) return HttpResponseRedirect(reverse('admin:%s_%s_change' % (opts.app_label, module_name), args=(pk_value,), current_app=self.admin_site.name)) elif "_addanother" in request.POST: - self.message_user(request, msg + ' ' + (_("You may add another %s below.") % force_unicode(verbose_name))) + self.message_user(request, msg + ' ' + (_("You may add another %s below.") % force_text(verbose_name))) return HttpResponseRedirect(reverse('admin:%s_%s_add' % (opts.app_label, module_name), current_app=self.admin_site.name)) @@ -995,7 +995,7 @@ class ModelAdmin(BaseModelAdmin): media = media + inline_admin_formset.media context = { - 'title': _('Add %s') % force_unicode(opts.verbose_name), + 'title': _('Add %s') % force_text(opts.verbose_name), 'adminform': adminForm, 'is_popup': "_popup" in request.REQUEST, 'media': media, @@ -1019,7 +1019,7 @@ class ModelAdmin(BaseModelAdmin): raise PermissionDenied if obj is None: - raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_unicode(opts.verbose_name), 'key': escape(object_id)}) + raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(opts.verbose_name), 'key': escape(object_id)}) if request.method == 'POST' and "_saveasnew" in request.POST: return self.add_view(request, form_url=reverse('admin:%s_%s_add' % @@ -1085,7 +1085,7 @@ class ModelAdmin(BaseModelAdmin): media = media + inline_admin_formset.media context = { - 'title': _('Change %s') % force_unicode(opts.verbose_name), + 'title': _('Change %s') % force_text(opts.verbose_name), 'adminform': adminForm, 'object_id': object_id, 'original': obj, @@ -1194,14 +1194,14 @@ class ModelAdmin(BaseModelAdmin): if changecount: if changecount == 1: - name = force_unicode(opts.verbose_name) + name = force_text(opts.verbose_name) else: - name = force_unicode(opts.verbose_name_plural) + name = force_text(opts.verbose_name_plural) msg = ungettext("%(count)s %(name)s was changed successfully.", "%(count)s %(name)s were changed successfully.", changecount) % {'count': changecount, 'name': name, - 'obj': force_unicode(obj)} + 'obj': force_text(obj)} self.message_user(request, msg) return HttpResponseRedirect(request.get_full_path()) @@ -1228,7 +1228,7 @@ class ModelAdmin(BaseModelAdmin): 'All %(total_count)s selected', cl.result_count) context = { - 'module_name': force_unicode(opts.verbose_name_plural), + 'module_name': force_text(opts.verbose_name_plural), 'selection_note': _('0 of %(cnt)s selected') % {'cnt': len(cl.result_list)}, 'selection_note_all': selection_note_all % {'total_count': cl.result_count}, 'title': cl.title, @@ -1263,7 +1263,7 @@ class ModelAdmin(BaseModelAdmin): raise PermissionDenied if obj is None: - raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_unicode(opts.verbose_name), 'key': escape(object_id)}) + raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(opts.verbose_name), 'key': escape(object_id)}) using = router.db_for_write(self.model) @@ -1275,11 +1275,11 @@ class ModelAdmin(BaseModelAdmin): if request.POST: # The user has already confirmed the deletion. if perms_needed: raise PermissionDenied - obj_display = force_unicode(obj) + obj_display = force_text(obj) self.log_deletion(request, obj, obj_display) self.delete_model(request, obj) - self.message_user(request, _('The %(name)s "%(obj)s" was deleted successfully.') % {'name': force_unicode(opts.verbose_name), 'obj': force_unicode(obj_display)}) + self.message_user(request, _('The %(name)s "%(obj)s" was deleted successfully.') % {'name': force_text(opts.verbose_name), 'obj': force_text(obj_display)}) if not self.has_change_permission(request, None): return HttpResponseRedirect(reverse('admin:index', @@ -1288,7 +1288,7 @@ class ModelAdmin(BaseModelAdmin): (opts.app_label, opts.module_name), current_app=self.admin_site.name)) - object_name = force_unicode(opts.verbose_name) + object_name = force_text(opts.verbose_name) if perms_needed or protected: title = _("Cannot delete %(name)s") % {"name": object_name} @@ -1326,9 +1326,9 @@ class ModelAdmin(BaseModelAdmin): # If no history was found, see whether this object even exists. obj = get_object_or_404(model, pk=unquote(object_id)) context = { - 'title': _('Change history: %s') % force_unicode(obj), + 'title': _('Change history: %s') % force_text(obj), 'action_list': action_list, - 'module_name': capfirst(force_unicode(opts.verbose_name_plural)), + 'module_name': capfirst(force_text(opts.verbose_name_plural)), 'object': obj, 'app_label': app_label, 'opts': opts, @@ -1415,7 +1415,7 @@ class InlineModelAdmin(BaseModelAdmin): if self.declared_fieldsets: return self.declared_fieldsets form = self.get_formset(request, obj).form - fields = form.base_fields.keys() + list(self.get_readonly_fields(request, obj)) + fields = list(form.base_fields) + list(self.get_readonly_fields(request, obj)) return [(None, {'fields': fields})] def queryset(self, request): diff --git a/django/contrib/admin/sites.py b/django/contrib/admin/sites.py index 515cc33eca..05773ceac0 100644 --- a/django/contrib/admin/sites.py +++ b/django/contrib/admin/sites.py @@ -10,6 +10,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.urlresolvers import reverse, NoReverseMatch from django.template.response import TemplateResponse from django.utils.safestring import mark_safe +from django.utils import six from django.utils.text import capfirst from django.utils.translation import ugettext as _ from django.views.decorators.cache import never_cache @@ -133,7 +134,7 @@ class AdminSite(object): """ Get all the enabled actions as an iterable of (name, func). """ - return self._actions.iteritems() + return six.iteritems(self._actions) def has_permission(self, request): """ @@ -239,7 +240,7 @@ class AdminSite(object): ) # Add in each model's views. - for model, model_admin in self._registry.iteritems(): + for model, model_admin in six.iteritems(self._registry): urlpatterns += patterns('', url(r'^%s/%s/' % (model._meta.app_label, model._meta.module_name), include(model_admin.urls)) @@ -370,7 +371,7 @@ class AdminSite(object): } # Sort the apps alphabetically. - app_list = app_dict.values() + app_list = list(six.itervalues(app_dict)) app_list.sort(key=lambda x: x['name']) # Sort the models alphabetically within each app. diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py index 0f15781fa9..1873d44989 100644 --- a/django/contrib/admin/templatetags/admin_list.py +++ b/django/contrib/admin/templatetags/admin_list.py @@ -12,9 +12,10 @@ from django.db import models from django.utils import formats from django.utils.html import format_html from django.utils.safestring import mark_safe +from django.utils import six from django.utils.text import capfirst from django.utils.translation import ugettext as _ -from django.utils.encoding import smart_unicode, force_unicode +from django.utils.encoding import smart_text, force_text from django.template import Library from django.template.loader import get_template from django.template.context import Context @@ -125,7 +126,7 @@ def result_headers(cl): if i in ordering_field_columns: sorted = True order_type = ordering_field_columns.get(i).lower() - sort_priority = ordering_field_columns.keys().index(i) + 1 + sort_priority = list(ordering_field_columns).index(i) + 1 th_classes.append('sorted %sending' % order_type) new_order_type = {'asc': 'desc', 'desc': 'asc'}[order_type] @@ -209,7 +210,7 @@ def items_for_result(cl, result, form): result_repr = display_for_field(value, f) if isinstance(f, (models.DateField, models.TimeField, models.ForeignKey)): row_class = mark_safe(' class="nowrap"') - if force_unicode(result_repr) == '': + if force_text(result_repr) == '': result_repr = mark_safe(' ') # If list_display_links not defined, add the link tag to the first field if (first and not cl.list_display_links) or field_name in cl.list_display_links: @@ -223,7 +224,7 @@ def items_for_result(cl, result, form): else: attr = pk value = result.serializable_value(attr) - result_id = repr(force_unicode(value))[1:] + result_id = repr(force_text(value))[1:] yield format_html('<{0}{1}><a href="{2}"{3}>{4}</a></{5}>', table_tag, row_class, @@ -240,10 +241,10 @@ def items_for_result(cl, result, form): field_name == cl.model._meta.pk.name and form[cl.model._meta.pk.name].is_hidden)): bf = form[field_name] - result_repr = mark_safe(force_unicode(bf.errors) + force_unicode(bf)) + result_repr = mark_safe(force_text(bf.errors) + force_text(bf)) yield format_html('<td{0}>{1}</td>', row_class, result_repr) if form and not form[cl.model._meta.pk.name].is_hidden: - yield format_html('<td>{0}</td>', force_unicode(form[cl.model._meta.pk.name])) + yield format_html('<td>{0}</td>', force_text(form[cl.model._meta.pk.name])) class ResultList(list): # Wrapper class used to return items in a list_editable @@ -266,7 +267,7 @@ def result_hidden_fields(cl): if cl.formset: for res, form in zip(cl.result_list, cl.formset.forms): if form[cl.model._meta.pk.name].is_hidden: - yield mark_safe(force_unicode(form[cl.model._meta.pk.name])) + yield mark_safe(force_text(form[cl.model._meta.pk.name])) @register.inclusion_tag("admin/change_list_results.html") def result_list(cl): diff --git a/django/contrib/admin/util.py b/django/contrib/admin/util.py index 16bdfe0566..ff90e1d007 100644 --- a/django/contrib/admin/util.py +++ b/django/contrib/admin/util.py @@ -12,7 +12,7 @@ from django.utils import formats from django.utils.html import format_html from django.utils.text import capfirst from django.utils import timezone -from django.utils.encoding import force_unicode, smart_unicode, smart_str +from django.utils.encoding import force_text, smart_text, smart_bytes from django.utils import six from django.utils.translation import ungettext from django.core.urlresolvers import reverse @@ -132,7 +132,7 @@ def get_deleted_objects(objs, opts, user, admin_site, using): # Don't display link to edit, because it either has no # admin or is edited inline. return '%s: %s' % (capfirst(opts.verbose_name), - force_unicode(obj)) + force_text(obj)) to_delete = collector.nested(format_callback) @@ -207,8 +207,8 @@ def model_format_dict(obj): else: opts = obj return { - 'verbose_name': force_unicode(opts.verbose_name), - 'verbose_name_plural': force_unicode(opts.verbose_name_plural) + 'verbose_name': force_text(opts.verbose_name), + 'verbose_name_plural': force_text(opts.verbose_name_plural) } @@ -274,10 +274,10 @@ def label_for_field(name, model, model_admin=None, return_attr=False): label = field.verbose_name except models.FieldDoesNotExist: if name == "__unicode__": - label = force_unicode(model._meta.verbose_name) + label = force_text(model._meta.verbose_name) attr = six.text_type elif name == "__str__": - label = smart_str(model._meta.verbose_name) + label = smart_bytes(model._meta.verbose_name) attr = bytes else: if callable(name): @@ -311,7 +311,7 @@ def help_text_for_field(name, model): help_text = model._meta.get_field_by_name(name)[0].help_text except models.FieldDoesNotExist: help_text = "" - return smart_unicode(help_text) + return smart_text(help_text) def display_for_field(value, field): @@ -335,7 +335,7 @@ def display_for_field(value, field): elif isinstance(field, models.FloatField): return formats.number_format(value) else: - return smart_unicode(value) + return smart_text(value) def display_for_value(value, boolean=False): @@ -353,7 +353,7 @@ def display_for_value(value, boolean=False): elif isinstance(value, six.integer_types + (decimal.Decimal, float)): return formats.number_format(value) else: - return smart_unicode(value) + return smart_text(value) class NotRelationField(Exception): diff --git a/django/contrib/admin/views/main.py b/django/contrib/admin/views/main.py index 85e03f3b75..3eabf3dbeb 100644 --- a/django/contrib/admin/views/main.py +++ b/django/contrib/admin/views/main.py @@ -6,7 +6,7 @@ from django.core.paginator import InvalidPage from django.db import models from django.db.models.fields import FieldDoesNotExist from django.utils.datastructures import SortedDict -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils.translation import ugettext, ugettext_lazy from django.utils.http import urlencode @@ -75,7 +75,7 @@ class ChangeList(object): title = ugettext('Select %s') else: title = ugettext('Select %s to change') - self.title = title % force_unicode(self.opts.verbose_name) + self.title = title % force_text(self.opts.verbose_name) self.pk_attname = self.lookup_opts.pk.attname def get_filters(self, request): @@ -94,7 +94,7 @@ class ChangeList(object): # 'key' will be used as a keyword argument later, so Python # requires it to be a string. del lookup_params[key] - lookup_params[smart_str(key)] = value + lookup_params[smart_bytes(key)] = value if not self.model_admin.lookup_allowed(key, value): raise SuspiciousOperation("Filtering by %s not allowed" % key) diff --git a/django/contrib/admin/widgets.py b/django/contrib/admin/widgets.py index 550ed0f7e2..1e0bc2d366 100644 --- a/django/contrib/admin/widgets.py +++ b/django/contrib/admin/widgets.py @@ -14,7 +14,7 @@ from django.utils.html import escape, format_html, format_html_join from django.utils.text import Truncator from django.utils.translation import ugettext as _ from django.utils.safestring import mark_safe -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils import six @@ -96,7 +96,7 @@ class AdminRadioFieldRenderer(RadioFieldRenderer): return format_html('<ul{0}>\n{1}\n</ul>', flatatt(self.attrs), format_html_join('\n', '<li>{0}</li>', - ((force_unicode(w),) for w in self))) + ((force_text(w),) for w in self))) class AdminRadioSelect(forms.RadioSelect): renderer = AdminRadioFieldRenderer @@ -197,7 +197,7 @@ class ManyToManyRawIdWidget(ForeignKeyRawIdWidget): # The related object is registered with the same AdminSite attrs['class'] = 'vManyToManyRawIdAdminField' if value: - value = ','.join([force_unicode(v) for v in value]) + value = ','.join([force_text(v) for v in value]) else: value = '' return super(ManyToManyRawIdWidget, self).render(name, value, attrs) @@ -221,7 +221,7 @@ class ManyToManyRawIdWidget(ForeignKeyRawIdWidget): if len(initial) != len(data): return True for pk1, pk2 in zip(initial, data): - if force_unicode(pk1) != force_unicode(pk2): + if force_text(pk1) != force_text(pk2): return True return False diff --git a/django/contrib/admindocs/utils.py b/django/contrib/admindocs/utils.py index 4bf1250ac6..0e10eb4fa3 100644 --- a/django/contrib/admindocs/utils.py +++ b/django/contrib/admindocs/utils.py @@ -6,7 +6,7 @@ from email.errors import HeaderParseError from django.utils.safestring import mark_safe from django.core.urlresolvers import reverse -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes try: import docutils.core import docutils.nodes @@ -66,7 +66,7 @@ def parse_rst(text, default_reference_context, thing_being_parsed=None): "link_base" : reverse('django-admindocs-docroot').rstrip('/') } if thing_being_parsed: - thing_being_parsed = smart_str("<%s>" % thing_being_parsed) + thing_being_parsed = smart_bytes("<%s>" % thing_being_parsed) parts = docutils.core.publish_parts(text, source_path=thing_being_parsed, destination_path=None, writer_name='html', settings_overrides=overrides) diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 5649398cc8..94963b4d39 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -14,6 +14,7 @@ from django.core import urlresolvers from django.contrib.admindocs import utils from django.contrib.sites.models import Site from django.utils.importlib import import_module +from django.utils import six from django.utils.translation import ugettext as _ from django.utils.safestring import mark_safe @@ -48,7 +49,7 @@ def template_tag_index(request): load_all_installed_template_libraries() tags = [] - app_libs = template.libraries.items() + app_libs = list(six.iteritems(template.libraries)) builtin_libs = [(None, lib) for lib in template.builtins] for module_name, library in builtin_libs + app_libs: for tag_name, tag_func in library.tags.items(): @@ -83,7 +84,7 @@ def template_filter_index(request): load_all_installed_template_libraries() filters = [] - app_libs = template.libraries.items() + app_libs = list(six.iteritems(template.libraries)) builtin_libs = [(None, lib) for lib in template.builtins] for module_name, library in builtin_libs + app_libs: for filter_name, filter_func in library.filters.items(): diff --git a/django/contrib/auth/admin.py b/django/contrib/auth/admin.py index ad61904041..ccf940d16d 100644 --- a/django/contrib/auth/admin.py +++ b/django/contrib/auth/admin.py @@ -12,6 +12,7 @@ from django.template.response import TemplateResponse from django.utils.html import escape from django.utils.decorators import method_decorator from django.utils.safestring import mark_safe +from django.utils import six from django.utils.translation import ugettext, ugettext_lazy as _ from django.views.decorators.csrf import csrf_protect from django.views.decorators.debug import sensitive_post_parameters @@ -128,7 +129,7 @@ class UserAdmin(admin.ModelAdmin): else: form = self.change_password_form(user) - fieldsets = [(None, {'fields': form.base_fields.keys()})] + fieldsets = [(None, {'fields': list(form.base_fields)})] adminForm = admin.helpers.AdminForm(form, fieldsets, {}) context = { diff --git a/django/contrib/auth/context_processors.py b/django/contrib/auth/context_processors.py index 3ffab01e94..1b6c2eedd0 100644 --- a/django/contrib/auth/context_processors.py +++ b/django/contrib/auth/context_processors.py @@ -11,8 +11,9 @@ class PermLookupDict(object): def __getitem__(self, perm_name): return self.user.has_perm("%s.%s" % (self.module_name, perm_name)) - def __nonzero__(self): + def __bool__(self): return self.user.has_module_perms(self.module_name) + __nonzero__ = __bool__ # Python 2 class PermWrapper(object): diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py index d17c41132e..dfd039f018 100644 --- a/django/contrib/auth/forms.py +++ b/django/contrib/auth/forms.py @@ -89,9 +89,9 @@ class UserCreationForm(forms.ModelForm): raise forms.ValidationError(self.error_messages['duplicate_username']) def clean_password2(self): - password1 = self.cleaned_data.get("password1", "") - password2 = self.cleaned_data["password2"] - if password1 != password2: + password1 = self.cleaned_data.get("password1") + password2 = self.cleaned_data.get("password2") + if password1 and password2 and password1 != password2: raise forms.ValidationError( self.error_messages['password_mismatch']) return password2 diff --git a/django/contrib/auth/hashers.py b/django/contrib/auth/hashers.py index 96ec40ba60..45c1f88ab2 100644 --- a/django/contrib/auth/hashers.py +++ b/django/contrib/auth/hashers.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import base64 import hashlib from django.dispatch import receiver @@ -7,7 +8,7 @@ from django.conf import settings from django.test.signals import setting_changed from django.utils import importlib from django.utils.datastructures import SortedDict -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.core.exceptions import ImproperlyConfigured from django.utils.crypto import ( pbkdf2, constant_time_compare, get_random_string) @@ -218,7 +219,7 @@ class PBKDF2PasswordHasher(BasePasswordHasher): if not iterations: iterations = self.iterations hash = pbkdf2(password, salt, iterations, digest=self.digest) - hash = hash.encode('base64').strip() + hash = base64.b64encode(hash).strip() return "%s$%d$%s$%s" % (self.algorithm, iterations, salt, hash) def verify(self, password, encoded): @@ -298,7 +299,7 @@ class SHA1PasswordHasher(BasePasswordHasher): def encode(self, password, salt): assert password assert salt and '$' not in salt - hash = hashlib.sha1(smart_str(salt + password)).hexdigest() + hash = hashlib.sha1(smart_bytes(salt + password)).hexdigest() return "%s$%s$%s" % (self.algorithm, salt, hash) def verify(self, password, encoded): @@ -326,7 +327,7 @@ class MD5PasswordHasher(BasePasswordHasher): def encode(self, password, salt): assert password assert salt and '$' not in salt - hash = hashlib.md5(smart_str(salt + password)).hexdigest() + hash = hashlib.md5(smart_bytes(salt + password)).hexdigest() return "%s$%s$%s" % (self.algorithm, salt, hash) def verify(self, password, encoded): @@ -360,7 +361,7 @@ class UnsaltedMD5PasswordHasher(BasePasswordHasher): return '' def encode(self, password, salt): - return hashlib.md5(smart_str(password)).hexdigest() + return hashlib.md5(smart_bytes(password)).hexdigest() def verify(self, password, encoded): encoded_2 = self.encode(password, '') diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py index 100acb6c5b..7abd2abcf4 100644 --- a/django/contrib/auth/management/__init__.py +++ b/django/contrib/auth/management/__init__.py @@ -9,6 +9,7 @@ import unicodedata from django.contrib.auth import models as auth_app from django.db.models import get_models, signals from django.contrib.auth.models import User +from django.utils.six.moves import input def _get_permission_codename(action, opts): @@ -66,10 +67,10 @@ def create_superuser(app, created_models, verbosity, db, **kwargs): msg = ("\nYou just installed Django's auth system, which means you " "don't have any superusers defined.\nWould you like to create one " "now? (yes/no): ") - confirm = raw_input(msg) + confirm = input(msg) while 1: if confirm not in ('yes', 'no'): - confirm = raw_input('Please enter either "yes" or "no": ') + confirm = input('Please enter either "yes" or "no": ') continue if confirm == 'yes': call_command("createsuperuser", interactive=True, database=db) diff --git a/django/contrib/auth/management/commands/createsuperuser.py b/django/contrib/auth/management/commands/createsuperuser.py index f3f1a7b671..6e0d0bc754 100644 --- a/django/contrib/auth/management/commands/createsuperuser.py +++ b/django/contrib/auth/management/commands/createsuperuser.py @@ -12,6 +12,7 @@ from django.contrib.auth.management import get_default_username from django.core import exceptions from django.core.management.base import BaseCommand, CommandError from django.db import DEFAULT_DB_ALIAS +from django.utils.six.moves import input from django.utils.translation import ugettext as _ RE_VALID_USERNAME = re.compile('[\w.@+-]+$') @@ -76,7 +77,7 @@ class Command(BaseCommand): input_msg = 'Username' if default_username: input_msg += ' (leave blank to use %r)' % default_username - username = raw_input(input_msg + ': ') + username = input(input_msg + ': ') if default_username and username == '': username = default_username if not RE_VALID_USERNAME.match(username): @@ -94,7 +95,7 @@ class Command(BaseCommand): # Get an email while 1: if not email: - email = raw_input('E-mail address: ') + email = input('E-mail address: ') try: is_valid_email(email) except exceptions.ValidationError: diff --git a/django/contrib/auth/tests/forms.py b/django/contrib/auth/tests/forms.py index 2ab895804f..594b55c633 100644 --- a/django/contrib/auth/tests/forms.py +++ b/django/contrib/auth/tests/forms.py @@ -8,7 +8,8 @@ from django.core import mail from django.forms.fields import Field, EmailField from django.test import TestCase from django.test.utils import override_settings -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text +from django.utils import six from django.utils import translation from django.utils.translation import ugettext as _ @@ -27,7 +28,7 @@ class UserCreationFormTest(TestCase): form = UserCreationForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form["username"].errors, - [force_unicode(form.error_messages['duplicate_username'])]) + [force_text(form.error_messages['duplicate_username'])]) def test_invalid_data(self): data = { @@ -38,7 +39,7 @@ class UserCreationFormTest(TestCase): form = UserCreationForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form["username"].errors, - [force_unicode(form.fields['username'].error_messages['invalid'])]) + [force_text(form.fields['username'].error_messages['invalid'])]) def test_password_verification(self): # The verification password is incorrect. @@ -50,13 +51,13 @@ class UserCreationFormTest(TestCase): form = UserCreationForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form["password2"].errors, - [force_unicode(form.error_messages['password_mismatch'])]) + [force_text(form.error_messages['password_mismatch'])]) def test_both_passwords(self): # One (or both) passwords weren't given data = {'username': 'jsmith'} form = UserCreationForm(data) - required_error = [force_unicode(Field.default_error_messages['required'])] + required_error = [force_text(Field.default_error_messages['required'])] self.assertFalse(form.is_valid()) self.assertEqual(form['password1'].errors, required_error) self.assertEqual(form['password2'].errors, required_error) @@ -65,6 +66,7 @@ class UserCreationFormTest(TestCase): form = UserCreationForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form['password1'].errors, required_error) + self.assertEqual(form['password2'].errors, []) def test_success(self): # The success case. @@ -94,7 +96,7 @@ class AuthenticationFormTest(TestCase): form = AuthenticationForm(None, data) self.assertFalse(form.is_valid()) self.assertEqual(form.non_field_errors(), - [force_unicode(form.error_messages['invalid_login'])]) + [force_text(form.error_messages['invalid_login'])]) def test_inactive_user(self): # The user is inactive. @@ -105,7 +107,7 @@ class AuthenticationFormTest(TestCase): form = AuthenticationForm(None, data) self.assertFalse(form.is_valid()) self.assertEqual(form.non_field_errors(), - [force_unicode(form.error_messages['inactive'])]) + [force_text(form.error_messages['inactive'])]) def test_inactive_user_i18n(self): with self.settings(USE_I18N=True): @@ -118,7 +120,7 @@ class AuthenticationFormTest(TestCase): form = AuthenticationForm(None, data) self.assertFalse(form.is_valid()) self.assertEqual(form.non_field_errors(), - [force_unicode(form.error_messages['inactive'])]) + [force_text(form.error_messages['inactive'])]) def test_success(self): # The success case @@ -146,7 +148,7 @@ class SetPasswordFormTest(TestCase): form = SetPasswordForm(user, data) self.assertFalse(form.is_valid()) self.assertEqual(form["new_password2"].errors, - [force_unicode(form.error_messages['password_mismatch'])]) + [force_text(form.error_messages['password_mismatch'])]) def test_success(self): user = User.objects.get(username='testclient') @@ -173,7 +175,7 @@ class PasswordChangeFormTest(TestCase): form = PasswordChangeForm(user, data) self.assertFalse(form.is_valid()) self.assertEqual(form["old_password"].errors, - [force_unicode(form.error_messages['password_incorrect'])]) + [force_text(form.error_messages['password_incorrect'])]) def test_password_verification(self): # The two new passwords do not match. @@ -186,7 +188,7 @@ class PasswordChangeFormTest(TestCase): form = PasswordChangeForm(user, data) self.assertFalse(form.is_valid()) self.assertEqual(form["new_password2"].errors, - [force_unicode(form.error_messages['password_mismatch'])]) + [force_text(form.error_messages['password_mismatch'])]) def test_success(self): # The success case. @@ -202,7 +204,7 @@ class PasswordChangeFormTest(TestCase): def test_field_order(self): # Regression test - check the order of fields: user = User.objects.get(username='testclient') - self.assertEqual(PasswordChangeForm(user, {}).fields.keys(), + self.assertEqual(list(PasswordChangeForm(user, {}).fields), ['old_password', 'new_password1', 'new_password2']) @@ -217,7 +219,7 @@ class UserChangeFormTest(TestCase): form = UserChangeForm(data, instance=user) self.assertFalse(form.is_valid()) self.assertEqual(form['username'].errors, - [force_unicode(form.fields['username'].error_messages['invalid'])]) + [force_text(form.fields['username'].error_messages['invalid'])]) def test_bug_14242(self): # A regression test, introduce by adding an optimization for the @@ -272,7 +274,7 @@ class PasswordResetFormTest(TestCase): form = PasswordResetForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form['email'].errors, - [force_unicode(EmailField.default_error_messages['invalid'])]) + [force_text(EmailField.default_error_messages['invalid'])]) def test_nonexistant_email(self): # Test nonexistant email address @@ -280,7 +282,7 @@ class PasswordResetFormTest(TestCase): form = PasswordResetForm(data) self.assertFalse(form.is_valid()) self.assertEqual(form.errors, - {'email': [force_unicode(form.error_messages['unknown'])]}) + {'email': [force_text(form.error_messages['unknown'])]}) def test_cleaned_data(self): # Regression test diff --git a/django/contrib/auth/tests/views.py b/django/contrib/auth/tests/views.py index e76e7dd10f..3c847f456a 100644 --- a/django/contrib/auth/tests/views.py +++ b/django/contrib/auth/tests/views.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import User from django.core import mail from django.core.urlresolvers import reverse, NoReverseMatch from django.http import QueryDict -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.html import escape from django.utils.http import urlquote from django.test import TestCase @@ -46,7 +46,7 @@ class AuthViewsTestCase(TestCase): self.assertTrue(SESSION_KEY in self.client.session) def assertContainsEscaped(self, response, text, **kwargs): - return self.assertContains(response, escape(force_unicode(text)), **kwargs) + return self.assertContains(response, escape(force_text(text)), **kwargs) class AuthViewNamedURLTests(AuthViewsTestCase): diff --git a/django/contrib/comments/forms.py b/django/contrib/comments/forms.py index 830e24bca9..bd254d2733 100644 --- a/django/contrib/comments/forms.py +++ b/django/contrib/comments/forms.py @@ -5,7 +5,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.contrib.comments.models import Comment from django.utils.crypto import salted_hmac, constant_time_compare -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.text import get_text_list from django.utils import timezone from django.utils.translation import ungettext, ugettext, ugettext_lazy as _ @@ -133,7 +133,7 @@ class CommentDetailsForm(CommentSecurityForm): """ return dict( content_type = ContentType.objects.get_for_model(self.target_object), - object_pk = force_unicode(self.target_object._get_pk_val()), + object_pk = force_text(self.target_object._get_pk_val()), user_name = self.cleaned_data["name"], user_email = self.cleaned_data["email"], user_url = self.cleaned_data["url"], diff --git a/django/contrib/comments/managers.py b/django/contrib/comments/managers.py index 499feee6c3..bc0fc5f332 100644 --- a/django/contrib/comments/managers.py +++ b/django/contrib/comments/managers.py @@ -1,6 +1,6 @@ from django.db import models from django.contrib.contenttypes.models import ContentType -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text class CommentManager(models.Manager): @@ -18,5 +18,5 @@ class CommentManager(models.Manager): ct = ContentType.objects.get_for_model(model) qs = self.get_query_set().filter(content_type=ct) if isinstance(model, models.Model): - qs = qs.filter(object_pk=force_unicode(model._get_pk_val())) + qs = qs.filter(object_pk=force_text(model._get_pk_val())) return qs diff --git a/django/contrib/comments/templatetags/comments.py b/django/contrib/comments/templatetags/comments.py index ce1825e529..4d4eb2322f 100644 --- a/django/contrib/comments/templatetags/comments.py +++ b/django/contrib/comments/templatetags/comments.py @@ -3,7 +3,7 @@ from django.template.loader import render_to_string from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.contrib import comments -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text register = template.Library() @@ -75,7 +75,7 @@ class BaseCommentNode(template.Node): qs = self.comment_model.objects.filter( content_type = ctype, - object_pk = smart_unicode(object_pk), + object_pk = smart_text(object_pk), site__pk = settings.SITE_ID, ) diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index d5062cabf3..29e93eefe7 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -17,7 +17,7 @@ from django.forms import ModelForm from django.forms.models import BaseModelFormSet, modelformset_factory, save_instance from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets from django.contrib.contenttypes.models import ContentType -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text class GenericForeignKey(object): """ @@ -169,7 +169,7 @@ class GenericRelation(RelatedField, Field): def value_to_string(self, obj): qs = getattr(obj, self.name).all() - return smart_unicode([instance._get_pk_val() for instance in qs]) + return smart_text([instance._get_pk_val() for instance in qs]) def m2m_db_table(self): return self.rel.to._meta.db_table diff --git a/django/contrib/contenttypes/management.py b/django/contrib/contenttypes/management.py index 6a23ef5287..9f287d494b 100644 --- a/django/contrib/contenttypes/management.py +++ b/django/contrib/contenttypes/management.py @@ -1,6 +1,8 @@ from django.contrib.contenttypes.models import ContentType from django.db.models import get_apps, get_models, signals -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text +from django.utils import six +from django.utils.six.moves import input def update_contenttypes(app, created_models, verbosity=2, **kwargs): """ @@ -24,17 +26,17 @@ def update_contenttypes(app, created_models, verbosity=2, **kwargs): ) to_remove = [ ct - for (model_name, ct) in content_types.iteritems() + for (model_name, ct) in six.iteritems(content_types) if model_name not in app_models ] cts = ContentType.objects.bulk_create([ ContentType( - name=smart_unicode(model._meta.verbose_name_raw), + name=smart_text(model._meta.verbose_name_raw), app_label=app_label, model=model_name, ) - for (model_name, model) in app_models.iteritems() + for (model_name, model) in six.iteritems(app_models) if model_name not in content_types ]) if verbosity >= 2: @@ -48,7 +50,7 @@ def update_contenttypes(app, created_models, verbosity=2, **kwargs): ' %s | %s' % (ct.app_label, ct.model) for ct in to_remove ]) - ok_to_delete = raw_input("""The following content types are stale and need to be deleted: + ok_to_delete = input("""The following content types are stale and need to be deleted: %s diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py index 867351f6c8..e6d547a491 100644 --- a/django/contrib/contenttypes/models.py +++ b/django/contrib/contenttypes/models.py @@ -1,6 +1,6 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode, force_unicode +from django.utils.encoding import smart_text, force_text class ContentTypeManager(models.Manager): @@ -37,13 +37,13 @@ class ContentTypeManager(models.Manager): try: ct = self._get_from_cache(opts) except KeyError: - # Load or create the ContentType entry. The smart_unicode() is + # Load or create the ContentType entry. The smart_text() is # needed around opts.verbose_name_raw because name_raw might be a # django.utils.functional.__proxy__ object. ct, created = self.get_or_create( app_label = opts.app_label, model = opts.object_name.lower(), - defaults = {'name': smart_unicode(opts.verbose_name_raw)}, + defaults = {'name': smart_text(opts.verbose_name_raw)}, ) self._add_to_cache(self.db, ct) @@ -86,7 +86,7 @@ class ContentTypeManager(models.Manager): ct = self.create( app_label=opts.app_label, model=opts.object_name.lower(), - name=smart_unicode(opts.verbose_name_raw), + name=smart_text(opts.verbose_name_raw), ) self._add_to_cache(self.db, ct) results[ct.model_class()] = ct @@ -147,7 +147,7 @@ class ContentType(models.Model): if not model or self.name != model._meta.verbose_name_raw: return self.name else: - return force_unicode(model._meta.verbose_name) + return force_text(model._meta.verbose_name) def model_class(self): "Returns the Python model class for this type of content." diff --git a/django/contrib/databrowse/datastructures.py b/django/contrib/databrowse/datastructures.py index 687aa87f03..810e039894 100644 --- a/django/contrib/databrowse/datastructures.py +++ b/django/contrib/databrowse/datastructures.py @@ -7,7 +7,7 @@ from __future__ import unicode_literals from django.db import models from django.utils import formats from django.utils.text import capfirst -from django.utils.encoding import smart_unicode, smart_str, iri_to_uri +from django.utils.encoding import smart_text, smart_bytes, iri_to_uri from django.db.models.query import QuerySet EMPTY_VALUE = '(None)' @@ -17,12 +17,12 @@ class EasyModel(object): def __init__(self, site, model): self.site = site self.model = model - self.model_list = site.registry.keys() + self.model_list = list(site.registry.keys()) self.verbose_name = model._meta.verbose_name self.verbose_name_plural = model._meta.verbose_name_plural def __repr__(self): - return '<EasyModel for %s>' % smart_str(self.model._meta.object_name) + return '<EasyModel for %s>' % smart_bytes(self.model._meta.object_name) def model_databrowse(self): "Returns the ModelDatabrowse class for this model." @@ -61,7 +61,7 @@ class EasyField(object): self.model, self.field = easy_model, field def __repr__(self): - return smart_str('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) + return smart_bytes('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) def choices(self): for value, label in self.field.choices: @@ -79,7 +79,7 @@ class EasyChoice(object): self.value, self.label = value, label def __repr__(self): - return smart_str('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) + return smart_bytes('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) def url(self): return '%s%s/%s/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.field.name, iri_to_uri(self.value)) @@ -89,10 +89,10 @@ class EasyInstance(object): self.model, self.instance = easy_model, instance def __repr__(self): - return smart_str('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val())) + return smart_bytes('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val())) def __unicode__(self): - val = smart_unicode(self.instance) + val = smart_text(self.instance) if len(val) > DISPLAY_SIZE: return val[:DISPLAY_SIZE] + '...' return val @@ -136,7 +136,7 @@ class EasyInstanceField(object): self.raw_value = getattr(instance.instance, field.name) def __repr__(self): - return smart_str('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) + return smart_bytes('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name)) def values(self): """ @@ -176,8 +176,6 @@ class EasyInstanceField(object): for plugin_name, plugin in self.model.model_databrowse().plugins.items(): urls = plugin.urls(plugin_name, self) if urls is not None: - #plugin_urls.append(urls) - values = self.values() return zip(self.values(), urls) if self.field.rel: m = EasyModel(self.model.site, self.field.rel.to) @@ -187,7 +185,7 @@ class EasyInstanceField(object): if value is None: continue url = '%s%s/%s/objects/%s/' % (self.model.site.root_url, m.model._meta.app_label, m.model._meta.module_name, iri_to_uri(value._get_pk_val())) - lst.append((smart_unicode(value), url)) + lst.append((smart_text(value), url)) else: lst = [(value, None) for value in self.values()] elif self.field.choices: @@ -196,10 +194,10 @@ class EasyInstanceField(object): url = '%s%s/%s/fields/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.name, iri_to_uri(self.raw_value)) lst.append((value, url)) elif isinstance(self.field, models.URLField): - val = self.values()[0] + val = list(self.values())[0] lst = [(val, iri_to_uri(val))] else: - lst = [(self.values()[0], None)] + lst = [(list(self.values())[0], None)] return lst class EasyQuerySet(QuerySet): diff --git a/django/contrib/databrowse/plugins/calendars.py b/django/contrib/databrowse/plugins/calendars.py index 7bdd1e0032..a548c33c8f 100644 --- a/django/contrib/databrowse/plugins/calendars.py +++ b/django/contrib/databrowse/plugins/calendars.py @@ -7,7 +7,7 @@ from django.contrib.databrowse.sites import DatabrowsePlugin from django.shortcuts import render_to_response from django.utils.html import format_html, format_html_join from django.utils.text import capfirst -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.views.generic import dates from django.utils import datetime_safe @@ -66,7 +66,7 @@ class CalendarPlugin(DatabrowsePlugin): return '' return format_html('<p class="filter"><strong>View calendar by:</strong> {0}</p>', format_html_join(', ', '<a href="calendars/{0}/">{1}</a>', - ((f.name, force_unicode(capfirst(f.verbose_name))) for f in fields.values()))) + ((f.name, force_text(capfirst(f.verbose_name))) for f in fields.values()))) def urls(self, plugin_name, easy_instance_field): if isinstance(easy_instance_field.field, models.DateField): @@ -96,7 +96,7 @@ class CalendarPlugin(DatabrowsePlugin): def homepage_view(self, request): easy_model = EasyModel(self.site, self.model) - field_list = self.fields.values() + field_list = list(self.fields.values()) field_list.sort(key=lambda k:k.verbose_name) return render_to_response('databrowse/calendar_homepage.html', { 'root_url': self.site.root_url, diff --git a/django/contrib/databrowse/plugins/fieldchoices.py b/django/contrib/databrowse/plugins/fieldchoices.py index c016385ffb..dc5e9aef14 100644 --- a/django/contrib/databrowse/plugins/fieldchoices.py +++ b/django/contrib/databrowse/plugins/fieldchoices.py @@ -8,7 +8,7 @@ from django.shortcuts import render_to_response from django.utils.html import format_html, format_html_join from django.utils.http import urlquote from django.utils.text import capfirst -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text class FieldChoicePlugin(DatabrowsePlugin): @@ -35,7 +35,7 @@ class FieldChoicePlugin(DatabrowsePlugin): return '' return format_html('<p class="filter"><strong>View by:</strong> {0}</p>', format_html_join(', ', '<a href="fields/{0}/">{1}</a>', - ((f.name, force_unicode(capfirst(f.verbose_name))) for f in fields.values()))) + ((f.name, force_text(capfirst(f.verbose_name))) for f in fields.values()))) def urls(self, plugin_name, easy_instance_field): if easy_instance_field.field in self.field_dict(easy_instance_field.model.model).values(): @@ -63,7 +63,7 @@ class FieldChoicePlugin(DatabrowsePlugin): def homepage_view(self, request): easy_model = EasyModel(self.site, self.model) - field_list = self.fields.values() + field_list = list(self.fields.values()) field_list.sort(key=lambda k: k.verbose_name) return render_to_response('databrowse/fieldchoice_homepage.html', {'root_url': self.site.root_url, 'model': easy_model, 'field_list': field_list}) diff --git a/django/contrib/formtools/tests/__init__.py b/django/contrib/formtools/tests/__init__.py index 3bccb55034..ee93479cbd 100644 --- a/django/contrib/formtools/tests/__init__.py +++ b/django/contrib/formtools/tests/__init__.py @@ -317,7 +317,7 @@ class WizardTests(TestCase): class WizardWithProcessStep(TestWizardClass): def process_step(self, request, form, step): - that.assertTrue(hasattr(form, 'cleaned_data')) + that.assertTrue(form.is_valid()) reached[0] = True wizard = WizardWithProcessStep([WizardPageOneForm, diff --git a/django/contrib/formtools/wizard/storage/base.py b/django/contrib/formtools/wizard/storage/base.py index 274e07ffbe..05c9f6f121 100644 --- a/django/contrib/formtools/wizard/storage/base.py +++ b/django/contrib/formtools/wizard/storage/base.py @@ -1,7 +1,8 @@ from django.core.files.uploadedfile import UploadedFile from django.utils.datastructures import MultiValueDict -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils.functional import lazy_property +from django.utils import six from django.contrib.formtools.wizard.storage.exceptions import NoFileStorageConfigured @@ -72,9 +73,9 @@ class BaseStorage(object): raise NoFileStorageConfigured files = {} - for field, field_dict in wizard_files.iteritems(): - field_dict = dict((smart_str(k), v) - for k, v in field_dict.iteritems()) + for field, field_dict in six.iteritems(wizard_files): + field_dict = dict((smart_bytes(k), v) + for k, v in six.iteritems(field_dict)) tmp_name = field_dict.pop('tmp_name') files[field] = UploadedFile( file=self.file_storage.open(tmp_name), **field_dict) @@ -87,7 +88,7 @@ class BaseStorage(object): if step not in self.data[self.step_files_key]: self.data[self.step_files_key][step] = {} - for field, field_file in (files or {}).iteritems(): + for field, field_file in six.iteritems(files or {}): tmp_filename = self.file_storage.save(field_file.name, field_file) file_dict = { 'tmp_name': tmp_filename, diff --git a/django/contrib/formtools/wizard/views.py b/django/contrib/formtools/wizard/views.py index 466af1cac9..ea41e86852 100644 --- a/django/contrib/formtools/wizard/views.py +++ b/django/contrib/formtools/wizard/views.py @@ -44,7 +44,7 @@ class StepsHelper(object): @property def all(self): "Returns the names of all steps/forms." - return self._wizard.get_form_list().keys() + return list(self._wizard.get_form_list()) @property def count(self): @@ -164,14 +164,14 @@ class WizardView(TemplateView): init_form_list[six.text_type(i)] = form # walk through the new created list of forms - for form in init_form_list.itervalues(): + for form in six.itervalues(init_form_list): if issubclass(form, formsets.BaseFormSet): # if the element is based on BaseFormSet (FormSet/ModelFormSet) # we need to override the form variable. form = form.form # check if any form contains a FileField, if yes, we need a # file_storage added to the wizardview (by subclassing). - for field in form.base_fields.itervalues(): + for field in six.itervalues(form.base_fields): if (isinstance(field, forms.FileField) and not hasattr(cls, 'file_storage')): raise NoFileStorageConfigured @@ -196,7 +196,7 @@ class WizardView(TemplateView): could use data from other (maybe previous forms). """ form_list = SortedDict() - for form_key, form_class in self.form_list.iteritems(): + for form_key, form_class in six.iteritems(self.form_list): # try to fetch the value from condition list, by default, the form # gets passed to the new list. condition = self.condition_dict.get(form_key, True) diff --git a/django/contrib/gis/db/backends/mysql/operations.py b/django/contrib/gis/db/backends/mysql/operations.py index c0e5aa6691..7152f4682d 100644 --- a/django/contrib/gis/db/backends/mysql/operations.py +++ b/django/contrib/gis/db/backends/mysql/operations.py @@ -3,6 +3,8 @@ from django.db.backends.mysql.base import DatabaseOperations from django.contrib.gis.db.backends.adapter import WKTAdapter from django.contrib.gis.db.backends.base import BaseSpatialOperations +from django.utils import six + class MySQLOperations(DatabaseOperations, BaseSpatialOperations): compiler_module = 'django.contrib.gis.db.backends.mysql.compiler' @@ -30,7 +32,7 @@ class MySQLOperations(DatabaseOperations, BaseSpatialOperations): 'within' : 'MBRWithin', } - gis_terms = dict([(term, None) for term in geometry_functions.keys() + ['isnull']]) + gis_terms = dict([(term, None) for term in list(geometry_functions) + ['isnull']]) def geo_db_type(self, f): return f.geom_type diff --git a/django/contrib/gis/db/backends/oracle/operations.py b/django/contrib/gis/db/backends/oracle/operations.py index 4e33942f7a..392feb129b 100644 --- a/django/contrib/gis/db/backends/oracle/operations.py +++ b/django/contrib/gis/db/backends/oracle/operations.py @@ -128,7 +128,7 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): geometry_functions.update(distance_functions) gis_terms = ['isnull'] - gis_terms += geometry_functions.keys() + gis_terms += list(geometry_functions) gis_terms = dict([(term, None) for term in gis_terms]) truncate_params = {'relate' : None} diff --git a/django/contrib/gis/db/backends/postgis/operations.py b/django/contrib/gis/db/backends/postgis/operations.py index a6340ce22b..434d8719cc 100644 --- a/django/contrib/gis/db/backends/postgis/operations.py +++ b/django/contrib/gis/db/backends/postgis/operations.py @@ -163,7 +163,9 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): 'contains' : PostGISFunction(prefix, 'Contains'), 'intersects' : PostGISFunction(prefix, 'Intersects'), 'relate' : (PostGISRelate, six.string_types), - } + 'coveredby' : PostGISFunction(prefix, 'CoveredBy'), + 'covers' : PostGISFunction(prefix, 'Covers'), + } # Valid distance types and substitutions dtypes = (Decimal, Distance, float) + six.integer_types @@ -178,33 +180,12 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): 'distance_gte' : (get_dist_ops('>='), dtypes), 'distance_lt' : (get_dist_ops('<'), dtypes), 'distance_lte' : (get_dist_ops('<='), dtypes), - } - - # Versions 1.2.2+ have KML serialization support. - if version < (1, 2, 2): - ASKML = False - else: - ASKML = 'ST_AsKML' - self.geometry_functions.update( - {'coveredby' : PostGISFunction(prefix, 'CoveredBy'), - 'covers' : PostGISFunction(prefix, 'Covers'), - }) - self.distance_functions['dwithin'] = (PostGISFunctionParam(prefix, 'DWithin'), dtypes) + 'dwithin' : (PostGISFunctionParam(prefix, 'DWithin'), dtypes) + } # Adding the distance functions to the geometries lookup. self.geometry_functions.update(self.distance_functions) - # The union aggregate and topology operation use the same signature - # in versions 1.3+. - if version < (1, 3, 0): - UNIONAGG = 'GeomUnion' - UNION = 'Union' - MAKELINE = False - else: - UNIONAGG = 'ST_Union' - UNION = 'ST_Union' - MAKELINE = 'ST_MakeLine' - # Only PostGIS versions 1.3.4+ have GeoJSON serialization support. if version < (1, 3, 4): GEOJSON = False @@ -236,8 +217,8 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): # Creating a dictionary lookup of all GIS terms for PostGIS. gis_terms = ['isnull'] - gis_terms += self.geometry_operators.keys() - gis_terms += self.geometry_functions.keys() + gis_terms += list(self.geometry_operators) + gis_terms += list(self.geometry_functions) self.gis_terms = dict([(term, None) for term in gis_terms]) self.area = prefix + 'Area' @@ -256,11 +237,11 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): self.geojson = GEOJSON self.gml = prefix + 'AsGML' self.intersection = prefix + 'Intersection' - self.kml = ASKML + self.kml = prefix + 'AsKML' self.length = prefix + 'Length' self.length3d = prefix + 'Length3D' self.length_spheroid = prefix + 'length_spheroid' - self.makeline = MAKELINE + self.makeline = prefix + 'MakeLine' self.mem_size = prefix + 'mem_size' self.num_geom = prefix + 'NumGeometries' self.num_points =prefix + 'npoints' @@ -275,8 +256,8 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): self.sym_difference = prefix + 'SymDifference' self.transform = prefix + 'Transform' self.translate = prefix + 'Translate' - self.union = UNION - self.unionagg = UNIONAGG + self.union = prefix + 'Union' + self.unionagg = prefix + 'Union' def check_aggregate_support(self, aggregate): """ diff --git a/django/contrib/gis/db/backends/spatialite/creation.py b/django/contrib/gis/db/backends/spatialite/creation.py index 31f2fca1bd..d0a5f82033 100644 --- a/django/contrib/gis/db/backends/spatialite/creation.py +++ b/django/contrib/gis/db/backends/spatialite/creation.py @@ -99,14 +99,14 @@ class SpatiaLiteCreation(DatabaseCreation): """ This routine loads up the SpatiaLite SQL file. """ - if self.connection.ops.spatial_version[:2] >= (3, 0): - # Spatialite >= 3.0.x -- No need to load any SQL file, calling + if self.connection.ops.spatial_version[:2] >= (2, 4): + # Spatialite >= 2.4 -- No need to load any SQL file, calling # InitSpatialMetaData() transparently creates the spatial metadata # tables cur = self.connection._cursor() cur.execute("SELECT InitSpatialMetaData()") else: - # Spatialite < 3.0.x -- Load the initial SQL + # Spatialite < 2.4 -- Load the initial SQL # Getting the location of the SpatiaLite SQL file, and confirming # it exists. diff --git a/django/contrib/gis/db/backends/spatialite/operations.py b/django/contrib/gis/db/backends/spatialite/operations.py index 1d7c4fab52..60fe0a8069 100644 --- a/django/contrib/gis/db/backends/spatialite/operations.py +++ b/django/contrib/gis/db/backends/spatialite/operations.py @@ -131,7 +131,7 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations): # Creating the GIS terms dictionary. gis_terms = ['isnull'] - gis_terms += self.geometry_functions.keys() + gis_terms += list(self.geometry_functions) self.gis_terms = dict([(term, None) for term in gis_terms]) if version >= (2, 4, 0): diff --git a/django/contrib/gis/db/models/query.py b/django/contrib/gis/db/models/query.py index dd2983aecc..cc61dfa4d2 100644 --- a/django/contrib/gis/db/models/query.py +++ b/django/contrib/gis/db/models/query.py @@ -1,5 +1,6 @@ from django.db import connections from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet +from django.utils import six from django.contrib.gis.db.models import aggregates from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField @@ -25,7 +26,7 @@ class GeoQuerySet(QuerySet): flat = kwargs.pop('flat', False) if kwargs: raise TypeError('Unexpected keyword arguments to values_list: %s' - % (kwargs.keys(),)) + % (list(kwargs),)) if flat and len(fields) > 1: raise TypeError("'flat' is not valid when values_list is called with more than one field.") return self._clone(klass=GeoValuesListQuerySet, setup=True, flat=flat, @@ -531,7 +532,7 @@ class GeoQuerySet(QuerySet): if settings.get('setup', True): default_args, geo_field = self._spatial_setup(att, desc=settings['desc'], field_name=field_name, geo_field_type=settings.get('geo_field_type', None)) - for k, v in default_args.iteritems(): settings['procedure_args'].setdefault(k, v) + for k, v in six.iteritems(default_args): settings['procedure_args'].setdefault(k, v) else: geo_field = settings['geo_field'] diff --git a/django/contrib/gis/db/models/sql/compiler.py b/django/contrib/gis/db/models/sql/compiler.py index d016357f1b..5c8d2647f7 100644 --- a/django/contrib/gis/db/models/sql/compiler.py +++ b/django/contrib/gis/db/models/sql/compiler.py @@ -3,6 +3,7 @@ from django.utils.six.moves import zip from django.db.backends.util import truncate_name, typecast_timestamp from django.db.models.sql import compiler from django.db.models.sql.constants import MULTI +from django.utils import six SQLCompiler = compiler.SQLCompiler @@ -24,7 +25,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): qn = self.quote_name_unless_alias qn2 = self.connection.ops.quote_name result = ['(%s) AS %s' % (self.get_extra_select_format(alias) % col[0], qn2(alias)) - for alias, col in self.query.extra_select.iteritems()] + for alias, col in six.iteritems(self.query.extra_select)] aliases = set(self.query.extra_select.keys()) if with_aliases: col_aliases = aliases.copy() @@ -170,7 +171,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): objects. """ values = [] - aliases = self.query.extra_select.keys() + aliases = list(self.query.extra_select) # Have to set a starting row number offset that is used for # determining the correct starting row index -- needed for diff --git a/django/contrib/gis/gdal/geometries.py b/django/contrib/gis/gdal/geometries.py index d752104e0a..373ece777d 100644 --- a/django/contrib/gis/gdal/geometries.py +++ b/django/contrib/gis/gdal/geometries.py @@ -40,7 +40,7 @@ """ # Python library requisites. import sys -from binascii import a2b_hex +from binascii import a2b_hex, b2a_hex from ctypes import byref, string_at, c_char_p, c_double, c_ubyte, c_void_p # Getting GDAL prerequisites @@ -322,8 +322,7 @@ class OGRGeometry(GDALBase): @property def hex(self): "Returns the hexadecimal representation of the WKB (a string)." - return str(self.wkb).encode('hex').upper() - #return b2a_hex(self.wkb).upper() + return b2a_hex(self.wkb).upper() @property def json(self): diff --git a/django/contrib/gis/geometry/test_data.py b/django/contrib/gis/geometry/test_data.py index f92740248e..505f0e4f4b 100644 --- a/django/contrib/gis/geometry/test_data.py +++ b/django/contrib/gis/geometry/test_data.py @@ -7,6 +7,7 @@ import json import os from django.contrib import gis +from django.utils import six # This global used to store reference geometry data. @@ -25,7 +26,7 @@ def tuplize(seq): def strconvert(d): "Converts all keys in dictionary to str type." - return dict([(str(k), v) for k, v in d.iteritems()]) + return dict([(str(k), v) for k, v in six.iteritems(d)]) def get_ds_file(name, ext): diff --git a/django/contrib/gis/measure.py b/django/contrib/gis/measure.py index ba7817e51c..6e074be355 100644 --- a/django/contrib/gis/measure.py +++ b/django/contrib/gis/measure.py @@ -143,7 +143,7 @@ class MeasureBase(object): def __rmul__(self, other): return self * other - def __div__(self, other): + def __truediv__(self, other): if isinstance(other, self.__class__): return self.standard / other.standard if isinstance(other, NUMERIC_TYPES): @@ -151,16 +151,19 @@ class MeasureBase(object): **{self.STANDARD_UNIT: (self.standard / other)}) else: raise TypeError('%(class)s must be divided with number or %(class)s' % {"class":pretty_name(self)}) + __div__ = __truediv__ # Python 2 compatibility - def __idiv__(self, other): + def __itruediv__(self, other): if isinstance(other, NUMERIC_TYPES): self.standard /= float(other) return self else: raise TypeError('%(class)s must be divided with number' % {"class":pretty_name(self)}) + __idiv__ = __itruediv__ # Python 2 compatibility - def __nonzero__(self): + def __bool__(self): return bool(self.standard) + __nonzero__ = __bool__ # Python 2 compatibility def default_units(self, kwargs): """ @@ -169,7 +172,7 @@ class MeasureBase(object): """ val = 0.0 default_unit = self.STANDARD_UNIT - for unit, value in kwargs.iteritems(): + for unit, value in six.iteritems(kwargs): if not isinstance(value, float): value = float(value) if unit in self.UNITS: val += self.UNITS[unit] * value @@ -305,12 +308,13 @@ class Area(MeasureBase): ALIAS = dict([(k, '%s%s' % (AREA_PREFIX, v)) for k, v in Distance.ALIAS.items()]) LALIAS = dict([(k.lower(), v) for k, v in ALIAS.items()]) - def __div__(self, other): + def __truediv__(self, other): if isinstance(other, NUMERIC_TYPES): return self.__class__(default_unit=self._default_unit, **{self.STANDARD_UNIT: (self.standard / other)}) else: raise TypeError('%(class)s must be divided by a number' % {"class":pretty_name(self)}) + __div__ = __truediv__ # Python 2 compatibility # Shortcuts diff --git a/django/contrib/gis/sitemaps/views.py b/django/contrib/gis/sitemaps/views.py index eb42d0cae8..8bcdba1b44 100644 --- a/django/contrib/gis/sitemaps/views.py +++ b/django/contrib/gis/sitemaps/views.py @@ -8,7 +8,8 @@ from django.core.paginator import EmptyPage, PageNotAnInteger from django.contrib.gis.db.models.fields import GeometryField from django.db import connections, DEFAULT_DB_ALIAS from django.db.models import get_model -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes +from django.utils import six from django.utils.translation import ugettext as _ from django.contrib.gis.shortcuts import render_to_kml, render_to_kmz @@ -46,7 +47,7 @@ def sitemap(request, sitemaps, section=None): raise Http404(_("No sitemap available for section: %r") % section) maps.append(sitemaps[section]) else: - maps = sitemaps.values() + maps = list(six.itervalues(sitemaps)) page = request.GET.get("p", 1) current_site = get_current_site(request) @@ -60,7 +61,7 @@ def sitemap(request, sitemaps, section=None): raise Http404(_("Page %s empty") % page) except PageNotAnInteger: raise Http404(_("No page '%s'") % page) - xml = smart_str(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})) + xml = smart_bytes(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})) return HttpResponse(xml, content_type='application/xml') def kml(request, label, model, field_name=None, compress=False, using=DEFAULT_DB_ALIAS): diff --git a/django/contrib/gis/tests/geoapp/test_regress.py b/django/contrib/gis/tests/geoapp/test_regress.py index a9d802d8f1..fffd7d3cab 100644 --- a/django/contrib/gis/tests/geoapp/test_regress.py +++ b/django/contrib/gis/tests/geoapp/test_regress.py @@ -12,7 +12,7 @@ from .models import City, PennsylvaniaCity, State, Truth class GeoRegressionTests(TestCase): - def test01_update(self): + def test_update(self): "Testing GeoQuerySet.update(). See #10411." pnt = City.objects.get(name='Pueblo').point bak = pnt.clone() @@ -24,7 +24,7 @@ class GeoRegressionTests(TestCase): City.objects.filter(name='Pueblo').update(point=bak) self.assertEqual(bak, City.objects.get(name='Pueblo').point) - def test02_kmz(self): + def test_kmz(self): "Testing `render_to_kmz` with non-ASCII data. See #11624." name = '\xc3\x85land Islands'.decode('iso-8859-1') places = [{'name' : name, @@ -35,7 +35,7 @@ class GeoRegressionTests(TestCase): @no_spatialite @no_mysql - def test03_extent(self): + def test_extent(self): "Testing `extent` on a table with a single point. See #11827." pnt = City.objects.get(name='Pueblo').point ref_ext = (pnt.x, pnt.y, pnt.x, pnt.y) @@ -43,14 +43,14 @@ class GeoRegressionTests(TestCase): for ref_val, val in zip(ref_ext, extent): self.assertAlmostEqual(ref_val, val, 4) - def test04_unicode_date(self): + def test_unicode_date(self): "Testing dates are converted properly, even on SpatiaLite. See #16408." founded = datetime(1857, 5, 23) mansfield = PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)', founded=founded) self.assertEqual(founded, PennsylvaniaCity.objects.dates('founded', 'day')[0]) - def test05_empty_count(self): + def test_empty_count(self): "Testing that PostGISAdapter.__eq__ does check empty strings. See #13670." # contrived example, but need a geo lookup paired with an id__in lookup pueblo = City.objects.get(name='Pueblo') @@ -60,12 +60,12 @@ class GeoRegressionTests(TestCase): # .count() should not throw TypeError in __eq__ self.assertEqual(cities_within_state.count(), 1) - def test06_defer_or_only_with_annotate(self): + def test_defer_or_only_with_annotate(self): "Regression for #16409. Make sure defer() and only() work with annotate()" self.assertIsInstance(list(City.objects.annotate(Count('point')).defer('name')), list) self.assertIsInstance(list(City.objects.annotate(Count('point')).only('name')), list) - def test07_boolean_conversion(self): + def test_boolean_conversion(self): "Testing Boolean value conversion with the spatial backend, see #15169." t1 = Truth.objects.create(val=True) t2 = Truth.objects.create(val=False) diff --git a/django/contrib/gis/tests/geoapp/tests.py b/django/contrib/gis/tests/geoapp/tests.py index bcdbe734ff..b06d6b5e1b 100644 --- a/django/contrib/gis/tests/geoapp/tests.py +++ b/django/contrib/gis/tests/geoapp/tests.py @@ -15,19 +15,24 @@ from django.utils import six from .models import Country, City, PennsylvaniaCity, State, Track +from .test_feeds import GeoFeedTest +from .test_regress import GeoRegressionTests +from .test_sitemaps import GeoSitemapTest + + if not spatialite: from .models import Feature, MinusOneSRID class GeoModelTest(TestCase): - def test01_fixtures(self): + def test_fixtures(self): "Testing geographic model initialization from fixtures." # Ensuring that data was loaded from initial data fixtures. self.assertEqual(2, Country.objects.count()) self.assertEqual(8, City.objects.count()) self.assertEqual(2, State.objects.count()) - def test02_proxy(self): + def test_proxy(self): "Testing Lazy-Geometry support (using the GeometryProxy)." ## Testing on a Point pnt = Point(0, 0) @@ -95,165 +100,97 @@ class GeoModelTest(TestCase): self.assertEqual(ply, State.objects.get(name='NullState').poly) ns.delete() - def test03a_kml(self): - "Testing KML output from the database using GeoQuerySet.kml()." - # Only PostGIS and Spatialite (>=2.4.0-RC4) support KML serialization - if not (postgis or (spatialite and connection.ops.kml)): - self.assertRaises(NotImplementedError, State.objects.all().kml, field_name='poly') - return - - # Should throw a TypeError when trying to obtain KML from a - # non-geometry field. - qs = City.objects.all() - self.assertRaises(TypeError, qs.kml, 'name') - - # The reference KML depends on the version of PostGIS used - # (the output stopped including altitude in 1.3.3). - if connection.ops.spatial_version >= (1, 3, 3): - ref_kml = '<Point><coordinates>-104.609252,38.255001</coordinates></Point>' - else: - ref_kml = '<Point><coordinates>-104.609252,38.255001,0</coordinates></Point>' - - # Ensuring the KML is as expected. - ptown1 = City.objects.kml(field_name='point', precision=9).get(name='Pueblo') - ptown2 = City.objects.kml(precision=9).get(name='Pueblo') - for ptown in [ptown1, ptown2]: - self.assertEqual(ref_kml, ptown.kml) - - def test03b_gml(self): - "Testing GML output from the database using GeoQuerySet.gml()." - if mysql or (spatialite and not connection.ops.gml) : - self.assertRaises(NotImplementedError, Country.objects.all().gml, field_name='mpoly') - return - - # Should throw a TypeError when tyring to obtain GML from a - # non-geometry field. - qs = City.objects.all() - self.assertRaises(TypeError, qs.gml, field_name='name') - ptown1 = City.objects.gml(field_name='point', precision=9).get(name='Pueblo') - ptown2 = City.objects.gml(precision=9).get(name='Pueblo') + @no_mysql + def test_lookup_insert_transform(self): + "Testing automatic transform for lookups and inserts." + # San Antonio in 'WGS84' (SRID 4326) + sa_4326 = 'POINT (-98.493183 29.424170)' + wgs_pnt = fromstr(sa_4326, srid=4326) # Our reference point in WGS84 + # Oracle doesn't have SRID 3084, using 41157. if oracle: - # No precision parameter for Oracle :-/ - gml_regex = re.compile(r'^<gml:Point srsName="SDO:4326" xmlns:gml="http://www.opengis.net/gml"><gml:coordinates decimal="\." cs="," ts=" ">-104.60925\d+,38.25500\d+ </gml:coordinates></gml:Point>') - elif spatialite: - # Spatialite has extra colon in SrsName - gml_regex = re.compile(r'^<gml:Point SrsName="EPSG::4326"><gml:coordinates decimal="\." cs="," ts=" ">-104.609251\d+,38.255001</gml:coordinates></gml:Point>') + # San Antonio in 'Texas 4205, Southern Zone (1983, meters)' (SRID 41157) + # Used the following Oracle SQL to get this value: + # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_CS.TRANSFORM(SDO_GEOMETRY('POINT (-98.493183 29.424170)', 4326), 41157)) FROM DUAL; + nad_wkt = 'POINT (300662.034646583 5416427.45974934)' + nad_srid = 41157 else: - gml_regex = re.compile(r'^<gml:Point srsName="EPSG:4326"><gml:coordinates>-104\.60925\d+,38\.255001</gml:coordinates></gml:Point>') - - for ptown in [ptown1, ptown2]: - self.assertTrue(gml_regex.match(ptown.gml)) - - - def test03c_geojson(self): - "Testing GeoJSON output from the database using GeoQuerySet.geojson()." - # Only PostGIS 1.3.4+ supports GeoJSON. - if not connection.ops.geojson: - self.assertRaises(NotImplementedError, Country.objects.all().geojson, field_name='mpoly') - return + # San Antonio in 'NAD83(HARN) / Texas Centric Lambert Conformal' (SRID 3084) + nad_wkt = 'POINT (1645978.362408288754523 6276356.025927528738976)' # Used ogr.py in gdal 1.4.1 for this transform + nad_srid = 3084 - if connection.ops.spatial_version >= (1, 4, 0): - pueblo_json = '{"type":"Point","coordinates":[-104.609252,38.255001]}' - houston_json = '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},"coordinates":[-95.363151,29.763374]}' - victoria_json = '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],"coordinates":[-123.305196,48.462611]}' - chicago_json = '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}' + # Constructing & querying with a point from a different SRID. Oracle + # `SDO_OVERLAPBDYINTERSECT` operates differently from + # `ST_Intersects`, so contains is used instead. + nad_pnt = fromstr(nad_wkt, srid=nad_srid) + if oracle: + tx = Country.objects.get(mpoly__contains=nad_pnt) else: - pueblo_json = '{"type":"Point","coordinates":[-104.60925200,38.25500100]}' - houston_json = '{"type":"Point","crs":{"type":"EPSG","properties":{"EPSG":4326}},"coordinates":[-95.36315100,29.76337400]}' - victoria_json = '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],"coordinates":[-123.30519600,48.46261100]}' - chicago_json = '{"type":"Point","crs":{"type":"EPSG","properties":{"EPSG":4326}},"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}' + tx = Country.objects.get(mpoly__intersects=nad_pnt) + self.assertEqual('Texas', tx.name) - # Precision argument should only be an integer - self.assertRaises(TypeError, City.objects.geojson, precision='foo') + # Creating San Antonio. Remember the Alamo. + sa = City.objects.create(name='San Antonio', point=nad_pnt) - # Reference queries and values. - # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 0) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Pueblo'; - self.assertEqual(pueblo_json, City.objects.geojson().get(name='Pueblo').geojson) + # Now verifying that San Antonio was transformed correctly + sa = City.objects.get(name='San Antonio') + self.assertAlmostEqual(wgs_pnt.x, sa.point.x, 6) + self.assertAlmostEqual(wgs_pnt.y, sa.point.y, 6) - # 1.3.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; - # 1.4.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; - # This time we want to include the CRS by using the `crs` keyword. - self.assertEqual(houston_json, City.objects.geojson(crs=True, model_att='json').get(name='Houston').json) + # If the GeometryField SRID is -1, then we shouldn't perform any + # transformation if the SRID of the input geometry is different. + # SpatiaLite does not support missing SRID values. + if not spatialite: + m1 = MinusOneSRID(geom=Point(17, 23, srid=4326)) + m1.save() + self.assertEqual(-1, m1.geom.srid) - # 1.3.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Victoria'; - # 1.4.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; - # This time we include the bounding box by using the `bbox` keyword. - self.assertEqual(victoria_json, City.objects.geojson(bbox=True).get(name='Victoria').geojson) + def test_createnull(self): + "Testing creating a model instance and the geometry being None" + c = City() + self.assertEqual(c.point, None) - # 1.(3|4).x: SELECT ST_AsGeoJson("geoapp_city"."point", 5, 3) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Chicago'; - # Finally, we set every available keyword. - self.assertEqual(chicago_json, City.objects.geojson(bbox=True, crs=True, precision=5).get(name='Chicago').geojson) + @no_spatialite # SpatiaLite does not support abstract geometry columns + def test_geometryfield(self): + "Testing the general GeometryField." + Feature(name='Point', geom=Point(1, 1)).save() + Feature(name='LineString', geom=LineString((0, 0), (1, 1), (5, 5))).save() + Feature(name='Polygon', geom=Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0)))).save() + Feature(name='GeometryCollection', + geom=GeometryCollection(Point(2, 2), LineString((0, 0), (2, 2)), + Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0))))).save() - def test03d_svg(self): - "Testing SVG output using GeoQuerySet.svg()." - if mysql or oracle: - self.assertRaises(NotImplementedError, City.objects.svg) - return + f_1 = Feature.objects.get(name='Point') + self.assertEqual(True, isinstance(f_1.geom, Point)) + self.assertEqual((1.0, 1.0), f_1.geom.tuple) + f_2 = Feature.objects.get(name='LineString') + self.assertEqual(True, isinstance(f_2.geom, LineString)) + self.assertEqual(((0.0, 0.0), (1.0, 1.0), (5.0, 5.0)), f_2.geom.tuple) - self.assertRaises(TypeError, City.objects.svg, precision='foo') - # SELECT AsSVG(geoapp_city.point, 0, 8) FROM geoapp_city WHERE name = 'Pueblo'; - svg1 = 'cx="-104.609252" cy="-38.255001"' - # Even though relative, only one point so it's practically the same except for - # the 'c' letter prefix on the x,y values. - svg2 = svg1.replace('c', '') - self.assertEqual(svg1, City.objects.svg().get(name='Pueblo').svg) - self.assertEqual(svg2, City.objects.svg(relative=5).get(name='Pueblo').svg) + f_3 = Feature.objects.get(name='Polygon') + self.assertEqual(True, isinstance(f_3.geom, Polygon)) + f_4 = Feature.objects.get(name='GeometryCollection') + self.assertEqual(True, isinstance(f_4.geom, GeometryCollection)) + self.assertEqual(f_3.geom, f_4.geom[2]) @no_mysql - def test04_transform(self): - "Testing the transform() GeoManager method." - # Pre-transformed points for Houston and Pueblo. - htown = fromstr('POINT(1947516.83115183 6322297.06040572)', srid=3084) - ptown = fromstr('POINT(992363.390841912 481455.395105533)', srid=2774) - prec = 3 # Precision is low due to version variations in PROJ and GDAL. - - # Asserting the result of the transform operation with the values in - # the pre-transformed points. Oracle does not have the 3084 SRID. - if not oracle: - h = City.objects.transform(htown.srid).get(name='Houston') - self.assertEqual(3084, h.point.srid) - self.assertAlmostEqual(htown.x, h.point.x, prec) - self.assertAlmostEqual(htown.y, h.point.y, prec) - - p1 = City.objects.transform(ptown.srid, field_name='point').get(name='Pueblo') - p2 = City.objects.transform(srid=ptown.srid).get(name='Pueblo') - for p in [p1, p2]: - self.assertEqual(2774, p.point.srid) - self.assertAlmostEqual(ptown.x, p.point.x, prec) - self.assertAlmostEqual(ptown.y, p.point.y, prec) + def test_inherited_geofields(self): + "Test GeoQuerySet methods on inherited Geometry fields." + # Creating a Pennsylvanian city. + mansfield = PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)') - @no_mysql - @no_spatialite # SpatiaLite does not have an Extent function - def test05_extent(self): - "Testing the `extent` GeoQuerySet method." - # Reference query: - # `SELECT ST_extent(point) FROM geoapp_city WHERE (name='Houston' or name='Dallas');` - # => BOX(-96.8016128540039 29.7633724212646,-95.3631439208984 32.7820587158203) - expected = (-96.8016128540039, 29.7633724212646, -95.3631439208984, 32.782058715820) + # All transformation SQL will need to be performed on the + # _parent_ table. + qs = PennsylvaniaCity.objects.transform(32128) - qs = City.objects.filter(name__in=('Houston', 'Dallas')) - extent = qs.extent() + self.assertEqual(1, qs.count()) + for pc in qs: self.assertEqual(32128, pc.point.srid) - for val, exp in zip(extent, expected): - self.assertAlmostEqual(exp, val, 4) - # Only PostGIS has support for the MakeLine aggregate. - @no_mysql - @no_oracle - @no_spatialite - def test06_make_line(self): - "Testing the `make_line` GeoQuerySet method." - # Ensuring that a `TypeError` is raised on models without PointFields. - self.assertRaises(TypeError, State.objects.make_line) - self.assertRaises(TypeError, Country.objects.make_line) - # Reference query: - # SELECT AsText(ST_MakeLine(geoapp_city.point)) FROM geoapp_city; - ref_line = GEOSGeometry('LINESTRING(-95.363151 29.763374,-96.801611 32.782057,-97.521157 34.464642,174.783117 -41.315268,-104.609252 38.255001,-95.23506 38.971823,-87.650175 41.850385,-123.305196 48.462611)', srid=4326) - self.assertEqual(ref_line, City.objects.make_line()) +class GeoLookupTest(TestCase): @no_mysql - def test09_disjoint(self): + def test_disjoint_lookup(self): "Testing the `disjoint` lookup type." ptown = City.objects.get(name='Pueblo') qs1 = City.objects.filter(point__disjoint=ptown.point) @@ -263,7 +200,7 @@ class GeoModelTest(TestCase): self.assertEqual(1, qs2.count()) self.assertEqual('Kansas', qs2[0].name) - def test10_contains_contained(self): + def test_contains_contained_lookups(self): "Testing the 'contained', 'contains', and 'bbcontains' lookup types." # Getting Texas, yes we were a country -- once ;) texas = Country.objects.get(name='Texas') @@ -308,86 +245,11 @@ class GeoModelTest(TestCase): self.assertEqual(1, len(qs)) self.assertEqual('Texas', qs[0].name) - @no_mysql - def test11_lookup_insert_transform(self): - "Testing automatic transform for lookups and inserts." - # San Antonio in 'WGS84' (SRID 4326) - sa_4326 = 'POINT (-98.493183 29.424170)' - wgs_pnt = fromstr(sa_4326, srid=4326) # Our reference point in WGS84 - - # Oracle doesn't have SRID 3084, using 41157. - if oracle: - # San Antonio in 'Texas 4205, Southern Zone (1983, meters)' (SRID 41157) - # Used the following Oracle SQL to get this value: - # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_CS.TRANSFORM(SDO_GEOMETRY('POINT (-98.493183 29.424170)', 4326), 41157)) FROM DUAL; - nad_wkt = 'POINT (300662.034646583 5416427.45974934)' - nad_srid = 41157 - else: - # San Antonio in 'NAD83(HARN) / Texas Centric Lambert Conformal' (SRID 3084) - nad_wkt = 'POINT (1645978.362408288754523 6276356.025927528738976)' # Used ogr.py in gdal 1.4.1 for this transform - nad_srid = 3084 - - # Constructing & querying with a point from a different SRID. Oracle - # `SDO_OVERLAPBDYINTERSECT` operates differently from - # `ST_Intersects`, so contains is used instead. - nad_pnt = fromstr(nad_wkt, srid=nad_srid) - if oracle: - tx = Country.objects.get(mpoly__contains=nad_pnt) - else: - tx = Country.objects.get(mpoly__intersects=nad_pnt) - self.assertEqual('Texas', tx.name) - - # Creating San Antonio. Remember the Alamo. - sa = City.objects.create(name='San Antonio', point=nad_pnt) - - # Now verifying that San Antonio was transformed correctly - sa = City.objects.get(name='San Antonio') - self.assertAlmostEqual(wgs_pnt.x, sa.point.x, 6) - self.assertAlmostEqual(wgs_pnt.y, sa.point.y, 6) - - # If the GeometryField SRID is -1, then we shouldn't perform any - # transformation if the SRID of the input geometry is different. - # SpatiaLite does not support missing SRID values. - if not spatialite: - m1 = MinusOneSRID(geom=Point(17, 23, srid=4326)) - m1.save() - self.assertEqual(-1, m1.geom.srid) - - @no_mysql - def test12_null_geometries(self): - "Testing NULL geometry support, and the `isnull` lookup type." - # Creating a state with a NULL boundary. - State.objects.create(name='Puerto Rico') - - # Querying for both NULL and Non-NULL values. - nullqs = State.objects.filter(poly__isnull=True) - validqs = State.objects.filter(poly__isnull=False) - - # Puerto Rico should be NULL (it's a commonwealth unincorporated territory) - self.assertEqual(1, len(nullqs)) - self.assertEqual('Puerto Rico', nullqs[0].name) - - # The valid states should be Colorado & Kansas - self.assertEqual(2, len(validqs)) - state_names = [s.name for s in validqs] - self.assertEqual(True, 'Colorado' in state_names) - self.assertEqual(True, 'Kansas' in state_names) - - # Saving another commonwealth w/a NULL geometry. - nmi = State.objects.create(name='Northern Mariana Islands', poly=None) - self.assertEqual(nmi.poly, None) - - # Assigning a geomery and saving -- then UPDATE back to NULL. - nmi.poly = 'POLYGON((0 0,1 0,1 1,1 0,0 0))' - nmi.save() - State.objects.filter(name='Northern Mariana Islands').update(poly=None) - self.assertEqual(None, State.objects.get(name='Northern Mariana Islands').poly) - # Only PostGIS has `left` and `right` lookup types. @no_mysql @no_oracle @no_spatialite - def test13_left_right(self): + def test_left_right_lookups(self): "Testing the 'left' and 'right' lookup types." # Left: A << B => true if xmax(A) < xmin(B) # Right: A >> B => true if xmin(A) > xmax(B) @@ -423,7 +285,7 @@ class GeoModelTest(TestCase): self.assertEqual(2, len(qs)) for c in qs: self.assertEqual(True, c.name in cities) - def test14_equals(self): + def test_equals_lookups(self): "Testing the 'same_as' and 'equals' lookup types." pnt = fromstr('POINT (-95.363151 29.763374)', srid=4326) c1 = City.objects.get(point=pnt) @@ -432,7 +294,37 @@ class GeoModelTest(TestCase): for c in [c1, c2, c3]: self.assertEqual('Houston', c.name) @no_mysql - def test15_relate(self): + def test_null_geometries(self): + "Testing NULL geometry support, and the `isnull` lookup type." + # Creating a state with a NULL boundary. + State.objects.create(name='Puerto Rico') + + # Querying for both NULL and Non-NULL values. + nullqs = State.objects.filter(poly__isnull=True) + validqs = State.objects.filter(poly__isnull=False) + + # Puerto Rico should be NULL (it's a commonwealth unincorporated territory) + self.assertEqual(1, len(nullqs)) + self.assertEqual('Puerto Rico', nullqs[0].name) + + # The valid states should be Colorado & Kansas + self.assertEqual(2, len(validqs)) + state_names = [s.name for s in validqs] + self.assertEqual(True, 'Colorado' in state_names) + self.assertEqual(True, 'Kansas' in state_names) + + # Saving another commonwealth w/a NULL geometry. + nmi = State.objects.create(name='Northern Mariana Islands', poly=None) + self.assertEqual(nmi.poly, None) + + # Assigning a geomery and saving -- then UPDATE back to NULL. + nmi.poly = 'POLYGON((0 0,1 0,1 1,1 0,0 0))' + nmi.save() + State.objects.filter(name='Northern Mariana Islands').update(poly=None) + self.assertEqual(None, State.objects.get(name='Northern Mariana Islands').poly) + + @no_mysql + def test_relate_lookup(self): "Testing the 'relate' lookup type." # To make things more interesting, we will have our Texas reference point in # different SRIDs. @@ -474,60 +366,12 @@ class GeoModelTest(TestCase): self.assertEqual('Texas', Country.objects.get(mpoly__relate=(pnt2, intersects_mask)).name) self.assertEqual('Lawrence', City.objects.get(point__relate=(ks.poly, intersects_mask)).name) - def test16_createnull(self): - "Testing creating a model instance and the geometry being None" - c = City() - self.assertEqual(c.point, None) - @no_mysql - def test17_unionagg(self): - "Testing the `unionagg` (aggregate union) GeoManager method." - tx = Country.objects.get(name='Texas').mpoly - # Houston, Dallas -- Oracle has different order. - union1 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)') - union2 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)') - qs = City.objects.filter(point__within=tx) - self.assertRaises(TypeError, qs.unionagg, 'name') - # Using `field_name` keyword argument in one query and specifying an - # order in the other (which should not be used because this is - # an aggregate method on a spatial column) - u1 = qs.unionagg(field_name='point') - u2 = qs.order_by('name').unionagg() - tol = 0.00001 - if oracle: - union = union2 - else: - union = union1 - self.assertEqual(True, union.equals_exact(u1, tol)) - self.assertEqual(True, union.equals_exact(u2, tol)) - qs = City.objects.filter(name='NotACity') - self.assertEqual(None, qs.unionagg(field_name='point')) - - @no_spatialite # SpatiaLite does not support abstract geometry columns - def test18_geometryfield(self): - "Testing the general GeometryField." - Feature(name='Point', geom=Point(1, 1)).save() - Feature(name='LineString', geom=LineString((0, 0), (1, 1), (5, 5))).save() - Feature(name='Polygon', geom=Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0)))).save() - Feature(name='GeometryCollection', - geom=GeometryCollection(Point(2, 2), LineString((0, 0), (2, 2)), - Polygon(LinearRing((0, 0), (0, 5), (5, 5), (5, 0), (0, 0))))).save() - - f_1 = Feature.objects.get(name='Point') - self.assertEqual(True, isinstance(f_1.geom, Point)) - self.assertEqual((1.0, 1.0), f_1.geom.tuple) - f_2 = Feature.objects.get(name='LineString') - self.assertEqual(True, isinstance(f_2.geom, LineString)) - self.assertEqual(((0.0, 0.0), (1.0, 1.0), (5.0, 5.0)), f_2.geom.tuple) - - f_3 = Feature.objects.get(name='Polygon') - self.assertEqual(True, isinstance(f_3.geom, Polygon)) - f_4 = Feature.objects.get(name='GeometryCollection') - self.assertEqual(True, isinstance(f_4.geom, GeometryCollection)) - self.assertEqual(f_3.geom, f_4.geom[2]) +class GeoQuerySetTest(TestCase): + # Please keep the tests in GeoQuerySet method's alphabetic order @no_mysql - def test19_centroid(self): + def test_centroid(self): "Testing the `centroid` GeoQuerySet method." qs = State.objects.exclude(poly__isnull=True).centroid() if oracle: @@ -540,60 +384,182 @@ class GeoModelTest(TestCase): self.assertEqual(True, s.poly.centroid.equals_exact(s.centroid, tol)) @no_mysql - def test20_pointonsurface(self): - "Testing the `point_on_surface` GeoQuerySet method." - # Reference values. - if oracle: - # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_GEOM.SDO_POINTONSURFACE(GEOAPP_COUNTRY.MPOLY, 0.05)) FROM GEOAPP_COUNTRY; - ref = {'New Zealand' : fromstr('POINT (174.616364 -36.100861)', srid=4326), - 'Texas' : fromstr('POINT (-103.002434 36.500397)', srid=4326), - } + def test_diff_intersection_union(self): + "Testing the `difference`, `intersection`, `sym_difference`, and `union` GeoQuerySet methods." + geom = Point(5, 23) + tol = 1 + qs = Country.objects.all().difference(geom).sym_difference(geom).union(geom) - elif postgis or spatialite: - # Using GEOSGeometry to compute the reference point on surface values - # -- since PostGIS also uses GEOS these should be the same. - ref = {'New Zealand' : Country.objects.get(name='New Zealand').mpoly.point_on_surface, - 'Texas' : Country.objects.get(name='Texas').mpoly.point_on_surface - } + # XXX For some reason SpatiaLite does something screwey with the Texas geometry here. Also, + # XXX it doesn't like the null intersection. + if spatialite: + qs = qs.exclude(name='Texas') + else: + qs = qs.intersection(geom) - for c in Country.objects.point_on_surface(): - if spatialite: - # XXX This seems to be a WKT-translation-related precision issue? - tol = 0.00001 + for c in qs: + if oracle: + # Should be able to execute the queries; however, they won't be the same + # as GEOS (because Oracle doesn't use GEOS internally like PostGIS or + # SpatiaLite). + pass else: - tol = 0.000000001 - self.assertEqual(True, ref[c.name].equals_exact(c.point_on_surface, tol)) + self.assertEqual(c.mpoly.difference(geom), c.difference) + if not spatialite: + self.assertEqual(c.mpoly.intersection(geom), c.intersection) + self.assertEqual(c.mpoly.sym_difference(geom), c.sym_difference) + self.assertEqual(c.mpoly.union(geom), c.union) + + @no_mysql + @no_spatialite # SpatiaLite does not have an Extent function + def test_extent(self): + "Testing the `extent` GeoQuerySet method." + # Reference query: + # `SELECT ST_extent(point) FROM geoapp_city WHERE (name='Houston' or name='Dallas');` + # => BOX(-96.8016128540039 29.7633724212646,-95.3631439208984 32.7820587158203) + expected = (-96.8016128540039, 29.7633724212646, -95.3631439208984, 32.782058715820) + + qs = City.objects.filter(name__in=('Houston', 'Dallas')) + extent = qs.extent() + + for val, exp in zip(extent, expected): + self.assertAlmostEqual(exp, val, 4) @no_mysql @no_oracle - def test21_scale(self): - "Testing the `scale` GeoQuerySet method." - xfac, yfac = 2, 3 - tol = 5 # XXX The low precision tolerance is for SpatiaLite - qs = Country.objects.scale(xfac, yfac, model_att='scaled') - for c in qs: - for p1, p2 in zip(c.mpoly, c.scaled): - for r1, r2 in zip(p1, p2): - for c1, c2 in zip(r1.coords, r2.coords): - self.assertAlmostEqual(c1[0] * xfac, c2[0], tol) - self.assertAlmostEqual(c1[1] * yfac, c2[1], tol) + @no_spatialite + def test_force_rhr(self): + "Testing GeoQuerySet.force_rhr()." + rings = ( ( (0, 0), (5, 0), (0, 5), (0, 0) ), + ( (1, 1), (1, 3), (3, 1), (1, 1) ), + ) + rhr_rings = ( ( (0, 0), (0, 5), (5, 0), (0, 0) ), + ( (1, 1), (3, 1), (1, 3), (1, 1) ), + ) + State.objects.create(name='Foo', poly=Polygon(*rings)) + s = State.objects.force_rhr().get(name='Foo') + self.assertEqual(rhr_rings, s.force_rhr.coords) @no_mysql @no_oracle - def test22_translate(self): - "Testing the `translate` GeoQuerySet method." - xfac, yfac = 5, -23 - qs = Country.objects.translate(xfac, yfac, model_att='translated') - for c in qs: - for p1, p2 in zip(c.mpoly, c.translated): - for r1, r2 in zip(p1, p2): - for c1, c2 in zip(r1.coords, r2.coords): - # XXX The low precision is for SpatiaLite - self.assertAlmostEqual(c1[0] + xfac, c2[0], 5) - self.assertAlmostEqual(c1[1] + yfac, c2[1], 5) + @no_spatialite + def test_geohash(self): + "Testing GeoQuerySet.geohash()." + if not connection.ops.geohash: return + # Reference query: + # SELECT ST_GeoHash(point) FROM geoapp_city WHERE name='Houston'; + # SELECT ST_GeoHash(point, 5) FROM geoapp_city WHERE name='Houston'; + ref_hash = '9vk1mfq8jx0c8e0386z6' + h1 = City.objects.geohash().get(name='Houston') + h2 = City.objects.geohash(precision=5).get(name='Houston') + self.assertEqual(ref_hash, h1.geohash) + self.assertEqual(ref_hash[:5], h2.geohash) + + def test_geojson(self): + "Testing GeoJSON output from the database using GeoQuerySet.geojson()." + # Only PostGIS 1.3.4+ supports GeoJSON. + if not connection.ops.geojson: + self.assertRaises(NotImplementedError, Country.objects.all().geojson, field_name='mpoly') + return + + if connection.ops.spatial_version >= (1, 4, 0): + pueblo_json = '{"type":"Point","coordinates":[-104.609252,38.255001]}' + houston_json = '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},"coordinates":[-95.363151,29.763374]}' + victoria_json = '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],"coordinates":[-123.305196,48.462611]}' + chicago_json = '{"type":"Point","crs":{"type":"name","properties":{"name":"EPSG:4326"}},"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}' + else: + pueblo_json = '{"type":"Point","coordinates":[-104.60925200,38.25500100]}' + houston_json = '{"type":"Point","crs":{"type":"EPSG","properties":{"EPSG":4326}},"coordinates":[-95.36315100,29.76337400]}' + victoria_json = '{"type":"Point","bbox":[-123.30519600,48.46261100,-123.30519600,48.46261100],"coordinates":[-123.30519600,48.46261100]}' + chicago_json = '{"type":"Point","crs":{"type":"EPSG","properties":{"EPSG":4326}},"bbox":[-87.65018,41.85039,-87.65018,41.85039],"coordinates":[-87.65018,41.85039]}' + + # Precision argument should only be an integer + self.assertRaises(TypeError, City.objects.geojson, precision='foo') + + # Reference queries and values. + # SELECT ST_AsGeoJson("geoapp_city"."point", 8, 0) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Pueblo'; + self.assertEqual(pueblo_json, City.objects.geojson().get(name='Pueblo').geojson) + + # 1.3.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; + # 1.4.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; + # This time we want to include the CRS by using the `crs` keyword. + self.assertEqual(houston_json, City.objects.geojson(crs=True, model_att='json').get(name='Houston').json) + + # 1.3.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 2) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Victoria'; + # 1.4.x: SELECT ST_AsGeoJson("geoapp_city"."point", 8, 1) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Houston'; + # This time we include the bounding box by using the `bbox` keyword. + self.assertEqual(victoria_json, City.objects.geojson(bbox=True).get(name='Victoria').geojson) + + # 1.(3|4).x: SELECT ST_AsGeoJson("geoapp_city"."point", 5, 3) FROM "geoapp_city" WHERE "geoapp_city"."name" = 'Chicago'; + # Finally, we set every available keyword. + self.assertEqual(chicago_json, City.objects.geojson(bbox=True, crs=True, precision=5).get(name='Chicago').geojson) + + def test_gml(self): + "Testing GML output from the database using GeoQuerySet.gml()." + if mysql or (spatialite and not connection.ops.gml) : + self.assertRaises(NotImplementedError, Country.objects.all().gml, field_name='mpoly') + return + + # Should throw a TypeError when tyring to obtain GML from a + # non-geometry field. + qs = City.objects.all() + self.assertRaises(TypeError, qs.gml, field_name='name') + ptown1 = City.objects.gml(field_name='point', precision=9).get(name='Pueblo') + ptown2 = City.objects.gml(precision=9).get(name='Pueblo') + + if oracle: + # No precision parameter for Oracle :-/ + gml_regex = re.compile(r'^<gml:Point srsName="SDO:4326" xmlns:gml="http://www.opengis.net/gml"><gml:coordinates decimal="\." cs="," ts=" ">-104.60925\d+,38.25500\d+ </gml:coordinates></gml:Point>') + elif spatialite: + # Spatialite has extra colon in SrsName + gml_regex = re.compile(r'^<gml:Point SrsName="EPSG::4326"><gml:coordinates decimal="\." cs="," ts=" ">-104.609251\d+,38.255001</gml:coordinates></gml:Point>') + else: + gml_regex = re.compile(r'^<gml:Point srsName="EPSG:4326"><gml:coordinates>-104\.60925\d+,38\.255001</gml:coordinates></gml:Point>') + + for ptown in [ptown1, ptown2]: + self.assertTrue(gml_regex.match(ptown.gml)) + + def test_kml(self): + "Testing KML output from the database using GeoQuerySet.kml()." + # Only PostGIS and Spatialite (>=2.4.0-RC4) support KML serialization + if not (postgis or (spatialite and connection.ops.kml)): + self.assertRaises(NotImplementedError, State.objects.all().kml, field_name='poly') + return + + # Should throw a TypeError when trying to obtain KML from a + # non-geometry field. + qs = City.objects.all() + self.assertRaises(TypeError, qs.kml, 'name') + + # The reference KML depends on the version of PostGIS used + # (the output stopped including altitude in 1.3.3). + if connection.ops.spatial_version >= (1, 3, 3): + ref_kml = '<Point><coordinates>-104.609252,38.255001</coordinates></Point>' + else: + ref_kml = '<Point><coordinates>-104.609252,38.255001,0</coordinates></Point>' + + # Ensuring the KML is as expected. + ptown1 = City.objects.kml(field_name='point', precision=9).get(name='Pueblo') + ptown2 = City.objects.kml(precision=9).get(name='Pueblo') + for ptown in [ptown1, ptown2]: + self.assertEqual(ref_kml, ptown.kml) + + # Only PostGIS has support for the MakeLine aggregate. + @no_mysql + @no_oracle + @no_spatialite + def test_make_line(self): + "Testing the `make_line` GeoQuerySet method." + # Ensuring that a `TypeError` is raised on models without PointFields. + self.assertRaises(TypeError, State.objects.make_line) + self.assertRaises(TypeError, Country.objects.make_line) + # Reference query: + # SELECT AsText(ST_MakeLine(geoapp_city.point)) FROM geoapp_city; + ref_line = GEOSGeometry('LINESTRING(-95.363151 29.763374,-96.801611 32.782057,-97.521157 34.464642,174.783117 -41.315268,-104.609252 38.255001,-95.23506 38.971823,-87.650175 41.850385,-123.305196 48.462611)', srid=4326) + self.assertEqual(ref_line, City.objects.make_line()) @no_mysql - def test23_numgeom(self): + def test_num_geom(self): "Testing the `num_geom` GeoQuerySet method." # Both 'countries' only have two geometries. for c in Country.objects.num_geom(): self.assertEqual(2, c.num_geom) @@ -607,7 +573,7 @@ class GeoModelTest(TestCase): @no_mysql @no_spatialite # SpatiaLite can only count vertices in LineStrings - def test24_numpoints(self): + def test_num_points(self): "Testing the `num_points` GeoQuerySet method." for c in Country.objects.num_points(): self.assertEqual(c.mpoly.num_points, c.num_points) @@ -617,49 +583,60 @@ class GeoModelTest(TestCase): for c in City.objects.num_points(): self.assertEqual(1, c.num_points) @no_mysql - def test25_geoset(self): - "Testing the `difference`, `intersection`, `sym_difference`, and `union` GeoQuerySet methods." - geom = Point(5, 23) - tol = 1 - qs = Country.objects.all().difference(geom).sym_difference(geom).union(geom) + def test_point_on_surface(self): + "Testing the `point_on_surface` GeoQuerySet method." + # Reference values. + if oracle: + # SELECT SDO_UTIL.TO_WKTGEOMETRY(SDO_GEOM.SDO_POINTONSURFACE(GEOAPP_COUNTRY.MPOLY, 0.05)) FROM GEOAPP_COUNTRY; + ref = {'New Zealand' : fromstr('POINT (174.616364 -36.100861)', srid=4326), + 'Texas' : fromstr('POINT (-103.002434 36.500397)', srid=4326), + } - # XXX For some reason SpatiaLite does something screwey with the Texas geometry here. Also, - # XXX it doesn't like the null intersection. - if spatialite: - qs = qs.exclude(name='Texas') - else: - qs = qs.intersection(geom) + elif postgis or spatialite: + # Using GEOSGeometry to compute the reference point on surface values + # -- since PostGIS also uses GEOS these should be the same. + ref = {'New Zealand' : Country.objects.get(name='New Zealand').mpoly.point_on_surface, + 'Texas' : Country.objects.get(name='Texas').mpoly.point_on_surface + } - for c in qs: - if oracle: - # Should be able to execute the queries; however, they won't be the same - # as GEOS (because Oracle doesn't use GEOS internally like PostGIS or - # SpatiaLite). - pass + for c in Country.objects.point_on_surface(): + if spatialite: + # XXX This seems to be a WKT-translation-related precision issue? + tol = 0.00001 else: - self.assertEqual(c.mpoly.difference(geom), c.difference) - if not spatialite: - self.assertEqual(c.mpoly.intersection(geom), c.intersection) - self.assertEqual(c.mpoly.sym_difference(geom), c.sym_difference) - self.assertEqual(c.mpoly.union(geom), c.union) + tol = 0.000000001 + self.assertEqual(True, ref[c.name].equals_exact(c.point_on_surface, tol)) @no_mysql - def test26_inherited_geofields(self): - "Test GeoQuerySet methods on inherited Geometry fields." - # Creating a Pennsylvanian city. - mansfield = PennsylvaniaCity.objects.create(name='Mansfield', county='Tioga', point='POINT(-77.071445 41.823881)') - - # All transformation SQL will need to be performed on the - # _parent_ table. - qs = PennsylvaniaCity.objects.transform(32128) + @no_spatialite + def test_reverse_geom(self): + "Testing GeoQuerySet.reverse_geom()." + coords = [ (-95.363151, 29.763374), (-95.448601, 29.713803) ] + Track.objects.create(name='Foo', line=LineString(coords)) + t = Track.objects.reverse_geom().get(name='Foo') + coords.reverse() + self.assertEqual(tuple(coords), t.reverse_geom.coords) + if oracle: + self.assertRaises(TypeError, State.objects.reverse_geom) - self.assertEqual(1, qs.count()) - for pc in qs: self.assertEqual(32128, pc.point.srid) + @no_mysql + @no_oracle + def test_scale(self): + "Testing the `scale` GeoQuerySet method." + xfac, yfac = 2, 3 + tol = 5 # XXX The low precision tolerance is for SpatiaLite + qs = Country.objects.scale(xfac, yfac, model_att='scaled') + for c in qs: + for p1, p2 in zip(c.mpoly, c.scaled): + for r1, r2 in zip(p1, p2): + for c1, c2 in zip(r1.coords, r2.coords): + self.assertAlmostEqual(c1[0] * xfac, c2[0], tol) + self.assertAlmostEqual(c1[1] * yfac, c2[1], tol) @no_mysql @no_oracle @no_spatialite - def test27_snap_to_grid(self): + def test_snap_to_grid(self): "Testing GeoQuerySet.snap_to_grid()." # Let's try and break snap_to_grid() with bad combinations of arguments. for bad_args in ((), range(3), range(5)): @@ -695,48 +672,78 @@ class GeoModelTest(TestCase): ref = fromstr('MULTIPOLYGON(((12.4 43.87,12.45 43.87,12.45 44.1,12.5 44.1,12.5 43.87,12.45 43.87,12.4 43.87)))') self.assertTrue(ref.equals_exact(Country.objects.snap_to_grid(0.05, 0.23, 0.5, 0.17).get(name='San Marino').snap_to_grid, tol)) - @no_mysql - @no_spatialite - def test28_reverse(self): - "Testing GeoQuerySet.reverse_geom()." - coords = [ (-95.363151, 29.763374), (-95.448601, 29.713803) ] - Track.objects.create(name='Foo', line=LineString(coords)) - t = Track.objects.reverse_geom().get(name='Foo') - coords.reverse() - self.assertEqual(tuple(coords), t.reverse_geom.coords) - if oracle: - self.assertRaises(TypeError, State.objects.reverse_geom) + def test_svg(self): + "Testing SVG output using GeoQuerySet.svg()." + if mysql or oracle: + self.assertRaises(NotImplementedError, City.objects.svg) + return + + self.assertRaises(TypeError, City.objects.svg, precision='foo') + # SELECT AsSVG(geoapp_city.point, 0, 8) FROM geoapp_city WHERE name = 'Pueblo'; + svg1 = 'cx="-104.609252" cy="-38.255001"' + # Even though relative, only one point so it's practically the same except for + # the 'c' letter prefix on the x,y values. + svg2 = svg1.replace('c', '') + self.assertEqual(svg1, City.objects.svg().get(name='Pueblo').svg) + self.assertEqual(svg2, City.objects.svg(relative=5).get(name='Pueblo').svg) @no_mysql - @no_oracle - @no_spatialite - def test29_force_rhr(self): - "Testing GeoQuerySet.force_rhr()." - rings = ( ( (0, 0), (5, 0), (0, 5), (0, 0) ), - ( (1, 1), (1, 3), (3, 1), (1, 1) ), - ) - rhr_rings = ( ( (0, 0), (0, 5), (5, 0), (0, 0) ), - ( (1, 1), (3, 1), (1, 3), (1, 1) ), - ) - State.objects.create(name='Foo', poly=Polygon(*rings)) - s = State.objects.force_rhr().get(name='Foo') - self.assertEqual(rhr_rings, s.force_rhr.coords) + def test_transform(self): + "Testing the transform() GeoQuerySet method." + # Pre-transformed points for Houston and Pueblo. + htown = fromstr('POINT(1947516.83115183 6322297.06040572)', srid=3084) + ptown = fromstr('POINT(992363.390841912 481455.395105533)', srid=2774) + prec = 3 # Precision is low due to version variations in PROJ and GDAL. + + # Asserting the result of the transform operation with the values in + # the pre-transformed points. Oracle does not have the 3084 SRID. + if not oracle: + h = City.objects.transform(htown.srid).get(name='Houston') + self.assertEqual(3084, h.point.srid) + self.assertAlmostEqual(htown.x, h.point.x, prec) + self.assertAlmostEqual(htown.y, h.point.y, prec) + + p1 = City.objects.transform(ptown.srid, field_name='point').get(name='Pueblo') + p2 = City.objects.transform(srid=ptown.srid).get(name='Pueblo') + for p in [p1, p2]: + self.assertEqual(2774, p.point.srid) + self.assertAlmostEqual(ptown.x, p.point.x, prec) + self.assertAlmostEqual(ptown.y, p.point.y, prec) @no_mysql @no_oracle - @no_spatialite - def test30_geohash(self): - "Testing GeoQuerySet.geohash()." - if not connection.ops.geohash: return - # Reference query: - # SELECT ST_GeoHash(point) FROM geoapp_city WHERE name='Houston'; - # SELECT ST_GeoHash(point, 5) FROM geoapp_city WHERE name='Houston'; - ref_hash = '9vk1mfq8jx0c8e0386z6' - h1 = City.objects.geohash().get(name='Houston') - h2 = City.objects.geohash(precision=5).get(name='Houston') - self.assertEqual(ref_hash, h1.geohash) - self.assertEqual(ref_hash[:5], h2.geohash) + def test_translate(self): + "Testing the `translate` GeoQuerySet method." + xfac, yfac = 5, -23 + qs = Country.objects.translate(xfac, yfac, model_att='translated') + for c in qs: + for p1, p2 in zip(c.mpoly, c.translated): + for r1, r2 in zip(p1, p2): + for c1, c2 in zip(r1.coords, r2.coords): + # XXX The low precision is for SpatiaLite + self.assertAlmostEqual(c1[0] + xfac, c2[0], 5) + self.assertAlmostEqual(c1[1] + yfac, c2[1], 5) -from .test_feeds import GeoFeedTest -from .test_regress import GeoRegressionTests -from .test_sitemaps import GeoSitemapTest + @no_mysql + def test_unionagg(self): + "Testing the `unionagg` (aggregate union) GeoQuerySet method." + tx = Country.objects.get(name='Texas').mpoly + # Houston, Dallas -- Oracle has different order. + union1 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)') + union2 = fromstr('MULTIPOINT(-96.801611 32.782057,-95.363151 29.763374)') + qs = City.objects.filter(point__within=tx) + self.assertRaises(TypeError, qs.unionagg, 'name') + # Using `field_name` keyword argument in one query and specifying an + # order in the other (which should not be used because this is + # an aggregate method on a spatial column) + u1 = qs.unionagg(field_name='point') + u2 = qs.order_by('name').unionagg() + tol = 0.00001 + if oracle: + union = union2 + else: + union = union1 + self.assertEqual(True, union.equals_exact(u1, tol)) + self.assertEqual(True, union.equals_exact(u2, tol)) + qs = City.objects.filter(name='NotACity') + self.assertEqual(None, qs.unionagg(field_name='point')) diff --git a/django/contrib/humanize/templatetags/humanize.py b/django/contrib/humanize/templatetags/humanize.py index 8f6c2602c9..7e8f163174 100644 --- a/django/contrib/humanize/templatetags/humanize.py +++ b/django/contrib/humanize/templatetags/humanize.py @@ -5,7 +5,7 @@ from datetime import date, datetime from django import template from django.conf import settings from django.template import defaultfilters -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.formats import number_format from django.utils.translation import pgettext, ungettext, ugettext as _ from django.utils.timezone import is_aware, utc @@ -41,7 +41,7 @@ def intcomma(value, use_l10n=True): return intcomma(value, False) else: return number_format(value, force_grouping=True) - orig = force_unicode(value) + orig = force_text(value) new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', orig) if orig == new: return new diff --git a/django/contrib/localflavor/au/forms.py b/django/contrib/localflavor/au/forms.py index 34170fabc8..d3a00e200c 100644 --- a/django/contrib/localflavor/au/forms.py +++ b/django/contrib/localflavor/au/forms.py @@ -10,7 +10,7 @@ from django.contrib.localflavor.au.au_states import STATE_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -44,7 +44,7 @@ class AUPhoneNumberField(Field): super(AUPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+|-)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+|-)', '', smart_text(value)) phone_match = PHONE_DIGITS_RE.search(value) if phone_match: return '%s' % phone_match.group(1) diff --git a/django/contrib/localflavor/br/forms.py b/django/contrib/localflavor/br/forms.py index f287d46a9a..0f957be37f 100644 --- a/django/contrib/localflavor/br/forms.py +++ b/django/contrib/localflavor/br/forms.py @@ -11,7 +11,7 @@ from django.contrib.localflavor.br.br_states import STATE_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, CharField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -35,7 +35,7 @@ class BRPhoneNumberField(Field): super(BRPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s-%s-%s' % (m.group(1), m.group(2), m.group(3)) @@ -68,10 +68,10 @@ class BRStateChoiceField(Field): value = super(BRStateChoiceField, self).clean(value) if value in EMPTY_VALUES: value = '' - value = smart_unicode(value) + value = smart_text(value) if value == '': return value - valid_values = set([smart_unicode(k) for k, v in self.widget.choices]) + valid_values = set([smart_text(k) for k, v in self.widget.choices]) if value not in valid_values: raise ValidationError(self.error_messages['invalid']) return value @@ -154,10 +154,10 @@ class BRCNPJField(Field): raise ValidationError(self.error_messages['max_digits']) orig_dv = value[-2:] - new_1dv = sum([i * int(value[idx]) for idx, i in enumerate(range(5, 1, -1) + range(9, 1, -1))]) + new_1dv = sum([i * int(value[idx]) for idx, i in enumerate(list(range(5, 1, -1)) + list(range(9, 1, -1)))]) new_1dv = DV_maker(new_1dv % 11) value = value[:-2] + str(new_1dv) + value[-1] - new_2dv = sum([i * int(value[idx]) for idx, i in enumerate(range(6, 1, -1) + range(9, 1, -1))]) + new_2dv = sum([i * int(value[idx]) for idx, i in enumerate(list(range(6, 1, -1)) + list(range(9, 1, -1)))]) new_2dv = DV_maker(new_2dv % 11) value = value[:-1] + str(new_2dv) if value[-2:] != orig_dv: diff --git a/django/contrib/localflavor/ca/forms.py b/django/contrib/localflavor/ca/forms.py index daa40044f9..4ebfb06c2b 100644 --- a/django/contrib/localflavor/ca/forms.py +++ b/django/contrib/localflavor/ca/forms.py @@ -9,7 +9,7 @@ import re from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, CharField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -53,7 +53,7 @@ class CAPhoneNumberField(Field): super(CAPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s-%s-%s' % (m.group(1), m.group(2), m.group(3)) diff --git a/django/contrib/localflavor/ch/forms.py b/django/contrib/localflavor/ch/forms.py index e844a3c57c..bf71eeea32 100644 --- a/django/contrib/localflavor/ch/forms.py +++ b/django/contrib/localflavor/ch/forms.py @@ -10,7 +10,7 @@ from django.contrib.localflavor.ch.ch_states import STATE_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -41,7 +41,7 @@ class CHPhoneNumberField(Field): super(CHPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\.|\s|/|-)', '', smart_unicode(value)) + value = re.sub('(\.|\s|/|-)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s %s %s %s' % (value[0:3], value[3:6], value[6:8], value[8:10]) diff --git a/django/contrib/localflavor/cl/forms.py b/django/contrib/localflavor/cl/forms.py index 5991176382..a5340141ce 100644 --- a/django/contrib/localflavor/cl/forms.py +++ b/django/contrib/localflavor/cl/forms.py @@ -8,7 +8,7 @@ from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import RegexField, Select from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from .cl_regions import REGION_CHOICES @@ -75,7 +75,7 @@ class CLRutField(RegexField): Turns the RUT into one normalized format. Returns a (rut, verifier) tuple. """ - rut = smart_unicode(rut).replace(' ', '').replace('.', '').replace('-', '') + rut = smart_text(rut).replace(' ', '').replace('.', '').replace('-', '') return rut[:-1], rut[-1].upper() def _format(self, code, verifier=None): diff --git a/django/contrib/localflavor/fr/forms.py b/django/contrib/localflavor/fr/forms.py index d836dd6397..8b841fff5f 100644 --- a/django/contrib/localflavor/fr/forms.py +++ b/django/contrib/localflavor/fr/forms.py @@ -9,7 +9,7 @@ from django.contrib.localflavor.fr.fr_department import DEPARTMENT_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import CharField, RegexField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -43,7 +43,7 @@ class FRPhoneNumberField(CharField): super(FRPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\.|\s)', '', smart_unicode(value)) + value = re.sub('(\.|\s)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s %s %s %s %s' % (value[0:2], value[2:4], value[4:6], value[6:8], value[8:10]) diff --git a/django/contrib/localflavor/hk/forms.py b/django/contrib/localflavor/hk/forms.py index 8cf9360e19..ab4f70f193 100644 --- a/django/contrib/localflavor/hk/forms.py +++ b/django/contrib/localflavor/hk/forms.py @@ -8,7 +8,7 @@ import re from django.core.validators import EMPTY_VALUES from django.forms import CharField from django.forms import ValidationError -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -53,7 +53,7 @@ class HKPhoneNumberField(CharField): if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+|\+)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+|\+)', '', smart_text(value)) m = hk_phone_digits_re.search(value) if not m: raise ValidationError(self.error_messages['invalid']) diff --git a/django/contrib/localflavor/hr/forms.py b/django/contrib/localflavor/hr/forms.py index eb4436a78c..b935fd8a3a 100644 --- a/django/contrib/localflavor/hr/forms.py +++ b/django/contrib/localflavor/hr/forms.py @@ -12,7 +12,7 @@ from django.contrib.localflavor.hr.hr_choices import ( from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, Select, RegexField -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -159,7 +159,7 @@ class HRLicensePlateField(Field): if value in EMPTY_VALUES: return '' - value = re.sub(r'[\s\-]+', '', smart_unicode(value.strip())).upper() + value = re.sub(r'[\s\-]+', '', smart_text(value.strip())).upper() matches = plate_re.search(value) if matches is None: @@ -225,7 +225,7 @@ class HRPhoneNumberField(Field): if value in EMPTY_VALUES: return '' - value = re.sub(r'[\-\s\(\)]', '', smart_unicode(value)) + value = re.sub(r'[\-\s\(\)]', '', smart_text(value)) matches = phone_re.search(value) if matches is None: diff --git a/django/contrib/localflavor/id/forms.py b/django/contrib/localflavor/id/forms.py index f22b06134e..2005dbc75c 100644 --- a/django/contrib/localflavor/id/forms.py +++ b/django/contrib/localflavor/id/forms.py @@ -11,7 +11,7 @@ from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, Select from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text postcode_re = re.compile(r'^[1-9]\d{4}$') @@ -77,10 +77,10 @@ class IDPhoneNumberField(Field): if value in EMPTY_VALUES: return '' - phone_number = re.sub(r'[\-\s\(\)]', '', smart_unicode(value)) + phone_number = re.sub(r'[\-\s\(\)]', '', smart_text(value)) if phone_re.search(phone_number): - return smart_unicode(value) + return smart_text(value) raise ValidationError(self.error_messages['invalid']) @@ -120,7 +120,7 @@ class IDLicensePlateField(Field): return '' plate_number = re.sub(r'\s+', ' ', - smart_unicode(value.strip())).upper() + smart_text(value.strip())).upper() matches = plate_re.search(plate_number) if matches is None: @@ -181,7 +181,7 @@ class IDNationalIdentityNumberField(Field): if value in EMPTY_VALUES: return '' - value = re.sub(r'[\s.]', '', smart_unicode(value)) + value = re.sub(r'[\s.]', '', smart_text(value)) if not nik_re.search(value): raise ValidationError(self.error_messages['invalid']) diff --git a/django/contrib/localflavor/in_/forms.py b/django/contrib/localflavor/in_/forms.py index b62ec7bdb2..5c1d009ef4 100644 --- a/django/contrib/localflavor/in_/forms.py +++ b/django/contrib/localflavor/in_/forms.py @@ -10,7 +10,7 @@ from django.contrib.localflavor.in_.in_states import STATES_NORMALIZED, STATE_CH from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, CharField, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -74,7 +74,7 @@ class INStateField(Field): pass else: try: - return smart_unicode(STATES_NORMALIZED[value.strip().lower()]) + return smart_text(STATES_NORMALIZED[value.strip().lower()]) except KeyError: pass raise ValidationError(self.error_messages['invalid']) @@ -107,7 +107,7 @@ class INPhoneNumberField(CharField): super(INPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = smart_unicode(value) + value = smart_text(value) m = phone_digits_re.match(value) if m: return '%s' % (value) diff --git a/django/contrib/localflavor/is_/forms.py b/django/contrib/localflavor/is_/forms.py index 7af9f51cfb..1ae3e012a1 100644 --- a/django/contrib/localflavor/is_/forms.py +++ b/django/contrib/localflavor/is_/forms.py @@ -9,7 +9,7 @@ from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import RegexField from django.forms.widgets import Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -58,7 +58,7 @@ class ISIdNumberField(RegexField): Takes in the value in canonical form and returns it in the common display format. """ - return smart_unicode(value[:6]+'-'+value[6:]) + return smart_text(value[:6]+'-'+value[6:]) class ISPhoneNumberField(RegexField): """ diff --git a/django/contrib/localflavor/it/forms.py b/django/contrib/localflavor/it/forms.py index 60b1eff951..916ce9bb3d 100644 --- a/django/contrib/localflavor/it/forms.py +++ b/django/contrib/localflavor/it/forms.py @@ -13,7 +13,7 @@ from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, Select from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text class ITZipCodeField(RegexField): @@ -85,4 +85,4 @@ class ITVatNumberField(Field): check_digit = vat_number_check_digit(vat_number[0:10]) if not vat_number[10] == check_digit: raise ValidationError(self.error_messages['invalid']) - return smart_unicode(vat_number) + return smart_text(vat_number) diff --git a/django/contrib/localflavor/it/util.py b/django/contrib/localflavor/it/util.py index ec1b7e3f83..e1aa9c0419 100644 --- a/django/contrib/localflavor/it/util.py +++ b/django/contrib/localflavor/it/util.py @@ -1,4 +1,4 @@ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text def ssn_check_digit(value): "Calculate Italian social security number check digit." @@ -34,11 +34,11 @@ def ssn_check_digit(value): def vat_number_check_digit(vat_number): "Calculate Italian VAT number check digit." - normalized_vat_number = smart_unicode(vat_number).zfill(10) + normalized_vat_number = smart_text(vat_number).zfill(10) total = 0 for i in range(0, 10, 2): total += int(normalized_vat_number[i]) for i in range(1, 11, 2): quotient , remainder = divmod(int(normalized_vat_number[i]) * 2, 10) total += quotient + remainder - return smart_unicode((10 - total % 10) % 10) + return smart_text((10 - total % 10) % 10) diff --git a/django/contrib/localflavor/nl/forms.py b/django/contrib/localflavor/nl/forms.py index bdd769bd39..a05dd38f7f 100644 --- a/django/contrib/localflavor/nl/forms.py +++ b/django/contrib/localflavor/nl/forms.py @@ -10,7 +10,7 @@ from django.contrib.localflavor.nl.nl_provinces import PROVINCE_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, Select -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -61,7 +61,7 @@ class NLPhoneNumberField(Field): if value in EMPTY_VALUES: return '' - phone_nr = re.sub('[\-\s\(\)]', '', smart_unicode(value)) + phone_nr = re.sub('[\-\s\(\)]', '', smart_text(value)) if len(phone_nr) == 10 and numeric_re.search(phone_nr): return value diff --git a/django/contrib/localflavor/pt/forms.py b/django/contrib/localflavor/pt/forms.py index b27fb577bd..01cdd101b2 100644 --- a/django/contrib/localflavor/pt/forms.py +++ b/django/contrib/localflavor/pt/forms.py @@ -8,7 +8,7 @@ import re from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ phone_digits_re = re.compile(r'^(\d{9}|(00|\+)\d*)$') @@ -29,7 +29,7 @@ class PTZipCodeField(RegexField): return '%s-%s' % (cleaned[:4],cleaned[4:]) else: return cleaned - + class PTPhoneNumberField(Field): """ Validate local Portuguese phone number (including international ones) @@ -43,7 +43,7 @@ class PTPhoneNumberField(Field): super(PTPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\.|\s)', '', smart_unicode(value)) + value = re.sub('(\.|\s)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s' % value diff --git a/django/contrib/localflavor/si/forms.py b/django/contrib/localflavor/si/forms.py index aa4b9dac5a..bab35935fd 100644 --- a/django/contrib/localflavor/si/forms.py +++ b/django/contrib/localflavor/si/forms.py @@ -41,7 +41,7 @@ class SIEMSOField(CharField): # Validate EMSO s = 0 int_values = [int(i) for i in value] - for a, b in zip(int_values, range(7, 1, -1) * 2): + for a, b in zip(int_values, list(range(7, 1, -1)) * 2): s += a * b chk = s % 11 if chk == 0: diff --git a/django/contrib/localflavor/tr/forms.py b/django/contrib/localflavor/tr/forms.py index 15bc1f99bb..c4f928e670 100644 --- a/django/contrib/localflavor/tr/forms.py +++ b/django/contrib/localflavor/tr/forms.py @@ -10,7 +10,7 @@ from django.contrib.localflavor.tr.tr_provinces import PROVINCE_CHOICES from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, Select, CharField -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -46,7 +46,7 @@ class TRPhoneNumberField(CharField): super(TRPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s%s' % (m.group(2), m.group(4)) diff --git a/django/contrib/localflavor/us/forms.py b/django/contrib/localflavor/us/forms.py index ef565163cd..437bb7c466 100644 --- a/django/contrib/localflavor/us/forms.py +++ b/django/contrib/localflavor/us/forms.py @@ -9,7 +9,7 @@ import re from django.core.validators import EMPTY_VALUES from django.forms import ValidationError from django.forms.fields import Field, RegexField, Select, CharField -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy as _ @@ -34,7 +34,7 @@ class USPhoneNumberField(CharField): super(USPhoneNumberField, self).clean(value) if value in EMPTY_VALUES: return '' - value = re.sub('(\(|\)|\s+)', '', smart_unicode(value)) + value = re.sub('(\(|\)|\s+)', '', smart_text(value)) m = phone_digits_re.search(value) if m: return '%s-%s-%s' % (m.group(1), m.group(2), m.group(3)) diff --git a/django/contrib/markup/templatetags/markup.py b/django/contrib/markup/templatetags/markup.py index 84251cf30a..af9c842f42 100644 --- a/django/contrib/markup/templatetags/markup.py +++ b/django/contrib/markup/templatetags/markup.py @@ -13,7 +13,7 @@ markup syntaxes to HTML; currently there is support for: from django import template from django.conf import settings -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import smart_bytes, force_text from django.utils.safestring import mark_safe register = template.Library() @@ -25,9 +25,9 @@ def textile(value): except ImportError: if settings.DEBUG: raise template.TemplateSyntaxError("Error in 'textile' filter: The Python textile library isn't installed.") - return force_unicode(value) + return force_text(value) else: - return mark_safe(force_unicode(textile.textile(smart_str(value), encoding='utf-8', output='utf-8'))) + return mark_safe(force_text(textile.textile(smart_bytes(value), encoding='utf-8', output='utf-8'))) @register.filter(is_safe=True) def markdown(value, arg=''): @@ -52,23 +52,23 @@ def markdown(value, arg=''): except ImportError: if settings.DEBUG: raise template.TemplateSyntaxError("Error in 'markdown' filter: The Python markdown library isn't installed.") - return force_unicode(value) + return force_text(value) else: markdown_vers = getattr(markdown, "version_info", 0) if markdown_vers < (2, 1): if settings.DEBUG: raise template.TemplateSyntaxError( "Error in 'markdown' filter: Django does not support versions of the Python markdown library < 2.1.") - return force_unicode(value) + return force_text(value) else: extensions = [e for e in arg.split(",") if e] if extensions and extensions[0] == "safe": extensions = extensions[1:] return mark_safe(markdown.markdown( - force_unicode(value), extensions, safe_mode=True, enable_attributes=False)) + force_text(value), extensions, safe_mode=True, enable_attributes=False)) else: return mark_safe(markdown.markdown( - force_unicode(value), extensions, safe_mode=False)) + force_text(value), extensions, safe_mode=False)) @register.filter(is_safe=True) def restructuredtext(value): @@ -77,8 +77,8 @@ def restructuredtext(value): except ImportError: if settings.DEBUG: raise template.TemplateSyntaxError("Error in 'restructuredtext' filter: The Python docutils library isn't installed.") - return force_unicode(value) + return force_text(value) else: docutils_settings = getattr(settings, "RESTRUCTUREDTEXT_FILTER_SETTINGS", {}) - parts = publish_parts(source=smart_str(value), writer_name="html4css1", settings_overrides=docutils_settings) - return mark_safe(force_unicode(parts["fragment"])) + parts = publish_parts(source=smart_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings) + return mark_safe(force_text(parts["fragment"])) diff --git a/django/contrib/messages/storage/base.py b/django/contrib/messages/storage/base.py index e80818e84e..5433bbff28 100644 --- a/django/contrib/messages/storage/base.py +++ b/django/contrib/messages/storage/base.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from django.conf import settings -from django.utils.encoding import force_unicode, StrAndUnicode +from django.utils.encoding import force_text, StrAndUnicode from django.contrib.messages import constants, utils @@ -26,22 +26,22 @@ class Message(StrAndUnicode): and ``extra_tags`` to unicode in case they are lazy translations. Known "safe" types (None, int, etc.) are not converted (see Django's - ``force_unicode`` implementation for details). + ``force_text`` implementation for details). """ - self.message = force_unicode(self.message, strings_only=True) - self.extra_tags = force_unicode(self.extra_tags, strings_only=True) + self.message = force_text(self.message, strings_only=True) + self.extra_tags = force_text(self.extra_tags, strings_only=True) def __eq__(self, other): return isinstance(other, Message) and self.level == other.level and \ self.message == other.message def __unicode__(self): - return force_unicode(self.message) + return force_text(self.message) def _get_tags(self): - label_tag = force_unicode(LEVEL_TAGS.get(self.level, ''), + label_tag = force_text(LEVEL_TAGS.get(self.level, ''), strings_only=True) - extra_tags = force_unicode(self.extra_tags, strings_only=True) + extra_tags = force_text(self.extra_tags, strings_only=True) if extra_tags and label_tag: return ' '.join([extra_tags, label_tag]) elif extra_tags: diff --git a/django/contrib/messages/storage/cookie.py b/django/contrib/messages/storage/cookie.py index 07620050c7..5f64ccd0c5 100644 --- a/django/contrib/messages/storage/cookie.py +++ b/django/contrib/messages/storage/cookie.py @@ -4,6 +4,7 @@ from django.conf import settings from django.contrib.messages.storage.base import BaseStorage, Message from django.http import SimpleCookie from django.utils.crypto import salted_hmac, constant_time_compare +from django.utils import six class MessageEncoder(json.JSONEncoder): @@ -33,7 +34,7 @@ class MessageDecoder(json.JSONDecoder): return [self.process_messages(item) for item in obj] if isinstance(obj, dict): return dict([(key, self.process_messages(value)) - for key, value in obj.iteritems()]) + for key, value in six.iteritems(obj)]) return obj def decode(self, s, **kwargs): diff --git a/django/contrib/sessions/backends/db.py b/django/contrib/sessions/backends/db.py index 3dd0d9516c..0cc17b44c3 100644 --- a/django/contrib/sessions/backends/db.py +++ b/django/contrib/sessions/backends/db.py @@ -1,7 +1,7 @@ from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation from django.db import IntegrityError, transaction, router -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils import timezone @@ -18,7 +18,7 @@ class SessionStore(SessionBase): session_key = self.session_key, expire_date__gt=timezone.now() ) - return self.decode(force_unicode(s.session_data)) + return self.decode(force_text(s.session_data)) except (Session.DoesNotExist, SuspiciousOperation): self.create() return {} diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 328b085f1e..7de2941122 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -16,6 +16,7 @@ from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.http import HttpResponse from django.test import TestCase, RequestFactory from django.test.utils import override_settings +from django.utils import six from django.utils import timezone from django.utils import unittest @@ -86,16 +87,16 @@ class SessionTestsMixin(object): self.assertFalse(self.session.modified) def test_values(self): - self.assertEqual(self.session.values(), []) + self.assertEqual(list(self.session.values()), []) self.assertTrue(self.session.accessed) self.session['some key'] = 1 - self.assertEqual(self.session.values(), [1]) + self.assertEqual(list(self.session.values()), [1]) def test_iterkeys(self): self.session['x'] = 1 self.session.modified = False self.session.accessed = False - i = self.session.iterkeys() + i = six.iterkeys(self.session) self.assertTrue(hasattr(i, '__iter__')) self.assertTrue(self.session.accessed) self.assertFalse(self.session.modified) @@ -105,7 +106,7 @@ class SessionTestsMixin(object): self.session['x'] = 1 self.session.modified = False self.session.accessed = False - i = self.session.itervalues() + i = six.itervalues(self.session) self.assertTrue(hasattr(i, '__iter__')) self.assertTrue(self.session.accessed) self.assertFalse(self.session.modified) @@ -115,7 +116,7 @@ class SessionTestsMixin(object): self.session['x'] = 1 self.session.modified = False self.session.accessed = False - i = self.session.iteritems() + i = six.iteritems(self.session) self.assertTrue(hasattr(i, '__iter__')) self.assertTrue(self.session.accessed) self.assertFalse(self.session.modified) @@ -125,9 +126,9 @@ class SessionTestsMixin(object): self.session['x'] = 1 self.session.modified = False self.session.accessed = False - self.assertEqual(self.session.items(), [('x', 1)]) + self.assertEqual(list(self.session.items()), [('x', 1)]) self.session.clear() - self.assertEqual(self.session.items(), []) + self.assertEqual(list(self.session.items()), []) self.assertTrue(self.session.accessed) self.assertTrue(self.session.modified) @@ -154,10 +155,10 @@ class SessionTestsMixin(object): self.session['a'], self.session['b'] = 'c', 'd' self.session.save() prev_key = self.session.session_key - prev_data = self.session.items() + prev_data = list(self.session.items()) self.session.cycle_key() self.assertNotEqual(self.session.session_key, prev_key) - self.assertEqual(self.session.items(), prev_data) + self.assertEqual(list(self.session.items()), prev_data) def test_invalid_key(self): # Submitting an invalid session key (either by guessing, or if the db has diff --git a/django/contrib/sitemaps/views.py b/django/contrib/sitemaps/views.py index b90a39e954..cfe3aa66a9 100644 --- a/django/contrib/sitemaps/views.py +++ b/django/contrib/sitemaps/views.py @@ -3,6 +3,7 @@ from django.core import urlresolvers from django.core.paginator import EmptyPage, PageNotAnInteger from django.http import Http404 from django.template.response import TemplateResponse +from django.utils import six def index(request, sitemaps, template_name='sitemap_index.xml', mimetype='application/xml', @@ -35,7 +36,7 @@ def sitemap(request, sitemaps, section=None, raise Http404("No sitemap available for section: %r" % section) maps = [sitemaps[section]] else: - maps = sitemaps.values() + maps = list(six.itervalues(sitemaps)) page = request.GET.get("p", 1) urls = [] diff --git a/django/contrib/staticfiles/finders.py b/django/contrib/staticfiles/finders.py index 766687cf7d..9b06c2cf60 100644 --- a/django/contrib/staticfiles/finders.py +++ b/django/contrib/staticfiles/finders.py @@ -6,6 +6,7 @@ from django.utils.datastructures import SortedDict from django.utils.functional import empty, memoize, LazyObject from django.utils.importlib import import_module from django.utils._os import safe_join +from django.utils import six from django.contrib.staticfiles import utils from django.contrib.staticfiles.storage import AppStaticStorage @@ -132,7 +133,7 @@ class AppDirectoriesFinder(BaseFinder): """ List all files in all app storages. """ - for storage in self.storages.itervalues(): + for storage in six.itervalues(self.storages): if storage.exists(''): # check if storage location exists for path in utils.get_files(storage, ignore_patterns): yield path, storage diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py index d3977213a9..7dac0ffb4c 100644 --- a/django/contrib/staticfiles/management/commands/collectstatic.py +++ b/django/contrib/staticfiles/management/commands/collectstatic.py @@ -6,8 +6,9 @@ from optparse import make_option from django.core.files.storage import FileSystemStorage from django.core.management.base import CommandError, NoArgsCommand -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.datastructures import SortedDict +from django.utils.six.moves import input from django.contrib.staticfiles import finders, storage @@ -148,7 +149,7 @@ class Command(NoArgsCommand): clear_display = 'This will overwrite existing files!' if self.interactive: - confirm = raw_input(""" + confirm = input(""" You have requested to collect static files at the destination location as specified in your settings%s @@ -198,9 +199,9 @@ Type 'yes' to continue, or 'no' to cancel: """ fpath = os.path.join(path, f) if self.dry_run: self.log("Pretending to delete '%s'" % - smart_unicode(fpath), level=1) + smart_text(fpath), level=1) else: - self.log("Deleting '%s'" % smart_unicode(fpath), level=1) + self.log("Deleting '%s'" % smart_text(fpath), level=1) self.storage.delete(fpath) for d in dirs: self.clear_dir(os.path.join(path, d)) diff --git a/django/contrib/staticfiles/management/commands/findstatic.py b/django/contrib/staticfiles/management/commands/findstatic.py index 772220b342..dc1e88d778 100644 --- a/django/contrib/staticfiles/management/commands/findstatic.py +++ b/django/contrib/staticfiles/management/commands/findstatic.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import os from optparse import make_option from django.core.management.base import LabelCommand -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.contrib.staticfiles import finders @@ -19,12 +19,12 @@ class Command(LabelCommand): def handle_label(self, path, **options): verbosity = int(options.get('verbosity', 1)) result = finders.find(path, all=options['all']) - path = smart_unicode(path) + path = smart_text(path) if result: if not isinstance(result, (list, tuple)): result = [result] output = '\n '.join( - (smart_unicode(os.path.realpath(path)) for path in result)) + (smart_text(os.path.realpath(path)) for path in result)) self.stdout.write("Found '%s' here:\n %s" % (path, output)) else: if verbosity >= 1: diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py index a0133e1c6a..2ca54dde71 100644 --- a/django/contrib/staticfiles/storage.py +++ b/django/contrib/staticfiles/storage.py @@ -16,7 +16,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files.base import ContentFile from django.core.files.storage import FileSystemStorage, get_storage_class from django.utils.datastructures import SortedDict -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils.functional import LazyObject from django.utils.importlib import import_module @@ -112,7 +112,7 @@ class CachedFilesMixin(object): return urlunsplit(unparsed_name) def cache_key(self, name): - return 'staticfiles:%s' % hashlib.md5(smart_str(name)).hexdigest() + return 'staticfiles:%s' % hashlib.md5(smart_bytes(name)).hexdigest() def url(self, name, force=False): """ @@ -248,9 +248,9 @@ class CachedFilesMixin(object): if hashed_file_exists: self.delete(hashed_name) # then save the processed result - content_file = ContentFile(smart_str(content)) + content_file = ContentFile(smart_bytes(content)) saved_name = self._save(hashed_name, content_file) - hashed_name = force_unicode(saved_name.replace('\\', '/')) + hashed_name = force_text(saved_name.replace('\\', '/')) processed = True else: # or handle the case in which neither processing nor @@ -258,7 +258,7 @@ class CachedFilesMixin(object): if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) - hashed_name = force_unicode(saved_name.replace('\\', '/')) + hashed_name = force_text(saved_name.replace('\\', '/')) # and then set the cache accordingly hashed_paths[self.cache_key(name)] = hashed_name diff --git a/django/contrib/syndication/views.py b/django/contrib/syndication/views.py index 3c84f1f60c..bce7ef7cfb 100644 --- a/django/contrib/syndication/views.py +++ b/django/contrib/syndication/views.py @@ -6,7 +6,7 @@ from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist from django.http import HttpResponse, Http404 from django.template import loader, TemplateDoesNotExist, RequestContext from django.utils import feedgenerator, tzinfo -from django.utils.encoding import force_unicode, iri_to_uri, smart_unicode +from django.utils.encoding import force_text, iri_to_uri, smart_text from django.utils.html import escape from django.utils.timezone import is_naive @@ -43,10 +43,10 @@ class Feed(object): def item_title(self, item): # Titles should be double escaped by default (see #6533) - return escape(force_unicode(item)) + return escape(force_text(item)) def item_description(self, item): - return force_unicode(item) + return force_text(item) def item_link(self, item): try: @@ -154,9 +154,9 @@ class Feed(object): enc_url = self.__get_dynamic_attr('item_enclosure_url', item) if enc_url: enc = feedgenerator.Enclosure( - url = smart_unicode(enc_url), - length = smart_unicode(self.__get_dynamic_attr('item_enclosure_length', item)), - mime_type = smart_unicode(self.__get_dynamic_attr('item_enclosure_mime_type', item)) + url = smart_text(enc_url), + length = smart_text(self.__get_dynamic_attr('item_enclosure_length', item)), + mime_type = smart_text(self.__get_dynamic_attr('item_enclosure_mime_type', item)) ) author_name = self.__get_dynamic_attr('item_author_name', item) if author_name is not None: diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index f7573b2e31..d527e44d8b 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -3,7 +3,7 @@ import warnings from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils.importlib import import_module class InvalidCacheBackendError(ImproperlyConfigured): @@ -23,7 +23,7 @@ def default_key_func(key, key_prefix, version): the `key_prefix'. KEY_FUNCTION can be used to specify an alternate function with custom key making behavior. """ - return ':'.join([key_prefix, str(version), smart_str(key)]) + return ':'.join([key_prefix, str(version), smart_bytes(key)]) def get_key_func(key_func): """ @@ -62,7 +62,7 @@ class BaseCache(object): except (ValueError, TypeError): self._cull_frequency = 3 - self.key_prefix = smart_str(params.get('KEY_PREFIX', '')) + self.key_prefix = smart_bytes(params.get('KEY_PREFIX', '')) self.version = params.get('VERSION', 1) self.key_func = get_key_func(params.get('KEY_FUNCTION', None)) diff --git a/django/core/context_processors.py b/django/core/context_processors.py index 325f64d224..a503270cf4 100644 --- a/django/core/context_processors.py +++ b/django/core/context_processors.py @@ -9,7 +9,7 @@ RequestContext. from django.conf import settings from django.middleware.csrf import get_token -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils.functional import lazy def csrf(request): @@ -25,7 +25,7 @@ def csrf(request): # instead of returning an empty dict. return b'NOTPROVIDED' else: - return smart_str(token) + return smart_bytes(token) _get_val = lazy(_get_val, str) return {'csrf_token': _get_val() } diff --git a/django/core/exceptions.py b/django/core/exceptions.py index e3d1dc9c7e..f0f14cffda 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -43,7 +43,7 @@ class ValidationError(Exception): """An error while validating data.""" def __init__(self, message, code=None, params=None): import operator - from django.utils.encoding import force_unicode + from django.utils.encoding import force_text """ ValidationError can be passed any object that can be printed (usually a string), a list of objects or a dictionary. @@ -54,11 +54,11 @@ class ValidationError(Exception): message = reduce(operator.add, message.values()) if isinstance(message, list): - self.messages = [force_unicode(msg) for msg in message] + self.messages = [force_text(msg) for msg in message] else: self.code = code self.params = params - message = force_unicode(message) + message = force_text(message) self.messages = [message] def __str__(self): diff --git a/django/core/files/base.py b/django/core/files/base.py index 04853fad0c..d0b25250a5 100644 --- a/django/core/files/base.py +++ b/django/core/files/base.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import os from io import BytesIO -from django.utils.encoding import smart_str, smart_unicode +from django.utils.encoding import smart_bytes, smart_text from django.core.files.utils import FileProxyMixin class File(FileProxyMixin): @@ -18,16 +18,17 @@ class File(FileProxyMixin): self.mode = file.mode def __str__(self): - return smart_str(self.name or '') + return smart_bytes(self.name or '') def __unicode__(self): - return smart_unicode(self.name or '') + return smart_text(self.name or '') def __repr__(self): return "<%s: %s>" % (self.__class__.__name__, self or "None") - def __nonzero__(self): + def __bool__(self): return bool(self.name) + __nonzero__ = __bool__ # Python 2 def __len__(self): return self.size @@ -135,8 +136,9 @@ class ContentFile(File): def __str__(self): return 'Raw content' - def __nonzero__(self): + def __bool__(self): return True + __nonzero__ = __bool__ # Python 2 def open(self, mode=None): self.seek(0) diff --git a/django/core/files/images.py b/django/core/files/images.py index 228a7118c5..7d7eac65db 100644 --- a/django/core/files/images.py +++ b/django/core/files/images.py @@ -47,13 +47,18 @@ def get_image_dimensions(file_or_path, close=False): file = open(file_or_path, 'rb') close = True try: + # Most of the time PIL only needs a small chunk to parse the image and + # get the dimensions, but with some TIFF files PIL needs to parse the + # whole file. + chunk_size = 1024 while 1: - data = file.read(1024) + data = file.read(chunk_size) if not data: break p.feed(data) if p.image: return p.image.size + chunk_size = chunk_size*2 return None finally: if close: diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 5179980513..7542dcda46 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -11,7 +11,7 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.core.files import locks, File from django.core.files.move import file_move_safe -from django.utils.encoding import force_unicode, filepath_to_uri +from django.utils.encoding import force_text, filepath_to_uri from django.utils.functional import LazyObject from django.utils.importlib import import_module from django.utils.text import get_valid_filename @@ -48,7 +48,7 @@ class Storage(object): name = self._save(name, content) # Store filenames with forward slashes, even on Windows - return force_unicode(name.replace('\\', '/')) + return force_text(name.replace('\\', '/')) # These methods are part of the public API, with default implementations. diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py index 97d53482e4..3a6c632975 100644 --- a/django/core/files/uploadedfile.py +++ b/django/core/files/uploadedfile.py @@ -8,7 +8,7 @@ from io import BytesIO from django.conf import settings from django.core.files.base import File from django.core.files import temp as tempfile -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile') @@ -30,7 +30,7 @@ class UploadedFile(File): self.charset = charset def __repr__(self): - return smart_str("<%s: %s (%s)>" % ( + return smart_bytes("<%s: %s (%s)>" % ( self.__class__.__name__, self.name, self.content_type)) def _get_name(self): diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py index 7fd7d19c4a..5a6825f0a7 100644 --- a/django/core/handlers/base.py +++ b/django/core/handlers/base.py @@ -4,7 +4,7 @@ import sys from django import http from django.core import signals -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.importlib import import_module from django.utils.log import getLogger from django.utils import six @@ -250,7 +250,7 @@ def get_script_name(environ): """ from django.conf import settings if settings.FORCE_SCRIPT_NAME is not None: - return force_unicode(settings.FORCE_SCRIPT_NAME) + return force_text(settings.FORCE_SCRIPT_NAME) # If Apache's mod_rewrite had a whack at the URL, Apache set either # SCRIPT_URL or REDIRECT_URL to the full resource URL before applying any @@ -261,5 +261,5 @@ def get_script_name(environ): if not script_url: script_url = environ.get('REDIRECT_URL', '') if script_url: - return force_unicode(script_url[:-len(environ.get('PATH_INFO', ''))]) - return force_unicode(environ.get('SCRIPT_NAME', '')) + return force_text(script_url[:-len(environ.get('PATH_INFO', ''))]) + return force_text(environ.get('SCRIPT_NAME', '')) diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py index 617068a21c..70b23f8515 100644 --- a/django/core/handlers/wsgi.py +++ b/django/core/handlers/wsgi.py @@ -9,7 +9,7 @@ from django.core import signals from django.core.handlers import base from django.core.urlresolvers import set_script_prefix from django.utils import datastructures -from django.utils.encoding import force_unicode, smart_str, iri_to_uri +from django.utils.encoding import force_text, smart_bytes, iri_to_uri from django.utils.log import getLogger logger = getLogger('django.request') @@ -127,7 +127,7 @@ class LimitedStream(object): class WSGIRequest(http.HttpRequest): def __init__(self, environ): script_name = base.get_script_name(environ) - path_info = force_unicode(environ.get('PATH_INFO', '/')) + path_info = force_text(environ.get('PATH_INFO', '/')) if not path_info or path_info == script_name: # Sometimes PATH_INFO exists, but is empty (e.g. accessing # the SCRIPT_NAME URL without a trailing slash). We really need to @@ -245,6 +245,6 @@ class WSGIHandler(base.BaseHandler): status = '%s %s' % (response.status_code, status_text) response_headers = [(str(k), str(v)) for k, v in response.items()] for c in response.cookies.values(): - response_headers.append((b'Set-Cookie', str(c.output(header='')))) - start_response(smart_str(status), response_headers) + response_headers.append((str('Set-Cookie'), str(c.output(header='')))) + start_response(smart_bytes(status), response_headers) return response diff --git a/django/core/mail/message.py b/django/core/mail/message.py index 629ad464f9..b332ffba04 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -11,11 +11,10 @@ from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from email.header import Header from email.utils import formatdate, getaddresses, formataddr, parseaddr -from io import BytesIO from django.conf import settings from django.core.mail.utils import DNS_NAME -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import force_text from django.utils import six @@ -79,38 +78,38 @@ ADDRESS_HEADERS = set([ def forbid_multi_line_headers(name, val, encoding): """Forbids multi-line headers, to prevent header injection.""" encoding = encoding or settings.DEFAULT_CHARSET - val = force_unicode(val) + val = force_text(val) if '\n' in val or '\r' in val: raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name)) try: - val = val.encode('ascii') + val.encode('ascii') except UnicodeEncodeError: if name.lower() in ADDRESS_HEADERS: val = ', '.join(sanitize_address(addr, encoding) for addr in getaddresses((val,))) else: - val = str(Header(val, encoding)) + val = Header(val, encoding).encode() else: if name.lower() == 'subject': - val = Header(val) - return smart_str(name), val + val = Header(val).encode() + return str(name), val def sanitize_address(addr, encoding): if isinstance(addr, six.string_types): - addr = parseaddr(force_unicode(addr)) + addr = parseaddr(force_text(addr)) nm, addr = addr - nm = str(Header(nm, encoding)) + nm = Header(nm, encoding).encode() try: - addr = addr.encode('ascii') + addr.encode('ascii') except UnicodeEncodeError: # IDN if '@' in addr: localpart, domain = addr.split('@', 1) localpart = str(Header(localpart, encoding)) - domain = domain.encode('idna') + domain = domain.encode('idna').decode('ascii') addr = '@'.join([localpart, domain]) else: - addr = str(Header(addr, encoding)) + addr = Header(addr, encoding).encode() return formataddr((nm, addr)) @@ -132,7 +131,7 @@ class SafeMIMEText(MIMEText): This overrides the default as_string() implementation to not mangle lines that begin with 'From '. See bug #13433 for details. """ - fp = BytesIO() + fp = six.StringIO() g = Generator(fp, mangle_from_ = False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -156,7 +155,7 @@ class SafeMIMEMultipart(MIMEMultipart): This overrides the default as_string() implementation to not mangle lines that begin with 'From '. See bug #13433 for details. """ - fp = BytesIO() + fp = six.StringIO() g = Generator(fp, mangle_from_ = False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -210,8 +209,7 @@ class EmailMessage(object): def message(self): encoding = self.encoding or settings.DEFAULT_CHARSET - msg = SafeMIMEText(smart_str(self.body, encoding), - self.content_subtype, encoding) + msg = SafeMIMEText(self.body, self.content_subtype, encoding) msg = self._create_message(msg) msg['Subject'] = self.subject msg['From'] = self.extra_headers.get('From', self.from_email) @@ -293,7 +291,7 @@ class EmailMessage(object): basetype, subtype = mimetype.split('/', 1) if basetype == 'text': encoding = self.encoding or settings.DEFAULT_CHARSET - attachment = SafeMIMEText(smart_str(content, encoding), subtype, encoding) + attachment = SafeMIMEText(content, subtype, encoding) else: # Encode non-text attachments with base64. attachment = MIMEBase(basetype, subtype) @@ -313,9 +311,11 @@ class EmailMessage(object): attachment = self._create_mime_attachment(content, mimetype) if filename: try: - filename = filename.encode('ascii') + filename.encode('ascii') except UnicodeEncodeError: - filename = ('utf-8', '', filename.encode('utf-8')) + if not six.PY3: + filename = filename.encode('utf-8') + filename = ('utf-8', '', filename) attachment.add_header('Content-Disposition', 'attachment', filename=filename) return attachment diff --git a/django/core/management/__init__.py b/django/core/management/__init__.py index 68048e5672..268cd63c38 100644 --- a/django/core/management/__init__.py +++ b/django/core/management/__init__.py @@ -8,6 +8,7 @@ import warnings from django.core.management.base import BaseCommand, CommandError, handle_default_options from django.core.management.color import color_style from django.utils.importlib import import_module +from django.utils import six # For backwards compatibility: get_version() used to be in this module. from django import get_version @@ -228,7 +229,7 @@ class ManagementUtility(object): "Available subcommands:", ] commands_dict = collections.defaultdict(lambda: []) - for name, app in get_commands().iteritems(): + for name, app in six.iteritems(get_commands()): if app == 'django.core': app = 'django' else: @@ -294,7 +295,7 @@ class ManagementUtility(object): except IndexError: curr = '' - subcommands = get_commands().keys() + ['help'] + subcommands = list(get_commands()) + ['help'] options = [('--help', None)] # subcommand diff --git a/django/core/management/base.py b/django/core/management/base.py index a204f6f0bc..5e630d5207 100644 --- a/django/core/management/base.py +++ b/django/core/management/base.py @@ -6,7 +6,6 @@ be executed through ``django-admin.py`` or ``manage.py``). import os import sys -from io import BytesIO from optparse import make_option, OptionParser import traceback @@ -14,6 +13,7 @@ import django from django.core.exceptions import ImproperlyConfigured from django.core.management.color import color_style from django.utils.encoding import smart_str +from django.utils.six import StringIO class CommandError(Exception): @@ -273,7 +273,7 @@ class BaseCommand(object): """ from django.core.management.validation import get_validation_errors - s = BytesIO() + s = StringIO() num_errors = get_validation_errors(s, app) if num_errors: s.seek(0) diff --git a/django/core/management/commands/compilemessages.py b/django/core/management/commands/compilemessages.py index fdc3535cf6..b7392b9173 100644 --- a/django/core/management/commands/compilemessages.py +++ b/django/core/management/commands/compilemessages.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import codecs import os import sys @@ -7,7 +9,7 @@ from django.core.management.base import BaseCommand, CommandError def has_bom(fn): with open(fn, 'rb') as f: sample = f.read(4) - return sample[:3] == '\xef\xbb\xbf' or \ + return sample[:3] == b'\xef\xbb\xbf' or \ sample.startswith(codecs.BOM_UTF16_LE) or \ sample.startswith(codecs.BOM_UTF16_BE) diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py index fd6dbbbd2c..411042ee76 100644 --- a/django/core/management/commands/createcachetable.py +++ b/django/core/management/commands/createcachetable.py @@ -4,7 +4,7 @@ from django.core.cache.backends.db import BaseDatabaseCache from django.core.management.base import LabelCommand, CommandError from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS from django.db.utils import DatabaseError -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text class Command(LabelCommand): @@ -60,7 +60,7 @@ class Command(LabelCommand): transaction.rollback_unless_managed(using=db) raise CommandError( "Cache table '%s' could not be created.\nThe error was: %s." % - (tablename, force_unicode(e))) + (tablename, force_text(e))) for statement in index_output: curs.execute(statement) transaction.commit_unless_managed(using=db) diff --git a/django/core/management/commands/diffsettings.py b/django/core/management/commands/diffsettings.py index 98b53b405d..aa7395e5ee 100644 --- a/django/core/management/commands/diffsettings.py +++ b/django/core/management/commands/diffsettings.py @@ -22,9 +22,7 @@ class Command(NoArgsCommand): default_settings = module_to_dict(global_settings) output = [] - keys = user_settings.keys() - keys.sort() - for key in keys: + for key in sorted(user_settings.keys()): if key not in default_settings: output.append("%s = %s ###" % (key, user_settings[key])) elif user_settings[key] != default_settings[key]: diff --git a/django/core/management/commands/flush.py b/django/core/management/commands/flush.py index ac7b7a3599..b8b78434ce 100644 --- a/django/core/management/commands/flush.py +++ b/django/core/management/commands/flush.py @@ -7,6 +7,7 @@ from django.core.management.base import NoArgsCommand, CommandError from django.core.management.color import no_style from django.core.management.sql import sql_flush, emit_post_sync_signal from django.utils.importlib import import_module +from django.utils.six.moves import input class Command(NoArgsCommand): @@ -45,7 +46,7 @@ class Command(NoArgsCommand): sql_list = sql_flush(self.style, connection, only_django=True, reset_sequences=reset_sequences) if interactive: - confirm = raw_input("""You have requested a flush of the database. + confirm = input("""You have requested a flush of the database. This will IRREVERSIBLY DESTROY all data currently in the %r database, and return each table to the state it was in after syncdb. Are you sure you want to do this? diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 34f8041d33..1896e53cee 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -14,7 +14,7 @@ from django.core.management.color import no_style from django.db import (connections, router, transaction, DEFAULT_DB_ALIAS, IntegrityError, DatabaseError) from django.db.models import get_apps -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from itertools import product try: @@ -189,7 +189,7 @@ class Command(BaseCommand): 'app_label': obj.object._meta.app_label, 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, - 'error_msg': force_unicode(e) + 'error_msg': force_text(e) },) raise diff --git a/django/core/serializers/__init__.py b/django/core/serializers/__init__.py index 09bcb651d5..cf7e66190f 100644 --- a/django/core/serializers/__init__.py +++ b/django/core/serializers/__init__.py @@ -18,6 +18,7 @@ To add your own serializers, use the SERIALIZATION_MODULES setting:: from django.conf import settings from django.utils import importlib +from django.utils import six from django.core.serializers.base import SerializerDoesNotExist # Built-in serializers @@ -75,12 +76,12 @@ def get_serializer(format): def get_serializer_formats(): if not _serializers: _load_serializers() - return _serializers.keys() + return list(_serializers) def get_public_serializer_formats(): if not _serializers: _load_serializers() - return [k for k, v in _serializers.iteritems() if not v.Serializer.internal_use_only] + return [k for k, v in six.iteritems(_serializers) if not v.Serializer.internal_use_only] def get_deserializer(format): if not _serializers: diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py index 19886f7d53..bdb43db9f3 100644 --- a/django/core/serializers/base.py +++ b/django/core/serializers/base.py @@ -5,7 +5,7 @@ Module for abstract serializer/unserializer base classes. from io import BytesIO from django.db import models -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils import six class SerializerDoesNotExist(KeyError): @@ -136,10 +136,12 @@ class Deserializer(object): def __iter__(self): return self - def next(self): + def __next__(self): """Iteration iterface -- return the next item in the stream""" raise NotImplementedError + next = __next__ # Python 2 compatibility + class DeserializedObject(object): """ A deserialized model. diff --git a/django/core/serializers/json.py b/django/core/serializers/json.py index 8b56d0e7b8..3bac24d33a 100644 --- a/django/core/serializers/json.py +++ b/django/core/serializers/json.py @@ -12,7 +12,7 @@ import json from django.core.serializers.base import DeserializationError from django.core.serializers.python import Serializer as PythonSerializer from django.core.serializers.python import Deserializer as PythonDeserializer -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils import six from django.utils.timezone import is_aware diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py index 333161c929..a1fff6f9bb 100644 --- a/django/core/serializers/python.py +++ b/django/core/serializers/python.py @@ -8,7 +8,8 @@ from __future__ import unicode_literals from django.conf import settings from django.core.serializers import base from django.db import models, DEFAULT_DB_ALIAS -from django.utils.encoding import smart_unicode, is_protected_type +from django.utils.encoding import smart_text, is_protected_type +from django.utils import six class Serializer(base.Serializer): """ @@ -33,8 +34,8 @@ class Serializer(base.Serializer): def get_dump_object(self, obj): return { - "pk": smart_unicode(obj._get_pk_val(), strings_only=True), - "model": smart_unicode(obj._meta), + "pk": smart_text(obj._get_pk_val(), strings_only=True), + "model": smart_text(obj._meta), "fields": self._current } @@ -64,7 +65,7 @@ class Serializer(base.Serializer): if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'): m2m_value = lambda value: value.natural_key() else: - m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True) + m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True) self._current[field.name] = [m2m_value(related) for related in getattr(obj, field.name).iterator()] @@ -87,9 +88,9 @@ def Deserializer(object_list, **options): m2m_data = {} # Handle each field - for (field_name, field_value) in d["fields"].iteritems(): + for (field_name, field_value) in six.iteritems(d["fields"]): if isinstance(field_value, str): - field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) + field_value = smart_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) @@ -97,19 +98,19 @@ def Deserializer(object_list, **options): if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') and not isinstance(value, six.text_type): return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk else: - return smart_unicode(field.rel.to._meta.pk.to_python(value)) + return smart_text(field.rel.to._meta.pk.to_python(value)) else: - m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) + m2m_convert = lambda v: smart_text(field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): - if hasattr(field_value, '__iter__'): + if hasattr(field_value, '__iter__') and not isinstance(field_value, six.text_type): obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that diff --git a/django/core/serializers/pyyaml.py b/django/core/serializers/pyyaml.py index ac0e6cf82d..9be1ea4492 100644 --- a/django/core/serializers/pyyaml.py +++ b/django/core/serializers/pyyaml.py @@ -12,7 +12,7 @@ from django.db import models from django.core.serializers.base import DeserializationError from django.core.serializers.python import Serializer as PythonSerializer from django.core.serializers.python import Deserializer as PythonDeserializer -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils import six diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index 9d9c023b64..666587dc77 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -8,7 +8,7 @@ from django.conf import settings from django.core.serializers import base from django.db import models, DEFAULT_DB_ALIAS from django.utils.xmlutils import SimplerXMLGenerator -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from xml.dom import pulldom class Serializer(base.Serializer): @@ -46,11 +46,11 @@ class Serializer(base.Serializer): self.indent(1) obj_pk = obj._get_pk_val() if obj_pk is None: - attrs = {"model": smart_unicode(obj._meta),} + attrs = {"model": smart_text(obj._meta),} else: attrs = { - "pk": smart_unicode(obj._get_pk_val()), - "model": smart_unicode(obj._meta), + "pk": smart_text(obj._get_pk_val()), + "model": smart_text(obj._meta), } self.xml.startElement("object", attrs) @@ -96,10 +96,10 @@ class Serializer(base.Serializer): # Iterable natural keys are rolled out as subelements for key_value in related: self.xml.startElement("natural", {}) - self.xml.characters(smart_unicode(key_value)) + self.xml.characters(smart_text(key_value)) self.xml.endElement("natural") else: - self.xml.characters(smart_unicode(related_att)) + self.xml.characters(smart_text(related_att)) else: self.xml.addQuickElement("None") self.xml.endElement("field") @@ -120,13 +120,13 @@ class Serializer(base.Serializer): self.xml.startElement("object", {}) for key_value in natural: self.xml.startElement("natural", {}) - self.xml.characters(smart_unicode(key_value)) + self.xml.characters(smart_text(key_value)) self.xml.endElement("natural") self.xml.endElement("object") else: def handle_m2m(value): self.xml.addQuickElement("object", attrs={ - 'pk' : smart_unicode(value._get_pk_val()) + 'pk' : smart_text(value._get_pk_val()) }) for relobj in getattr(obj, field.name).iterator(): handle_m2m(relobj) @@ -141,7 +141,7 @@ class Serializer(base.Serializer): self.xml.startElement("field", { "name" : field.name, "rel" : field.rel.__class__.__name__, - "to" : smart_unicode(field.rel.to._meta), + "to" : smart_text(field.rel.to._meta), }) class Deserializer(base.Deserializer): @@ -154,13 +154,15 @@ class Deserializer(base.Deserializer): self.event_stream = pulldom.parse(self.stream) self.db = options.pop('using', DEFAULT_DB_ALIAS) - def next(self): + def __next__(self): for event, node in self.event_stream: if event == "START_ELEMENT" and node.nodeName == "object": self.event_stream.expandNode(node) return self._handle_object(node) raise StopIteration + next = __next__ # Python 2 compatibility + def _handle_object(self, node): """ Convert an <object> node to a DeserializedObject. diff --git a/django/core/signing.py b/django/core/signing.py index cd9759e536..9ab8c5b8b0 100644 --- a/django/core/signing.py +++ b/django/core/signing.py @@ -41,7 +41,7 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils import baseconv from django.utils.crypto import constant_time_compare, salted_hmac -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils.importlib import import_module @@ -135,7 +135,7 @@ def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, ma """ Reverse of dumps(), raises BadSignature if signature fails """ - base64d = smart_str( + base64d = smart_bytes( TimestampSigner(key, salt=salt).unsign(s, max_age=max_age)) decompress = False if base64d[0] == '.': @@ -159,16 +159,16 @@ class Signer(object): return base64_hmac(self.salt + 'signer', value, self.key) def sign(self, value): - value = smart_str(value) + value = smart_bytes(value) return '%s%s%s' % (value, self.sep, self.signature(value)) def unsign(self, signed_value): - signed_value = smart_str(signed_value) + signed_value = smart_bytes(signed_value) if not self.sep in signed_value: raise BadSignature('No "%s" found in value' % self.sep) value, sig = signed_value.rsplit(self.sep, 1) if constant_time_compare(sig, self.signature(value)): - return force_unicode(value) + return force_text(value) raise BadSignature('Signature "%s" does not match' % sig) @@ -178,7 +178,7 @@ class TimestampSigner(Signer): return baseconv.base62.encode(int(time.time())) def sign(self, value): - value = smart_str('%s%s%s' % (value, self.sep, self.timestamp())) + value = smart_bytes('%s%s%s' % (value, self.sep, self.timestamp())) return '%s%s%s' % (value, self.sep, self.signature(value)) def unsign(self, value, max_age=None): diff --git a/django/core/urlresolvers.py b/django/core/urlresolvers.py index 7397cf3b3d..2fe744e8eb 100644 --- a/django/core/urlresolvers.py +++ b/django/core/urlresolvers.py @@ -14,7 +14,7 @@ from threading import local from django.http import Http404 from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist from django.utils.datastructures import MultiValueDict -from django.utils.encoding import iri_to_uri, force_unicode, smart_str +from django.utils.encoding import iri_to_uri, force_text, smart_bytes from django.utils.functional import memoize, lazy from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule @@ -163,7 +163,7 @@ class LocaleRegexProvider(object): if isinstance(self._regex, six.string_types): regex = self._regex else: - regex = force_unicode(self._regex) + regex = force_text(self._regex) try: compiled_regex = re.compile(regex, re.UNICODE) except re.error as e: @@ -190,7 +190,7 @@ class RegexURLPattern(LocaleRegexProvider): self.name = name def __repr__(self): - return smart_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern)) + return smart_bytes('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern)) def add_prefix(self, prefix): """ @@ -240,7 +240,7 @@ class RegexURLResolver(LocaleRegexProvider): self._app_dict = {} def __repr__(self): - return smart_str('<%s %s (%s:%s) %s>' % (self.__class__.__name__, self.urlconf_name, self.app_name, self.namespace, self.regex.pattern)) + return smart_bytes('<%s %s (%s:%s) %s>' % (self.__class__.__name__, self.urlconf_name, self.app_name, self.namespace, self.regex.pattern)) def _populate(self): lookups = MultiValueDict() @@ -373,10 +373,10 @@ class RegexURLResolver(LocaleRegexProvider): if args: if len(args) != len(params) + len(prefix_args): continue - unicode_args = [force_unicode(val) for val in args] + unicode_args = [force_text(val) for val in args] candidate = (prefix_norm + result) % dict(zip(prefix_args + params, unicode_args)) else: - if set(kwargs.keys() + defaults.keys()) != set(params + defaults.keys() + prefix_args): + if set(kwargs.keys()) | set(defaults.keys()) != set(params) | set(defaults.keys()) | set(prefix_args): continue matches = True for k, v in defaults.items(): @@ -385,7 +385,7 @@ class RegexURLResolver(LocaleRegexProvider): break if not matches: continue - unicode_kwargs = dict([(k, force_unicode(v)) for (k, v) in kwargs.items()]) + unicode_kwargs = dict([(k, force_text(v)) for (k, v) in kwargs.items()]) candidate = (prefix_norm + result) % unicode_kwargs if re.search('^%s%s' % (_prefix, pattern), candidate, re.UNICODE): return candidate diff --git a/django/core/validators.py b/django/core/validators.py index 03ff8be3bc..456fa3cec5 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -8,7 +8,7 @@ except ImportError: # Python 2 from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.ipv6 import is_valid_ipv6_address from django.utils import six @@ -36,7 +36,7 @@ class RegexValidator(object): """ Validates that the input matches the regular expression. """ - if not self.regex.search(smart_unicode(value)): + if not self.regex.search(smart_text(value)): raise ValidationError(self.message, code=self.code) class URLValidator(RegexValidator): @@ -54,7 +54,7 @@ class URLValidator(RegexValidator): except ValidationError as e: # Trivial case failed. Try for possible IDN domain if value: - value = smart_unicode(value) + value = smart_text(value) scheme, netloc, path, query, fragment = urlsplit(value) try: netloc = netloc.encode('idna') # IDN -> ACE @@ -138,7 +138,7 @@ def ip_address_validators(protocol, unpack_ipv4): return ip_address_validator_map[protocol.lower()] except KeyError: raise ValueError("The protocol '%s' is unknown. Supported: %s" - % (protocol, ip_address_validator_map.keys())) + % (protocol, list(ip_address_validator_map))) comma_separated_int_list_re = re.compile('^[\d,]+$') validate_comma_separated_integer_list = RegexValidator(comma_separated_int_list_re, _('Enter only digits separated by commas.'), 'invalid') diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index b9c642d093..5719f51445 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -621,16 +621,16 @@ class BaseDatabaseOperations(object): exists for database backends to provide a better implementation according to their own quoting schemes. """ - from django.utils.encoding import smart_unicode, force_unicode + from django.utils.encoding import smart_text, force_text # Convert params to contain Unicode values. - to_unicode = lambda s: force_unicode(s, strings_only=True, errors='replace') + to_unicode = lambda s: force_text(s, strings_only=True, errors='replace') if isinstance(params, (list, tuple)): u_params = tuple([to_unicode(val) for val in params]) else: u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()]) - return smart_unicode(sql) % u_params + return smart_text(sql) % u_params def last_insert_id(self, cursor, table_name, pk_name): """ @@ -814,8 +814,8 @@ class BaseDatabaseOperations(object): def prep_for_like_query(self, x): """Prepares a value for use in a LIKE query.""" - from django.utils.encoding import smart_unicode - return smart_unicode(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_") + from django.utils.encoding import smart_text + return smart_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_") # Same as prep_for_like_query(), but called for "iexact" matches, which # need not necessarily be implemented using "LIKE" in the backend. diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py index 4dffd78f44..0cc01cc876 100644 --- a/django/db/backends/creation.py +++ b/django/db/backends/creation.py @@ -3,6 +3,7 @@ import time from django.conf import settings from django.db.utils import load_backend +from django.utils.six.moves import input # The prefix to put on the default database name when creating # the test database. @@ -331,7 +332,7 @@ class BaseDatabaseCreation(object): sys.stderr.write( "Got an error creating the test database: %s\n" % e) if not autoclobber: - confirm = raw_input( + confirm = input( "Type 'yes' if you would like to try deleting the test " "database '%s', or 'no' to cancel: " % test_database_name) if autoclobber or confirm == 'yes': diff --git a/django/db/backends/mysql/introspection.py b/django/db/backends/mysql/introspection.py index e6f18b819f..6aab0b99ab 100644 --- a/django/db/backends/mysql/introspection.py +++ b/django/db/backends/mysql/introspection.py @@ -1,4 +1,5 @@ from django.db.backends import BaseDatabaseIntrospection +from django.utils import six from MySQLdb import ProgrammingError, OperationalError from MySQLdb.constants import FIELD_TYPE import re @@ -79,7 +80,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection): """ Returns the name of the primary key column for the given table """ - for column in self.get_indexes(cursor, table_name).iteritems(): + for column in six.iteritems(self.get_indexes(cursor, table_name)): if column[1]['primary_key']: return column[0] return None diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index b08113fed7..89cad12b6c 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -53,7 +53,7 @@ from django.db.backends.signals import connection_created from django.db.backends.oracle.client import DatabaseClient from django.db.backends.oracle.creation import DatabaseCreation from django.db.backends.oracle.introspection import DatabaseIntrospection -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import smart_bytes, force_text from django.utils import six from django.utils import timezone @@ -64,9 +64,9 @@ IntegrityError = Database.IntegrityError # Check whether cx_Oracle was compiled with the WITH_UNICODE option. This will # also be True in Python 3.0. if int(Database.version.split('.', 1)[0]) >= 5 and not hasattr(Database, 'UNICODE'): - convert_unicode = force_unicode + convert_unicode = force_text else: - convert_unicode = smart_str + convert_unicode = smart_bytes class DatabaseFeatures(BaseDatabaseFeatures): @@ -162,7 +162,7 @@ WHEN (new.%(col_name)s IS NULL) if isinstance(value, Database.LOB): value = value.read() if field and field.get_internal_type() == 'TextField': - value = force_unicode(value) + value = force_text(value) # Oracle stores empty strings as null. We need to undo this in # order to adhere to the Django convention of using the empty @@ -245,7 +245,7 @@ WHEN (new.%(col_name)s IS NULL) def process_clob(self, value): if value is None: return '' - return force_unicode(value.read()) + return force_text(value.read()) def quote_name(self, name): # SQL92 requires delimited (quoted) names to be case-sensitive. When @@ -595,9 +595,9 @@ class OracleParam(object): param = param.astimezone(timezone.utc).replace(tzinfo=None) if hasattr(param, 'bind_parameter'): - self.smart_str = param.bind_parameter(cursor) + self.smart_bytes = param.bind_parameter(cursor) else: - self.smart_str = convert_unicode(param, cursor.charset, + self.smart_bytes = convert_unicode(param, cursor.charset, strings_only) if hasattr(param, 'input_size'): # If parameter has `input_size` attribute, use that. @@ -676,7 +676,7 @@ class FormatStylePlaceholderCursor(object): self.setinputsizes(*sizes) def _param_generator(self, params): - return [p.smart_str for p in params] + return [p.smart_bytes for p in params] def execute(self, query, params=None): if params is None: @@ -774,9 +774,11 @@ class CursorIterator(object): def __iter__(self): return self - def next(self): + def __next__(self): return _rowfactory(next(self.iter), self.cursor) + next = __next__ # Python 2 compatibility + def _rowfactory(row, cursor): # Cast numeric values as the appropriate Python type based upon the @@ -831,7 +833,7 @@ def to_unicode(s): unchanged). """ if isinstance(s, six.string_types): - return force_unicode(s) + return force_text(s) return s diff --git a/django/db/backends/oracle/creation.py b/django/db/backends/oracle/creation.py index 2f096f735a..d9bf3dfea2 100644 --- a/django/db/backends/oracle/creation.py +++ b/django/db/backends/oracle/creation.py @@ -1,6 +1,7 @@ import sys import time from django.db.backends.creation import BaseDatabaseCreation +from django.utils.six.moves import input TEST_DATABASE_PREFIX = 'test_' PASSWORD = 'Im_a_lumberjack' @@ -65,7 +66,7 @@ class DatabaseCreation(BaseDatabaseCreation): except Exception as e: sys.stderr.write("Got an error creating the test database: %s\n" % e) if not autoclobber: - confirm = raw_input("It appears the test database, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_NAME) + confirm = input("It appears the test database, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_NAME) if autoclobber or confirm == 'yes': try: if verbosity >= 1: @@ -87,7 +88,7 @@ class DatabaseCreation(BaseDatabaseCreation): except Exception as e: sys.stderr.write("Got an error creating the test user: %s\n" % e) if not autoclobber: - confirm = raw_input("It appears the test user, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_USER) + confirm = input("It appears the test user, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_USER) if autoclobber or confirm == 'yes': try: if verbosity >= 1: diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 0a97449789..0880079189 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -54,15 +54,15 @@ def adapt_datetime_with_timezone_support(value): default_timezone = timezone.get_default_timezone() value = timezone.make_aware(value, default_timezone) value = value.astimezone(timezone.utc).replace(tzinfo=None) - return value.isoformat(b" ") + return value.isoformat(str(" ")) -Database.register_converter(b"bool", lambda s: str(s) == '1') -Database.register_converter(b"time", parse_time) -Database.register_converter(b"date", parse_date) -Database.register_converter(b"datetime", parse_datetime_with_timezone_support) -Database.register_converter(b"timestamp", parse_datetime_with_timezone_support) -Database.register_converter(b"TIMESTAMP", parse_datetime_with_timezone_support) -Database.register_converter(b"decimal", util.typecast_decimal) +Database.register_converter(str("bool"), lambda s: str(s) == '1') +Database.register_converter(str("time"), parse_time) +Database.register_converter(str("date"), parse_date) +Database.register_converter(str("datetime"), parse_datetime_with_timezone_support) +Database.register_converter(str("timestamp"), parse_datetime_with_timezone_support) +Database.register_converter(str("TIMESTAMP"), parse_datetime_with_timezone_support) +Database.register_converter(str("decimal"), util.typecast_decimal) Database.register_adapter(datetime.datetime, adapt_datetime_with_timezone_support) Database.register_adapter(decimal.Decimal, util.rev_typecast_decimal) if Database.version_info >= (2, 4, 1): diff --git a/django/db/backends/sqlite3/creation.py b/django/db/backends/sqlite3/creation.py index efdc457be0..c022b56c85 100644 --- a/django/db/backends/sqlite3/creation.py +++ b/django/db/backends/sqlite3/creation.py @@ -1,6 +1,7 @@ import os import sys from django.db.backends.creation import BaseDatabaseCreation +from django.utils.six.moves import input class DatabaseCreation(BaseDatabaseCreation): # SQLite doesn't actually support most of these types, but it "does the right @@ -53,7 +54,7 @@ class DatabaseCreation(BaseDatabaseCreation): print("Destroying old test database '%s'..." % self.connection.alias) if os.access(test_database_name, os.F_OK): if not autoclobber: - confirm = raw_input("Type 'yes' if you would like to try deleting the test database '%s', or 'no' to cancel: " % test_database_name) + confirm = input("Type 'yes' if you would like to try deleting the test database '%s', or 'no' to cancel: " % test_database_name) if autoclobber or confirm == 'yes': try: os.remove(test_database_name) diff --git a/django/db/models/base.py b/django/db/models/base.py index 002e2aff65..569b8e876c 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -23,11 +23,35 @@ from django.db.models import signals from django.db.models.loading import register_models, get_model from django.utils.translation import ugettext_lazy as _ from django.utils.functional import curry -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import smart_bytes, force_text from django.utils import six from django.utils.text import get_text_list, capfirst +def subclass_exception(name, parents, module, attached_to=None): + """ + Create exception subclass. Used by ModelBase below. + + If 'attached_to' is supplied, the exception will be created in a way that + allows it to be pickled, assuming the returned exception class will be added + as an attribute to the 'attached_to' class. + """ + class_dict = {'__module__': module} + if attached_to is not None: + def __reduce__(self): + # Exceptions are special - they've got state that isn't + # in self.__dict__. We assume it is all in self.args. + return (unpickle_inner_exception, (attached_to, name), self.args) + + def __setstate__(self, args): + self.args = args + + class_dict['__reduce__'] = __reduce__ + class_dict['__setstate__'] = __setstate__ + + return type(name, parents, class_dict) + + class ModelBase(type): """ Metaclass for all models. @@ -63,12 +87,12 @@ class ModelBase(type): new_class.add_to_class('_meta', Options(meta, **kwargs)) if not abstract: - new_class.add_to_class('DoesNotExist', subclass_exception(b'DoesNotExist', + new_class.add_to_class('DoesNotExist', subclass_exception(str('DoesNotExist'), tuple(x.DoesNotExist for x in parents if hasattr(x, '_meta') and not x._meta.abstract) or (ObjectDoesNotExist,), module, attached_to=new_class)) - new_class.add_to_class('MultipleObjectsReturned', subclass_exception(b'MultipleObjectsReturned', + new_class.add_to_class('MultipleObjectsReturned', subclass_exception(str('MultipleObjectsReturned'), tuple(x.MultipleObjectsReturned for x in parents if hasattr(x, '_meta') and not x._meta.abstract) or (MultipleObjectsReturned,), @@ -364,14 +388,14 @@ class Model(six.with_metaclass(ModelBase, object)): setattr(self, field.attname, val) if kwargs: - for prop in kwargs.keys(): + for prop in list(kwargs): try: if isinstance(getattr(self.__class__, prop), property): setattr(self, prop, kwargs.pop(prop)) except AttributeError: pass if kwargs: - raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0]) + raise TypeError("'%s' is an invalid keyword argument for this function" % list(kwargs)[0]) super(Model, self).__init__() signals.post_init.send(sender=self.__class__, instance=self) @@ -380,11 +404,11 @@ class Model(six.with_metaclass(ModelBase, object)): u = six.text_type(self) except (UnicodeEncodeError, UnicodeDecodeError): u = '[Bad Unicode data]' - return smart_str('<%s: %s>' % (self.__class__.__name__, u)) + return smart_bytes('<%s: %s>' % (self.__class__.__name__, u)) def __str__(self): if hasattr(self, '__unicode__'): - return force_unicode(self).encode('utf-8') + return force_text(self).encode('utf-8') return '%s object' % self.__class__.__name__ def __eq__(self, other): @@ -605,14 +629,14 @@ class Model(six.with_metaclass(ModelBase, object)): def _get_FIELD_display(self, field): value = getattr(self, field.attname) - return force_unicode(dict(field.flatchoices).get(value, value), strings_only=True) + return force_text(dict(field.flatchoices).get(value, value), strings_only=True) def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs): if not self.pk: raise ValueError("get_next/get_previous cannot be used on unsaved objects.") op = is_next and 'gt' or 'lt' order = not is_next and '-' or '' - param = smart_str(getattr(self, field.attname)) + param = smart_bytes(getattr(self, field.attname)) q = Q(**{'%s__%s' % (field.name, op): param}) q = q|Q(**{field.name: param, 'pk__%s' % op: self.pk}) qs = self.__class__._default_manager.using(self._state.db).filter(**kwargs).filter(q).order_by('%s%s' % (order, field.name), '%spk' % order) @@ -929,29 +953,6 @@ def model_unpickle(model, attrs): return cls.__new__(cls) model_unpickle.__safe_for_unpickle__ = True -def subclass_exception(name, parents, module, attached_to=None): - """ - Create exception subclass. - - If 'attached_to' is supplied, the exception will be created in a way that - allows it to be pickled, assuming the returned exception class will be added - as an attribute to the 'attached_to' class. - """ - class_dict = {'__module__': module} - if attached_to is not None: - def __reduce__(self): - # Exceptions are special - they've got state that isn't - # in self.__dict__. We assume it is all in self.args. - return (unpickle_inner_exception, (attached_to, name), self.args) - - def __setstate__(self, args): - self.args = args - - class_dict['__reduce__'] = __reduce__ - class_dict['__setstate__'] = __setstate__ - - return type(name, parents, class_dict) - def unpickle_inner_exception(klass, exception_name): # Get the exception class from the class it is attached to: exception = getattr(klass, exception_name) diff --git a/django/db/models/deletion.py b/django/db/models/deletion.py index d8bb8f2e66..4449b75a81 100644 --- a/django/db/models/deletion.py +++ b/django/db/models/deletion.py @@ -4,6 +4,7 @@ from operator import attrgetter from django.db import connections, transaction, IntegrityError from django.db.models import signals, sql from django.utils.datastructures import SortedDict +from django.utils import six class ProtectedError(IntegrityError): @@ -157,7 +158,7 @@ class Collector(object): # Recursively collect concrete model's parent models, but not their # related objects. These will be found by meta.get_all_related_objects() concrete_model = model._meta.concrete_model - for ptr in concrete_model._meta.parents.itervalues(): + for ptr in six.itervalues(concrete_model._meta.parents): if ptr: parent_objs = [getattr(obj, ptr.name) for obj in new_objs] self.collect(parent_objs, source=model, @@ -199,14 +200,14 @@ class Collector(object): ) def instances_with_model(self): - for model, instances in self.data.iteritems(): + for model, instances in six.iteritems(self.data): for obj in instances: yield model, obj def sort(self): sorted_models = [] concrete_models = set() - models = self.data.keys() + models = list(self.data) while len(sorted_models) < len(models): found = False for model in models: @@ -241,24 +242,24 @@ class Collector(object): ) # update fields - for model, instances_for_fieldvalues in self.field_updates.iteritems(): + for model, instances_for_fieldvalues in six.iteritems(self.field_updates): query = sql.UpdateQuery(model) - for (field, value), instances in instances_for_fieldvalues.iteritems(): + for (field, value), instances in six.iteritems(instances_for_fieldvalues): query.update_batch([obj.pk for obj in instances], {field.name: value}, self.using) # reverse instance collections - for instances in self.data.itervalues(): + for instances in six.itervalues(self.data): instances.reverse() # delete batches - for model, batches in self.batches.iteritems(): + for model, batches in six.iteritems(self.batches): query = sql.DeleteQuery(model) - for field, instances in batches.iteritems(): + for field, instances in six.iteritems(batches): query.delete_batch([obj.pk for obj in instances], self.using, field) # delete instances - for model, instances in self.data.iteritems(): + for model, instances in six.iteritems(self.data): query = sql.DeleteQuery(model) pk_list = [obj.pk for obj in instances] query.delete_batch(pk_list, self.using) @@ -271,10 +272,10 @@ class Collector(object): ) # update collected instances - for model, instances_for_fieldvalues in self.field_updates.iteritems(): - for (field, value), instances in instances_for_fieldvalues.iteritems(): + for model, instances_for_fieldvalues in six.iteritems(self.field_updates): + for (field, value), instances in six.iteritems(instances_for_fieldvalues): for obj in instances: setattr(obj, field.attname, value) - for model, instances in self.data.iteritems(): + for model, instances in six.iteritems(self.data): for instance in instances: setattr(instance, model._meta.pk.attname, None) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 9606b1b843..d07851bbf5 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -19,7 +19,7 @@ from django.utils.functional import curry, total_ordering from django.utils.text import capfirst from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from django.utils.encoding import smart_unicode, force_unicode +from django.utils.encoding import smart_text, force_text from django.utils.ipv6 import clean_ipv6_address from django.utils import six @@ -135,6 +135,8 @@ class Field(object): return self.creation_counter < other.creation_counter return NotImplemented + __hash__ = object.__hash__ + def __deepcopy__(self, memodict): # We don't have to deepcopy very much here, since most things are not # intended to be altered after initial creation. @@ -179,7 +181,8 @@ class Field(object): if not self.editable: # Skip validation for non-editable fields. return - if self._choices and value: + + if self._choices and value not in validators.EMPTY_VALUES: for option_key, option_value in self.choices: if isinstance(option_value, (list, tuple)): # This is an optgroup, so look inside the group for @@ -385,7 +388,7 @@ class Field(object): if self.has_default(): if callable(self.default): return self.default() - return force_unicode(self.default, strings_only=True) + return force_text(self.default, strings_only=True) if (not self.empty_strings_allowed or (self.null and not connection.features.interprets_empty_strings_as_nulls)): return None @@ -403,11 +406,11 @@ class Field(object): rel_model = self.rel.to if hasattr(self.rel, 'get_related_field'): lst = [(getattr(x, self.rel.get_related_field().attname), - smart_unicode(x)) + smart_text(x)) for x in rel_model._default_manager.complex_filter( self.rel.limit_choices_to)] else: - lst = [(x._get_pk_val(), smart_unicode(x)) + lst = [(x._get_pk_val(), smart_text(x)) for x in rel_model._default_manager.complex_filter( self.rel.limit_choices_to)] return first_choice + lst @@ -434,7 +437,7 @@ class Field(object): Returns a string value of this field from the passed obj. This is used by the serialization framework. """ - return smart_unicode(self._get_val_from_obj(obj)) + return smart_text(self._get_val_from_obj(obj)) def bind(self, fieldmapping, original, bound_field_class): return bound_field_class(self, fieldmapping, original) @@ -486,7 +489,7 @@ class Field(object): # Many of the subclass-specific formfield arguments (min_value, # max_value) don't apply for choice fields, so be sure to only pass # the values that TypedChoiceField will understand. - for k in kwargs.keys(): + for k in list(kwargs): if k not in ('coerce', 'empty_value', 'choices', 'required', 'widget', 'label', 'initial', 'help_text', 'error_messages', 'show_hidden_initial'): @@ -628,7 +631,7 @@ class CharField(Field): def to_python(self, value): if isinstance(value, six.string_types) or value is None: return value - return smart_unicode(value) + return smart_text(value) def get_prep_value(self, value): return self.to_python(value) @@ -1188,7 +1191,7 @@ class TextField(Field): def get_prep_value(self, value): if isinstance(value, six.string_types) or value is None: return value - return smart_unicode(value) + return smart_text(value) def formfield(self, **kwargs): defaults = {'widget': forms.Textarea} diff --git a/django/db/models/fields/files.py b/django/db/models/fields/files.py index b51ef1d5d6..ad4c36ca0d 100644 --- a/django/db/models/fields/files.py +++ b/django/db/models/fields/files.py @@ -8,7 +8,7 @@ from django.core.files.base import File from django.core.files.storage import default_storage from django.core.files.images import ImageFile from django.db.models import signals -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils import six from django.utils.translation import ugettext_lazy as _ @@ -280,7 +280,7 @@ class FileField(Field): setattr(cls, self.name, self.descriptor_class(self)) def get_directory_name(self): - return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to)))) + return os.path.normpath(force_text(datetime.datetime.now().strftime(smart_bytes(self.upload_to)))) def get_filename(self, filename): return os.path.normpath(self.storage.get_valid_name(os.path.basename(filename))) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 2a2502b54f..08cc0a747f 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -9,7 +9,7 @@ from django.db.models.related import RelatedObject from django.db.models.query import QuerySet from django.db.models.query_utils import QueryWrapper from django.db.models.deletion import CASCADE -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils import six from django.utils.translation import ugettext_lazy as _, string_concat from django.utils.functional import curry, cached_property @@ -241,7 +241,7 @@ class SingleRelatedObjectDescriptor(object): rel_obj_attr = attrgetter(self.related.field.attname) instance_attr = lambda obj: obj._get_pk_val() instances_dict = dict((instance_attr(inst), inst) for inst in instances) - params = {'%s__pk__in' % self.related.field.name: instances_dict.keys()} + params = {'%s__pk__in' % self.related.field.name: list(instances_dict)} qs = self.get_query_set(instance=instances[0]).filter(**params) # Since we're going to assign directly in the cache, # we must manage the reverse relation cache manually. @@ -335,9 +335,9 @@ class ReverseSingleRelatedObjectDescriptor(object): instance_attr = attrgetter(self.field.attname) instances_dict = dict((instance_attr(inst), inst) for inst in instances) if other_field.rel: - params = {'%s__pk__in' % self.field.rel.field_name: instances_dict.keys()} + params = {'%s__pk__in' % self.field.rel.field_name: list(instances_dict)} else: - params = {'%s__in' % self.field.rel.field_name: instances_dict.keys()} + params = {'%s__in' % self.field.rel.field_name: list(instances_dict)} qs = self.get_query_set(instance=instances[0]).filter(**params) # Since we're going to assign directly in the cache, # we must manage the reverse relation cache manually. @@ -488,7 +488,7 @@ class ForeignRelatedObjectsDescriptor(object): instance_attr = attrgetter(attname) instances_dict = dict((instance_attr(inst), inst) for inst in instances) db = self._db or router.db_for_read(self.model, instance=instances[0]) - query = {'%s__%s__in' % (rel_field.name, attname): instances_dict.keys()} + query = {'%s__%s__in' % (rel_field.name, attname): list(instances_dict)} qs = super(RelatedManager, self).get_query_set().using(db).filter(**query) # Since we just bypassed this class' get_query_set(), we must manage # the reverse relation manually. @@ -999,7 +999,7 @@ class ForeignKey(RelatedField, Field): if not self.blank and self.choices: choice_list = self.get_choices_default() if len(choice_list) == 2: - return smart_unicode(choice_list[1][0]) + return smart_text(choice_list[1][0]) return Field.value_to_string(self, obj) def contribute_to_class(self, cls, name): @@ -1205,7 +1205,7 @@ class ManyToManyField(RelatedField, Field): choices_list = self.get_choices_default() if len(choices_list) == 1: data = [choices_list[0][0]] - return smart_unicode(data) + return smart_text(data) def contribute_to_class(self, cls, name): # To support multiple relations to self, it's useful to have a non-None diff --git a/django/db/models/loading.py b/django/db/models/loading.py index d651584e7a..7a9cb2cb41 100644 --- a/django/db/models/loading.py +++ b/django/db/models/loading.py @@ -5,6 +5,7 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.datastructures import SortedDict from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule +from django.utils import six import imp import sys @@ -193,9 +194,9 @@ class AppCache(object): else: if only_installed: app_list = [self.app_models.get(app_label, SortedDict()) - for app_label in self.app_labels.iterkeys()] + for app_label in six.iterkeys(self.app_labels)] else: - app_list = self.app_models.itervalues() + app_list = six.itervalues(self.app_models) model_list = [] for app in app_list: model_list.extend( diff --git a/django/db/models/options.py b/django/db/models/options.py index 7308a15c6b..2e8ccb49ce 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -8,7 +8,7 @@ from django.db.models.fields import AutoField, FieldDoesNotExist from django.db.models.fields.proxy import OrderWrt from django.db.models.loading import get_models, app_cache_ready from django.utils.translation import activate, deactivate_all, get_language, string_concat -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text, smart_bytes from django.utils.datastructures import SortedDict from django.utils import six @@ -127,7 +127,7 @@ class Options(object): if self.parents: # Promote the first parent link in lieu of adding yet another # field. - field = next(self.parents.itervalues()) + field = next(six.itervalues(self.parents)) # Look for a local field with the same name as the # first parent link. If a local field has already been # created, use it instead of promoting the parent @@ -147,13 +147,13 @@ class Options(object): # self.duplicate_targets will map each duplicate field column to the # columns it duplicates. collections = {} - for column, target in self.duplicate_targets.iteritems(): + for column, target in six.iteritems(self.duplicate_targets): try: collections[target].add(column) except KeyError: collections[target] = set([column]) self.duplicate_targets = {} - for elt in collections.itervalues(): + for elt in six.itervalues(collections): if len(elt) == 1: continue for column in elt: @@ -199,7 +199,7 @@ class Options(object): return '<Options for %s>' % self.object_name def __str__(self): - return "%s.%s" % (smart_str(self.app_label), smart_str(self.module_name)) + return "%s.%s" % (smart_bytes(self.app_label), smart_bytes(self.module_name)) def verbose_name_raw(self): """ @@ -209,7 +209,7 @@ class Options(object): """ lang = get_language() deactivate_all() - raw = force_unicode(self.verbose_name) + raw = force_text(self.verbose_name) activate(lang) return raw verbose_name_raw = property(verbose_name_raw) @@ -258,7 +258,7 @@ class Options(object): self._m2m_cache except AttributeError: self._fill_m2m_cache() - return self._m2m_cache.keys() + return list(self._m2m_cache) many_to_many = property(_many_to_many) def get_m2m_with_model(self): @@ -269,7 +269,7 @@ class Options(object): self._m2m_cache except AttributeError: self._fill_m2m_cache() - return self._m2m_cache.items() + return list(six.iteritems(self._m2m_cache)) def _fill_m2m_cache(self): cache = SortedDict() @@ -326,8 +326,7 @@ class Options(object): cache = self._name_map except AttributeError: cache = self.init_name_map() - names = cache.keys() - names.sort() + names = sorted(cache.keys()) # Internal-only names end with "+" (symmetrical m2m related names being # the main example). Trim them. return [val for val in names if not val.endswith('+')] @@ -417,7 +416,7 @@ class Options(object): cache = self._fill_related_many_to_many_cache() if local_only: return [k for k, v in cache.items() if not v] - return cache.keys() + return list(cache) def get_all_related_m2m_objects_with_model(self): """ @@ -428,7 +427,7 @@ class Options(object): cache = self._related_many_to_many_cache except AttributeError: cache = self._fill_related_many_to_many_cache() - return cache.items() + return list(six.iteritems(cache)) def _fill_related_many_to_many_cache(self): cache = SortedDict() diff --git a/django/db/models/query.py b/django/db/models/query.py index 8b6b42b7b1..e8d6ae2a7b 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -120,7 +120,7 @@ class QuerySet(object): if len(self._result_cache) <= pos: self._fill_cache() - def __nonzero__(self): + def __bool__(self): if self._prefetch_related_lookups and not self._prefetch_done: # We need all the results in order to be able to do the prefetch # in one go. To minimize code duplication, we use the __len__ @@ -134,6 +134,7 @@ class QuerySet(object): except StopIteration: return False return True + __nonzero__ = __bool__ # Python 2 def __contains__(self, val): # The 'in' operator works without this method, due to __iter__. This @@ -245,8 +246,8 @@ class QuerySet(object): requested = None max_depth = self.query.max_depth - extra_select = self.query.extra_select.keys() - aggregate_select = self.query.aggregate_select.keys() + extra_select = list(self.query.extra_select) + aggregate_select = list(self.query.aggregate_select) only_load = self.query.get_loaded_field_names() if not fill_cache: @@ -593,7 +594,7 @@ class QuerySet(object): flat = kwargs.pop('flat', False) if kwargs: raise TypeError('Unexpected keyword arguments to values_list: %s' - % (kwargs.keys(),)) + % (list(kwargs),)) if flat and len(fields) > 1: raise TypeError("'flat' is not valid when values_list is called with more than one field.") return self._clone(klass=ValuesListQuerySet, setup=True, flat=flat, @@ -693,7 +694,7 @@ class QuerySet(object): depth = kwargs.pop('depth', 0) if kwargs: raise TypeError('Unexpected keyword arguments to select_related: %s' - % (kwargs.keys(),)) + % (list(kwargs),)) obj = self._clone() if fields: if depth: @@ -751,7 +752,7 @@ class QuerySet(object): obj = self._clone() - obj._setup_aggregate_query(kwargs.keys()) + obj._setup_aggregate_query(list(kwargs)) # Add the aggregates to the query for (alias, aggregate_expr) in kwargs.items(): @@ -966,9 +967,9 @@ class ValuesQuerySet(QuerySet): def iterator(self): # Purge any extra columns that haven't been explicitly asked for - extra_names = self.query.extra_select.keys() + extra_names = list(self.query.extra_select) field_names = self.field_names - aggregate_names = self.query.aggregate_select.keys() + aggregate_names = list(self.query.aggregate_select) names = extra_names + field_names + aggregate_names @@ -1097,9 +1098,9 @@ class ValuesListQuerySet(ValuesQuerySet): # When extra(select=...) or an annotation is involved, the extra # cols are always at the start of the row, and we need to reorder # the fields to match the order in self._fields. - extra_names = self.query.extra_select.keys() + extra_names = list(self.query.extra_select) field_names = self.field_names - aggregate_names = self.query.aggregate_select.keys() + aggregate_names = list(self.query.aggregate_select) names = extra_names + field_names + aggregate_names @@ -1527,7 +1528,7 @@ class RawQuerySet(object): # Associate fields to values if skip: model_init_kwargs = {} - for attname, pos in model_init_field_names.iteritems(): + for attname, pos in six.iteritems(model_init_field_names): model_init_kwargs[attname] = values[pos] instance = model_cls(**model_init_kwargs) else: diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py index 60bdb2bcb4..c1a690a524 100644 --- a/django/db/models/query_utils.py +++ b/django/db/models/query_utils.py @@ -8,6 +8,7 @@ circular import difficulties. from __future__ import unicode_literals from django.db.backends import util +from django.utils import six from django.utils import tree @@ -40,7 +41,7 @@ class Q(tree.Node): default = AND def __init__(self, *args, **kwargs): - super(Q, self).__init__(children=list(args) + kwargs.items()) + super(Q, self).__init__(children=list(args) + list(six.iteritems(kwargs))) def _combine(self, other, conn): if not isinstance(other, Q): @@ -114,7 +115,7 @@ class DeferredAttribute(object): def _check_parent_chain(self, instance, name): """ - Check if the field value can be fetched from a parent field already + Check if the field value can be fetched from a parent field already loaded in the instance. This can be done if the to-be fetched field is a primary key field. """ diff --git a/django/db/models/related.py b/django/db/models/related.py index 90995d749f..a0dcec7132 100644 --- a/django/db/models/related.py +++ b/django/db/models/related.py @@ -1,4 +1,4 @@ -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.db.models.fields import BLANK_CHOICE_DASH class BoundRelatedObject(object): @@ -34,9 +34,9 @@ class RelatedObject(object): if limit_to_currently_related: queryset = queryset.complex_filter( {'%s__isnull' % self.parent_model._meta.module_name: False}) - lst = [(x._get_pk_val(), smart_unicode(x)) for x in queryset] + lst = [(x._get_pk_val(), smart_text(x)) for x in queryset] return first_choice + lst - + def get_db_prep_lookup(self, lookup_type, value, connection, prepared=False): # Defer to the actual field definition for db prep return self.field.get_db_prep_lookup(lookup_type, value, diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 7a0afa349d..1311ea546c 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -9,6 +9,7 @@ from django.db.models.sql.datastructures import EmptyResultSet from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.query import get_order_dir, Query from django.db.utils import DatabaseError +from django.utils import six class SQLCompiler(object): @@ -82,7 +83,7 @@ class SQLCompiler(object): where, w_params = self.query.where.as_sql(qn=qn, connection=self.connection) having, h_params = self.query.having.as_sql(qn=qn, connection=self.connection) params = [] - for val in self.query.extra_select.itervalues(): + for val in six.itervalues(self.query.extra_select): params.extend(val[1]) result = ['SELECT'] @@ -177,7 +178,7 @@ class SQLCompiler(object): """ qn = self.quote_name_unless_alias qn2 = self.connection.ops.quote_name - result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in self.query.extra_select.iteritems()] + result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in six.iteritems(self.query.extra_select)] aliases = set(self.query.extra_select.keys()) if with_aliases: col_aliases = aliases.copy() @@ -553,7 +554,7 @@ class SQLCompiler(object): group_by = self.query.group_by or [] extra_selects = [] - for extra_select, extra_params in self.query.extra_select.itervalues(): + for extra_select, extra_params in six.itervalues(self.query.extra_select): extra_selects.append(extra_select) params.extend(extra_params) cols = (group_by + self.query.select + diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 53dad608bf..be257a5410 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -10,8 +10,9 @@ all about the internals of models in order to get the information it needs. import copy from django.utils.datastructures import SortedDict -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.tree import Node +from django.utils import six from django.db import connections, DEFAULT_DB_ALIAS from django.db.models import signals from django.db.models.expressions import ExpressionNode @@ -602,22 +603,22 @@ class Query(object): # slight complexity here is handling fields that exist on parent # models. workset = {} - for model, values in seen.iteritems(): + for model, values in six.iteritems(seen): for field, m in model._meta.get_fields_with_model(): if field in values: continue add_to_dict(workset, m or model, field) - for model, values in must_include.iteritems(): + for model, values in six.iteritems(must_include): # If we haven't included a model in workset, we don't add the # corresponding must_include fields for that model, since an # empty set means "include all fields". That's why there's no # "else" branch here. if model in workset: workset[model].update(values) - for model, values in workset.iteritems(): + for model, values in six.iteritems(workset): callback(target, model, values) else: - for model, values in must_include.iteritems(): + for model, values in six.iteritems(must_include): if model in seen: seen[model].update(values) else: @@ -631,7 +632,7 @@ class Query(object): for model in orig_opts.get_parent_list(): if model not in seen: seen[model] = set() - for model, values in seen.iteritems(): + for model, values in six.iteritems(seen): callback(target, model, values) @@ -770,7 +771,7 @@ class Query(object): for k, aliases in self.join_map.items(): aliases = tuple([change_map.get(a, a) for a in aliases]) self.join_map[k] = aliases - for old_alias, new_alias in change_map.iteritems(): + for old_alias, new_alias in six.iteritems(change_map): alias_data = self.alias_map[old_alias] alias_data = alias_data._replace(rhs_alias=new_alias) self.alias_refcount[new_alias] = self.alias_refcount[old_alias] @@ -792,7 +793,7 @@ class Query(object): self.included_inherited_models[key] = change_map[alias] # 3. Update any joins that refer to the old alias. - for alias, data in self.alias_map.iteritems(): + for alias, data in six.iteritems(self.alias_map): lhs = data.lhs_alias if lhs in change_map: data = data._replace(lhs_alias=change_map[lhs]) @@ -842,7 +843,7 @@ class Query(object): count. Note that after execution, the reference counts are zeroed, so tables added in compiler will not be seen by this method. """ - return len([1 for count in self.alias_refcount.itervalues() if count]) + return len([1 for count in six.itervalues(self.alias_refcount) if count]) def join(self, connection, always_create=False, exclusions=(), promote=False, outer_if_first=False, nullable=False, reuse=None): @@ -1302,7 +1303,7 @@ class Query(object): field, model, direct, m2m = opts.get_field_by_name(f.name) break else: - names = opts.get_all_field_names() + self.aggregate_select.keys() + names = opts.get_all_field_names() + list(self.aggregate_select) raise FieldError("Cannot resolve keyword %r into field. " "Choices are: %s" % (name, ", ".join(names))) @@ -1571,7 +1572,7 @@ class Query(object): # Tag.objects.exclude(parent__parent__name='t1'), a tag with no parent # would otherwise be overlooked). active_positions = [pos for (pos, count) in - enumerate(query.alias_refcount.itervalues()) if count] + enumerate(six.itervalues(query.alias_refcount)) if count] if active_positions[-1] > 1: self.add_filter(('%s__isnull' % prefix, False), negate=True, trim=True, can_reuse=can_reuse) @@ -1660,8 +1661,8 @@ class Query(object): # from the model on which the lookup failed. raise else: - names = sorted(opts.get_all_field_names() + self.extra.keys() - + self.aggregate_select.keys()) + names = sorted(opts.get_all_field_names() + list(self.extra) + + list(self.aggregate_select)) raise FieldError("Cannot resolve keyword %r into field. " "Choices are: %s" % (name, ", ".join(names))) self.remove_inherited_models() @@ -1775,7 +1776,7 @@ class Query(object): else: param_iter = iter([]) for name, entry in select.items(): - entry = force_unicode(entry) + entry = force_text(entry) entry_params = [] pos = entry.find("%s") while pos != -1: diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py index 7b92394e90..937505b9b0 100644 --- a/django/db/models/sql/subqueries.py +++ b/django/db/models/sql/subqueries.py @@ -10,7 +10,8 @@ from django.db.models.sql.query import Query from django.db.models.sql.where import AND, Constraint from django.utils.datastructures import SortedDict from django.utils.functional import Promise -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text +from django.utils import six __all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery', @@ -87,7 +88,7 @@ class UpdateQuery(Query): querysets. """ values_seq = [] - for name, val in values.iteritems(): + for name, val in six.iteritems(values): field, model, direct, m2m = self.model._meta.get_field_by_name(name) if not direct or m2m: raise FieldError('Cannot update model field %r (only non-relations and foreign keys permitted).' % field) @@ -104,7 +105,7 @@ class UpdateQuery(Query): saving models. """ # Check that no Promise object passes to the query. Refs #10498. - values_seq = [(value[0], value[1], force_unicode(value[2])) + values_seq = [(value[0], value[1], force_text(value[2])) if isinstance(value[2], Promise) else value for value in values_seq] self.values.extend(values_seq) @@ -129,7 +130,7 @@ class UpdateQuery(Query): if not self.related_updates: return [] result = [] - for model, values in self.related_updates.iteritems(): + for model, values in six.iteritems(self.related_updates): query = UpdateQuery(model) query.values = values if self.related_ids is not None: @@ -170,7 +171,7 @@ class InsertQuery(Query): for obj in objs: value = getattr(obj, field.attname) if isinstance(value, Promise): - setattr(obj, field.attname, force_unicode(value)) + setattr(obj, field.attname, force_text(value)) self.objs = objs self.raw = raw diff --git a/django/dispatch/saferef.py b/django/dispatch/saferef.py index 2a2c8739fb..d8728e219a 100644 --- a/django/dispatch/saferef.py +++ b/django/dispatch/saferef.py @@ -152,9 +152,10 @@ class BoundMethodWeakref(object): __repr__ = __str__ - def __nonzero__( self ): + def __bool__( self ): """Whether we are still a valid reference""" return self() is not None + __nonzero__ = __bool__ # Python 2 def __eq__(self, other): """Compare with another reference""" diff --git a/django/forms/extras/widgets.py b/django/forms/extras/widgets.py index 4e11a4ee06..c5ca1424c8 100644 --- a/django/forms/extras/widgets.py +++ b/django/forms/extras/widgets.py @@ -79,7 +79,7 @@ class SelectDateWidget(Widget): year_val, month_val, day_val = [int(v) for v in match.groups()] choices = [(i, i) for i in self.years] year_html = self.create_select(name, self.year_field, value, year_val, choices) - choices = MONTHS.items() + choices = list(six.iteritems(MONTHS)) month_html = self.create_select(name, self.month_field, value, month_val, choices) choices = [(i, i) for i in range(1, 32)] day_html = self.create_select(name, self.day_field, value, day_val, choices) diff --git a/django/forms/fields.py b/django/forms/fields.py index c4a675da74..7f0d26d1aa 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -23,7 +23,7 @@ from django.forms.widgets import (TextInput, PasswordInput, HiddenInput, NullBooleanSelect, SelectMultiple, DateInput, DateTimeInput, TimeInput, SplitDateTimeWidget, SplitHiddenDateTimeWidget, FILE_INPUT_CONTRADICTION) from django.utils import formats -from django.utils.encoding import smart_unicode, force_unicode +from django.utils.encoding import smart_text, force_text from django.utils.ipv6 import clean_ipv6_address from django.utils import six from django.utils.translation import ugettext_lazy as _ @@ -78,13 +78,13 @@ class Field(object): # validators -- List of addtional validators to use # localize -- Boolean that specifies if the field should be localized. if label is not None: - label = smart_unicode(label) + label = smart_text(label) self.required, self.label, self.initial = required, label, initial self.show_hidden_initial = show_hidden_initial if help_text is None: self.help_text = '' else: - self.help_text = smart_unicode(help_text) + self.help_text = smart_text(help_text) widget = widget or self.widget if isinstance(widget, type): widget = widget() @@ -195,7 +195,7 @@ class CharField(Field): "Returns a Unicode object." if value in validators.EMPTY_VALUES: return '' - return smart_unicode(value) + return smart_text(value) def widget_attrs(self, widget): attrs = super(CharField, self).widget_attrs(widget) @@ -288,7 +288,7 @@ class DecimalField(Field): return None if self.localize: value = formats.sanitize_separators(value) - value = smart_unicode(value).strip() + value = smart_text(value).strip() try: value = Decimal(value) except DecimalException: @@ -333,7 +333,7 @@ class BaseTemporalField(Field): def to_python(self, value): # Try to coerce the value to unicode. - unicode_value = force_unicode(value, strings_only=True) + unicode_value = force_text(value, strings_only=True) if isinstance(unicode_value, six.text_type): value = unicode_value.strip() # If unicode, try to strptime against each input format. @@ -560,20 +560,10 @@ class ImageField(FileField): file = BytesIO(data['content']) try: - # load() is the only method that can spot a truncated JPEG, - # but it cannot be called sanely after verify() - trial_image = Image.open(file) - trial_image.load() - - # Since we're about to use the file again we have to reset the - # file object if possible. - if hasattr(file, 'seek') and callable(file.seek): - file.seek(0) - - # verify() is the only method that can spot a corrupt PNG, - # but it must be called immediately after the constructor - trial_image = Image.open(file) - trial_image.verify() + # load() could spot a truncated JPEG, but it loads the entire + # image in memory, which is a DoS vector. See #3848 and #18520. + # verify() must be called immediately after the constructor. + Image.open(file).verify() except ImportError: # Under PyPy, it is possible to import PIL. However, the underlying # _imaging C module isn't available, so an ImportError will be @@ -702,7 +692,7 @@ class ChoiceField(Field): "Returns a Unicode object." if value in validators.EMPTY_VALUES: return '' - return smart_unicode(value) + return smart_text(value) def validate(self, value): """ @@ -718,10 +708,10 @@ class ChoiceField(Field): if isinstance(v, (list, tuple)): # This is an optgroup, so look inside the group for options for k2, v2 in v: - if value == smart_unicode(k2): + if value == smart_text(k2): return True else: - if value == smart_unicode(k): + if value == smart_text(k): return True return False @@ -762,7 +752,7 @@ class MultipleChoiceField(ChoiceField): return [] elif not isinstance(value, (list, tuple)): raise ValidationError(self.error_messages['invalid_list']) - return [smart_unicode(val) for val in value] + return [smart_text(val) for val in value] def validate(self, value): """ diff --git a/django/forms/forms.py b/django/forms/forms.py index 4bc3ee9d26..45b758202a 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -12,7 +12,7 @@ from django.forms.util import flatatt, ErrorDict, ErrorList from django.forms.widgets import Media, media_property, TextInput, Textarea from django.utils.datastructures import SortedDict from django.utils.html import conditional_escape, format_html -from django.utils.encoding import StrAndUnicode, smart_unicode, force_unicode +from django.utils.encoding import StrAndUnicode, smart_text, force_text from django.utils.safestring import mark_safe from django.utils import six @@ -38,7 +38,7 @@ def get_declared_fields(bases, attrs, with_base_fields=True): used. The distinction is useful in ModelForm subclassing. Also integrates any additional media definitions """ - fields = [(field_name, attrs.pop(field_name)) for field_name, obj in attrs.items() if isinstance(obj, Field)] + fields = [(field_name, attrs.pop(field_name)) for field_name, obj in list(six.iteritems(attrs)) if isinstance(obj, Field)] fields.sort(key=lambda x: x[1].creation_counter) # If this class is subclassing another Form, add that Form's fields. @@ -47,11 +47,11 @@ def get_declared_fields(bases, attrs, with_base_fields=True): if with_base_fields: for base in bases[::-1]: if hasattr(base, 'base_fields'): - fields = base.base_fields.items() + fields + fields = list(six.iteritems(base.base_fields)) + fields else: for base in bases[::-1]: if hasattr(base, 'declared_fields'): - fields = base.declared_fields.items() + fields + fields = list(six.iteritems(base.declared_fields)) + fields return SortedDict(fields) @@ -150,7 +150,7 @@ class BaseForm(StrAndUnicode): bf_errors = self.error_class([conditional_escape(error) for error in bf.errors]) # Escape and cache in local variable. if bf.is_hidden: if bf_errors: - top_errors.extend(['(Hidden field %s) %s' % (name, force_unicode(e)) for e in bf_errors]) + top_errors.extend(['(Hidden field %s) %s' % (name, force_text(e)) for e in bf_errors]) hidden_fields.append(six.text_type(bf)) else: # Create a 'class="..."' atribute if the row should have any @@ -160,10 +160,10 @@ class BaseForm(StrAndUnicode): html_class_attr = ' class="%s"' % css_classes if errors_on_separate_row and bf_errors: - output.append(error_row % force_unicode(bf_errors)) + output.append(error_row % force_text(bf_errors)) if bf.label: - label = conditional_escape(force_unicode(bf.label)) + label = conditional_escape(force_text(bf.label)) # Only add the suffix if the label does not end in # punctuation. if self.label_suffix: @@ -174,20 +174,20 @@ class BaseForm(StrAndUnicode): label = '' if field.help_text: - help_text = help_text_html % force_unicode(field.help_text) + help_text = help_text_html % force_text(field.help_text) else: help_text = '' output.append(normal_row % { - 'errors': force_unicode(bf_errors), - 'label': force_unicode(label), + 'errors': force_text(bf_errors), + 'label': force_text(label), 'field': six.text_type(bf), 'help_text': help_text, 'html_class_attr': html_class_attr }) if top_errors: - output.insert(0, error_row % force_unicode(top_errors)) + output.insert(0, error_row % force_text(top_errors)) if hidden_fields: # Insert any hidden fields in the last row. str_hidden = ''.join(hidden_fields) @@ -271,8 +271,6 @@ class BaseForm(StrAndUnicode): self._clean_fields() self._clean_form() self._post_clean() - if self._errors: - del self.cleaned_data def _clean_fields(self): for name, field in self.fields.items(): @@ -537,8 +535,8 @@ class BoundField(StrAndUnicode): associated Form has specified auto_id. Returns an empty string otherwise. """ auto_id = self.form.auto_id - if auto_id and '%s' in smart_unicode(auto_id): - return smart_unicode(auto_id) % self.html_name + if auto_id and '%s' in smart_text(auto_id): + return smart_text(auto_id) % self.html_name elif auto_id: return self.html_name return '' diff --git a/django/forms/formsets.py b/django/forms/formsets.py index 240cf71f43..4ea8dc4ca9 100644 --- a/django/forms/formsets.py +++ b/django/forms/formsets.py @@ -65,9 +65,10 @@ class BaseFormSet(StrAndUnicode): def __len__(self): return len(self.forms) - def __nonzero__(self): + def __bool__(self): """All formsets have a management form which is not included in the length""" return True + __nonzero__ = __bool__ # Python 2 def _management_form(self): """Returns the ManagementForm instance for this FormSet.""" @@ -365,7 +366,7 @@ def formset_factory(form, formset=BaseFormSet, extra=1, can_order=False, attrs = {'form': form, 'extra': extra, 'can_order': can_order, 'can_delete': can_delete, 'max_num': max_num} - return type(form.__name__ + b'FormSet', (formset,), attrs) + return type(form.__name__ + str('FormSet'), (formset,), attrs) def all_valid(formsets): """Returns true if every formset in formsets is valid.""" diff --git a/django/forms/models.py b/django/forms/models.py index 4d56f1d38a..0b07d31d9a 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -13,11 +13,12 @@ from django.forms.formsets import BaseFormSet, formset_factory from django.forms.util import ErrorList from django.forms.widgets import (SelectMultiple, HiddenInput, MultipleHiddenInput, media_property) -from django.utils.encoding import smart_unicode, force_unicode +from django.utils.encoding import smart_text, force_text from django.utils.datastructures import SortedDict from django.utils import six from django.utils.text import get_text_list, capfirst from django.utils.translation import ugettext_lazy as _, ugettext +from django.utils import six __all__ = ( @@ -206,7 +207,7 @@ class ModelFormMetaclass(type): fields = fields_for_model(opts.model, opts.fields, opts.exclude, opts.widgets, formfield_callback) # make sure opts.fields doesn't specify an invalid field - none_model_fields = [k for k, v in fields.iteritems() if not v] + none_model_fields = [k for k, v in six.iteritems(fields) if not v] missing_fields = set(none_model_fields) - \ set(declared_fields.keys()) if missing_fields: @@ -389,10 +390,10 @@ def modelform_factory(model, form=ModelForm, fields=None, exclude=None, parent = (object,) if hasattr(form, 'Meta'): parent = (form.Meta, object) - Meta = type(b'Meta', parent, attrs) + Meta = type(str('Meta'), parent, attrs) # Give this new form class a reasonable name. - class_name = model.__name__ + b'Form' + class_name = model.__name__ + str('Form') # Class attributes for the new form class. form_class_attrs = { @@ -506,7 +507,7 @@ class BaseModelFormSet(BaseFormSet): all_unique_checks = set() all_date_checks = set() for form in self.forms: - if not hasattr(form, 'cleaned_data'): + if not form.is_valid(): continue exclude = form._get_validation_exclusions() unique_checks, date_checks = form.instance._get_unique_checks(exclude=exclude) @@ -518,21 +519,21 @@ class BaseModelFormSet(BaseFormSet): for uclass, unique_check in all_unique_checks: seen_data = set() for form in self.forms: - # if the form doesn't have cleaned_data then we ignore it, - # it's already invalid - if not hasattr(form, "cleaned_data"): + if not form.is_valid(): continue # get data for each field of each of unique_check row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data]) if row_data and not None in row_data: - # if we've aready seen it then we have a uniqueness failure + # if we've already seen it then we have a uniqueness failure if row_data in seen_data: # poke error messages into the right places and mark # the form as invalid errors.append(self.get_unique_error_message(unique_check)) form._errors[NON_FIELD_ERRORS] = self.error_class([self.get_form_error()]) - del form.cleaned_data - break + # remove the data from the cleaned_data dict since it was invalid + for field in unique_check: + if field in form.cleaned_data: + del form.cleaned_data[field] # mark the data as seen seen_data.add(row_data) # iterate over each of the date checks now @@ -540,9 +541,7 @@ class BaseModelFormSet(BaseFormSet): seen_data = set() uclass, lookup, field, unique_for = date_check for form in self.forms: - # if the form doesn't have cleaned_data then we ignore it, - # it's already invalid - if not hasattr(self, 'cleaned_data'): + if not form.is_valid(): continue # see if we have data for both fields if (form.cleaned_data and form.cleaned_data[field] is not None @@ -556,14 +555,15 @@ class BaseModelFormSet(BaseFormSet): else: date_data = (getattr(form.cleaned_data[unique_for], lookup),) data = (form.cleaned_data[field],) + date_data - # if we've aready seen it then we have a uniqueness failure + # if we've already seen it then we have a uniqueness failure if data in seen_data: # poke error messages into the right places and mark # the form as invalid errors.append(self.get_date_error_message(date_check)) form._errors[NON_FIELD_ERRORS] = self.error_class([self.get_form_error()]) - del form.cleaned_data - break + # remove the data from the cleaned_data dict since it was invalid + del form.cleaned_data[field] + # mark the data as seen seen_data.add(data) if errors: raise ValidationError(errors) @@ -875,7 +875,7 @@ class InlineForeignKeyField(Field): orig = getattr(self.parent_instance, self.to_field) else: orig = self.parent_instance.pk - if force_unicode(value) != force_unicode(orig): + if force_text(value) != force_text(orig): raise ValidationError(self.error_messages['invalid_choice']) return self.parent_instance @@ -953,7 +953,7 @@ class ModelChoiceField(ChoiceField): generate the labels for the choices presented by this object. Subclasses can override this method to customize the display of the choices. """ - return smart_unicode(obj) + return smart_text(obj) def _get_choices(self): # If self._choices is set, then somebody must have manually set @@ -1025,9 +1025,9 @@ class ModelMultipleChoiceField(ModelChoiceField): except ValueError: raise ValidationError(self.error_messages['invalid_pk_value'] % pk) qs = self.queryset.filter(**{'%s__in' % key: value}) - pks = set([force_unicode(getattr(o, key)) for o in qs]) + pks = set([force_text(getattr(o, key)) for o in qs]) for val in value: - if force_unicode(val) not in pks: + if force_text(val) not in pks: raise ValidationError(self.error_messages['invalid_choice'] % val) # Since this overrides the inherited ModelChoiceField.clean # we run custom validators here @@ -1035,6 +1035,6 @@ class ModelMultipleChoiceField(ModelChoiceField): return qs def prepare_value(self, value): - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') and not isinstance(value, six.text_type): return [super(ModelMultipleChoiceField, self).prepare_value(v) for v in value] return super(ModelMultipleChoiceField, self).prepare_value(value) diff --git a/django/forms/util.py b/django/forms/util.py index 8cf03d38af..cd6b52df6f 100644 --- a/django/forms/util.py +++ b/django/forms/util.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.conf import settings from django.utils.html import format_html, format_html_join -from django.utils.encoding import StrAndUnicode, force_unicode +from django.utils.encoding import StrAndUnicode, force_text from django.utils.safestring import mark_safe from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -35,12 +35,12 @@ class ErrorDict(dict, StrAndUnicode): if not self: return '' return format_html('<ul class="errorlist">{0}</ul>', format_html_join('', '<li>{0}{1}</li>', - ((k, force_unicode(v)) + ((k, force_text(v)) for k, v in self.items()) )) def as_text(self): - return '\n'.join(['* %s\n%s' % (k, '\n'.join([' * %s' % force_unicode(i) for i in v])) for k, v in self.items()]) + return '\n'.join(['* %s\n%s' % (k, '\n'.join([' * %s' % force_text(i) for i in v])) for k, v in self.items()]) class ErrorList(list, StrAndUnicode): """ @@ -53,16 +53,16 @@ class ErrorList(list, StrAndUnicode): if not self: return '' return format_html('<ul class="errorlist">{0}</ul>', format_html_join('', '<li>{0}</li>', - ((force_unicode(e),) for e in self) + ((force_text(e),) for e in self) ) ) def as_text(self): if not self: return '' - return '\n'.join(['* %s' % force_unicode(e) for e in self]) + return '\n'.join(['* %s' % force_text(e) for e in self]) def __repr__(self): - return repr([force_unicode(e) for e in self]) + return repr([force_text(e) for e in self]) # Utilities for time zone support in DateTimeField et al. diff --git a/django/forms/widgets.py b/django/forms/widgets.py index 6b1be37ec2..be9ac8eb8f 100644 --- a/django/forms/widgets.py +++ b/django/forms/widgets.py @@ -17,7 +17,7 @@ from django.forms.util import flatatt, to_current_timezone from django.utils.datastructures import MultiValueDict, MergeDict from django.utils.html import conditional_escape, format_html, format_html_join from django.utils.translation import ugettext, ugettext_lazy -from django.utils.encoding import StrAndUnicode, force_unicode +from django.utils.encoding import StrAndUnicode, force_text from django.utils.safestring import mark_safe from django.utils import six from django.utils import datetime_safe, formats @@ -63,8 +63,7 @@ class Media(StrAndUnicode): def render_css(self): # To keep rendering order consistent, we can't just iterate over items(). # We need to sort the keys, and iterate over the sorted list. - media = self._css.keys() - media.sort() + media = sorted(self._css.keys()) return chain(*[ [format_html('<link href="{0}" type="text/css" media="{1}" rel="stylesheet" />', self.absolute_path(path), medium) for path in self._css[medium]] @@ -110,9 +109,10 @@ class Media(StrAndUnicode): def media_property(cls): def _media(self): # Get the media property of the superclass, if it exists - if hasattr(super(cls, self), 'media'): - base = super(cls, self).media - else: + sup_cls = super(cls, self) + try: + base = sup_cls.media + except AttributeError: base = Media() # Get the media definition for this class @@ -223,7 +223,7 @@ class Widget(six.with_metaclass(MediaDefiningClass)): initial_value = '' else: initial_value = initial - if force_unicode(initial_value) != force_unicode(data_value): + if force_text(initial_value) != force_text(data_value): return True return False @@ -257,7 +257,7 @@ class Input(Widget): final_attrs = self.build_attrs(attrs, type=self.input_type, name=name) if value != '': # Only add the 'value' attribute if a value is non-empty. - final_attrs['value'] = force_unicode(self._format_value(value)) + final_attrs['value'] = force_text(self._format_value(value)) return format_html('<input{0} />', flatatt(final_attrs)) class TextInput(Input): @@ -294,7 +294,7 @@ class MultipleHiddenInput(HiddenInput): id_ = final_attrs.get('id', None) inputs = [] for i, v in enumerate(value): - input_attrs = dict(value=force_unicode(v), **final_attrs) + input_attrs = dict(value=force_text(v), **final_attrs) if id_: # An ID attribute was given. Add a numeric index as a suffix # so that the inputs don't all have the same ID attribute. @@ -361,7 +361,7 @@ class ClearableFileInput(FileInput): template = self.template_with_initial substitutions['initial'] = format_html('<a href="{0}">{1}</a>', value.url, - force_unicode(value)) + force_text(value)) if not self.is_required: checkbox_name = self.clear_checkbox_name(name) checkbox_id = self.clear_checkbox_id(checkbox_name) @@ -398,7 +398,7 @@ class Textarea(Widget): final_attrs = self.build_attrs(attrs, name=name) return format_html('<textarea{0}>{1}</textarea>', flatatt(final_attrs), - force_unicode(value)) + force_text(value)) class DateInput(Input): input_type = 'text' @@ -515,7 +515,7 @@ class CheckboxInput(Widget): final_attrs['checked'] = 'checked' if not (value is True or value is False or value is None or value == ''): # Only add the 'value' attribute if a value is non-empty. - final_attrs['value'] = force_unicode(value) + final_attrs['value'] = force_text(value) return format_html('<input{0} />', flatatt(final_attrs)) def value_from_datadict(self, data, files, name): @@ -556,7 +556,7 @@ class Select(Widget): return mark_safe('\n'.join(output)) def render_option(self, selected_choices, option_value, option_label): - option_value = force_unicode(option_value) + option_value = force_text(option_value) if option_value in selected_choices: selected_html = mark_safe(' selected="selected"') if not self.allow_multiple_selected: @@ -567,15 +567,15 @@ class Select(Widget): return format_html('<option value="{0}"{1}>{2}</option>', option_value, selected_html, - force_unicode(option_label)) + force_text(option_label)) def render_options(self, choices, selected_choices): # Normalize to strings. - selected_choices = set(force_unicode(v) for v in selected_choices) + selected_choices = set(force_text(v) for v in selected_choices) output = [] for option_value, option_label in chain(self.choices, choices): if isinstance(option_label, (list, tuple)): - output.append(format_html('<optgroup label="{0}">', force_unicode(option_value))) + output.append(format_html('<optgroup label="{0}">', force_text(option_value))) for option in option_label: output.append(self.render_option(selected_choices, *option)) output.append('</optgroup>') @@ -643,8 +643,8 @@ class SelectMultiple(Select): data = [] if len(initial) != len(data): return True - initial_set = set([force_unicode(value) for value in initial]) - data_set = set([force_unicode(value) for value in data]) + initial_set = set([force_text(value) for value in initial]) + data_set = set([force_text(value) for value in data]) return data_set != initial_set class RadioInput(SubWidget): @@ -656,8 +656,8 @@ class RadioInput(SubWidget): def __init__(self, name, value, attrs, choice, index): self.name, self.value = name, value self.attrs = attrs - self.choice_value = force_unicode(choice[0]) - self.choice_label = force_unicode(choice[1]) + self.choice_value = force_text(choice[0]) + self.choice_label = force_text(choice[1]) self.index = index def __unicode__(self): @@ -671,7 +671,7 @@ class RadioInput(SubWidget): label_for = format_html(' for="{0}_{1}"', self.attrs['id'], self.index) else: label_for = '' - choice_label = force_unicode(self.choice_label) + choice_label = force_text(self.choice_label) return format_html('<label{0}>{1} {2}</label>', label_for, self.tag(), choice_label) def is_checked(self): @@ -709,7 +709,7 @@ class RadioFieldRenderer(StrAndUnicode): """Outputs a <ul> for this set of radio fields.""" return format_html('<ul>\n{0}\n</ul>', format_html_join('\n', '<li>{0}</li>', - [(force_unicode(w),) for w in self] + [(force_text(w),) for w in self] )) class RadioSelect(Select): @@ -729,7 +729,7 @@ class RadioSelect(Select): def get_renderer(self, name, value, attrs=None, choices=()): """Returns an instance of the renderer.""" if value is None: value = '' - str_value = force_unicode(value) # Normalize to string. + str_value = force_text(value) # Normalize to string. final_attrs = self.build_attrs(attrs) choices = list(chain(self.choices, choices)) return self.renderer(name, str_value, final_attrs, choices) @@ -753,7 +753,7 @@ class CheckboxSelectMultiple(SelectMultiple): final_attrs = self.build_attrs(attrs, name=name) output = ['<ul>'] # Normalize to strings - str_values = set([force_unicode(v) for v in value]) + str_values = set([force_text(v) for v in value]) for i, (option_value, option_label) in enumerate(chain(self.choices, choices)): # If an ID attribute was given, add a numeric index as a suffix, # so that the checkboxes don't all have the same ID attribute. @@ -764,9 +764,9 @@ class CheckboxSelectMultiple(SelectMultiple): label_for = '' cb = CheckboxInput(final_attrs, check_test=lambda value: value in str_values) - option_value = force_unicode(option_value) + option_value = force_text(option_value) rendered_cb = cb.render(name, option_value) - option_label = force_unicode(option_label) + option_label = force_text(option_label) output.append(format_html('<li><label{0}>{1} {2}</label></li>', label_for, rendered_cb, option_label)) output.append('</ul>') diff --git a/django/http/__init__.py b/django/http/__init__.py index 6f9d70eebd..05824ad1d9 100644 --- a/django/http/__init__.py +++ b/django/http/__init__.py @@ -11,10 +11,10 @@ import warnings from io import BytesIO from pprint import pformat try: - from urllib.parse import quote, parse_qsl, urlencode, urljoin + from urllib.parse import quote, parse_qsl, urlencode, urljoin, urlparse except ImportError: # Python 2 from urllib import quote, urlencode - from urlparse import parse_qsl, urljoin + from urlparse import parse_qsl, urljoin, urlparse from django.utils.six.moves import http_cookies # Some versions of Python 2.7 and later won't need this encoding bug fix: @@ -22,7 +22,7 @@ _cookie_encodes_correctly = http_cookies.SimpleCookie().value_encode(';') == ('; # See ticket #13007, http://bugs.python.org/issue2193 and http://trac.edgewall.org/ticket/2256 _tc = http_cookies.SimpleCookie() try: - _tc.load(b'foo:bar=1') + _tc.load(str('foo:bar=1')) _cookie_allows_colon_in_names = True except http_cookies.CookieError: _cookie_allows_colon_in_names = False @@ -61,14 +61,14 @@ else: if not _cookie_allows_colon_in_names: def load(self, rawdata): self.bad_cookies = set() - super(SimpleCookie, self).load(smart_str(rawdata)) + super(SimpleCookie, self).load(smart_bytes(rawdata)) for key in self.bad_cookies: del self[key] # override private __set() method: # (needed for using our Morsel, and for laxness with CookieError def _BaseCookie__set(self, key, real_value, coded_value): - key = smart_str(key) + key = smart_bytes(key) try: M = self.get(key, Morsel()) M.set(key, real_value, coded_value) @@ -80,12 +80,12 @@ else: from django.conf import settings from django.core import signing -from django.core.exceptions import ImproperlyConfigured +from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.core.files import uploadhandler from django.http.multipartparser import MultiPartParser from django.http.utils import * from django.utils.datastructures import MultiValueDict, ImmutableList -from django.utils.encoding import smart_str, iri_to_uri, force_unicode +from django.utils.encoding import smart_bytes, iri_to_uri, force_text from django.utils.http import cookie_date from django.utils import six from django.utils import timezone @@ -137,7 +137,7 @@ def build_request_repr(request, path_override=None, GET_override=None, except: meta = '<could not parse>' path = path_override if path_override is not None else request.path - return smart_str('<%s\npath:%s,\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % + return smart_bytes('<%s\npath:%s,\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % (request.__class__.__name__, path, six.text_type(get), @@ -385,8 +385,8 @@ class QueryDict(MultiValueDict): encoding = settings.DEFAULT_CHARSET self.encoding = encoding for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True - self.appendlist(force_unicode(key, encoding, errors='replace'), - force_unicode(value, encoding, errors='replace')) + self.appendlist(force_text(key, encoding, errors='replace'), + force_text(value, encoding, errors='replace')) self._mutable = mutable def _get_encoding(self): @@ -481,13 +481,13 @@ class QueryDict(MultiValueDict): """ output = [] if safe: - safe = smart_str(safe, self.encoding) + safe = smart_bytes(safe, self.encoding) encode = lambda k, v: '%s=%s' % ((quote(k, safe), quote(v, safe))) else: encode = lambda k, v: urlencode({k: v}) for k, list_ in self.lists(): - k = smart_str(k, self.encoding) - output.extend([encode(k, smart_str(v, self.encoding)) + k = smart_bytes(k, self.encoding) + output.extend([encode(k, smart_bytes(v, self.encoding)) for v in list_]) return '&'.join(output) @@ -549,7 +549,12 @@ class HttpResponse(object): for value in values: if isinstance(value, six.text_type): try: - value = value.encode('us-ascii') + if not six.PY3: + value = value.encode('us-ascii') + else: + # In Python 3, use a string in headers, + # but ensure in only contains ASCII characters. + value.encode('us-ascii') except UnicodeError as e: e.reason += ', HTTP response headers must be in US-ASCII format' raise @@ -648,10 +653,10 @@ class HttpResponse(object): def _get_content(self): if self.has_header('Content-Encoding'): return b''.join([str(e) for e in self._container]) - return b''.join([smart_str(e, self._charset) for e in self._container]) + return b''.join([smart_bytes(e, self._charset) for e in self._container]) def _set_content(self, value): - if hasattr(value, '__iter__'): + if hasattr(value, '__iter__') and not isinstance(value, (bytes, six.text_type)): self._container = value self._base_content_is_iter = True else: @@ -664,12 +669,14 @@ class HttpResponse(object): self._iterator = iter(self._container) return self - def next(self): + def __next__(self): chunk = next(self._iterator) if isinstance(chunk, six.text_type): chunk = chunk.encode(self._charset) return str(chunk) + next = __next__ # Python 2 compatibility + def close(self): if hasattr(self._container, 'close'): self._container.close() @@ -689,19 +696,21 @@ class HttpResponse(object): raise Exception("This %s instance cannot tell its position" % self.__class__) return sum([len(chunk) for chunk in self]) -class HttpResponseRedirect(HttpResponse): - status_code = 302 +class HttpResponseRedirectBase(HttpResponse): + allowed_schemes = ['http', 'https', 'ftp'] def __init__(self, redirect_to): - super(HttpResponseRedirect, self).__init__() + parsed = urlparse(redirect_to) + if parsed.scheme and parsed.scheme not in self.allowed_schemes: + raise SuspiciousOperation("Unsafe redirect to URL with protocol '%s'" % parsed.scheme) + super(HttpResponseRedirectBase, self).__init__() self['Location'] = iri_to_uri(redirect_to) -class HttpResponsePermanentRedirect(HttpResponse): - status_code = 301 +class HttpResponseRedirect(HttpResponseRedirectBase): + status_code = 302 - def __init__(self, redirect_to): - super(HttpResponsePermanentRedirect, self).__init__() - self['Location'] = iri_to_uri(redirect_to) +class HttpResponsePermanentRedirect(HttpResponseRedirectBase): + status_code = 301 class HttpResponseNotModified(HttpResponse): status_code = 304 @@ -733,7 +742,7 @@ def get_host(request): return request.get_host() # It's neither necessary nor appropriate to use -# django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus, +# django.utils.encoding.smart_text for parsing URLs and form inputs. Thus, # this slightly more restricted function. def str_to_unicode(s, encoding): """ diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 0e28a55c3a..9c66827cbb 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -10,7 +10,7 @@ import cgi from django.conf import settings from django.core.exceptions import SuspiciousOperation from django.utils.datastructures import MultiValueDict -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils import six from django.utils.text import unescape_entities from django.core.files.uploadhandler import StopUpload, SkipFile, StopFutureHandlers @@ -151,7 +151,7 @@ class MultiPartParser(object): transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() - field_name = force_unicode(field_name, encoding, errors='replace') + field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # This is a post field, we can just set it in the post @@ -165,13 +165,13 @@ class MultiPartParser(object): data = field_stream.read() self._post.appendlist(field_name, - force_unicode(data, encoding, errors='replace')) + force_text(data, encoding, errors='replace')) elif item_type == FILE: # This is a file, use the handler... file_name = disposition.get('filename') if not file_name: continue - file_name = force_unicode(file_name, encoding, errors='replace') + file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type = meta_data.get('content-type', ('',))[0].strip() @@ -245,7 +245,7 @@ class MultiPartParser(object): file_obj = handler.file_complete(counters[i]) if file_obj: # If it returns a file object, then set the files dict. - self._files.appendlist(force_unicode(old_field_name, + self._files.appendlist(force_text(old_field_name, self._encoding, errors='replace'), file_obj) @@ -305,7 +305,7 @@ class LazyStream(object): out = b''.join(parts()) return out - def next(self): + def __next__(self): """ Used when the exact number of bytes to read is unimportant. @@ -322,6 +322,8 @@ class LazyStream(object): self.position += len(output) return output + next = __next__ # Python 2 compatibility + def close(self): """ Used to invalidate/disable this lazy stream. @@ -376,7 +378,7 @@ class ChunkIter(object): self.flo = flo self.chunk_size = chunk_size - def next(self): + def __next__(self): try: data = self.flo.read(self.chunk_size) except InputStreamExhausted: @@ -386,6 +388,8 @@ class ChunkIter(object): else: raise StopIteration() + next = __next__ # Python 2 compatibility + def __iter__(self): return self @@ -400,12 +404,14 @@ class InterBoundaryIter(object): def __iter__(self): return self - def next(self): + def __next__(self): try: return LazyStream(BoundaryIter(self._stream, self._boundary)) except InputStreamExhausted: raise StopIteration() + next = __next__ # Python 2 compatibility + class BoundaryIter(object): """ A Producer that is sensitive to boundaries. @@ -441,7 +447,7 @@ class BoundaryIter(object): def __iter__(self): return self - def next(self): + def __next__(self): if self._done: raise StopIteration() @@ -482,6 +488,8 @@ class BoundaryIter(object): stream.unget(chunk[-rollback:]) return chunk[:-rollback] + next = __next__ # Python 2 compatibility + def _find_boundary(self, data, eof = False): """ Finds a multipart boundary in data. diff --git a/django/template/base.py b/django/template/base.py index 489b1681e0..661d8c092a 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -11,7 +11,7 @@ from django.utils.importlib import import_module from django.utils.itercompat import is_iterable from django.utils.text import (smart_split, unescape_string_literal, get_text_list) -from django.utils.encoding import smart_unicode, force_unicode, smart_str +from django.utils.encoding import smart_text, force_text, smart_bytes from django.utils.translation import ugettext_lazy, pgettext_lazy from django.utils.safestring import (SafeData, EscapeData, mark_safe, mark_for_escaping) @@ -89,7 +89,7 @@ class VariableDoesNotExist(Exception): return six.text_type(self).encode('utf-8') def __unicode__(self): - return self.msg % tuple([force_unicode(p, errors='replace') + return self.msg % tuple([force_text(p, errors='replace') for p in self.params]) class InvalidTemplateLibrary(Exception): @@ -117,7 +117,7 @@ class Template(object): def __init__(self, template_string, origin=None, name='<Unknown Template>'): try: - template_string = smart_unicode(template_string) + template_string = smart_text(template_string) except UnicodeDecodeError: raise TemplateEncodingError("Templates can only be constructed " "from unicode or UTF-8 strings.") @@ -831,7 +831,7 @@ class NodeList(list): bit = self.render_node(node, context) else: bit = node - bits.append(force_unicode(bit)) + bits.append(force_text(bit)) return mark_safe(''.join(bits)) def get_nodes_by_type(self, nodetype): @@ -849,7 +849,7 @@ class TextNode(Node): self.s = s def __repr__(self): - return "<Text Node: '%s'>" % smart_str(self.s[:25], 'ascii', + return "<Text Node: '%s'>" % smart_bytes(self.s[:25], 'ascii', errors='replace') def render(self, context): @@ -863,7 +863,7 @@ def _render_value_in_context(value, context): """ value = template_localtime(value, use_tz=context.use_tz) value = localize(value, use_l10n=context.use_l10n) - value = force_unicode(value) + value = force_text(value) if ((context.autoescape and not isinstance(value, SafeData)) or isinstance(value, EscapeData)): return escape(value) @@ -961,7 +961,7 @@ def parse_bits(parser, bits, params, varargs, varkw, defaults, kwarg = token_kwargs([bit], parser) if kwarg: # The kwarg was successfully extracted - param, value = kwarg.items()[0] + param, value = list(six.iteritems(kwarg))[0] if param not in params and varkw is None: # An unexpected keyword argument was supplied raise TemplateSyntaxError( diff --git a/django/template/debug.py b/django/template/debug.py index 61674034d6..c7ac007b48 100644 --- a/django/template/debug.py +++ b/django/template/debug.py @@ -1,5 +1,5 @@ from django.template.base import Lexer, Parser, tag_re, NodeList, VariableNode, TemplateSyntaxError -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.html import escape from django.utils.safestring import SafeData, EscapeData from django.utils.formats import localize @@ -84,7 +84,7 @@ class DebugVariableNode(VariableNode): output = self.filter_expression.resolve(context) output = template_localtime(output, use_tz=context.use_tz) output = localize(output, use_l10n=context.use_l10n) - output = force_unicode(output) + output = force_text(output) except UnicodeDecodeError: return '' except Exception as e: diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index fa799cd46f..16801fd573 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -12,7 +12,7 @@ from django.template.base import Variable, Library, VariableDoesNotExist from django.conf import settings from django.utils import formats from django.utils.dateformat import format, time_format -from django.utils.encoding import force_unicode, iri_to_uri +from django.utils.encoding import force_text, iri_to_uri from django.utils.html import (conditional_escape, escapejs, fix_ampersands, escape, urlize as urlize_impl, linebreaks, strip_tags) from django.utils.http import urlquote @@ -38,7 +38,7 @@ def stringfilter(func): def _dec(*args, **kwargs): if args: args = list(args) - args[0] = force_unicode(args[0]) + args[0] = force_text(args[0]) if (isinstance(args[0], SafeData) and getattr(_dec._decorated_function, 'is_safe', False)): return mark_safe(func(*args, **kwargs)) @@ -139,7 +139,7 @@ def floatformat(text, arg=-1): """ try: - input_val = force_unicode(text) + input_val = force_text(text) d = Decimal(input_val) except UnicodeEncodeError: return '' @@ -147,7 +147,7 @@ def floatformat(text, arg=-1): if input_val in special_floats: return input_val try: - d = Decimal(force_unicode(float(text))) + d = Decimal(force_text(float(text))) except (ValueError, InvalidOperation, TypeError, UnicodeEncodeError): return '' try: @@ -192,7 +192,7 @@ def floatformat(text, arg=-1): @stringfilter def iriencode(value): """Escapes an IRI value for use in a URL.""" - return force_unicode(iri_to_uri(value)) + return force_text(iri_to_uri(value)) @register.filter(is_safe=True, needs_autoescape=True) @stringfilter @@ -462,7 +462,7 @@ def safeseq(value): individually, as safe, after converting them to unicode. Returns a list with the results. """ - return [mark_safe(force_unicode(obj)) for obj in value] + return [mark_safe(force_text(obj)) for obj in value] @register.filter(is_safe=True) @stringfilter @@ -521,7 +521,7 @@ def join(value, arg, autoescape=None): """ Joins a list with a string, like Python's ``str.join(list)``. """ - value = map(force_unicode, value) + value = map(force_text, value) if autoescape: value = [conditional_escape(v) for v in value] try: @@ -661,7 +661,7 @@ def unordered_list(value, autoescape=None): sublist = '\n%s<ul>\n%s\n%s</ul>\n%s' % (indent, sublist, indent, indent) output.append('%s<li>%s%s</li>' % (indent, - escaper(force_unicode(title)), sublist)) + escaper(force_text(title)), sublist)) i += 1 return '\n'.join(output) value, converted = convert_old_style_list(value) @@ -901,4 +901,4 @@ def pprint(value): try: return pformat(value) except Exception as e: - return "Error in formatting: %s" % force_unicode(e, errors="replace") + return "Error in formatting: %s" % force_text(e, errors="replace") diff --git a/django/template/defaulttags.py b/django/template/defaulttags.py index fb45fe722e..14391f08e1 100644 --- a/django/template/defaulttags.py +++ b/django/template/defaulttags.py @@ -14,9 +14,10 @@ from django.template.base import (Node, NodeList, Template, Context, Library, VARIABLE_ATTRIBUTE_SEPARATOR, get_library, token_kwargs, kwarg_re) from django.template.smartif import IfParser, Literal from django.template.defaultfilters import date -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.safestring import mark_safe from django.utils.html import format_html +from django.utils import six from django.utils import timezone register = Library() @@ -103,7 +104,7 @@ class FirstOfNode(Node): for var in self.vars: value = var.resolve(context, True) if value: - return smart_unicode(value) + return smart_text(value) return '' class ForNode(Node): @@ -392,7 +393,7 @@ class URLNode(Node): def render(self, context): from django.core.urlresolvers import reverse, NoReverseMatch args = [arg.resolve(context) for arg in self.args] - kwargs = dict([(smart_unicode(k, 'ascii'), v.resolve(context)) + kwargs = dict([(smart_text(k, 'ascii'), v.resolve(context)) for k, v in self.kwargs.items()]) view_name = self.view_name.resolve(context) @@ -473,7 +474,7 @@ class WithNode(Node): def render(self, context): values = dict([(key, val.resolve(context)) for key, val in - self.extra_context.iteritems()]) + six.iteritems(self.extra_context)]) context.update(values) output = self.nodelist.render(context) context.pop() @@ -1188,7 +1189,7 @@ def templatetag(parser, token): if tag not in TemplateTagNode.mapping: raise TemplateSyntaxError("Invalid templatetag argument: '%s'." " Must be one of: %s" % - (tag, TemplateTagNode.mapping.keys())) + (tag, list(TemplateTagNode.mapping))) return TemplateTagNode(tag) @register.tag diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py index b63938abb0..d295d058d0 100644 --- a/django/template/loader_tags.py +++ b/django/template/loader_tags.py @@ -3,6 +3,7 @@ from django.template.base import TemplateSyntaxError, Library, Node, TextNode,\ token_kwargs, Variable from django.template.loader import get_template from django.utils.safestring import mark_safe +from django.utils import six register = Library() @@ -17,7 +18,7 @@ class BlockContext(object): self.blocks = {} def add_blocks(self, blocks): - for name, block in blocks.iteritems(): + for name, block in six.iteritems(blocks): if name in self.blocks: self.blocks[name].insert(0, block) else: @@ -130,7 +131,7 @@ class BaseIncludeNode(Node): def render_template(self, template, context): values = dict([(name, var.resolve(context)) for name, var - in self.extra_context.iteritems()]) + in six.iteritems(self.extra_context)]) if self.isolated_context: return template.render(context.new(values)) context.update(values) diff --git a/django/template/loaders/app_directories.py b/django/template/loaders/app_directories.py index 6e0f079de7..887f8a0b72 100644 --- a/django/template/loaders/app_directories.py +++ b/django/template/loaders/app_directories.py @@ -12,9 +12,11 @@ from django.template.base import TemplateDoesNotExist from django.template.loader import BaseLoader from django.utils._os import safe_join from django.utils.importlib import import_module +from django.utils import six # At compile time, cache the directories to search. -fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +if not six.PY3: + fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() app_template_dirs = [] for app in settings.INSTALLED_APPS: try: @@ -23,7 +25,9 @@ for app in settings.INSTALLED_APPS: raise ImproperlyConfigured('ImportError %s: %s' % (app, e.args[0])) template_dir = os.path.join(os.path.dirname(mod.__file__), 'templates') if os.path.isdir(template_dir): - app_template_dirs.append(template_dir.decode(fs_encoding)) + if not six.PY3: + template_dir = template_dir.decode(fs_encoding) + app_template_dirs.append(template_dir) # It won't change, so convert it to a tuple to save memory. app_template_dirs = tuple(app_template_dirs) diff --git a/django/templatetags/i18n.py b/django/templatetags/i18n.py index 509ab6707d..30eb6b5f76 100644 --- a/django/templatetags/i18n.py +++ b/django/templatetags/i18n.py @@ -425,7 +425,7 @@ def do_block_translate(parser, token): options[option] = value if 'count' in options: - countervar, counter = options['count'].items()[0] + countervar, counter = list(six.iteritems(options['count']))[0] else: countervar, counter = None, None if 'context' in options: diff --git a/django/templatetags/l10n.py b/django/templatetags/l10n.py index 1eac1de071..667de2470e 100644 --- a/django/templatetags/l10n.py +++ b/django/templatetags/l10n.py @@ -1,7 +1,7 @@ from django.template import Node from django.template import TemplateSyntaxError, Library from django.utils import formats -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text register = Library() @@ -11,7 +11,7 @@ def localize(value): Forces a value to be rendered as a localized value, regardless of the value of ``settings.USE_L10N``. """ - return force_unicode(formats.localize(value, use_l10n=True)) + return force_text(formats.localize(value, use_l10n=True)) @register.filter(is_safe=False) def unlocalize(value): @@ -19,7 +19,7 @@ def unlocalize(value): Forces a value to be rendered as a non-localized value, regardless of the value of ``settings.USE_L10N``. """ - return force_unicode(value) + return force_text(value) class LocalizeNode(Node): def __init__(self, nodelist, use_l10n): diff --git a/django/test/_doctest.py b/django/test/_doctest.py index 316c785f33..82d4a6d08e 100644 --- a/django/test/_doctest.py +++ b/django/test/_doctest.py @@ -105,7 +105,7 @@ import unittest, difflib, pdb, tempfile import warnings from django.utils import six -from django.utils.six import StringIO +from django.utils.six.moves import StringIO, xrange if sys.platform.startswith('java'): # On Jython, isclass() reports some modules as classes. Patch it. @@ -501,11 +501,31 @@ class DocTest: # This lets us sort tests by name: + def _cmpkey(self): + return (self.name, self.filename, self.lineno, id(self)) def __cmp__(self, other): if not isinstance(other, DocTest): return -1 - return cmp((self.name, self.filename, self.lineno, id(self)), - (other.name, other.filename, other.lineno, id(other))) + return cmp(self._cmpkey(), other._cmpkey()) + + def __lt__(self, other): + return self._cmpkey() < other._cmpkey() + + def __le__(self, other): + return self._cmpkey() <= other._cmpkey() + + def __gt__(self, other): + return self._cmpkey() > other._cmpkey() + + def __ge__(self, other): + return self._cmpkey() >= other._cmpkey() + + def __eq__(self, other): + return self._cmpkey() == other._cmpkey() + + def __ne__(self, other): + return self._cmpkey() != other._cmpkey() + ###################################################################### ## 3. DocTestParser @@ -1229,6 +1249,57 @@ class DocTestRunner: # __patched_linecache_getlines). filename = '<doctest %s[%d]>' % (test.name, examplenum) + # Doctest and Py3 issue: + # If the current example that we wish to run is going to fail + # because it expects a leading u"", then use an alternate displayhook + original_displayhook = sys.displayhook + + if six.PY3: + # only set alternate displayhook if Python 3.x or after + lines = [] + def py3_displayhook(value): + if value is None: + # None should not be considered at all + return original_displayhook(value) + + # Collect the repr output in one variable + s = repr(value) + # Strip b"" and u"" prefixes from the repr and expected output + # TODO: better way of stripping the prefixes? + expected = example.want + expected = expected.strip() # be wary of newlines + s = s.replace("u", "") + s = s.replace("b", "") + expected = expected.replace("u", "") + expected = expected.replace("b", "") + # single quote vs. double quote should not matter + # default all quote marks to double quote + s = s.replace("'", '"') + expected = expected.replace("'", '"') + + # In case of multi-line expected result + lines.append(s) + + # let them match + if s == expected: # be wary of false positives here + # they should be the same, print expected value + sys.stdout.write("%s\n" % example.want.strip()) + + # multi-line expected output, doctest uses loop + elif len(expected.split("\n")) == len(lines): + if "\n".join(lines) == expected: + sys.stdout.write("%s\n" % example.want.strip()) + else: + sys.stdout.write("%s\n" % repr(value)) + elif len(expected.split("\n")) != len(lines): + # we are not done looping yet, do not print anything! + pass + + else: + sys.stdout.write("%s\n" % repr(value)) + + sys.displayhook = py3_displayhook + # Run the example in the given context (globs), and record # any exception that gets raised. (But don't intercept # keyboard interrupts.) @@ -1243,9 +1314,14 @@ class DocTestRunner: except: exception = sys.exc_info() self.debugger.set_continue() # ==== Example Finished ==== + finally: + # restore the original displayhook + sys.displayhook = original_displayhook got = self._fakeout.getvalue() # the actual output self._fakeout.truncate(0) + # Python 3.1 requires seek after truncate + self._fakeout.seek(0) outcome = FAILURE # guilty until proved innocent or insane # If the example executed without raising any exceptions, @@ -1256,10 +1332,21 @@ class DocTestRunner: # The example raised an exception: check if it was expected. else: - exc_info = sys.exc_info() - exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] + exc_msg = traceback.format_exception_only(*exception[:2])[-1] + if six.PY3: + # module name will be in group(1) and the expected + # exception message will be in group(2) + m = re.match(r'(.*)\.(\w+:.+\s)', exc_msg) + # make sure there's a match + if m != None: + f_name = m.group(1) + # check to see if m.group(1) contains the module name + if f_name == exception[0].__module__: + # strip the module name from exc_msg + exc_msg = m.group(2) + if not quiet: - got += _exception_traceback(exc_info) + got += _exception_traceback(exception) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1289,7 +1376,7 @@ class DocTestRunner: elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exc_info) + exception) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -1629,8 +1716,8 @@ class DebugRunner(DocTestRunner): ... {}, 'foo', 'foo.py', 0) >>> try: ... runner.run(test) - ... except UnexpectedException as failure: - ... pass + ... except UnexpectedException as e: + ... failure = e >>> failure.test is test True @@ -1657,8 +1744,8 @@ class DebugRunner(DocTestRunner): >>> try: ... runner.run(test) - ... except DocTestFailure as failure: - ... pass + ... except DocTestFailure as e: + ... failure = e DocTestFailure objects provide access to the test: @@ -2167,8 +2254,8 @@ class DocTestCase(unittest.TestCase): >>> case = DocTestCase(test) >>> try: ... case.debug() - ... except UnexpectedException as failure: - ... pass + ... except UnexpectedException as e: + ... failure = e The UnexpectedException contains the test, the example, and the original exception: @@ -2196,8 +2283,8 @@ class DocTestCase(unittest.TestCase): >>> try: ... case.debug() - ... except DocTestFailure as failure: - ... pass + ... except DocTestFailure as e: + ... failure = e DocTestFailure objects provide access to the test: @@ -2646,7 +2733,7 @@ __test__ = {"_TestClass": _TestClass, "whitespace normalization": r""" If the whitespace normalization flag is used, then differences in whitespace are ignored. - >>> print(range(30)) #doctest: +NORMALIZE_WHITESPACE + >>> print(list(xrange(30))) #doctest: +NORMALIZE_WHITESPACE [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] diff --git a/django/test/client.py b/django/test/client.py index a18b7f8853..a9ae7f5db1 100644 --- a/django/test/client.py +++ b/django/test/client.py @@ -19,7 +19,7 @@ from django.http import SimpleCookie, HttpRequest, QueryDict from django.template import TemplateDoesNotExist from django.test import signals from django.utils.functional import curry -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils.http import urlencode from django.utils.importlib import import_module from django.utils.itercompat import is_iterable @@ -108,7 +108,7 @@ def encode_multipart(boundary, data): as an application/octet-stream; otherwise, str(value) will be sent. """ lines = [] - to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET) + to_str = lambda s: smart_bytes(s, settings.DEFAULT_CHARSET) # Not by any means perfect, but good enough for our purposes. is_file = lambda thing: hasattr(thing, "read") and callable(thing.read) @@ -145,7 +145,7 @@ def encode_multipart(boundary, data): return '\r\n'.join(lines) def encode_file(boundary, key, file): - to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET) + to_str = lambda s: smart_bytes(s, settings.DEFAULT_CHARSET) content_type = mimetypes.guess_type(file.name)[0] if content_type is None: content_type = 'application/octet-stream' @@ -194,9 +194,9 @@ class RequestFactory(object): 'SERVER_NAME': 'testserver', 'SERVER_PORT': '80', 'SERVER_PROTOCOL': 'HTTP/1.1', - 'wsgi.version': (1,0), + 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', - 'wsgi.input': FakePayload(''), + 'wsgi.input': FakePayload(b''), 'wsgi.errors': self.errors, 'wsgi.multiprocess': True, 'wsgi.multithread': False, @@ -220,7 +220,7 @@ class RequestFactory(object): charset = match.group(1) else: charset = settings.DEFAULT_CHARSET - return smart_str(data, encoding=charset) + return smart_bytes(data, encoding=charset) def _get_path(self, parsed): # If there are parameters, add them @@ -291,7 +291,7 @@ class RequestFactory(object): def generic(self, method, path, data='', content_type='application/octet-stream', **extra): parsed = urlparse(path) - data = smart_str(data, settings.DEFAULT_CHARSET) + data = smart_bytes(data, settings.DEFAULT_CHARSET) r = { 'PATH_INFO': self._get_path(parsed), 'QUERY_STRING': parsed[4], diff --git a/django/test/html.py b/django/test/html.py index 2a1421bf17..acdb4ffd14 100644 --- a/django/test/html.py +++ b/django/test/html.py @@ -5,7 +5,7 @@ Comparing two html documents. from __future__ import unicode_literals import re -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.html_parser import HTMLParser, HTMLParseError from django.utils import six @@ -25,7 +25,7 @@ class Element(object): def append(self, element): if isinstance(element, six.string_types): - element = force_unicode(element) + element = force_text(element) element = normalize_whitespace(element) if self.children: if isinstance(self.children[-1], six.string_types): @@ -83,6 +83,8 @@ class Element(object): return False return True + __hash__ = object.__hash__ + def __ne__(self, element): return not self.__eq__(element) diff --git a/django/test/testcases.py b/django/test/testcases.py index b60188bf30..56ba56caf1 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -41,7 +41,7 @@ from django.test.utils import (get_warnings_state, restore_warnings_state, override_settings) from django.test.utils import ContextList from django.utils import unittest as ut2 -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import smart_bytes, force_text from django.utils import six from django.utils.unittest.util import safe_repr from django.views.static import serve @@ -398,7 +398,7 @@ class SimpleTestCase(ut2.TestCase): optional.clean(input) self.assertEqual(context_manager.exception.messages, errors) # test required inputs - error_required = [force_unicode(required.error_messages['required'])] + error_required = [force_text(required.error_messages['required'])] for e in EMPTY_VALUES: with self.assertRaises(ValidationError) as context_manager: required.clean(e) @@ -647,7 +647,7 @@ class TransactionTestCase(SimpleTestCase): self.assertEqual(response.status_code, status_code, msg_prefix + "Couldn't retrieve content: Response code was %d" " (expected %d)" % (response.status_code, status_code)) - enc_text = smart_str(text, response._charset) + enc_text = smart_bytes(text, response._charset) content = response.content if html: content = assert_and_parse_html(self, content, None, @@ -683,7 +683,7 @@ class TransactionTestCase(SimpleTestCase): self.assertEqual(response.status_code, status_code, msg_prefix + "Couldn't retrieve content: Response code was %d" " (expected %d)" % (response.status_code, status_code)) - enc_text = smart_str(text, response._charset) + enc_text = smart_bytes(text, response._charset) content = response.content if html: content = assert_and_parse_html(self, content, None, @@ -796,9 +796,10 @@ class TransactionTestCase(SimpleTestCase): " the response" % template_name) def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True): + items = six.moves.map(transform, qs) if not ordered: - return self.assertEqual(set(map(transform, qs)), set(values)) - return self.assertEqual(map(transform, qs), values) + return self.assertEqual(set(items), set(values)) + return self.assertEqual(list(items), values) def assertNumQueries(self, num, func=None, *args, **kwargs): using = kwargs.pop("using", DEFAULT_DB_ALIAS) diff --git a/django/utils/_os.py b/django/utils/_os.py index 884689fefc..9eb5e5e8ea 100644 --- a/django/utils/_os.py +++ b/django/utils/_os.py @@ -1,7 +1,8 @@ import os import stat from os.path import join, normcase, normpath, abspath, isabs, sep -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text +from django.utils import six try: WindowsError = WindowsError @@ -10,13 +11,12 @@ except NameError: pass -# Define our own abspath function that can handle joining -# unicode paths to a current working directory that has non-ASCII -# characters in it. This isn't necessary on Windows since the -# Windows version of abspath handles this correctly. The Windows -# abspath also handles drive letters differently than the pure -# Python implementation, so it's best not to replace it. -if os.name == 'nt': +# Under Python 2, define our own abspath function that can handle joining +# unicode paths to a current working directory that has non-ASCII characters +# in it. This isn't necessary on Windows since the Windows version of abspath +# handles this correctly. It also handles drive letters differently than the +# pure Python implementation, so it's best not to replace it. +if six.PY3 or os.name == 'nt': abspathu = abspath else: def abspathu(path): @@ -37,8 +37,8 @@ def safe_join(base, *paths): The final path must be located inside of the base path component (otherwise a ValueError is raised). """ - base = force_unicode(base) - paths = [force_unicode(p) for p in paths] + base = force_text(base) + paths = [force_text(p) for p in paths] final_path = abspathu(join(base, *paths)) base_path = abspathu(base) base_path_len = len(base_path) diff --git a/django/utils/archive.py b/django/utils/archive.py index 6b5d73290f..829b55dd28 100644 --- a/django/utils/archive.py +++ b/django/utils/archive.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ import os import shutil -import sys import tarfile import zipfile @@ -147,11 +146,11 @@ class TarArchive(BaseArchive): else: try: extracted = self._archive.extractfile(member) - except (KeyError, AttributeError): + except (KeyError, AttributeError) as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) - print ("In the tar file %s the member %s is invalid: %s" % - (name, member.name, sys.exc_info()[1])) + print("In the tar file %s the member %s is invalid: %s" % + (name, member.name, exc)) else: dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): diff --git a/django/utils/cache.py b/django/utils/cache.py index 8b81a2ffa2..42b0de4ce6 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -23,7 +23,7 @@ import time from django.conf import settings from django.core.cache import get_cache -from django.utils.encoding import iri_to_uri, force_unicode +from django.utils.encoding import iri_to_uri, force_text from django.utils.http import http_date from django.utils.timezone import get_current_timezone_name from django.utils.translation import get_language @@ -169,7 +169,7 @@ def _i18n_cache_key_suffix(request, cache_key): # Windows is known to use non-standard, locale-dependant names. # User-defined tzinfo classes may return absolutely anything. # Hence this paranoid conversion to create a valid cache key. - tz_name = force_unicode(get_current_timezone_name(), errors='ignore') + tz_name = force_text(get_current_timezone_name(), errors='ignore') cache_key += '.%s' % tz_name.encode('ascii', 'ignore').replace(' ', '_') return cache_key diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 9d46bdd793..70a07e7fde 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -23,7 +23,7 @@ except NotImplementedError: using_sysrandom = False from django.conf import settings -from django.utils.encoding import smart_str +from django.utils.encoding import smart_bytes from django.utils.six.moves import xrange @@ -50,7 +50,7 @@ def salted_hmac(key_salt, value, secret=None): # line is redundant and could be replaced by key = key_salt + secret, since # the hmac module does the same thing for keys longer than the block size. # However, we need to ensure that we *always* do this. - return hmac.new(key, msg=value, digestmod=hashlib.sha1) + return hmac.new(key, msg=smart_bytes(value), digestmod=hashlib.sha1) def get_random_string(length=12, @@ -99,7 +99,7 @@ def _bin_to_long(x): This is a clever optimization for fast xor vector math """ - return int(x.encode('hex'), 16) + return int(binascii.hexlify(x), 16) def _long_to_bin(x, hex_format_string): @@ -112,13 +112,14 @@ def _long_to_bin(x, hex_format_string): def _fast_hmac(key, msg, digest): """ - A trimmed down version of Python's HMAC implementation + A trimmed down version of Python's HMAC implementation. + + This function operates on bytes. """ dig1, dig2 = digest(), digest() - key = smart_str(key) if len(key) > dig1.block_size: key = digest(key).digest() - key += chr(0) * (dig1.block_size - len(key)) + key += b'\x00' * (dig1.block_size - len(key)) dig1.update(key.translate(_trans_36)) dig1.update(msg) dig2.update(key.translate(_trans_5c)) @@ -141,8 +142,8 @@ def pbkdf2(password, salt, iterations, dklen=0, digest=None): assert iterations > 0 if not digest: digest = hashlib.sha256 - password = smart_str(password) - salt = smart_str(salt) + password = smart_bytes(password) + salt = smart_bytes(salt) hlen = digest().digest_size if not dklen: dklen = hlen diff --git a/django/utils/datastructures.py b/django/utils/datastructures.py index bbd31ad36c..ad17573104 100644 --- a/django/utils/datastructures.py +++ b/django/utils/datastructures.py @@ -129,7 +129,7 @@ class SortedDict(dict): data = list(data) super(SortedDict, self).__init__(data) if isinstance(data, dict): - self.keyOrder = list(six.iterkeys(data)) + self.keyOrder = list(data) else: self.keyOrder = [] seen = set() diff --git a/django/utils/dateformat.py b/django/utils/dateformat.py index c9a6138aed..6a91a370e5 100644 --- a/django/utils/dateformat.py +++ b/django/utils/dateformat.py @@ -20,7 +20,7 @@ import datetime from django.utils.dates import MONTHS, MONTHS_3, MONTHS_ALT, MONTHS_AP, WEEKDAYS, WEEKDAYS_ABBR from django.utils.tzinfo import LocalTimezone from django.utils.translation import ugettext as _ -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils import six from django.utils.timezone import is_aware, is_naive @@ -30,9 +30,9 @@ re_escaped = re.compile(r'\\(.)') class Formatter(object): def format(self, formatstr): pieces = [] - for i, piece in enumerate(re_formatchars.split(force_unicode(formatstr))): + for i, piece in enumerate(re_formatchars.split(force_text(formatstr))): if i % 2: - pieces.append(force_unicode(getattr(self, piece)())) + pieces.append(force_text(getattr(self, piece)())) elif piece: pieces.append(re_escaped.sub(r'\1', piece)) return ''.join(pieces) diff --git a/django/utils/dateparse.py b/django/utils/dateparse.py index 532bb259c3..032eb493b6 100644 --- a/django/utils/dateparse.py +++ b/django/utils/dateparse.py @@ -7,6 +7,7 @@ import datetime import re +from django.utils import six from django.utils.timezone import utc from django.utils.tzinfo import FixedOffset @@ -34,7 +35,7 @@ def parse_date(value): """ match = date_re.match(value) if match: - kw = dict((k, int(v)) for k, v in match.groupdict().iteritems()) + kw = dict((k, int(v)) for k, v in six.iteritems(match.groupdict())) return datetime.date(**kw) def parse_time(value): @@ -53,7 +54,7 @@ def parse_time(value): kw = match.groupdict() if kw['microsecond']: kw['microsecond'] = kw['microsecond'].ljust(6, '0') - kw = dict((k, int(v)) for k, v in kw.iteritems() if v is not None) + kw = dict((k, int(v)) for k, v in six.iteritems(kw) if v is not None) return datetime.time(**kw) def parse_datetime(value): @@ -80,6 +81,6 @@ def parse_datetime(value): if tzinfo[0] == '-': offset = -offset tzinfo = FixedOffset(offset) - kw = dict((k, int(v)) for k, v in kw.iteritems() if v is not None) + kw = dict((k, int(v)) for k, v in six.iteritems(kw) if v is not None) kw['tzinfo'] = tzinfo return datetime.datetime(**kw) diff --git a/django/utils/dictconfig.py b/django/utils/dictconfig.py index b4d6d66b3c..c7b39819b5 100644 --- a/django/utils/dictconfig.py +++ b/django/utils/dictconfig.py @@ -363,7 +363,7 @@ class DictConfigurator(BaseConfigurator): #which were in the previous configuration but #which are not in the new configuration. root = logging.root - existing = root.manager.loggerDict.keys() + existing = list(root.manager.loggerDict) #The list needs to be sorted so that we can #avoid disabling child loggers of explicitly #named loggers. With a sorted list it is easier diff --git a/django/utils/encoding.py b/django/utils/encoding.py index f2295444bf..eb60cfde8b 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -24,9 +24,13 @@ class DjangoUnicodeDecodeError(UnicodeDecodeError): class StrAndUnicode(object): """ - A class whose __str__ returns its __unicode__ as a UTF-8 bytestring. + A class that derives __str__ from __unicode__. - Useful as a mix-in. + On Python 2, __str__ returns the output of __unicode__ encoded as a UTF-8 + bytestring. On Python 3, __str__ returns the output of __unicode__. + + Useful as a mix-in. If you support Python 2 and 3 with a single code base, + you can inherit this mix-in and just define __unicode__. """ if six.PY3: def __str__(self): @@ -35,37 +39,36 @@ class StrAndUnicode(object): def __str__(self): return self.__unicode__().encode('utf-8') -def smart_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): +def smart_text(s, encoding='utf-8', strings_only=False, errors='strict'): """ - Returns a unicode object representing 's'. Treats bytestrings using the - 'encoding' codec. + Returns a text object representing 's' -- unicode on Python 2 and str on + Python 3. Treats bytestrings using the 'encoding' codec. If strings_only is True, don't convert (some) non-string-like objects. """ if isinstance(s, Promise): # The input is the result of a gettext_lazy() call. return s - return force_unicode(s, encoding, strings_only, errors) + return force_text(s, encoding, strings_only, errors) def is_protected_type(obj): """Determine if the object instance is of a protected type. Objects of protected types are preserved as-is when passed to - force_unicode(strings_only=True). + force_text(strings_only=True). """ return isinstance(obj, six.integer_types + (type(None), float, Decimal, datetime.datetime, datetime.date, datetime.time)) -def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): +def force_text(s, encoding='utf-8', strings_only=False, errors='strict'): """ - Similar to smart_unicode, except that lazy instances are resolved to + Similar to smart_text, except that lazy instances are resolved to strings, rather than kept as lazy objects. If strings_only is True, don't convert (some) non-string-like objects. """ - # Handle the common case first, saves 30-40% in performance when s - # is an instance of unicode. This function gets called often in that - # setting. + # Handle the common case first, saves 30-40% when s is an instance of + # six.text_type. This function gets called often in that setting. if isinstance(s, six.text_type): return s if strings_only and is_protected_type(s): @@ -92,9 +95,9 @@ def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): # without raising a further exception. We do an # approximation to what the Exception's standard str() # output should be. - s = ' '.join([force_unicode(arg, encoding, strings_only, + s = ' '.join([force_text(arg, encoding, strings_only, errors) for arg in s]) - elif not isinstance(s, six.text_type): + else: # Note: We use .decode() here, instead of six.text_type(s, encoding, # errors), so that if s is a SafeString, it ends up being a # SafeUnicode at the end. @@ -108,21 +111,26 @@ def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): # working unicode method. Try to handle this without raising a # further exception by individually forcing the exception args # to unicode. - s = ' '.join([force_unicode(arg, encoding, strings_only, + s = ' '.join([force_text(arg, encoding, strings_only, errors) for arg in s]) return s -def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): +def smart_bytes(s, encoding='utf-8', strings_only=False, errors='strict'): """ Returns a bytestring version of 's', encoded as specified in 'encoding'. If strings_only is True, don't convert (some) non-string-like objects. """ + if isinstance(s, bytes): + if encoding == 'utf-8': + return s + else: + return s.decode('utf-8', errors).encode(encoding, errors) if strings_only and (s is None or isinstance(s, int)): return s if isinstance(s, Promise): return six.text_type(s).encode(encoding, errors) - elif not isinstance(s, six.string_types): + if not isinstance(s, six.string_types): try: if six.PY3: return six.text_type(s).encode(encoding) @@ -133,15 +141,25 @@ def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): # An Exception subclass containing non-ASCII data that doesn't # know how to print itself properly. We shouldn't raise a # further exception. - return ' '.join([smart_str(arg, encoding, strings_only, + return ' '.join([smart_bytes(arg, encoding, strings_only, errors) for arg in s]) return six.text_type(s).encode(encoding, errors) - elif isinstance(s, six.text_type): - return s.encode(encoding, errors) - elif s and encoding != 'utf-8': - return s.decode('utf-8', errors).encode(encoding, errors) else: - return s + return s.encode(encoding, errors) + +if six.PY3: + smart_str = smart_text +else: + smart_str = smart_bytes + # backwards compatibility for Python 2 + smart_unicode = smart_text + force_unicode = force_text + +smart_str.__doc__ = """\ +Apply smart_text in Python 3 and smart_bytes in Python 2. + +This is suitable for writing to sys.stdout (for instance). +""" def iri_to_uri(iri): """ @@ -168,7 +186,7 @@ def iri_to_uri(iri): # converted. if iri is None: return iri - return quote(smart_str(iri), safe=b"/#%[]=:;$&()+,!?*@'~") + return quote(smart_bytes(iri), safe=b"/#%[]=:;$&()+,!?*@'~") def filepath_to_uri(path): """Convert an file system path to a URI portion that is suitable for @@ -187,7 +205,7 @@ def filepath_to_uri(path): return path # I know about `os.sep` and `os.altsep` but I want to leave # some flexibility for hardcoding separators. - return quote(smart_str(path).replace("\\", "/"), safe=b"/~!*()'") + return quote(smart_bytes(path).replace("\\", "/"), safe=b"/~!*()'") # The encoding of the default system locale but falls back to the # given fallback encoding if the encoding is unsupported by python or could diff --git a/django/utils/feedgenerator.py b/django/utils/feedgenerator.py index 6498aaf57c..f9126a6782 100644 --- a/django/utils/feedgenerator.py +++ b/django/utils/feedgenerator.py @@ -29,8 +29,10 @@ try: except ImportError: # Python 2 from urlparse import urlparse from django.utils.xmlutils import SimplerXMLGenerator -from django.utils.encoding import force_unicode, iri_to_uri +from django.utils.encoding import force_text, iri_to_uri from django.utils import datetime_safe +from django.utils import six +from django.utils.six import StringIO from django.utils.timezone import is_aware def rfc2822_date(date): @@ -44,25 +46,29 @@ def rfc2822_date(date): dow = days[date.weekday()] month = months[date.month - 1] time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month)) + if not six.PY3: # strftime returns a byte string in Python 2 + time_str = time_str.decode('utf-8') if is_aware(date): offset = date.tzinfo.utcoffset(date) timezone = (offset.days * 24 * 60) + (offset.seconds // 60) hour, minute = divmod(timezone, 60) - return time_str + "%+03d%02d" % (hour, minute) + return time_str + '%+03d%02d' % (hour, minute) else: return time_str + '-0000' def rfc3339_date(date): # Support datetime objects older than 1900 date = datetime_safe.new_datetime(date) + time_str = date.strftime('%Y-%m-%dT%H:%M:%S') + if not six.PY3: # strftime returns a byte string in Python 2 + time_str = time_str.decode('utf-8') if is_aware(date): - time_str = date.strftime('%Y-%m-%dT%H:%M:%S') offset = date.tzinfo.utcoffset(date) timezone = (offset.days * 24 * 60) + (offset.seconds // 60) hour, minute = divmod(timezone, 60) - return time_str + "%+03d:%02d" % (hour, minute) + return time_str + '%+03d:%02d' % (hour, minute) else: - return date.strftime('%Y-%m-%dT%H:%M:%SZ') + return time_str + 'Z' def get_tag_uri(url, date): """ @@ -81,12 +87,12 @@ class SyndicationFeed(object): def __init__(self, title, link, description, language=None, author_email=None, author_name=None, author_link=None, subtitle=None, categories=None, feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs): - to_unicode = lambda s: force_unicode(s, strings_only=True) + to_unicode = lambda s: force_text(s, strings_only=True) if categories: - categories = [force_unicode(c) for c in categories] + categories = [force_text(c) for c in categories] if ttl is not None: # Force ints to unicode - ttl = force_unicode(ttl) + ttl = force_text(ttl) self.feed = { 'title': to_unicode(title), 'link': iri_to_uri(link), @@ -114,12 +120,12 @@ class SyndicationFeed(object): objects except pubdate, which is a datetime.datetime object, and enclosure, which is an instance of the Enclosure class. """ - to_unicode = lambda s: force_unicode(s, strings_only=True) + to_unicode = lambda s: force_text(s, strings_only=True) if categories: categories = [to_unicode(c) for c in categories] if ttl is not None: # Force ints to unicode - ttl = force_unicode(ttl) + ttl = force_text(ttl) item = { 'title': to_unicode(title), 'link': iri_to_uri(link), @@ -178,8 +184,7 @@ class SyndicationFeed(object): """ Returns the feed in the given encoding as a string. """ - from io import BytesIO - s = BytesIO() + s = StringIO() self.write(s, encoding) return s.getvalue() @@ -237,7 +242,7 @@ class RssFeed(SyndicationFeed): handler.addQuickElement("category", cat) if self.feed['feed_copyright'] is not None: handler.addQuickElement("copyright", self.feed['feed_copyright']) - handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()).decode('utf-8')) + handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date())) if self.feed['ttl'] is not None: handler.addQuickElement("ttl", self.feed['ttl']) @@ -271,7 +276,7 @@ class Rss201rev2Feed(RssFeed): handler.addQuickElement("dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}) if item['pubdate'] is not None: - handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']).decode('utf-8')) + handler.addQuickElement("pubDate", rfc2822_date(item['pubdate'])) if item['comments'] is not None: handler.addQuickElement("comments", item['comments']) if item['unique_id'] is not None: @@ -314,7 +319,7 @@ class Atom1Feed(SyndicationFeed): if self.feed['feed_url'] is not None: handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']}) handler.addQuickElement("id", self.feed['id']) - handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()).decode('utf-8')) + handler.addQuickElement("updated", rfc3339_date(self.latest_post_date())) if self.feed['author_name'] is not None: handler.startElement("author", {}) handler.addQuickElement("name", self.feed['author_name']) @@ -340,7 +345,7 @@ class Atom1Feed(SyndicationFeed): handler.addQuickElement("title", item['title']) handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"}) if item['pubdate'] is not None: - handler.addQuickElement("updated", rfc3339_date(item['pubdate']).decode('utf-8')) + handler.addQuickElement("updated", rfc3339_date(item['pubdate'])) # Author information. if item['author_name'] is not None: diff --git a/django/utils/functional.py b/django/utils/functional.py index 2dca182b99..085ec40b63 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -93,13 +93,19 @@ def lazy(func, *resultclasses): if hasattr(cls, k): continue setattr(cls, k, meth) - cls._delegate_str = bytes in resultclasses - cls._delegate_unicode = six.text_type in resultclasses - assert not (cls._delegate_str and cls._delegate_unicode), "Cannot call lazy() with both str and unicode return types." - if cls._delegate_unicode: - cls.__unicode__ = cls.__unicode_cast - elif cls._delegate_str: - cls.__str__ = cls.__str_cast + cls._delegate_bytes = bytes in resultclasses + cls._delegate_text = six.text_type in resultclasses + assert not (cls._delegate_bytes and cls._delegate_text), "Cannot call lazy() with both bytes and text return types." + if cls._delegate_text: + if six.PY3: + cls.__str__ = cls.__text_cast + else: + cls.__unicode__ = cls.__text_cast + elif cls._delegate_bytes: + if six.PY3: + cls.__bytes__ = cls.__bytes_cast + else: + cls.__str__ = cls.__bytes_cast __prepare_class__ = classmethod(__prepare_class__) def __promise__(cls, klass, funcname, method): @@ -120,17 +126,17 @@ def lazy(func, *resultclasses): return __wrapper__ __promise__ = classmethod(__promise__) - def __unicode_cast(self): + def __text_cast(self): return func(*self.__args, **self.__kw) - def __str_cast(self): - return str(func(*self.__args, **self.__kw)) + def __bytes_cast(self): + return bytes(func(*self.__args, **self.__kw)) def __cast(self): - if self._delegate_str: - return self.__str_cast() - elif self._delegate_unicode: - return self.__unicode_cast() + if self._delegate_bytes: + return self.__bytes_cast() + elif self._delegate_text: + return self.__text_cast() else: return func(*self.__args, **self.__kw) @@ -144,10 +150,12 @@ def lazy(func, *resultclasses): other = other.__cast() return self.__cast() < other + __hash__ = object.__hash__ + def __mod__(self, rhs): - if self._delegate_str: - return str(self) % rhs - elif self._delegate_unicode: + if self._delegate_bytes and not six.PY3: + return bytes(self) % rhs + elif self._delegate_text: return six.text_type(self) % rhs else: raise AssertionError('__mod__ not supported for non-string types') @@ -178,7 +186,7 @@ def allow_lazy(func, *resultclasses): """ @wraps(func) def wrapper(*args, **kwargs): - for arg in list(args) + kwargs.values(): + for arg in list(args) + list(six.itervalues(kwargs)): if isinstance(arg, Promise): break else: @@ -234,6 +242,9 @@ class LazyObject(object): __dir__ = new_method_proxy(dir) +# Workaround for http://bugs.python.org/issue12370 +_super = super + class SimpleLazyObject(LazyObject): """ A lazy object initialised from any function. @@ -251,13 +262,17 @@ class SimpleLazyObject(LazyObject): value. """ self.__dict__['_setupfunc'] = func - super(SimpleLazyObject, self).__init__() + _super(SimpleLazyObject, self).__init__() def _setup(self): self._wrapped = self._setupfunc() - __str__ = new_method_proxy(bytes) - __unicode__ = new_method_proxy(six.text_type) + if six.PY3: + __bytes__ = new_method_proxy(bytes) + __str__ = new_method_proxy(str) + else: + __str__ = new_method_proxy(str) + __unicode__ = new_method_proxy(unicode) def __deepcopy__(self, memo): if self._wrapped is empty: @@ -284,7 +299,8 @@ class SimpleLazyObject(LazyObject): __class__ = property(new_method_proxy(operator.attrgetter("__class__"))) __eq__ = new_method_proxy(operator.eq) __hash__ = new_method_proxy(hash) - __nonzero__ = new_method_proxy(bool) + __bool__ = new_method_proxy(bool) # Python 3 + __nonzero__ = __bool__ # Python 2 class lazy_property(property): @@ -312,8 +328,8 @@ def partition(predicate, values): Splits the values into two sets, based on the return value of the function (True/False). e.g.: - >>> partition(lambda: x > 3, range(5)) - [1, 2, 3], [4] + >>> partition(lambda x: x > 3, range(5)) + [0, 1, 2, 3], [4] """ results = ([], []) for item in values: diff --git a/django/utils/html.py b/django/utils/html.py index ca3340ccb1..7dd53a1353 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -11,7 +11,7 @@ except ImportError: # Python 2 from urlparse import urlsplit, urlunsplit from django.utils.safestring import SafeData, mark_safe -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import smart_bytes, force_text from django.utils.functional import allow_lazy from django.utils import six from django.utils.text import normalize_newlines @@ -33,13 +33,13 @@ link_target_attribute_re = re.compile(r'(<a [^>]*?)target=[^\s>]+') html_gunk_re = re.compile(r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE) hard_coded_bullets_re = re.compile(r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL) trailing_empty_content_re = re.compile(r'(?:<p>(?: |\s|<br \/>)*?</p>\s*)+\Z') -del x # Temporary variable + def escape(text): """ Returns the given text with ampersands, quotes and angle brackets encoded for use in HTML. """ - return mark_safe(force_unicode(text).replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''')) + return mark_safe(force_text(text).replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''')) escape = allow_lazy(escape, six.text_type) _base_js_escapes = ( @@ -63,7 +63,7 @@ _js_escapes = (_base_js_escapes + def escapejs(value): """Hex encodes characters for use in JavaScript strings.""" for bad, good in _js_escapes: - value = mark_safe(force_unicode(value).replace(bad, good)) + value = mark_safe(force_text(value).replace(bad, good)) return value escapejs = allow_lazy(escapejs, six.text_type) @@ -84,7 +84,7 @@ def format_html(format_string, *args, **kwargs): """ args_safe = map(conditional_escape, args) kwargs_safe = dict([(k, conditional_escape(v)) for (k, v) in - kwargs.iteritems()]) + six.iteritems(kwargs)]) return mark_safe(format_string.format(*args_safe, **kwargs_safe)) def format_html_join(sep, format_string, args_generator): @@ -120,22 +120,22 @@ linebreaks = allow_lazy(linebreaks, six.text_type) def strip_tags(value): """Returns the given HTML with all tags stripped.""" - return re.sub(r'<[^>]*?>', '', force_unicode(value)) + return re.sub(r'<[^>]*?>', '', force_text(value)) strip_tags = allow_lazy(strip_tags) def strip_spaces_between_tags(value): """Returns the given HTML with spaces between tags removed.""" - return re.sub(r'>\s+<', '><', force_unicode(value)) + return re.sub(r'>\s+<', '><', force_text(value)) strip_spaces_between_tags = allow_lazy(strip_spaces_between_tags, six.text_type) def strip_entities(value): """Returns the given HTML with all entities (&something;) stripped.""" - return re.sub(r'&(?:\w+|#\d+);', '', force_unicode(value)) + return re.sub(r'&(?:\w+|#\d+);', '', force_text(value)) strip_entities = allow_lazy(strip_entities, six.text_type) def fix_ampersands(value): """Returns the given HTML with all unencoded ampersands encoded correctly.""" - return unencoded_ampersands_re.sub('&', force_unicode(value)) + return unencoded_ampersands_re.sub('&', force_text(value)) fix_ampersands = allow_lazy(fix_ampersands, six.text_type) def smart_urlquote(url): @@ -153,9 +153,9 @@ def smart_urlquote(url): # contains a % not followed by two hexadecimal digits. See #9655. if '%' not in url or unquoted_percents_re.search(url): # See http://bugs.python.org/issue2637 - url = quote(smart_str(url), safe=b'!*\'();:@&=+$,/?#[]~') + url = quote(smart_bytes(url), safe=b'!*\'();:@&=+$,/?#[]~') - return force_unicode(url) + return force_text(url) def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): """ @@ -176,7 +176,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): """ trim_url = lambda x, limit=trim_url_limit: limit is not None and (len(x) > limit and ('%s...' % x[:max(0, limit - 3)])) or x safe_input = isinstance(text, SafeData) - words = word_split_re.split(force_unicode(text)) + words = word_split_re.split(force_text(text)) for i, word in enumerate(words): match = None if '.' in word or '@' in word or ':' in word: @@ -245,7 +245,7 @@ def clean_html(text): bottom of the text. """ from django.utils.text import normalize_newlines - text = normalize_newlines(force_unicode(text)) + text = normalize_newlines(force_text(text)) text = re.sub(r'<(/?)\s*b\s*>', '<\\1strong>', text) text = re.sub(r'<(/?)\s*i\s*>', '<\\1em>', text) text = fix_ampersands(text) diff --git a/django/utils/http.py b/django/utils/http.py index f3a3dce58c..22e81a33d7 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -13,7 +13,7 @@ except ImportError: # Python 2 from email.utils import formatdate from django.utils.datastructures import MultiValueDict -from django.utils.encoding import smart_str, force_unicode +from django.utils.encoding import force_text, smart_str from django.utils.functional import allow_lazy from django.utils import six @@ -37,7 +37,7 @@ def urlquote(url, safe='/'): can safely be used as part of an argument to a subsequent iri_to_uri() call without double-quoting occurring. """ - return force_unicode(urllib_parse.quote(smart_str(url), smart_str(safe))) + return force_text(urllib_parse.quote(smart_str(url), smart_str(safe))) urlquote = allow_lazy(urlquote, six.text_type) def urlquote_plus(url, safe=''): @@ -47,7 +47,7 @@ def urlquote_plus(url, safe=''): returned string can safely be used as part of an argument to a subsequent iri_to_uri() call without double-quoting occurring. """ - return force_unicode(urllib_parse.quote_plus(smart_str(url), smart_str(safe))) + return force_text(urllib_parse.quote_plus(smart_str(url), smart_str(safe))) urlquote_plus = allow_lazy(urlquote_plus, six.text_type) def urlunquote(quoted_url): @@ -55,7 +55,7 @@ def urlunquote(quoted_url): A wrapper for Python's urllib.unquote() function that can operate on the result of django.utils.http.urlquote(). """ - return force_unicode(urllib_parse.unquote(smart_str(quoted_url))) + return force_text(urllib_parse.unquote(smart_str(quoted_url))) urlunquote = allow_lazy(urlunquote, six.text_type) def urlunquote_plus(quoted_url): @@ -63,7 +63,7 @@ def urlunquote_plus(quoted_url): A wrapper for Python's urllib.unquote_plus() function that can operate on the result of django.utils.http.urlquote_plus(). """ - return force_unicode(urllib_parse.unquote_plus(smart_str(quoted_url))) + return force_text(urllib_parse.unquote_plus(smart_str(quoted_url))) urlunquote_plus = allow_lazy(urlunquote_plus, six.text_type) def urlencode(query, doseq=0): @@ -167,8 +167,9 @@ def base36_to_int(s): if len(s) > 13: raise ValueError("Base36 input too large") value = int(s, 36) - # ... then do a final check that the value will fit into an int. - if value > sys.maxint: + # ... then do a final check that the value will fit into an int to avoid + # returning a long (#15067). The long type was removed in Python 3. + if not six.PY3 and value > sys.maxint: raise ValueError("Base36 input too large") return value @@ -178,8 +179,13 @@ def int_to_base36(i): """ digits = "0123456789abcdefghijklmnopqrstuvwxyz" factor = 0 - if not 0 <= i <= sys.maxint: - raise ValueError("Base36 conversion input too large or incorrect type.") + if i < 0: + raise ValueError("Negative base36 conversion input.") + if not six.PY3: + if not isinstance(i, six.integer_types): + raise TypeError("Non-integer base36 conversion input.") + if i > sys.maxint: + raise ValueError("Base36 conversion input too large.") # Find starting factor while True: factor += 1 diff --git a/django/utils/regex_helper.py b/django/utils/regex_helper.py index 8953a21e95..7b40d141de 100644 --- a/django/utils/regex_helper.py +++ b/django/utils/regex_helper.py @@ -8,6 +8,7 @@ should be good enough for a large class of URLS, however. from __future__ import unicode_literals from django.utils import six +from django.utils.six.moves import zip # Mapping of an escape character to a representative of that class. So, e.g., # "\w" is replaced by "x" in a reverse URL. A value of None means to ignore @@ -44,8 +45,8 @@ class NonCapture(list): def normalize(pattern): """ - Given a reg-exp pattern, normalizes it to a list of forms that suffice for - reverse matching. This does the following: + Given a reg-exp pattern, normalizes it to an iterable of forms that + suffice for reverse matching. This does the following: (1) For any repeating sections, keeps the minimum number of occurrences permitted (this means zero for optional groups). @@ -80,7 +81,7 @@ def normalize(pattern): try: ch, escaped = next(pattern_iter) except StopIteration: - return zip([''], [[]]) + return [('', [])] try: while True: @@ -193,9 +194,9 @@ def normalize(pattern): pass except NotImplementedError: # A case of using the disjunctive form. No results for you! - return zip([''], [[]]) + return [('', [])] - return zip(*flatten_result(result)) + return list(zip(*flatten_result(result))) def next_char(input_iter): """ diff --git a/django/utils/safestring.py b/django/utils/safestring.py index 1599fc2a66..bfaefd07ee 100644 --- a/django/utils/safestring.py +++ b/django/utils/safestring.py @@ -96,7 +96,7 @@ def mark_safe(s): """ if isinstance(s, SafeData): return s - if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_str): + if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes): return SafeString(s) if isinstance(s, (six.text_type, Promise)): return SafeUnicode(s) @@ -112,7 +112,7 @@ def mark_for_escaping(s): """ if isinstance(s, (SafeData, EscapeData)): return s - if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_str): + if isinstance(s, bytes) or (isinstance(s, Promise) and s._delegate_bytes): return EscapeString(s) if isinstance(s, (six.text_type, Promise)): return EscapeUnicode(s) diff --git a/django/utils/six.py b/django/utils/six.py index e226bba09e..ceb5d70e58 100644 --- a/django/utils/six.py +++ b/django/utils/six.py @@ -113,6 +113,7 @@ class _MovedItems(types.ModuleType): _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("reload_module", "__builtin__", "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), diff --git a/django/utils/termcolors.py b/django/utils/termcolors.py index 1eebaa2316..4f74b564a5 100644 --- a/django/utils/termcolors.py +++ b/django/utils/termcolors.py @@ -2,6 +2,8 @@ termcolors.py """ +from django.utils import six + color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white') foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) background = dict([(color_names[x], '4%s' % x) for x in range(8)]) @@ -41,7 +43,7 @@ def colorize(text='', opts=(), **kwargs): code_list = [] if text == '' and len(opts) == 1 and opts[0] == 'reset': return '\x1b[%sm' % RESET - for k, v in kwargs.iteritems(): + for k, v in six.iteritems(kwargs): if k == 'fg': code_list.append(foreground[v]) elif k == 'bg': diff --git a/django/utils/text.py b/django/utils/text.py index 43056aa634..0838d79c65 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -7,13 +7,13 @@ from gzip import GzipFile from django.utils.six.moves import html_entities from io import BytesIO -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.functional import allow_lazy, SimpleLazyObject from django.utils import six from django.utils.translation import ugettext_lazy, ugettext as _, pgettext # Capitalizes the first letter of a string. -capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:] +capfirst = lambda x: x and force_text(x)[0].upper() + force_text(x)[1:] capfirst = allow_lazy(capfirst, six.text_type) # Set up regular expressions @@ -26,7 +26,7 @@ def wrap(text, width): A word-wrap function that preserves existing line breaks and most spaces in the text. Expects that existing line breaks are posix newlines. """ - text = force_unicode(text) + text = force_text(text) def _generator(): it = iter(text.split(' ')) word = next(it) @@ -55,14 +55,14 @@ class Truncator(SimpleLazyObject): An object used to truncate text, either by characters or words. """ def __init__(self, text): - super(Truncator, self).__init__(lambda: force_unicode(text)) + super(Truncator, self).__init__(lambda: force_text(text)) def add_truncation_text(self, text, truncate=None): if truncate is None: truncate = pgettext( 'String to return when truncating text', '%(truncated_text)s...') - truncate = force_unicode(truncate) + truncate = force_text(truncate) if '%(truncated_text)s' in truncate: return truncate % {'truncated_text': text} # The truncation text didn't contain the %(truncated_text)s string @@ -226,7 +226,7 @@ def get_valid_filename(s): >>> get_valid_filename("john's portrait in 2004.jpg") 'johns_portrait_in_2004.jpg' """ - s = force_unicode(s).strip().replace(' ', '_') + s = force_text(s).strip().replace(' ', '_') return re.sub(r'(?u)[^-\w.]', '', s) get_valid_filename = allow_lazy(get_valid_filename, six.text_type) @@ -244,20 +244,20 @@ def get_text_list(list_, last_word=ugettext_lazy('or')): '' """ if len(list_) == 0: return '' - if len(list_) == 1: return force_unicode(list_[0]) + if len(list_) == 1: return force_text(list_[0]) return '%s %s %s' % ( # Translators: This string is used as a separator between list elements - _(', ').join([force_unicode(i) for i in list_][:-1]), - force_unicode(last_word), force_unicode(list_[-1])) + _(', ').join([force_text(i) for i in list_][:-1]), + force_text(last_word), force_text(list_[-1])) get_text_list = allow_lazy(get_text_list, six.text_type) def normalize_newlines(text): - return force_unicode(re.sub(r'\r\n|\r|\n', '\n', text)) + return force_text(re.sub(r'\r\n|\r|\n', '\n', text)) normalize_newlines = allow_lazy(normalize_newlines, six.text_type) def recapitalize(text): "Recapitalizes text, placing caps after end-of-sentence punctuation." - text = force_unicode(text).lower() + text = force_text(text).lower() capsRE = re.compile(r'(?:^|(?<=[\.\?\!] ))([a-z])') text = capsRE.sub(lambda x: x.group(1).upper(), text) return text @@ -330,7 +330,7 @@ def smart_split(text): >>> list(smart_split(r'A "\"funky\" style" test.')) ['A', '"\\"funky\\" style"', 'test.'] """ - text = force_unicode(text) + text = force_text(text) for bit in smart_split_re.finditer(text): yield bit.group(0) smart_split = allow_lazy(smart_split, six.text_type) diff --git a/django/utils/translation/__init__.py b/django/utils/translation/__init__.py index d31a7aebf1..febde404a5 100644 --- a/django/utils/translation/__init__.py +++ b/django/utils/translation/__init__.py @@ -3,7 +3,7 @@ Internationalization support. """ from __future__ import unicode_literals -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.functional import lazy from django.utils import six @@ -139,7 +139,7 @@ def _string_concat(*strings): Lazy variant of string concatenation, needed for translations that are constructed from multiple parts. """ - return ''.join([force_unicode(s) for s in strings]) + return ''.join([force_text(s) for s in strings]) string_concat = lazy(_string_concat, six.text_type) def get_language_info(lang_code): diff --git a/django/utils/translation/trans_null.py b/django/utils/translation/trans_null.py index 0fabc707ce..5c51468204 100644 --- a/django/utils/translation/trans_null.py +++ b/django/utils/translation/trans_null.py @@ -3,7 +3,7 @@ # settings.USE_I18N = False can use this module rather than trans_real.py. from django.conf import settings -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.safestring import mark_safe, SafeData def ngettext(singular, plural, number): @@ -12,7 +12,7 @@ def ngettext(singular, plural, number): ngettext_lazy = ngettext def ungettext(singular, plural, number): - return force_unicode(ngettext(singular, plural, number)) + return force_text(ngettext(singular, plural, number)) def pgettext(context, message): return ugettext(message) @@ -44,7 +44,7 @@ def gettext(message): return result def ugettext(message): - return force_unicode(gettext(message)) + return force_text(gettext(message)) gettext_noop = gettext_lazy = _ = gettext diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 9ebcbf5441..9e6eadcd48 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -9,7 +9,9 @@ import gettext as gettext_module from threading import local from django.utils.importlib import import_module +from django.utils.encoding import smart_str, smart_text from django.utils.safestring import mark_safe, SafeData +from django.utils import six from django.utils.six import StringIO @@ -259,12 +261,14 @@ def do_translate(message, translation_function): def gettext(message): return do_translate(message, 'gettext') -def ugettext(message): - return do_translate(message, 'ugettext') +if six.PY3: + ugettext = gettext +else: + def ugettext(message): + return do_translate(message, 'ugettext') def pgettext(context, message): - result = do_translate( - "%s%s%s" % (context, CONTEXT_SEPARATOR, message), 'ugettext') + result = ugettext("%s%s%s" % (context, CONTEXT_SEPARATOR, message)) if CONTEXT_SEPARATOR in result: # Translation not found result = message @@ -297,20 +301,23 @@ def ngettext(singular, plural, number): """ return do_ntranslate(singular, plural, number, 'ngettext') -def ungettext(singular, plural, number): - """ - Returns a unicode strings of the translation of either the singular or - plural, based on the number. - """ - return do_ntranslate(singular, plural, number, 'ungettext') +if six.PY3: + ungettext = ngettext +else: + def ungettext(singular, plural, number): + """ + Returns a unicode strings of the translation of either the singular or + plural, based on the number. + """ + return do_ntranslate(singular, plural, number, 'ungettext') def npgettext(context, singular, plural, number): - result = do_ntranslate("%s%s%s" % (context, CONTEXT_SEPARATOR, singular), - "%s%s%s" % (context, CONTEXT_SEPARATOR, plural), - number, 'ungettext') + result = ungettext("%s%s%s" % (context, CONTEXT_SEPARATOR, singular), + "%s%s%s" % (context, CONTEXT_SEPARATOR, plural), + number) if CONTEXT_SEPARATOR in result: # Translation not found - result = do_ntranslate(singular, plural, number, 'ungettext') + result = ungettext(singular, plural, number) return result def all_locale_paths(): @@ -440,7 +447,7 @@ def templatize(src, origin=None): from django.conf import settings from django.template import (Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK) - src = src.decode(settings.FILE_CHARSET) + src = smart_text(src, settings.FILE_CHARSET) out = StringIO() message_context = None intrans = False @@ -455,7 +462,7 @@ def templatize(src, origin=None): content = ''.join(comment) translators_comment_start = None for lineno, line in enumerate(content.splitlines(True)): - if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK): + if line.lstrip().startswith(smart_text(TRANSLATOR_COMMENT_MARK)): translators_comment_start = lineno for lineno, line in enumerate(content.splitlines(True)): if translators_comment_start is not None and lineno >= translators_comment_start: @@ -570,7 +577,7 @@ def templatize(src, origin=None): out.write(' # %s' % t.contents) else: out.write(blankout(t.contents, 'X')) - return out.getvalue().encode('utf-8') + return smart_str(out.getvalue()) def parse_accept_lang_header(lang_string): """ diff --git a/django/utils/tree.py b/django/utils/tree.py index 36b5977942..717181d2b9 100644 --- a/django/utils/tree.py +++ b/django/utils/tree.py @@ -68,11 +68,12 @@ class Node(object): """ return len(self.children) - def __nonzero__(self): + def __bool__(self): """ For truth value testing. """ return bool(self.children) + __nonzero__ = __bool__ # Python 2 def __contains__(self, other): """ diff --git a/django/utils/tzinfo.py b/django/utils/tzinfo.py index 05f4aa6d2f..208b7e7191 100644 --- a/django/utils/tzinfo.py +++ b/django/utils/tzinfo.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import time from datetime import timedelta, tzinfo -from django.utils.encoding import smart_unicode, smart_str, DEFAULT_LOCALE_ENCODING +from django.utils.encoding import smart_text, smart_str, DEFAULT_LOCALE_ENCODING # Python's doc say: "A tzinfo subclass must have an __init__() method that can # be called with no arguments". FixedOffset and LocalTimezone don't honor this @@ -72,7 +72,7 @@ class LocalTimezone(tzinfo): def tzname(self, dt): try: - return smart_unicode(time.tzname[self._isdst(dt)], + return smart_text(time.tzname[self._isdst(dt)], DEFAULT_LOCALE_ENCODING) except UnicodeDecodeError: return None diff --git a/django/views/debug.py b/django/views/debug.py index 8e81b8239b..b275ef9e73 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -14,7 +14,7 @@ from django.template import Template, Context, TemplateDoesNotExist from django.template.defaultfilters import force_escape, pprint from django.utils.html import escape from django.utils.importlib import import_module -from django.utils.encoding import smart_unicode, smart_str +from django.utils.encoding import smart_text, smart_bytes from django.utils import six HIDDEN_SETTINGS = re.compile('API|TOKEN|KEY|SECRET|PASS|PROFANITIES_LIST|SIGNATURE') @@ -111,7 +111,7 @@ class ExceptionReporterFilter(object): return request.POST def get_traceback_frame_variables(self, request, tb_frame): - return tb_frame.f_locals.items() + return list(six.iteritems(tb_frame.f_locals)) class SafeExceptionReporterFilter(ExceptionReporterFilter): """ @@ -256,7 +256,7 @@ class ExceptionReporter(object): end = getattr(self.exc_value, 'end', None) if start is not None and end is not None: unicode_str = self.exc_value.args[1] - unicode_hint = smart_unicode(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace') + unicode_hint = smart_text(unicode_str[max(start-5, 0):min(end+5, len(unicode_str))], 'ascii', errors='replace') from django import get_version c = { 'is_email': self.is_email, @@ -278,7 +278,7 @@ class ExceptionReporter(object): if self.exc_type: c['exception_type'] = self.exc_type.__name__ if self.exc_value: - c['exception_value'] = smart_unicode(self.exc_value, errors='replace') + c['exception_value'] = smart_text(self.exc_value, errors='replace') if frames: c['lastframe'] = frames[-1] return c @@ -440,7 +440,7 @@ def technical_404_response(request, exception): 'root_urlconf': settings.ROOT_URLCONF, 'request_path': request.path_info[1:], # Trim leading slash 'urlpatterns': tried, - 'reason': smart_str(exception, errors='replace'), + 'reason': smart_bytes(exception, errors='replace'), 'request': request, 'settings': get_safe_settings(), }) diff --git a/django/views/generic/base.py b/django/views/generic/base.py index 69751727bb..6554e898ca 100644 --- a/django/views/generic/base.py +++ b/django/views/generic/base.py @@ -6,6 +6,7 @@ from django.core.exceptions import ImproperlyConfigured from django.template.response import TemplateResponse from django.utils.log import getLogger from django.utils.decorators import classonlymethod +from django.utils import six logger = getLogger('django.request') @@ -35,7 +36,7 @@ class View(object): """ # Go through keyword arguments, and either save their values to our # instance, or raise an error. - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): setattr(self, key, value) @classonlymethod diff --git a/django/views/generic/dates.py b/django/views/generic/dates.py index 22a72bc433..08b7318738 100644 --- a/django/views/generic/dates.py +++ b/django/views/generic/dates.py @@ -5,7 +5,7 @@ from django.conf import settings from django.db import models from django.core.exceptions import ImproperlyConfigured from django.http import Http404 -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.functional import cached_property from django.utils.translation import ugettext as _ from django.utils import timezone @@ -365,7 +365,7 @@ class BaseDateListView(MultipleObjectMixin, DateMixin, View): is_empty = len(qs) == 0 if paginate_by is None else not qs.exists() if is_empty: raise Http404(_("No %(verbose_name_plural)s available") % { - 'verbose_name_plural': force_unicode(qs.model._meta.verbose_name_plural) + 'verbose_name_plural': force_text(qs.model._meta.verbose_name_plural) }) return qs @@ -380,7 +380,7 @@ class BaseDateListView(MultipleObjectMixin, DateMixin, View): date_list = queryset.dates(date_field, date_type)[::-1] if date_list is not None and not date_list and not allow_empty: - name = force_unicode(queryset.model._meta.verbose_name_plural) + name = force_text(queryset.model._meta.verbose_name_plural) raise Http404(_("No %(verbose_name_plural)s available") % {'verbose_name_plural': name}) diff --git a/django/views/i18n.py b/django/views/i18n.py index b0f64f4902..00ef224254 100644 --- a/django/views/i18n.py +++ b/django/views/i18n.py @@ -6,7 +6,7 @@ from django.conf import settings from django.utils import importlib from django.utils.translation import check_for_language, activate, to_locale, get_language from django.utils.text import javascript_quote -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.utils.formats import get_format_modules, get_format from django.utils import six @@ -54,9 +54,9 @@ def get_formats(): src = [] for k, v in result.items(): if isinstance(v, (six.string_types, int)): - src.append("formats['%s'] = '%s';\n" % (javascript_quote(k), javascript_quote(smart_unicode(v)))) + src.append("formats['%s'] = '%s';\n" % (javascript_quote(k), javascript_quote(smart_text(v)))) elif isinstance(v, (tuple, list)): - v = [javascript_quote(smart_unicode(value)) for value in v] + v = [javascript_quote(smart_text(value)) for value in v] src.append("formats['%s'] = ['%s'];\n" % (javascript_quote(k), "', '".join(v))) return ''.join(src) |