diff options
| author | Andrew Godwin <andrew@aeracode.org> | 2013-06-07 11:15:34 +0100 |
|---|---|---|
| committer | Andrew Godwin <andrew@aeracode.org> | 2013-06-07 11:15:34 +0100 |
| commit | 3c296382b8dea5de7f4e1e11b66bd7cecaf2ee51 (patch) | |
| tree | 0ca12593be82971691ffca01a836d00d3fcb3bd4 /django | |
| parent | 7609e0b42e0014a6ad0adf9dafc7018cb268070e (diff) | |
| parent | 357d62d9f2972bf1bc21e5835c12c849143e06af (diff) | |
Merge remote-tracking branch 'core/master' into schema-alteration
Conflicts:
django/db/models/fields/related.py
Diffstat (limited to 'django')
154 files changed, 3032 insertions, 2006 deletions
diff --git a/django/__init__.py b/django/__init__.py index 873c328add..5a1c74efa7 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 6, 0, 'alpha', 0) +VERSION = (1, 6, 0, 'alpha', 1) def get_version(*args, **kwargs): # Don't litter django/__init__.py with all the get_version stuff. diff --git a/django/conf/__init__.py b/django/conf/__init__.py index f876c490c8..61584391cd 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -127,7 +127,10 @@ class Settings(BaseSettings): try: mod = importlib.import_module(self.SETTINGS_MODULE) except ImportError as e: - raise ImportError("Could not import settings '%s' (Is it on sys.path?): %s" % (self.SETTINGS_MODULE, e)) + raise ImportError( + "Could not import settings '%s' (Is it on sys.path? Is there an import error in the settings file?): %s" + % (self.SETTINGS_MODULE, e) + ) # Settings that should be converted into tuples if they're mistakenly entered # as strings. diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 53aef351c0..596f4ae78a 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -131,7 +131,7 @@ LANGUAGES = ( ) # Languages using BiDi (right-to-left) layout -LANGUAGES_BIDI = ("he", "ar", "fa") +LANGUAGES_BIDI = ("he", "ar", "fa", "ur") # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. diff --git a/django/conf/locale/en/LC_MESSAGES/django.po b/django/conf/locale/en/LC_MESSAGES/django.po index 4c94d5a00a..371f0af2ab 100644 --- a/django/conf/locale/en/LC_MESSAGES/django.po +++ b/django/conf/locale/en/LC_MESSAGES/django.po @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: Django\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2013-05-02 16:17+0200\n" +"POT-Creation-Date: 2013-05-25 14:27+0200\n" "PO-Revision-Date: 2010-05-13 15:35+0200\n" "Last-Translator: Django team\n" "Language-Team: English <en@li.org>\n" @@ -337,7 +337,7 @@ msgstr "" msgid "Enter a valid value." msgstr "" -#: core/validators.py:53 forms/fields.py:640 +#: core/validators.py:53 forms/fields.py:639 msgid "Enter a valid URL." msgstr "" @@ -362,7 +362,7 @@ msgstr "" msgid "Enter a valid IPv4 or IPv6 address." msgstr "" -#: core/validators.py:175 db/models/fields/__init__.py:704 +#: core/validators.py:175 db/models/fields/__init__.py:706 msgid "Enter only digits separated by commas." msgstr "" @@ -408,7 +408,7 @@ msgstr[1] "" msgid "%(field_name)s must be unique for %(date_field)s %(lookup)s." msgstr "" -#: db/models/base.py:905 forms/models.py:605 +#: db/models/base.py:905 forms/models.py:643 msgid "and" msgstr "" @@ -435,156 +435,156 @@ msgstr "" msgid "Field of type: %(field_type)s" msgstr "" -#: db/models/fields/__init__.py:568 db/models/fields/__init__.py:1034 +#: db/models/fields/__init__.py:570 db/models/fields/__init__.py:1036 msgid "Integer" msgstr "" -#: db/models/fields/__init__.py:572 db/models/fields/__init__.py:1032 +#: db/models/fields/__init__.py:574 db/models/fields/__init__.py:1034 #, python-format msgid "'%s' value must be an integer." msgstr "" -#: db/models/fields/__init__.py:620 +#: db/models/fields/__init__.py:622 #, python-format msgid "'%s' value must be either True or False." msgstr "" -#: db/models/fields/__init__.py:622 +#: db/models/fields/__init__.py:624 msgid "Boolean (Either True or False)" msgstr "" -#: db/models/fields/__init__.py:671 +#: db/models/fields/__init__.py:673 #, python-format msgid "String (up to %(max_length)s)" msgstr "" -#: db/models/fields/__init__.py:699 +#: db/models/fields/__init__.py:701 msgid "Comma-separated integers" msgstr "" -#: db/models/fields/__init__.py:713 +#: db/models/fields/__init__.py:715 #, python-format msgid "'%s' value has an invalid date format. It must be in YYYY-MM-DD format." msgstr "" -#: db/models/fields/__init__.py:715 db/models/fields/__init__.py:803 +#: db/models/fields/__init__.py:717 db/models/fields/__init__.py:805 #, python-format msgid "" "'%s' value has the correct format (YYYY-MM-DD) but it is an invalid date." msgstr "" -#: db/models/fields/__init__.py:718 +#: db/models/fields/__init__.py:720 msgid "Date (without time)" msgstr "" -#: db/models/fields/__init__.py:801 +#: db/models/fields/__init__.py:803 #, python-format msgid "" "'%s' value has an invalid format. It must be in YYYY-MM-DD HH:MM[:ss[." "uuuuuu]][TZ] format." msgstr "" -#: db/models/fields/__init__.py:805 +#: db/models/fields/__init__.py:807 #, python-format msgid "" "'%s' value has the correct format (YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ]) but " "it is an invalid date/time." msgstr "" -#: db/models/fields/__init__.py:809 +#: db/models/fields/__init__.py:811 msgid "Date (with time)" msgstr "" -#: db/models/fields/__init__.py:898 +#: db/models/fields/__init__.py:900 #, python-format msgid "'%s' value must be a decimal number." msgstr "" -#: db/models/fields/__init__.py:900 +#: db/models/fields/__init__.py:902 msgid "Decimal number" msgstr "" -#: db/models/fields/__init__.py:957 +#: db/models/fields/__init__.py:959 msgid "Email address" msgstr "" -#: db/models/fields/__init__.py:976 +#: db/models/fields/__init__.py:978 msgid "File path" msgstr "" -#: db/models/fields/__init__.py:1003 +#: db/models/fields/__init__.py:1005 #, python-format msgid "'%s' value must be a float." msgstr "" -#: db/models/fields/__init__.py:1005 +#: db/models/fields/__init__.py:1007 msgid "Floating point number" msgstr "" -#: db/models/fields/__init__.py:1066 +#: db/models/fields/__init__.py:1068 msgid "Big (8 byte) integer" msgstr "" -#: db/models/fields/__init__.py:1080 +#: db/models/fields/__init__.py:1082 msgid "IPv4 address" msgstr "" -#: db/models/fields/__init__.py:1096 +#: db/models/fields/__init__.py:1098 msgid "IP address" msgstr "" -#: db/models/fields/__init__.py:1139 +#: db/models/fields/__init__.py:1141 #, python-format msgid "'%s' value must be either None, True or False." msgstr "" -#: db/models/fields/__init__.py:1141 +#: db/models/fields/__init__.py:1143 msgid "Boolean (Either True, False or None)" msgstr "" -#: db/models/fields/__init__.py:1190 +#: db/models/fields/__init__.py:1192 msgid "Positive integer" msgstr "" -#: db/models/fields/__init__.py:1201 +#: db/models/fields/__init__.py:1203 msgid "Positive small integer" msgstr "" -#: db/models/fields/__init__.py:1212 +#: db/models/fields/__init__.py:1214 #, python-format msgid "Slug (up to %(max_length)s)" msgstr "" -#: db/models/fields/__init__.py:1230 +#: db/models/fields/__init__.py:1232 msgid "Small integer" msgstr "" -#: db/models/fields/__init__.py:1236 +#: db/models/fields/__init__.py:1238 msgid "Text" msgstr "" -#: db/models/fields/__init__.py:1254 +#: db/models/fields/__init__.py:1256 #, python-format msgid "" "'%s' value has an invalid format. It must be in HH:MM[:ss[.uuuuuu]] format." msgstr "" -#: db/models/fields/__init__.py:1256 +#: db/models/fields/__init__.py:1258 #, python-format msgid "" "'%s' value has the correct format (HH:MM[:ss[.uuuuuu]]) but it is an invalid " "time." msgstr "" -#: db/models/fields/__init__.py:1259 +#: db/models/fields/__init__.py:1261 msgid "Time" msgstr "" -#: db/models/fields/__init__.py:1321 +#: db/models/fields/__init__.py:1323 msgid "URL" msgstr "" -#: db/models/fields/__init__.py:1338 +#: db/models/fields/__init__.py:1340 msgid "Raw binary data" msgstr "" @@ -596,55 +596,50 @@ msgstr "" msgid "Image" msgstr "" -#: db/models/fields/related.py:1133 +#: db/models/fields/related.py:1118 #, python-format msgid "Model %(model)s with pk %(pk)r does not exist." msgstr "" -#: db/models/fields/related.py:1135 +#: db/models/fields/related.py:1120 msgid "Foreign Key (type determined by related field)" msgstr "" -#: db/models/fields/related.py:1272 +#: db/models/fields/related.py:1257 msgid "One-to-one relationship" msgstr "" -#: db/models/fields/related.py:1339 +#: db/models/fields/related.py:1324 msgid "Many-to-many relationship" msgstr "" -#: db/models/fields/related.py:1366 -msgid "" -"Hold down \"Control\", or \"Command\" on a Mac, to select more than one." -msgstr "" - #: forms/fields.py:56 msgid "This field is required." msgstr "" -#: forms/fields.py:225 +#: forms/fields.py:227 msgid "Enter a whole number." msgstr "" -#: forms/fields.py:266 forms/fields.py:294 +#: forms/fields.py:268 forms/fields.py:296 msgid "Enter a number." msgstr "" -#: forms/fields.py:296 +#: forms/fields.py:298 #, python-format msgid "Ensure that there are no more than %(max)s digit in total." msgid_plural "Ensure that there are no more than %(max)s digits in total." msgstr[0] "" msgstr[1] "" -#: forms/fields.py:300 +#: forms/fields.py:302 #, python-format msgid "Ensure that there are no more than %(max)s decimal place." msgid_plural "Ensure that there are no more than %(max)s decimal places." msgstr[0] "" msgstr[1] "" -#: forms/fields.py:304 +#: forms/fields.py:306 #, python-format msgid "" "Ensure that there are no more than %(max)s digit before the decimal point." @@ -653,31 +648,31 @@ msgid_plural "" msgstr[0] "" msgstr[1] "" -#: forms/fields.py:406 forms/fields.py:1058 +#: forms/fields.py:408 forms/fields.py:1064 msgid "Enter a valid date." msgstr "" -#: forms/fields.py:430 forms/fields.py:1059 +#: forms/fields.py:432 forms/fields.py:1065 msgid "Enter a valid time." msgstr "" -#: forms/fields.py:451 +#: forms/fields.py:454 msgid "Enter a valid date/time." msgstr "" -#: forms/fields.py:525 +#: forms/fields.py:531 msgid "No file was submitted. Check the encoding type on the form." msgstr "" -#: forms/fields.py:526 +#: forms/fields.py:532 msgid "No file was submitted." msgstr "" -#: forms/fields.py:527 +#: forms/fields.py:533 msgid "The submitted file is empty." msgstr "" -#: forms/fields.py:529 +#: forms/fields.py:535 #, python-format msgid "Ensure this filename has at most %(max)d character (it has %(length)d)." msgid_plural "" @@ -685,22 +680,22 @@ msgid_plural "" msgstr[0] "" msgstr[1] "" -#: forms/fields.py:532 +#: forms/fields.py:538 msgid "Please either submit a file or check the clear checkbox, not both." msgstr "" -#: forms/fields.py:593 +#: forms/fields.py:599 msgid "" "Upload a valid image. The file you uploaded was either not an image or a " "corrupted image." msgstr "" -#: forms/fields.py:746 forms/fields.py:824 forms/models.py:1042 +#: forms/fields.py:749 forms/fields.py:828 forms/models.py:1096 #, python-format msgid "Select a valid choice. %(value)s is not one of the available choices." msgstr "" -#: forms/fields.py:825 forms/fields.py:928 forms/models.py:1041 +#: forms/fields.py:829 forms/fields.py:933 forms/models.py:1095 msgid "Enter a list of values." msgstr "" @@ -709,53 +704,60 @@ msgstr "" msgid "(Hidden field %(name)s) %(error)s" msgstr "" -#: forms/formsets.py:305 +#: forms/formsets.py:310 #, python-format -msgid "Please submit %s or fewer forms." -msgstr "" +msgid "Please submit %d or fewer forms." +msgid_plural "Please submit %d or fewer forms." +msgstr[0] "" +msgstr[1] "" -#: forms/formsets.py:331 forms/formsets.py:333 +#: forms/formsets.py:337 forms/formsets.py:339 msgid "Order" msgstr "" -#: forms/formsets.py:335 +#: forms/formsets.py:341 msgid "Delete" msgstr "" -#: forms/models.py:599 +#: forms/models.py:637 #, python-format msgid "Please correct the duplicate data for %(field)s." msgstr "" -#: forms/models.py:603 +#: forms/models.py:641 #, python-format msgid "Please correct the duplicate data for %(field)s, which must be unique." msgstr "" -#: forms/models.py:609 +#: forms/models.py:647 #, python-format msgid "" "Please correct the duplicate data for %(field_name)s which must be unique " "for the %(lookup)s in %(date_field)s." msgstr "" -#: forms/models.py:617 +#: forms/models.py:655 msgid "Please correct the duplicate values below." msgstr "" -#: forms/models.py:883 +#: forms/models.py:937 msgid "The inline foreign key did not match the parent instance primary key." msgstr "" -#: forms/models.py:947 +#: forms/models.py:1001 msgid "Select a valid choice. That choice is not one of the available choices." msgstr "" -#: forms/models.py:1044 +#: forms/models.py:1098 #, python-format msgid "\"%(pk)s\" is not a valid value for a primary key." msgstr "" +#: forms/models.py:1109 +msgid "" +"Hold down \"Control\", or \"Command\" on a Mac, to select more than one." +msgstr "" + #: forms/util.py:84 #, python-format msgid "" @@ -791,34 +793,34 @@ msgstr "" msgid "yes,no,maybe" msgstr "" -#: template/defaultfilters.py:813 template/defaultfilters.py:824 +#: template/defaultfilters.py:813 template/defaultfilters.py:825 #, python-format msgid "%(size)d byte" msgid_plural "%(size)d bytes" msgstr[0] "" msgstr[1] "" -#: template/defaultfilters.py:826 +#: template/defaultfilters.py:827 #, python-format msgid "%s KB" msgstr "" -#: template/defaultfilters.py:828 +#: template/defaultfilters.py:829 #, python-format msgid "%s MB" msgstr "" -#: template/defaultfilters.py:830 +#: template/defaultfilters.py:831 #, python-format msgid "%s GB" msgstr "" -#: template/defaultfilters.py:832 +#: template/defaultfilters.py:833 #, python-format msgid "%s TB" msgstr "" -#: template/defaultfilters.py:833 +#: template/defaultfilters.py:835 #, python-format msgid "%s PB" msgstr "" @@ -1119,6 +1121,16 @@ msgctxt "alt. month" msgid "December" msgstr "" +#: utils/image.py:105 +#, python-format +msgid "Neither Pillow nor PIL could be imported: %s" +msgstr "" + +#: utils/image.py:127 +#, python-format +msgid "The '_imaging' module for the PIL could not be imported: %s" +msgstr "" + #: utils/text.py:70 #, python-format msgctxt "String to return when truncating text" @@ -1130,53 +1142,53 @@ msgid "or" msgstr "" #. Translators: This string is used as a separator between list elements -#: utils/text.py:242 utils/timesince.py:54 +#: utils/text.py:242 utils/timesince.py:55 msgid ", " msgstr "" -#: utils/timesince.py:22 +#: utils/timesince.py:23 #, python-format msgid "%d year" msgid_plural "%d years" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:23 +#: utils/timesince.py:24 #, python-format msgid "%d month" msgid_plural "%d months" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:24 +#: utils/timesince.py:25 #, python-format msgid "%d week" msgid_plural "%d weeks" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:25 +#: utils/timesince.py:26 #, python-format msgid "%d day" msgid_plural "%d days" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:26 +#: utils/timesince.py:27 #, python-format msgid "%d hour" msgid_plural "%d hours" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:27 +#: utils/timesince.py:28 #, python-format msgid "%d minute" msgid_plural "%d minutes" msgstr[0] "" msgstr[1] "" -#: utils/timesince.py:43 +#: utils/timesince.py:44 msgid "0 minutes" msgstr "" diff --git a/django/conf/urls/__init__.py b/django/conf/urls/__init__.py index 04fb1dff59..c0340c0543 100644 --- a/django/conf/urls/__init__.py +++ b/django/conf/urls/__init__.py @@ -5,8 +5,9 @@ from django.utils.importlib import import_module from django.utils import six -__all__ = ['handler403', 'handler404', 'handler500', 'include', 'patterns', 'url'] +__all__ = ['handler400', 'handler403', 'handler404', 'handler500', 'include', 'patterns', 'url'] +handler400 = 'django.views.defaults.bad_request' handler403 = 'django.views.defaults.permission_denied' handler404 = 'django.views.defaults.page_not_found' handler500 = 'django.views.defaults.server_error' diff --git a/django/contrib/admin/exceptions.py b/django/contrib/admin/exceptions.py new file mode 100644 index 0000000000..2e094c6da1 --- /dev/null +++ b/django/contrib/admin/exceptions.py @@ -0,0 +1,6 @@ +from django.core.exceptions import SuspiciousOperation + + +class DisallowedModelAdminLookup(SuspiciousOperation): + """Invalid filter was passed to admin view via URL querystring""" + pass diff --git a/django/contrib/admin/filters.py b/django/contrib/admin/filters.py index ff66d3e3f3..4131494515 100644 --- a/django/contrib/admin/filters.py +++ b/django/contrib/admin/filters.py @@ -216,7 +216,7 @@ class RelatedFieldListFilter(FieldListFilter): } FieldListFilter.register(lambda f: ( - hasattr(f, 'rel') and bool(f.rel) or + bool(f.rel) if hasattr(f, 'rel') else isinstance(f, models.related.RelatedObject)), RelatedFieldListFilter) diff --git a/django/contrib/admin/helpers.py b/django/contrib/admin/helpers.py index adc2302587..320d3267a7 100644 --- a/django/contrib/admin/helpers.py +++ b/django/contrib/admin/helpers.py @@ -131,7 +131,7 @@ class AdminField(object): classes.append('required') if not self.is_first: classes.append('inline') - attrs = classes and {'class': ' '.join(classes)} or {} + attrs = {'class': ' '.join(classes)} if classes else {} return self.field.label_tag(contents=mark_safe(contents), attrs=attrs) def errors(self): @@ -144,7 +144,7 @@ class AdminReadonlyField(object): # {{ field.name }} must be a useful class name to identify the field. # For convenience, store other field-related data here too. if callable(field): - class_name = field.__name__ != '<lambda>' and field.__name__ or '' + class_name = field.__name__ if field.__name__ != '<lambda>' else '' else: class_name = field self.field = { diff --git a/django/contrib/admin/locale/en/LC_MESSAGES/django.po b/django/contrib/admin/locale/en/LC_MESSAGES/django.po index 964b0a509b..8994d24cbb 100644 --- a/django/contrib/admin/locale/en/LC_MESSAGES/django.po +++ b/django/contrib/admin/locale/en/LC_MESSAGES/django.po @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: Django\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2013-05-02 16:18+0200\n" +"POT-Creation-Date: 2013-05-25 14:19+0200\n" "PO-Revision-Date: 2010-05-13 15:35+0200\n" "Last-Translator: Django team\n" "Language-Team: English <en@li.org>\n" @@ -18,12 +18,12 @@ msgstr "" msgid "Successfully deleted %(count)d %(items)s." msgstr "" -#: actions.py:61 options.py:1365 +#: actions.py:61 options.py:1418 #, python-format msgid "Cannot delete %(name)s" msgstr "" -#: actions.py:63 options.py:1367 +#: actions.py:63 options.py:1420 msgid "Are you sure?" msgstr "" @@ -130,157 +130,157 @@ msgstr "" msgid "LogEntry Object" msgstr "" -#: options.py:163 options.py:192 +#: options.py:173 options.py:202 msgid "None" msgstr "" -#: options.py:710 +#: options.py:763 #, python-format msgid "Changed %s." msgstr "" -#: options.py:710 options.py:720 options.py:1514 +#: options.py:763 options.py:773 options.py:1570 msgid "and" msgstr "" -#: options.py:715 +#: options.py:768 #, python-format msgid "Added %(name)s \"%(object)s\"." msgstr "" -#: options.py:719 +#: options.py:772 #, python-format msgid "Changed %(list)s for %(name)s \"%(object)s\"." msgstr "" -#: options.py:724 +#: options.py:777 #, python-format msgid "Deleted %(name)s \"%(object)s\"." msgstr "" -#: options.py:728 +#: options.py:781 msgid "No fields changed." msgstr "" -#: options.py:831 options.py:874 +#: options.py:884 options.py:927 #, python-format msgid "" "The %(name)s \"%(obj)s\" was added successfully. You may edit it again below." msgstr "" -#: options.py:849 +#: options.py:902 #, python-format msgid "" "The %(name)s \"%(obj)s\" was added successfully. You may add another " "%(name)s below." msgstr "" -#: options.py:853 +#: options.py:906 #, python-format msgid "The %(name)s \"%(obj)s\" was added successfully." msgstr "" -#: options.py:867 +#: options.py:920 #, python-format msgid "" "The %(name)s \"%(obj)s\" was changed successfully. You may edit it again " "below." msgstr "" -#: options.py:881 +#: options.py:934 #, python-format msgid "" "The %(name)s \"%(obj)s\" was changed successfully. You may add another " "%(name)s below." msgstr "" -#: options.py:887 +#: options.py:940 #, python-format msgid "The %(name)s \"%(obj)s\" was changed successfully." msgstr "" -#: options.py:965 options.py:1225 +#: options.py:1018 options.py:1278 msgid "" "Items must be selected in order to perform actions on them. No items have " "been changed." msgstr "" -#: options.py:984 +#: options.py:1037 msgid "No action selected." msgstr "" -#: options.py:1064 +#: options.py:1117 #, python-format msgid "Add %s" msgstr "" -#: options.py:1088 options.py:1333 +#: options.py:1141 options.py:1386 #, python-format msgid "%(name)s object with primary key %(key)r does not exist." msgstr "" -#: options.py:1154 +#: options.py:1207 #, python-format msgid "Change %s" msgstr "" -#: options.py:1204 +#: options.py:1257 msgid "Database error" msgstr "" -#: options.py:1267 +#: options.py:1320 #, python-format msgid "%(count)s %(name)s was changed successfully." msgid_plural "%(count)s %(name)s were changed successfully." msgstr[0] "" msgstr[1] "" -#: options.py:1294 +#: options.py:1347 #, python-format msgid "%(total_count)s selected" msgid_plural "All %(total_count)s selected" msgstr[0] "" msgstr[1] "" -#: options.py:1299 +#: options.py:1352 #, python-format msgid "0 of %(cnt)s selected" msgstr "" -#: options.py:1350 +#: options.py:1403 #, python-format msgid "The %(name)s \"%(obj)s\" was deleted successfully." msgstr "" -#: options.py:1406 +#: options.py:1459 #, python-format msgid "Change history: %s" msgstr "" #. Translators: Model verbose name and instance representation, suitable to be an item in a list -#: options.py:1508 +#: options.py:1564 #, python-format msgid "%(class_name)s %(instance)s" msgstr "" -#: options.py:1515 +#: options.py:1571 #, python-format msgid "" "Deleting %(class_name)s %(instance)s would require deleting the following " "protected related objects: %(related_objects)s" msgstr "" -#: sites.py:324 tests.py:71 templates/admin/login.html:48 +#: sites.py:318 tests.py:71 templates/admin/login.html:48 #: templates/registration/password_reset_complete.html:19 #: views/decorators.py:24 msgid "Log in" msgstr "" -#: sites.py:392 +#: sites.py:386 msgid "Site administration" msgstr "" -#: sites.py:446 +#: sites.py:440 #, python-format msgid "%s administration" msgstr "" @@ -429,9 +429,14 @@ msgstr "" #: templates/admin/auth/user/change_password.html:27 #: templates/registration/password_change_form.html:20 msgid "Please correct the error below." -msgid_plural "Please correct the errors below." -msgstr[0] "" -msgstr[1] "" +msgstr "" + +#: templates/admin/change_form.html:44 templates/admin/change_list.html:67 +#: templates/admin/login.html:17 +#: templates/admin/auth/user/change_password.html:27 +#: templates/registration/password_change_form.html:20 +msgid "Please correct the errors below." +msgstr "" #: templates/admin/change_list.html:58 #, python-format @@ -811,16 +816,16 @@ msgstr "" msgid "All dates" msgstr "" -#: views/main.py:37 +#: views/main.py:35 msgid "(None)" msgstr "" -#: views/main.py:86 +#: views/main.py:84 #, python-format msgid "Select %s" msgstr "" -#: views/main.py:88 +#: views/main.py:86 #, python-format msgid "Select %s to change" msgstr "" diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index 7373837bb0..34583ebf74 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -1,5 +1,6 @@ import copy -from functools import update_wrapper, partial +import operator +from functools import partial, reduce, update_wrapper from django import forms from django.conf import settings @@ -9,7 +10,8 @@ from django.forms.models import (modelform_factory, modelformset_factory, from django.contrib.contenttypes.models import ContentType from django.contrib.admin import widgets, helpers from django.contrib.admin.util import (unquote, flatten_fieldsets, get_deleted_objects, - model_format_dict, NestedObjects) + model_format_dict, NestedObjects, lookup_needs_distinct) +from django.contrib.admin import validation from django.contrib.admin.templatetags.admin_static import static from django.contrib import messages from django.views.decorators.csrf import csrf_protect @@ -22,6 +24,7 @@ from django.db.models.related import RelatedObject from django.db.models.fields import BLANK_CHOICE_DASH, FieldDoesNotExist from django.db.models.sql.constants import QUERY_TERMS from django.http import Http404, HttpResponse, HttpResponseRedirect +from django.http.response import HttpResponseBase from django.shortcuts import get_object_or_404 from django.template.response import SimpleTemplateResponse, TemplateResponse from django.utils.decorators import method_decorator @@ -87,6 +90,14 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): readonly_fields = () ordering = None + # validation + validator_class = validation.BaseValidator + + @classmethod + def validate(cls, model): + validator = cls.validator_class() + validator.validate(cls, model) + def __init__(self): overrides = FORMFIELD_FOR_DBFIELD_DEFAULTS.copy() overrides.update(self.formfield_overrides) @@ -246,6 +257,34 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): """ return self.prepopulated_fields + def get_search_results(self, request, queryset, search_term): + # Apply keyword searches. + def construct_search(field_name): + if field_name.startswith('^'): + return "%s__istartswith" % field_name[1:] + elif field_name.startswith('='): + return "%s__iexact" % field_name[1:] + elif field_name.startswith('@'): + return "%s__search" % field_name[1:] + else: + return "%s__icontains" % field_name + + use_distinct = False + if self.search_fields and search_term: + orm_lookups = [construct_search(str(search_field)) + for search_field in self.search_fields] + for bit in search_term.split(): + or_queries = [models.Q(**{orm_lookup: bit}) + for orm_lookup in orm_lookups] + queryset = queryset.filter(reduce(operator.or_, or_queries)) + if not use_distinct: + for search_spec in orm_lookups: + if lookup_needs_distinct(self.opts, search_spec): + use_distinct = True + break + + return queryset, use_distinct + def get_queryset(self, request): """ Returns a QuerySet of all model instances that can be edited by the @@ -371,6 +410,9 @@ class ModelAdmin(BaseModelAdmin): actions_on_bottom = False actions_selection_counter = True + # validation + validator_class = validation.ModelAdminValidator + def __init__(self, model, admin_site): self.model = model self.opts = model._meta @@ -456,7 +498,7 @@ class ModelAdmin(BaseModelAdmin): "Hook for specifying fieldsets for the add form." if self.declared_fieldsets: return self.declared_fieldsets - form = self.get_form(request, obj) + form = self.get_form(request, obj, fields=None) fields = list(form.base_fields) + list(self.get_readonly_fields(request, obj)) return [(None, {'fields': fields})] @@ -465,10 +507,10 @@ class ModelAdmin(BaseModelAdmin): Returns a Form class for use in the admin add view. This is used by add_view and change_view. """ - if self.declared_fieldsets: - fields = flatten_fieldsets(self.declared_fieldsets) + if 'fields' in kwargs: + fields = kwargs.pop('fields') else: - fields = None + fields = flatten_fieldsets(self.get_fieldsets(request, obj)) if self.exclude is None: exclude = [] else: @@ -985,10 +1027,10 @@ class ModelAdmin(BaseModelAdmin): response = func(self, request, queryset) - # Actions may return an HttpResponse, which will be used as the - # response from the POST. If not, we'll be a good little HTTP - # citizen and redirect back to the changelist page. - if isinstance(response, HttpResponse): + # Actions may return an HttpResponse-like object, which will be + # used as the response from the POST. If not, we'll be a good + # little HTTP citizen and redirect back to the changelist page. + if isinstance(response, HttpResponseBase): return response else: return HttpResponseRedirect(request.get_full_path()) @@ -1447,6 +1489,9 @@ class InlineModelAdmin(BaseModelAdmin): verbose_name_plural = None can_delete = True + # validation + validator_class = validation.InlineValidator + def __init__(self, parent_model, admin_site): self.admin_site = admin_site self.parent_model = parent_model @@ -1467,12 +1512,20 @@ class InlineModelAdmin(BaseModelAdmin): js.extend(['SelectBox.js', 'SelectFilter2.js']) return forms.Media(js=[static('admin/js/%s' % url) for url in js]) + def get_extra(self, request, obj=None, **kwargs): + """Hook for customizing the number of extra inline forms.""" + return self.extra + + def get_max_num(self, request, obj=None, **kwargs): + """Hook for customizing the max number of extra inline forms.""" + return self.max_num + def get_formset(self, request, obj=None, **kwargs): """Returns a BaseInlineFormSet class for use in admin add/change views.""" - if self.declared_fieldsets: - fields = flatten_fieldsets(self.declared_fieldsets) + if 'fields' in kwargs: + fields = kwargs.pop('fields') else: - fields = None + fields = flatten_fieldsets(self.get_fieldsets(request, obj)) if self.exclude is None: exclude = [] else: @@ -1493,8 +1546,8 @@ class InlineModelAdmin(BaseModelAdmin): "fields": fields, "exclude": exclude, "formfield_callback": partial(self.formfield_for_dbfield, request=request), - "extra": self.extra, - "max_num": self.max_num, + "extra": self.get_extra(request, obj, **kwargs), + "max_num": self.get_max_num(request, obj, **kwargs), "can_delete": can_delete, } @@ -1544,7 +1597,7 @@ class InlineModelAdmin(BaseModelAdmin): def get_fieldsets(self, request, obj=None): if self.declared_fieldsets: return self.declared_fieldsets - form = self.get_formset(request, obj).form + form = self.get_formset(request, obj, fields=None).form fields = list(form.base_fields) + list(self.get_readonly_fields(request, obj)) return [(None, {'fields': fields})] diff --git a/django/contrib/admin/sites.py b/django/contrib/admin/sites.py index 414d1b4f72..e0f43dfbfe 100644 --- a/django/contrib/admin/sites.py +++ b/django/contrib/admin/sites.py @@ -66,12 +66,6 @@ class AdminSite(object): if not admin_class: admin_class = ModelAdmin - # Don't import the humongous validation code unless required - if admin_class and settings.DEBUG: - from django.contrib.admin.validation import validate - else: - validate = lambda model, adminclass: None - if isinstance(model_or_iterable, ModelBase): model_or_iterable = [model_or_iterable] for model in model_or_iterable: @@ -94,8 +88,8 @@ class AdminSite(object): options['__module__'] = __name__ admin_class = type("%sAdmin" % model.__name__, (admin_class,), options) - # Validate (which might be a no-op) - validate(admin_class, model) + if admin_class is not ModelAdmin and settings.DEBUG: + admin_class.validate(model) # Instantiate the admin class to save in the registry self._registry[model] = admin_class(model, self) diff --git a/django/contrib/admin/templates/admin/auth/user/change_password.html b/django/contrib/admin/templates/admin/auth/user/change_password.html index 9d1b917b61..2a1b4d3c90 100644 --- a/django/contrib/admin/templates/admin/auth/user/change_password.html +++ b/django/contrib/admin/templates/admin/auth/user/change_password.html @@ -24,7 +24,7 @@ {% if is_popup %}<input type="hidden" name="_popup" value="1" />{% endif %} {% if form.errors %} <p class="errornote"> - {% blocktrans count counter=form.errors.items|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %} + {% if form.errors.items|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} </p> {% endif %} diff --git a/django/contrib/admin/templates/admin/change_form.html b/django/contrib/admin/templates/admin/change_form.html index daf37753dc..4accf80c46 100644 --- a/django/contrib/admin/templates/admin/change_form.html +++ b/django/contrib/admin/templates/admin/change_form.html @@ -41,7 +41,7 @@ {% if save_on_top %}{% block submit_buttons_top %}{% submit_row %}{% endblock %}{% endif %} {% if errors %} <p class="errornote"> - {% blocktrans count counter=errors|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %} + {% if errors|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} </p> {{ adminform.form.non_field_errors }} {% endif %} diff --git a/django/contrib/admin/templates/admin/change_list.html b/django/contrib/admin/templates/admin/change_list.html index c72b6630a3..5d1a6b2714 100644 --- a/django/contrib/admin/templates/admin/change_list.html +++ b/django/contrib/admin/templates/admin/change_list.html @@ -64,7 +64,7 @@ {% endblock %} {% if cl.formset.errors %} <p class="errornote"> - {% blocktrans count cl.formset.errors|length as counter %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %} + {% if cl.formset.errors|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} </p> {{ cl.formset.non_form_errors }} {% endif %} diff --git a/django/contrib/admin/templates/admin/includes/fieldset.html b/django/contrib/admin/templates/admin/includes/fieldset.html index 09bc971d2f..24b069cb2f 100644 --- a/django/contrib/admin/templates/admin/includes/fieldset.html +++ b/django/contrib/admin/templates/admin/includes/fieldset.html @@ -7,7 +7,7 @@ <div class="form-row{% if line.fields|length_is:'1' and line.errors %} errors{% endif %}{% for field in line %}{% if field.field.name %} field-{{ field.field.name }}{% endif %}{% endfor %}"> {% if line.fields|length_is:'1' %}{{ line.errors }}{% endif %} {% for field in line %} - <div{% if not line.fields|length_is:'1' %} class="field-box{% if field.field.name %} field-{{ field.field.name }}{% endif %}{% if not field.is_readonly and field.errors %} errors{% endif %}"{% endif %}> + <div{% if not line.fields|length_is:'1' %} class="field-box{% if field.field.name %} field-{{ field.field.name }}{% endif %}{% if not field.is_readonly and field.errors %} errors{% endif %}"{% elif field.is_checkbox %} class="checkbox-row"{% endif %}> {% if not line.fields|length_is:'1' and not field.is_readonly %}{{ field.errors }}{% endif %} {% if field.is_checkbox %} {{ field.field }}{{ field.label_tag }} diff --git a/django/contrib/admin/templates/admin/login.html b/django/contrib/admin/templates/admin/login.html index 4690363891..1371514d43 100644 --- a/django/contrib/admin/templates/admin/login.html +++ b/django/contrib/admin/templates/admin/login.html @@ -14,7 +14,7 @@ {% block content %} {% if form.errors and not form.non_field_errors and not form.this_is_the_login_form.errors %} <p class="errornote"> -{% blocktrans count counter=form.errors.items|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %} +{% if form.errors.items|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} </p> {% endif %} diff --git a/django/contrib/admin/templates/registration/password_change_form.html b/django/contrib/admin/templates/registration/password_change_form.html index 5cb34739df..f7316a739f 100644 --- a/django/contrib/admin/templates/registration/password_change_form.html +++ b/django/contrib/admin/templates/registration/password_change_form.html @@ -17,7 +17,7 @@ <div> {% if form.errors %} <p class="errornote"> - {% blocktrans count counter=form.errors.items|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %} + {% if form.errors.items|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} </p> {% endif %} diff --git a/django/contrib/admin/templates/registration/password_reset_email.html b/django/contrib/admin/templates/registration/password_reset_email.html index a220f12033..44ae5850b1 100644 --- a/django/contrib/admin/templates/registration/password_reset_email.html +++ b/django/contrib/admin/templates/registration/password_reset_email.html @@ -3,7 +3,7 @@ {% trans "Please go to the following page and choose a new password:" %} {% block reset_link %} -{{ protocol }}://{{ domain }}{% url 'django.contrib.auth.views.password_reset_confirm' uidb36=uid token=token %} +{{ protocol }}://{{ domain }}{% url 'password_reset_confirm' uidb36=uid token=token %} {% endblock %} {% trans "Your username, in case you've forgotten:" %} {{ user.get_username }} diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py index 18a45a006f..965352e0f5 100644 --- a/django/contrib/admin/templatetags/admin_list.py +++ b/django/contrib/admin/templatetags/admin_list.py @@ -62,7 +62,7 @@ def pagination(cl): # ON_EACH_SIDE links at either end of the "current page" link. page_range = [] if page_num > (ON_EACH_SIDE + ON_ENDS): - page_range.extend(range(0, ON_EACH_SIDE - 1)) + page_range.extend(range(0, ON_ENDS)) page_range.append(DOT) page_range.extend(range(page_num - ON_EACH_SIDE, page_num + 1)) else: diff --git a/django/contrib/admin/templatetags/log.py b/django/contrib/admin/templatetags/log.py index f70ee2731d..1b9e6aa7ef 100644 --- a/django/contrib/admin/templatetags/log.py +++ b/django/contrib/admin/templatetags/log.py @@ -53,4 +53,4 @@ def get_admin_log(parser, token): if tokens[4] != 'for_user': raise template.TemplateSyntaxError( "Fourth argument to 'get_admin_log' must be 'for_user'") - return AdminLogNode(limit=tokens[1], varname=tokens[3], user=(len(tokens) > 5 and tokens[5] or None)) + return AdminLogNode(limit=tokens[1], varname=tokens[3], user=(tokens[5] if len(tokens) > 5 else None)) diff --git a/django/contrib/admin/util.py b/django/contrib/admin/util.py index 97858e688e..a53cd367b4 100644 --- a/django/contrib/admin/util.py +++ b/django/contrib/admin/util.py @@ -37,9 +37,9 @@ def prepare_lookup_value(key, value): # if key ends with __in, split parameter into separate values if key.endswith('__in'): value = value.split(',') - # if key ends with __isnull, special case '' and false + # if key ends with __isnull, special case '' and the string literals 'false' and '0' if key.endswith('__isnull'): - if value.lower() in ('', 'false'): + if value.lower() in ('', 'false', '0'): value = False else: value = True @@ -269,8 +269,9 @@ def lookup_field(name, obj, model_admin=None): def label_for_field(name, model, model_admin=None, return_attr=False): """ - Returns a sensible label for a field name. The name can be a callable or the - name of an object attributes, as well as a genuine fields. If return_attr is + Returns a sensible label for a field name. The name can be a callable, + property (but not created with @property decorator) or the name of an + object's attribute, as well as a genuine fields. If return_attr is True, the resolved attribute (which could be a callable) is also returned. This will be None if (and only if) the name refers to a field. """ @@ -303,6 +304,10 @@ def label_for_field(name, model, model_admin=None, return_attr=False): if hasattr(attr, "short_description"): label = attr.short_description + elif (isinstance(attr, property) and + hasattr(attr, "fget") and + hasattr(attr.fget, "short_description")): + label = attr.fget.short_description elif callable(attr): if attr.__name__ == "<lambda>": label = "--" @@ -315,6 +320,7 @@ def label_for_field(name, model, model_admin=None, return_attr=False): else: return label + def help_text_for_field(name, model): try: help_text = model._meta.get_field_by_name(name)[0].help_text diff --git a/django/contrib/admin/validation.py b/django/contrib/admin/validation.py index 8d65f96cf1..222d433e53 100644 --- a/django/contrib/admin/validation.py +++ b/django/contrib/admin/validation.py @@ -3,358 +3,405 @@ from django.db import models from django.db.models.fields import FieldDoesNotExist from django.forms.models import (BaseModelForm, BaseModelFormSet, fields_for_model, _get_foreign_key) -from django.contrib.admin import ListFilter, FieldListFilter from django.contrib.admin.util import get_fields_from_path, NotRelationField -from django.contrib.admin.options import (flatten_fieldsets, BaseModelAdmin, - ModelAdmin, HORIZONTAL, VERTICAL) +""" +Does basic ModelAdmin option validation. Calls custom validation +classmethod in the end if it is provided in cls. The signature of the +custom validation classmethod should be: def validate(cls, model). +""" -__all__ = ['validate'] +__all__ = ['BaseValidator', 'InlineValidator'] -def validate(cls, model): - """ - Does basic ModelAdmin option validation. Calls custom validation - classmethod in the end if it is provided in cls. The signature of the - custom validation classmethod should be: def validate(cls, model). - """ - # Before we can introspect models, they need to be fully loaded so that - # inter-relations are set up correctly. We force that here. - models.get_apps() - opts = model._meta - validate_base(cls, model) +class BaseValidator(object): + def __init__(self): + # Before we can introspect models, they need to be fully loaded so that + # inter-relations are set up correctly. We force that here. + models.get_apps() - # list_display - if hasattr(cls, 'list_display'): - check_isseq(cls, 'list_display', cls.list_display) - for idx, field in enumerate(cls.list_display): - if not callable(field): - if not hasattr(cls, field): - if not hasattr(model, field): - try: - opts.get_field(field) - except models.FieldDoesNotExist: - raise ImproperlyConfigured("%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r." - % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) - else: - # getattr(model, field) could be an X_RelatedObjectsDescriptor - f = fetch_attr(cls, model, opts, "list_display[%d]" % idx, field) - if isinstance(f, models.ManyToManyField): - raise ImproperlyConfigured("'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported." - % (cls.__name__, idx, field)) + def validate(self, cls, model): + for m in dir(self): + if m.startswith('validate_'): + getattr(self, m)(cls, model) - # list_display_links - if hasattr(cls, 'list_display_links'): - check_isseq(cls, 'list_display_links', cls.list_display_links) - for idx, field in enumerate(cls.list_display_links): - if field not in cls.list_display: - raise ImproperlyConfigured("'%s.list_display_links[%d]' " - "refers to '%s' which is not defined in 'list_display'." - % (cls.__name__, idx, field)) + def check_field_spec(self, cls, model, flds, label): + """ + Validate the fields specification in `flds` from a ModelAdmin subclass + `cls` for the `model` model. Use `label` for reporting problems to the user. - # list_filter - if hasattr(cls, 'list_filter'): - check_isseq(cls, 'list_filter', cls.list_filter) - for idx, item in enumerate(cls.list_filter): - # There are three options for specifying a filter: - # 1: 'field' - a basic field filter, possibly w/ relationships (eg, 'field__rel') - # 2: ('field', SomeFieldListFilter) - a field-based list filter class - # 3: SomeListFilter - a non-field list filter class - if callable(item) and not isinstance(item, models.Field): - # If item is option 3, it should be a ListFilter... - if not issubclass(item, ListFilter): - raise ImproperlyConfigured("'%s.list_filter[%d]' is '%s'" - " which is not a descendant of ListFilter." - % (cls.__name__, idx, item.__name__)) - # ... but not a FieldListFilter. - if issubclass(item, FieldListFilter): - raise ImproperlyConfigured("'%s.list_filter[%d]' is '%s'" - " which is of type FieldListFilter but is not" - " associated with a field name." - % (cls.__name__, idx, item.__name__)) - else: - if isinstance(item, (tuple, list)): - # item is option #2 - field, list_filter_class = item - if not issubclass(list_filter_class, FieldListFilter): - raise ImproperlyConfigured("'%s.list_filter[%d][1]'" - " is '%s' which is not of type FieldListFilter." - % (cls.__name__, idx, list_filter_class.__name__)) - else: - # item is option #1 - field = item - # Validate the field string + The fields specification can be a ``fields`` option or a ``fields`` + sub-option from a ``fieldsets`` option component. + """ + for fields in flds: + # The entry in fields might be a tuple. If it is a standalone + # field, make it into a tuple to make processing easier. + if type(fields) != tuple: + fields = (fields,) + for field in fields: + if field in cls.readonly_fields: + # Stuff can be put in fields that isn't actually a + # model field if it's in readonly_fields, + # readonly_fields will handle the validation of such + # things. + continue try: - get_fields_from_path(model, field) - except (NotRelationField, FieldDoesNotExist): - raise ImproperlyConfigured("'%s.list_filter[%d]' refers to '%s'" - " which does not refer to a Field." + f = model._meta.get_field(field) + except models.FieldDoesNotExist: + # If we can't find a field on the model that matches, it could be an + # extra field on the form; nothing to check so move on to the next field. + continue + if isinstance(f, models.ManyToManyField) and not f.rel.through._meta.auto_created: + raise ImproperlyConfigured("'%s.%s' " + "can't include the ManyToManyField field '%s' because " + "'%s' manually specifies a 'through' model." % ( + cls.__name__, label, field, field)) + + def validate_raw_id_fields(self, cls, model): + " Validate that raw_id_fields only contains field names that are listed on the model. " + if hasattr(cls, 'raw_id_fields'): + check_isseq(cls, 'raw_id_fields', cls.raw_id_fields) + for idx, field in enumerate(cls.raw_id_fields): + f = get_field(cls, model, 'raw_id_fields', field) + if not isinstance(f, (models.ForeignKey, models.ManyToManyField)): + raise ImproperlyConfigured("'%s.raw_id_fields[%d]', '%s' must " + "be either a ForeignKey or ManyToManyField." % (cls.__name__, idx, field)) - # list_per_page = 100 - if hasattr(cls, 'list_per_page') and not isinstance(cls.list_per_page, int): - raise ImproperlyConfigured("'%s.list_per_page' should be a integer." - % cls.__name__) + def validate_fields(self, cls, model): + " Validate that fields only refer to existing fields, doesn't contain duplicates. " + # fields + if cls.fields: # default value is None + check_isseq(cls, 'fields', cls.fields) + self.check_field_spec(cls, model, cls.fields, 'fields') + if cls.fieldsets: + raise ImproperlyConfigured('Both fieldsets and fields are specified in %s.' % cls.__name__) + if len(cls.fields) > len(set(cls.fields)): + raise ImproperlyConfigured('There are duplicate field(s) in %s.fields' % cls.__name__) - # list_max_show_all - if hasattr(cls, 'list_max_show_all') and not isinstance(cls.list_max_show_all, int): - raise ImproperlyConfigured("'%s.list_max_show_all' should be an integer." - % cls.__name__) + def validate_fieldsets(self, cls, model): + " Validate that fieldsets is properly formatted and doesn't contain duplicates. " + from django.contrib.admin.options import flatten_fieldsets + if cls.fieldsets: # default value is None + check_isseq(cls, 'fieldsets', cls.fieldsets) + for idx, fieldset in enumerate(cls.fieldsets): + check_isseq(cls, 'fieldsets[%d]' % idx, fieldset) + if len(fieldset) != 2: + raise ImproperlyConfigured("'%s.fieldsets[%d]' does not " + "have exactly two elements." % (cls.__name__, idx)) + check_isdict(cls, 'fieldsets[%d][1]' % idx, fieldset[1]) + if 'fields' not in fieldset[1]: + raise ImproperlyConfigured("'fields' key is required in " + "%s.fieldsets[%d][1] field options dict." + % (cls.__name__, idx)) + self.check_field_spec(cls, model, fieldset[1]['fields'], "fieldsets[%d][1]['fields']" % idx) + flattened_fieldsets = flatten_fieldsets(cls.fieldsets) + if len(flattened_fieldsets) > len(set(flattened_fieldsets)): + raise ImproperlyConfigured('There are duplicate field(s) in %s.fieldsets' % cls.__name__) - # list_editable - if hasattr(cls, 'list_editable') and cls.list_editable: - check_isseq(cls, 'list_editable', cls.list_editable) - for idx, field_name in enumerate(cls.list_editable): - try: - field = opts.get_field_by_name(field_name)[0] - except models.FieldDoesNotExist: - raise ImproperlyConfigured("'%s.list_editable[%d]' refers to a " - "field, '%s', not defined on %s.%s." - % (cls.__name__, idx, field_name, model._meta.app_label, model.__name__)) - if field_name not in cls.list_display: - raise ImproperlyConfigured("'%s.list_editable[%d]' refers to " - "'%s' which is not defined in 'list_display'." - % (cls.__name__, idx, field_name)) - if field_name in cls.list_display_links: - raise ImproperlyConfigured("'%s' cannot be in both '%s.list_editable'" - " and '%s.list_display_links'" - % (field_name, cls.__name__, cls.__name__)) - if not cls.list_display_links and cls.list_display[0] in cls.list_editable: - raise ImproperlyConfigured("'%s.list_editable[%d]' refers to" - " the first field in list_display, '%s', which can't be" - " used unless list_display_links is set." - % (cls.__name__, idx, cls.list_display[0])) - if not field.editable: - raise ImproperlyConfigured("'%s.list_editable[%d]' refers to a " - "field, '%s', which isn't editable through the admin." - % (cls.__name__, idx, field_name)) + def validate_exclude(self, cls, model): + " Validate that exclude is a sequence without duplicates. " + if cls.exclude: # default value is None + check_isseq(cls, 'exclude', cls.exclude) + if len(cls.exclude) > len(set(cls.exclude)): + raise ImproperlyConfigured('There are duplicate field(s) in %s.exclude' % cls.__name__) - # search_fields = () - if hasattr(cls, 'search_fields'): - check_isseq(cls, 'search_fields', cls.search_fields) + def validate_form(self, cls, model): + " Validate that form subclasses BaseModelForm. " + if hasattr(cls, 'form') and not issubclass(cls.form, BaseModelForm): + raise ImproperlyConfigured("%s.form does not inherit from " + "BaseModelForm." % cls.__name__) - # date_hierarchy = None - if cls.date_hierarchy: - f = get_field(cls, model, opts, 'date_hierarchy', cls.date_hierarchy) - if not isinstance(f, (models.DateField, models.DateTimeField)): - raise ImproperlyConfigured("'%s.date_hierarchy is " - "neither an instance of DateField nor DateTimeField." - % cls.__name__) + def validate_filter_vertical(self, cls, model): + " Validate that filter_vertical is a sequence of field names. " + if hasattr(cls, 'filter_vertical'): + check_isseq(cls, 'filter_vertical', cls.filter_vertical) + for idx, field in enumerate(cls.filter_vertical): + f = get_field(cls, model, 'filter_vertical', field) + if not isinstance(f, models.ManyToManyField): + raise ImproperlyConfigured("'%s.filter_vertical[%d]' must be " + "a ManyToManyField." % (cls.__name__, idx)) - # ordering = None - if cls.ordering: - check_isseq(cls, 'ordering', cls.ordering) - for idx, field in enumerate(cls.ordering): - if field == '?' and len(cls.ordering) != 1: - raise ImproperlyConfigured("'%s.ordering' has the random " - "ordering marker '?', but contains other fields as " - "well. Please either remove '?' or the other fields." - % cls.__name__) - if field == '?': - continue - if field.startswith('-'): - field = field[1:] - # Skip ordering in the format field1__field2 (FIXME: checking - # this format would be nice, but it's a little fiddly). - if '__' in field: - continue - get_field(cls, model, opts, 'ordering[%d]' % idx, field) + def validate_filter_horizontal(self, cls, model): + " Validate that filter_horizontal is a sequence of field names. " + if hasattr(cls, 'filter_horizontal'): + check_isseq(cls, 'filter_horizontal', cls.filter_horizontal) + for idx, field in enumerate(cls.filter_horizontal): + f = get_field(cls, model, 'filter_horizontal', field) + if not isinstance(f, models.ManyToManyField): + raise ImproperlyConfigured("'%s.filter_horizontal[%d]' must be " + "a ManyToManyField." % (cls.__name__, idx)) - if hasattr(cls, "readonly_fields"): - check_readonly_fields(cls, model, opts) + def validate_radio_fields(self, cls, model): + " Validate that radio_fields is a dictionary of choice or foreign key fields. " + from django.contrib.admin.options import HORIZONTAL, VERTICAL + if hasattr(cls, 'radio_fields'): + check_isdict(cls, 'radio_fields', cls.radio_fields) + for field, val in cls.radio_fields.items(): + f = get_field(cls, model, 'radio_fields', field) + if not (isinstance(f, models.ForeignKey) or f.choices): + raise ImproperlyConfigured("'%s.radio_fields['%s']' " + "is neither an instance of ForeignKey nor does " + "have choices set." % (cls.__name__, field)) + if not val in (HORIZONTAL, VERTICAL): + raise ImproperlyConfigured("'%s.radio_fields['%s']' " + "is neither admin.HORIZONTAL nor admin.VERTICAL." + % (cls.__name__, field)) - # list_select_related = False - # save_as = False - # save_on_top = False - for attr in ('list_select_related', 'save_as', 'save_on_top'): - if not isinstance(getattr(cls, attr), bool): - raise ImproperlyConfigured("'%s.%s' should be a boolean." - % (cls.__name__, attr)) + def validate_prepopulated_fields(self, cls, model): + " Validate that prepopulated_fields if a dictionary containing allowed field types. " + # prepopulated_fields + if hasattr(cls, 'prepopulated_fields'): + check_isdict(cls, 'prepopulated_fields', cls.prepopulated_fields) + for field, val in cls.prepopulated_fields.items(): + f = get_field(cls, model, 'prepopulated_fields', field) + if isinstance(f, (models.DateTimeField, models.ForeignKey, + models.ManyToManyField)): + raise ImproperlyConfigured("'%s.prepopulated_fields['%s']' " + "is either a DateTimeField, ForeignKey or " + "ManyToManyField. This isn't allowed." + % (cls.__name__, field)) + check_isseq(cls, "prepopulated_fields['%s']" % field, val) + for idx, f in enumerate(val): + get_field(cls, model, "prepopulated_fields['%s'][%d]" % (field, idx), f) + def validate_ordering(self, cls, model): + " Validate that ordering refers to existing fields or is random. " + # ordering = None + if cls.ordering: + check_isseq(cls, 'ordering', cls.ordering) + for idx, field in enumerate(cls.ordering): + if field == '?' and len(cls.ordering) != 1: + raise ImproperlyConfigured("'%s.ordering' has the random " + "ordering marker '?', but contains other fields as " + "well. Please either remove '?' or the other fields." + % cls.__name__) + if field == '?': + continue + if field.startswith('-'): + field = field[1:] + # Skip ordering in the format field1__field2 (FIXME: checking + # this format would be nice, but it's a little fiddly). + if '__' in field: + continue + get_field(cls, model, 'ordering[%d]' % idx, field) - # inlines = [] - if hasattr(cls, 'inlines'): - check_isseq(cls, 'inlines', cls.inlines) - for idx, inline in enumerate(cls.inlines): - if not issubclass(inline, BaseModelAdmin): - raise ImproperlyConfigured("'%s.inlines[%d]' does not inherit " - "from BaseModelAdmin." % (cls.__name__, idx)) - if not inline.model: - raise ImproperlyConfigured("'model' is a required attribute " - "of '%s.inlines[%d]'." % (cls.__name__, idx)) - if not issubclass(inline.model, models.Model): - raise ImproperlyConfigured("'%s.inlines[%d].model' does not " - "inherit from models.Model." % (cls.__name__, idx)) - validate_base(inline, inline.model) - validate_inline(inline, cls, model) + def validate_readonly_fields(self, cls, model): + " Validate that readonly_fields refers to proper attribute or field. " + if hasattr(cls, "readonly_fields"): + check_isseq(cls, "readonly_fields", cls.readonly_fields) + for idx, field in enumerate(cls.readonly_fields): + if not callable(field): + if not hasattr(cls, field): + if not hasattr(model, field): + try: + model._meta.get_field(field) + except models.FieldDoesNotExist: + raise ImproperlyConfigured("%s.readonly_fields[%d], %r is not a callable or an attribute of %r or found in the model %r." + % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) -def validate_inline(cls, parent, parent_model): - # model is already verified to exist and be a Model - if cls.fk_name: # default value is None - f = get_field(cls, cls.model, cls.model._meta, 'fk_name', cls.fk_name) - if not isinstance(f, models.ForeignKey): - raise ImproperlyConfigured("'%s.fk_name is not an instance of " - "models.ForeignKey." % cls.__name__) +class ModelAdminValidator(BaseValidator): + def validate_save_as(self, cls, model): + " Validate save_as is a boolean. " + check_type(cls, 'save_as', bool) - fk = _get_foreign_key(parent_model, cls.model, fk_name=cls.fk_name, can_fail=True) + def validate_save_on_top(self, cls, model): + " Validate save_on_top is a boolean. " + check_type(cls, 'save_on_top', bool) - # extra = 3 - if not isinstance(cls.extra, int): - raise ImproperlyConfigured("'%s.extra' should be a integer." - % cls.__name__) + def validate_inlines(self, cls, model): + " Validate inline model admin classes. " + from django.contrib.admin.options import BaseModelAdmin + if hasattr(cls, 'inlines'): + check_isseq(cls, 'inlines', cls.inlines) + for idx, inline in enumerate(cls.inlines): + if not issubclass(inline, BaseModelAdmin): + raise ImproperlyConfigured("'%s.inlines[%d]' does not inherit " + "from BaseModelAdmin." % (cls.__name__, idx)) + if not inline.model: + raise ImproperlyConfigured("'model' is a required attribute " + "of '%s.inlines[%d]'." % (cls.__name__, idx)) + if not issubclass(inline.model, models.Model): + raise ImproperlyConfigured("'%s.inlines[%d].model' does not " + "inherit from models.Model." % (cls.__name__, idx)) + inline.validate(inline.model) + self.check_inline(inline, model) - # max_num = None - max_num = getattr(cls, 'max_num', None) - if max_num is not None and not isinstance(max_num, int): - raise ImproperlyConfigured("'%s.max_num' should be an integer or None (default)." - % cls.__name__) + def check_inline(self, cls, parent_model): + " Validate inline class's fk field is not excluded. " + fk = _get_foreign_key(parent_model, cls.model, fk_name=cls.fk_name, can_fail=True) + if hasattr(cls, 'exclude') and cls.exclude: + if fk and fk.name in cls.exclude: + raise ImproperlyConfigured("%s cannot exclude the field " + "'%s' - this is the foreign key to the parent model " + "%s.%s." % (cls.__name__, fk.name, parent_model._meta.app_label, parent_model.__name__)) - # formset - if hasattr(cls, 'formset') and not issubclass(cls.formset, BaseModelFormSet): - raise ImproperlyConfigured("'%s.formset' does not inherit from " - "BaseModelFormSet." % cls.__name__) + def validate_list_display(self, cls, model): + " Validate that list_display only contains fields or usable attributes. " + if hasattr(cls, 'list_display'): + check_isseq(cls, 'list_display', cls.list_display) + for idx, field in enumerate(cls.list_display): + if not callable(field): + if not hasattr(cls, field): + if not hasattr(model, field): + try: + model._meta.get_field(field) + except models.FieldDoesNotExist: + raise ImproperlyConfigured("%s.list_display[%d], %r is not a callable or an attribute of %r or found in the model %r." + % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) + else: + # getattr(model, field) could be an X_RelatedObjectsDescriptor + f = fetch_attr(cls, model, "list_display[%d]" % idx, field) + if isinstance(f, models.ManyToManyField): + raise ImproperlyConfigured("'%s.list_display[%d]', '%s' is a ManyToManyField which is not supported." + % (cls.__name__, idx, field)) - # exclude - if hasattr(cls, 'exclude') and cls.exclude: - if fk and fk.name in cls.exclude: - raise ImproperlyConfigured("%s cannot exclude the field " - "'%s' - this is the foreign key to the parent model " - "%s.%s." % (cls.__name__, fk.name, parent_model._meta.app_label, parent_model.__name__)) + def validate_list_display_links(self, cls, model): + " Validate that list_display_links is a unique subset of list_display. " + if hasattr(cls, 'list_display_links'): + check_isseq(cls, 'list_display_links', cls.list_display_links) + for idx, field in enumerate(cls.list_display_links): + if field not in cls.list_display: + raise ImproperlyConfigured("'%s.list_display_links[%d]' " + "refers to '%s' which is not defined in 'list_display'." + % (cls.__name__, idx, field)) - if hasattr(cls, "readonly_fields"): - check_readonly_fields(cls, cls.model, cls.model._meta) + def validate_list_filter(self, cls, model): + """ + Validate that list_filter is a sequence of one of three options: + 1: 'field' - a basic field filter, possibly w/ relationships (eg, 'field__rel') + 2: ('field', SomeFieldListFilter) - a field-based list filter class + 3: SomeListFilter - a non-field list filter class + """ + from django.contrib.admin import ListFilter, FieldListFilter + if hasattr(cls, 'list_filter'): + check_isseq(cls, 'list_filter', cls.list_filter) + for idx, item in enumerate(cls.list_filter): + if callable(item) and not isinstance(item, models.Field): + # If item is option 3, it should be a ListFilter... + if not issubclass(item, ListFilter): + raise ImproperlyConfigured("'%s.list_filter[%d]' is '%s'" + " which is not a descendant of ListFilter." + % (cls.__name__, idx, item.__name__)) + # ... but not a FieldListFilter. + if issubclass(item, FieldListFilter): + raise ImproperlyConfigured("'%s.list_filter[%d]' is '%s'" + " which is of type FieldListFilter but is not" + " associated with a field name." + % (cls.__name__, idx, item.__name__)) + else: + if isinstance(item, (tuple, list)): + # item is option #2 + field, list_filter_class = item + if not issubclass(list_filter_class, FieldListFilter): + raise ImproperlyConfigured("'%s.list_filter[%d][1]'" + " is '%s' which is not of type FieldListFilter." + % (cls.__name__, idx, list_filter_class.__name__)) + else: + # item is option #1 + field = item + # Validate the field string + try: + get_fields_from_path(model, field) + except (NotRelationField, FieldDoesNotExist): + raise ImproperlyConfigured("'%s.list_filter[%d]' refers to '%s'" + " which does not refer to a Field." + % (cls.__name__, idx, field)) -def validate_fields_spec(cls, model, opts, flds, label): - """ - Validate the fields specification in `flds` from a ModelAdmin subclass - `cls` for the `model` model. `opts` is `model`'s Meta inner class. - Use `label` for reporting problems to the user. + def validate_list_select_related(self, cls, model): + " Validate that list_select_related is a boolean, a list or a tuple. " + list_select_related = getattr(cls, 'list_select_related', None) + if list_select_related: + types = (bool, tuple, list) + if not isinstance(list_select_related, types): + raise ImproperlyConfigured("'%s.list_select_related' should be " + "either a bool, a tuple or a list" % + cls.__name__) - The fields specification can be a ``fields`` option or a ``fields`` - sub-option from a ``fieldsets`` option component. - """ - for fields in flds: - # The entry in fields might be a tuple. If it is a standalone - # field, make it into a tuple to make processing easier. - if type(fields) != tuple: - fields = (fields,) - for field in fields: - if field in cls.readonly_fields: - # Stuff can be put in fields that isn't actually a - # model field if it's in readonly_fields, - # readonly_fields will handle the validation of such - # things. - continue - try: - f = opts.get_field(field) - except models.FieldDoesNotExist: - # If we can't find a field on the model that matches, it could be an - # extra field on the form; nothing to check so move on to the next field. - continue - if isinstance(f, models.ManyToManyField) and not f.rel.through._meta.auto_created: - raise ImproperlyConfigured("'%s.%s' " - "can't include the ManyToManyField field '%s' because " - "'%s' manually specifies a 'through' model." % ( - cls.__name__, label, field, field)) + def validate_list_per_page(self, cls, model): + " Validate that list_per_page is an integer. " + check_type(cls, 'list_per_page', int) -def validate_base(cls, model): - opts = model._meta + def validate_list_max_show_all(self, cls, model): + " Validate that list_max_show_all is an integer. " + check_type(cls, 'list_max_show_all', int) - # raw_id_fields - if hasattr(cls, 'raw_id_fields'): - check_isseq(cls, 'raw_id_fields', cls.raw_id_fields) - for idx, field in enumerate(cls.raw_id_fields): - f = get_field(cls, model, opts, 'raw_id_fields', field) - if not isinstance(f, (models.ForeignKey, models.ManyToManyField)): - raise ImproperlyConfigured("'%s.raw_id_fields[%d]', '%s' must " - "be either a ForeignKey or ManyToManyField." - % (cls.__name__, idx, field)) + def validate_list_editable(self, cls, model): + """ + Validate that list_editable is a sequence of editable fields from + list_display without first element. + """ + if hasattr(cls, 'list_editable') and cls.list_editable: + check_isseq(cls, 'list_editable', cls.list_editable) + for idx, field_name in enumerate(cls.list_editable): + try: + field = model._meta.get_field_by_name(field_name)[0] + except models.FieldDoesNotExist: + raise ImproperlyConfigured("'%s.list_editable[%d]' refers to a " + "field, '%s', not defined on %s.%s." + % (cls.__name__, idx, field_name, model._meta.app_label, model.__name__)) + if field_name not in cls.list_display: + raise ImproperlyConfigured("'%s.list_editable[%d]' refers to " + "'%s' which is not defined in 'list_display'." + % (cls.__name__, idx, field_name)) + if field_name in cls.list_display_links: + raise ImproperlyConfigured("'%s' cannot be in both '%s.list_editable'" + " and '%s.list_display_links'" + % (field_name, cls.__name__, cls.__name__)) + if not cls.list_display_links and cls.list_display[0] in cls.list_editable: + raise ImproperlyConfigured("'%s.list_editable[%d]' refers to" + " the first field in list_display, '%s', which can't be" + " used unless list_display_links is set." + % (cls.__name__, idx, cls.list_display[0])) + if not field.editable: + raise ImproperlyConfigured("'%s.list_editable[%d]' refers to a " + "field, '%s', which isn't editable through the admin." + % (cls.__name__, idx, field_name)) - # fields - if cls.fields: # default value is None - check_isseq(cls, 'fields', cls.fields) - validate_fields_spec(cls, model, opts, cls.fields, 'fields') - if cls.fieldsets: - raise ImproperlyConfigured('Both fieldsets and fields are specified in %s.' % cls.__name__) - if len(cls.fields) > len(set(cls.fields)): - raise ImproperlyConfigured('There are duplicate field(s) in %s.fields' % cls.__name__) + def validate_search_fields(self, cls, model): + " Validate search_fields is a sequence. " + if hasattr(cls, 'search_fields'): + check_isseq(cls, 'search_fields', cls.search_fields) - # fieldsets - if cls.fieldsets: # default value is None - check_isseq(cls, 'fieldsets', cls.fieldsets) - for idx, fieldset in enumerate(cls.fieldsets): - check_isseq(cls, 'fieldsets[%d]' % idx, fieldset) - if len(fieldset) != 2: - raise ImproperlyConfigured("'%s.fieldsets[%d]' does not " - "have exactly two elements." % (cls.__name__, idx)) - check_isdict(cls, 'fieldsets[%d][1]' % idx, fieldset[1]) - if 'fields' not in fieldset[1]: - raise ImproperlyConfigured("'fields' key is required in " - "%s.fieldsets[%d][1] field options dict." - % (cls.__name__, idx)) - validate_fields_spec(cls, model, opts, fieldset[1]['fields'], "fieldsets[%d][1]['fields']" % idx) - flattened_fieldsets = flatten_fieldsets(cls.fieldsets) - if len(flattened_fieldsets) > len(set(flattened_fieldsets)): - raise ImproperlyConfigured('There are duplicate field(s) in %s.fieldsets' % cls.__name__) + def validate_date_hierarchy(self, cls, model): + " Validate that date_hierarchy refers to DateField or DateTimeField. " + if cls.date_hierarchy: + f = get_field(cls, model, 'date_hierarchy', cls.date_hierarchy) + if not isinstance(f, (models.DateField, models.DateTimeField)): + raise ImproperlyConfigured("'%s.date_hierarchy is " + "neither an instance of DateField nor DateTimeField." + % cls.__name__) - # exclude - if cls.exclude: # default value is None - check_isseq(cls, 'exclude', cls.exclude) - if len(cls.exclude) > len(set(cls.exclude)): - raise ImproperlyConfigured('There are duplicate field(s) in %s.exclude' % cls.__name__) - # form - if hasattr(cls, 'form') and not issubclass(cls.form, BaseModelForm): - raise ImproperlyConfigured("%s.form does not inherit from " - "BaseModelForm." % cls.__name__) +class InlineValidator(BaseValidator): + def validate_fk_name(self, cls, model): + " Validate that fk_name refers to a ForeignKey. " + if cls.fk_name: # default value is None + f = get_field(cls, model, 'fk_name', cls.fk_name) + if not isinstance(f, models.ForeignKey): + raise ImproperlyConfigured("'%s.fk_name is not an instance of " + "models.ForeignKey." % cls.__name__) - # filter_vertical - if hasattr(cls, 'filter_vertical'): - check_isseq(cls, 'filter_vertical', cls.filter_vertical) - for idx, field in enumerate(cls.filter_vertical): - f = get_field(cls, model, opts, 'filter_vertical', field) - if not isinstance(f, models.ManyToManyField): - raise ImproperlyConfigured("'%s.filter_vertical[%d]' must be " - "a ManyToManyField." % (cls.__name__, idx)) + def validate_extra(self, cls, model): + " Validate that extra is an integer. " + check_type(cls, 'extra', int) - # filter_horizontal - if hasattr(cls, 'filter_horizontal'): - check_isseq(cls, 'filter_horizontal', cls.filter_horizontal) - for idx, field in enumerate(cls.filter_horizontal): - f = get_field(cls, model, opts, 'filter_horizontal', field) - if not isinstance(f, models.ManyToManyField): - raise ImproperlyConfigured("'%s.filter_horizontal[%d]' must be " - "a ManyToManyField." % (cls.__name__, idx)) + def validate_max_num(self, cls, model): + " Validate that max_num is an integer. " + check_type(cls, 'max_num', int) - # radio_fields - if hasattr(cls, 'radio_fields'): - check_isdict(cls, 'radio_fields', cls.radio_fields) - for field, val in cls.radio_fields.items(): - f = get_field(cls, model, opts, 'radio_fields', field) - if not (isinstance(f, models.ForeignKey) or f.choices): - raise ImproperlyConfigured("'%s.radio_fields['%s']' " - "is neither an instance of ForeignKey nor does " - "have choices set." % (cls.__name__, field)) - if not val in (HORIZONTAL, VERTICAL): - raise ImproperlyConfigured("'%s.radio_fields['%s']' " - "is neither admin.HORIZONTAL nor admin.VERTICAL." - % (cls.__name__, field)) + def validate_formset(self, cls, model): + " Validate formset is a subclass of BaseModelFormSet. " + if hasattr(cls, 'formset') and not issubclass(cls.formset, BaseModelFormSet): + raise ImproperlyConfigured("'%s.formset' does not inherit from " + "BaseModelFormSet." % cls.__name__) - # prepopulated_fields - if hasattr(cls, 'prepopulated_fields'): - check_isdict(cls, 'prepopulated_fields', cls.prepopulated_fields) - for field, val in cls.prepopulated_fields.items(): - f = get_field(cls, model, opts, 'prepopulated_fields', field) - if isinstance(f, (models.DateTimeField, models.ForeignKey, - models.ManyToManyField)): - raise ImproperlyConfigured("'%s.prepopulated_fields['%s']' " - "is either a DateTimeField, ForeignKey or " - "ManyToManyField. This isn't allowed." - % (cls.__name__, field)) - check_isseq(cls, "prepopulated_fields['%s']" % field, val) - for idx, f in enumerate(val): - get_field(cls, model, opts, "prepopulated_fields['%s'][%d]" % (field, idx), f) + +def check_type(cls, attr, type_): + if getattr(cls, attr, None) is not None and not isinstance(getattr(cls, attr), type_): + raise ImproperlyConfigured("'%s.%s' should be a %s." + % (cls.__name__, attr, type_.__name__ )) def check_isseq(cls, label, obj): if not isinstance(obj, (list, tuple)): @@ -364,16 +411,16 @@ def check_isdict(cls, label, obj): if not isinstance(obj, dict): raise ImproperlyConfigured("'%s.%s' must be a dictionary." % (cls.__name__, label)) -def get_field(cls, model, opts, label, field): +def get_field(cls, model, label, field): try: - return opts.get_field(field) + return model._meta.get_field(field) except models.FieldDoesNotExist: raise ImproperlyConfigured("'%s.%s' refers to field '%s' that is missing from model '%s.%s'." % (cls.__name__, label, field, model._meta.app_label, model.__name__)) -def fetch_attr(cls, model, opts, label, field): +def fetch_attr(cls, model, label, field): try: - return opts.get_field(field) + return model._meta.get_field(field) except models.FieldDoesNotExist: pass try: @@ -381,15 +428,3 @@ def fetch_attr(cls, model, opts, label, field): except AttributeError: raise ImproperlyConfigured("'%s.%s' refers to '%s' that is neither a field, method or property of model '%s.%s'." % (cls.__name__, label, field, model._meta.app_label, model.__name__)) - -def check_readonly_fields(cls, model, opts): - check_isseq(cls, "readonly_fields", cls.readonly_fields) - for idx, field in enumerate(cls.readonly_fields): - if not callable(field): - if not hasattr(cls, field): - if not hasattr(model, field): - try: - opts.get_field(field) - except models.FieldDoesNotExist: - raise ImproperlyConfigured("%s.readonly_fields[%d], %r is not a callable or an attribute of %r or found in the model %r." - % (cls.__name__, idx, field, cls.__name__, model._meta.object_name)) diff --git a/django/contrib/admin/views/main.py b/django/contrib/admin/views/main.py index 050d4776d0..8ea7e10fc0 100644 --- a/django/contrib/admin/views/main.py +++ b/django/contrib/admin/views/main.py @@ -1,7 +1,5 @@ -import operator import sys import warnings -from functools import reduce from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured from django.core.paginator import InvalidPage @@ -16,6 +14,7 @@ from django.utils.translation import ugettext, ugettext_lazy from django.utils.http import urlencode from django.contrib.admin import FieldListFilter +from django.contrib.admin.exceptions import DisallowedModelAdminLookup from django.contrib.admin.options import IncorrectLookupParameters from django.contrib.admin.util import (quote, get_fields_from_path, lookup_needs_distinct, prepare_lookup_value) @@ -130,7 +129,7 @@ class ChangeList(six.with_metaclass(RenameChangeListMethods)): lookup_params[force_str(key)] = value if not self.model_admin.lookup_allowed(key, value): - raise SuspiciousOperation("Filtering by %s not allowed" % key) + raise DisallowedModelAdminLookup("Filtering by %s not allowed" % key) filter_specs = [] if self.list_filter: @@ -331,7 +330,7 @@ class ChangeList(six.with_metaclass(RenameChangeListMethods)): def get_queryset(self, request): # First, we collect all the declared list filters. (self.filter_specs, self.has_filters, remaining_lookup_params, - use_distinct) = self.get_filters(request) + filters_use_distinct) = self.get_filters(request) # Then, we let every list filter modify the queryset to its liking. qs = self.root_queryset @@ -357,56 +356,46 @@ class ChangeList(six.with_metaclass(RenameChangeListMethods)): # ValueError, ValidationError, or ?. raise IncorrectLookupParameters(e) - # Use select_related() if one of the list_display options is a field - # with a relationship and the provided queryset doesn't already have - # select_related defined. if not qs.query.select_related: - if self.list_select_related: - qs = qs.select_related() - else: - for field_name in self.list_display: - try: - field = self.lookup_opts.get_field(field_name) - except models.FieldDoesNotExist: - pass - else: - if isinstance(field.rel, models.ManyToOneRel): - qs = qs.select_related() - break + qs = self.apply_select_related(qs) # Set ordering. ordering = self.get_ordering(request, qs) qs = qs.order_by(*ordering) - # Apply keyword searches. - def construct_search(field_name): - if field_name.startswith('^'): - return "%s__istartswith" % field_name[1:] - elif field_name.startswith('='): - return "%s__iexact" % field_name[1:] - elif field_name.startswith('@'): - return "%s__search" % field_name[1:] - else: - return "%s__icontains" % field_name - - if self.search_fields and self.query: - orm_lookups = [construct_search(str(search_field)) - for search_field in self.search_fields] - for bit in self.query.split(): - or_queries = [models.Q(**{orm_lookup: bit}) - for orm_lookup in orm_lookups] - qs = qs.filter(reduce(operator.or_, or_queries)) - if not use_distinct: - for search_spec in orm_lookups: - if lookup_needs_distinct(self.lookup_opts, search_spec): - use_distinct = True - break + # Apply search results + qs, search_use_distinct = self.model_admin.get_search_results( + request, qs, self.query) - if use_distinct: + # Remove duplicates from results, if necessary + if filters_use_distinct | search_use_distinct: return qs.distinct() else: return qs + def apply_select_related(self, qs): + if self.list_select_related is True: + return qs.select_related() + + if self.list_select_related is False: + if self.has_related_field_in_list_display(): + return qs.select_related() + + if self.list_select_related: + return qs.select_related(*self.list_select_related) + return qs + + def has_related_field_in_list_display(self): + for field_name in self.list_display: + try: + field = self.lookup_opts.get_field(field_name) + except models.FieldDoesNotExist: + pass + else: + if isinstance(field.rel, models.ManyToOneRel): + return True + return False + def url_for_result(self, result): pk = getattr(result, self.pk_attname) return reverse('admin:%s_%s_change' % (self.opts.app_label, diff --git a/django/contrib/admindocs/locale/en/LC_MESSAGES/django.po b/django/contrib/admindocs/locale/en/LC_MESSAGES/django.po index 2e4dcdf13d..5a4dcd0872 100644 --- a/django/contrib/admindocs/locale/en/LC_MESSAGES/django.po +++ b/django/contrib/admindocs/locale/en/LC_MESSAGES/django.po @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: Django\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2013-05-02 16:18+0200\n" +"POT-Creation-Date: 2013-06-02 00:30-0400\n" "PO-Revision-Date: 2010-05-13 15:35+0200\n" "Last-Translator: Django team\n" "Language-Team: English <en@li.org>\n" @@ -60,12 +60,13 @@ msgstr "" msgid "number of %s" msgstr "" -#: views.py:270 +#. Translators: %s is an object type name +#: views.py:271 #, python-format -msgid "Fields on %s objects" +msgid "Attributes on %s objects" msgstr "" -#: views.py:362 +#: views.py:363 #, python-format msgid "%s does not appear to be a urlpattern object" msgstr "" @@ -83,7 +84,9 @@ msgid "Home" msgstr "" #: templates/admin_doc/bookmarklets.html:7 templates/admin_doc/index.html:7 +#: templates/admin_doc/index.html.py:10 templates/admin_doc/index.html:14 #: templates/admin_doc/missing_docutils.html:7 +#: templates/admin_doc/missing_docutils.html:14 #: templates/admin_doc/model_detail.html:15 #: templates/admin_doc/model_index.html:9 #: templates/admin_doc/template_detail.html:7 @@ -94,7 +97,7 @@ msgstr "" msgid "Documentation" msgstr "" -#: templates/admin_doc/bookmarklets.html:8 +#: templates/admin_doc/bookmarklets.html:8 templates/admin_doc/index.html:29 msgid "Bookmarklets" msgstr "" @@ -149,28 +152,204 @@ msgstr "" msgid "As above, but opens the admin page in a new window." msgstr "" -#: templates/admin_doc/model_detail.html:16 -#: templates/admin_doc/model_index.html:10 -msgid "Models" +#: templates/admin_doc/index.html:17 +#: templates/admin_doc/template_tag_index.html:9 +msgid "Tags" msgstr "" -#: templates/admin_doc/template_detail.html:8 -msgid "Templates" +#: templates/admin_doc/index.html:18 +msgid "List of all the template tags and their functions." msgstr "" +#: templates/admin_doc/index.html:20 #: templates/admin_doc/template_filter_index.html:9 msgid "Filters" msgstr "" -#: templates/admin_doc/template_tag_index.html:9 -msgid "Tags" +#: templates/admin_doc/index.html:21 +msgid "" +"Filters are actions which can be applied to variables in a template to alter " +"the output." +msgstr "" + +#: templates/admin_doc/index.html:23 templates/admin_doc/model_detail.html:16 +#: templates/admin_doc/model_index.html:10 +#: templates/admin_doc/model_index.html:14 +msgid "Models" msgstr "" -#: templates/admin_doc/view_detail.html:8 +#: templates/admin_doc/index.html:24 +msgid "" +"Models are descriptions of all the objects in the system and their " +"associated fields. Each model has a list of fields which can be accessed as " +"template variables" +msgstr "" + +#: templates/admin_doc/index.html:26 templates/admin_doc/view_detail.html:8 #: templates/admin_doc/view_index.html:9 +#: templates/admin_doc/view_index.html:12 msgid "Views" msgstr "" +#: templates/admin_doc/index.html:27 +msgid "" +"Each page on the public site is generated by a view. The view defines which " +"template is used to generate the page and which objects are available to " +"that template." +msgstr "" + +#: templates/admin_doc/index.html:30 +msgid "Tools for your browser to quickly access admin functionality." +msgstr "" + +#: templates/admin_doc/missing_docutils.html:10 +msgid "Please install docutils" +msgstr "" + +#: templates/admin_doc/missing_docutils.html:17 +#, python-format +msgid "" +"The admin documentation system requires Python's <a href=\"%(link)s" +"\">docutils</a> library." +msgstr "" + +#: templates/admin_doc/missing_docutils.html:19 +#, python-format +msgid "" +"Please ask your administrators to install <a href=\"%(link)s\">docutils</a>." +msgstr "" + +#: templates/admin_doc/model_detail.html:21 +#, python-format +msgid "Model: %(name)s" +msgstr "" + +#: templates/admin_doc/model_detail.html:35 +msgid "Field" +msgstr "" + +#: templates/admin_doc/model_detail.html:36 +msgid "Type" +msgstr "" + +#: templates/admin_doc/model_detail.html:37 +msgid "Description" +msgstr "" + +#: templates/admin_doc/model_detail.html:52 +msgid "Back to Models Documentation" +msgstr "" + +#: templates/admin_doc/model_index.html:18 +msgid "Model documentation" +msgstr "" + +#: templates/admin_doc/model_index.html:43 +msgid "Model groups" +msgstr "" + +#: templates/admin_doc/template_detail.html:8 +msgid "Templates" +msgstr "" + +#: templates/admin_doc/template_detail.html:13 +#, python-format +msgid "Template: %(name)s" +msgstr "" + +#: templates/admin_doc/template_detail.html:16 +#, python-format +msgid "Template: \"%(name)s\"" +msgstr "" + +#: templates/admin_doc/template_detail.html:20 +#, python-format +msgid "Search path for template \"%(name)s\" on %(grouper)s:" +msgstr "" + +#: templates/admin_doc/template_detail.html:23 +msgid "(does not exist)" +msgstr "" + +#: templates/admin_doc/template_detail.html:28 +msgid "Back to Documentation" +msgstr "" + +#: templates/admin_doc/template_filter_index.html:12 +msgid "Template filters" +msgstr "" + +#: templates/admin_doc/template_filter_index.html:16 +msgid "Template filter documentation" +msgstr "" + +#: templates/admin_doc/template_filter_index.html:22 +#: templates/admin_doc/template_filter_index.html:43 +msgid "Built-in filters" +msgstr "" + +#: templates/admin_doc/template_filter_index.html:23 +#, python-format +msgid "" +"To use these filters, put <code>%(code)s</code> in your template before " +"using the filter." +msgstr "" + +#: templates/admin_doc/template_tag_index.html:12 +msgid "Template tags" +msgstr "" + +#: templates/admin_doc/template_tag_index.html:16 +msgid "Template tag documentation" +msgstr "" + +#: templates/admin_doc/template_tag_index.html:22 +#: templates/admin_doc/template_tag_index.html:43 +msgid "Built-in tags" +msgstr "" + +#: templates/admin_doc/template_tag_index.html:23 +#, python-format +msgid "" +"To use these tags, put <code>%(code)s</code> in your template before using " +"the tag." +msgstr "" + +#: templates/admin_doc/view_detail.html:12 +#, python-format +msgid "View: %(name)s" +msgstr "" + +#: templates/admin_doc/view_detail.html:23 +msgid "Context:" +msgstr "" + +#: templates/admin_doc/view_detail.html:28 +msgid "Templates:" +msgstr "" + +#: templates/admin_doc/view_detail.html:32 +msgid "Back to Views Documentation" +msgstr "" + +#: templates/admin_doc/view_index.html:16 +msgid "View documentation" +msgstr "" + +#: templates/admin_doc/view_index.html:22 +msgid "Jump to site" +msgstr "" + +#: templates/admin_doc/view_index.html:35 +#, python-format +msgid "Views by URL on %(name)s" +msgstr "" + +#: templates/admin_doc/view_index.html:40 +#, python-format +msgid "View function: %(name)s" +msgstr "" + #: tests/test_fields.py:29 msgid "Boolean (Either True or False)" msgstr "" diff --git a/django/contrib/admindocs/middleware.py b/django/contrib/admindocs/middleware.py new file mode 100644 index 0000000000..ee3fe2cb2f --- /dev/null +++ b/django/contrib/admindocs/middleware.py @@ -0,0 +1,23 @@ +from django.conf import settings +from django import http + +class XViewMiddleware(object): + """ + Adds an X-View header to internal HEAD requests -- used by the documentation system. + """ + def process_view(self, request, view_func, view_args, view_kwargs): + """ + If the request method is HEAD and either the IP is internal or the + user is a logged-in staff member, quickly return with an x-header + indicating the view function. This is used by the documentation module + to lookup the view function for an arbitrary page. + """ + assert hasattr(request, 'user'), ( + "The XView middleware requires authentication middleware to be " + "installed. Edit your MIDDLEWARE_CLASSES setting to insert " + "'django.contrib.auth.middleware.AuthenticationMiddleware'.") + if request.method == 'HEAD' and (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or + (request.user.is_active and request.user.is_staff)): + response = http.HttpResponse() + response['X-View'] = "%s.%s" % (view_func.__module__, view_func.__name__) + return response diff --git a/django/contrib/admindocs/templates/admin_doc/index.html b/django/contrib/admindocs/templates/admin_doc/index.html index 5347341dc0..2af43d78c8 100644 --- a/django/contrib/admindocs/templates/admin_doc/index.html +++ b/django/contrib/admindocs/templates/admin_doc/index.html @@ -7,27 +7,27 @@ › {% trans 'Documentation' %}</a> </div> {% endblock %} -{% block title %}Documentation{% endblock %} +{% block title %}{% trans 'Documentation' %}{% endblock %} {% block content %} -<h1>Documentation</h1> +<h1>{% trans 'Documentation' %}</h1> <div id="content-main"> - <h3><a href="tags/">Tags</a></h3> - <p>List of all the template tags and their functions.</p> + <h3><a href="tags/">{% trans 'Tags' %}</a></h3> + <p>{% trans 'List of all the template tags and their functions.' %}</p> - <h3><a href="filters/">Filters</a></h3> - <p>Filters are actions which can be applied to variables in a template to alter the output.</p> + <h3><a href="filters/">{% trans 'Filters' %}</a></h3> + <p>{% trans 'Filters are actions which can be applied to variables in a template to alter the output.' %}</p> - <h3><a href="models/">Models</a></h3> - <p>Models are descriptions of all the objects in the system and their associated fields. Each model has a list of fields which can be accessed as template variables.</p> + <h3><a href="models/">{% trans 'Models' %}</a></h3> + <p>{% trans 'Models are descriptions of all the objects in the system and their associated fields. Each model has a list of fields which can be accessed as template variables' %}.</p> - <h3><a href="views/">Views</a></h3> - <p>Each page on the public site is generated by a view. The view defines which template is used to generate the page and which objects are available to that template.</p> + <h3><a href="views/">{% trans 'Views' %}</a></h3> + <p>{% trans 'Each page on the public site is generated by a view. The view defines which template is used to generate the page and which objects are available to that template.' %}</p> - <h3><a href="bookmarklets/">Bookmarklets</a></h3> - <p>Tools for your browser to quickly access admin functionality.</p> + <h3><a href="bookmarklets/">{% trans 'Bookmarklets' %}</a></h3> + <p>{% trans 'Tools for your browser to quickly access admin functionality.' %}</p> </div> {% endblock %} diff --git a/django/contrib/admindocs/templates/admin_doc/missing_docutils.html b/django/contrib/admindocs/templates/admin_doc/missing_docutils.html index f8a68ce04c..bd790f779a 100644 --- a/django/contrib/admindocs/templates/admin_doc/missing_docutils.html +++ b/django/contrib/admindocs/templates/admin_doc/missing_docutils.html @@ -7,16 +7,16 @@ › {% trans 'Documentation' %}</a> </div> {% endblock %} -{% block title %}Please install docutils{% endblock %} +{% block title %}{% trans 'Please install docutils' %}{% endblock %} {% block content %} -<h1>Documentation</h1> +<h1>{% trans 'Documentation' %}</h1> <div id="content-main"> - <h3>The admin documentation system requires Python's <a href="http://docutils.sf.net/">docutils</a> library.</h3> + <h3>{% blocktrans with "http://docutils.sf.net/" as link %}The admin documentation system requires Python's <a href="{{ link }}">docutils</a> library.{% endblocktrans %}</h3> - <p>Please ask your administrators to install <a href="http://docutils.sf.net/">docutils</a>.</p> + <p>{% blocktrans with "http://docutils.sf.net/" as link %}Please ask your administrators to install <a href="{{ link }}">docutils</a>.{% endblocktrans %}</p> </div> {% endblock %} diff --git a/django/contrib/admindocs/templates/admin_doc/model_detail.html b/django/contrib/admindocs/templates/admin_doc/model_detail.html index 9fb4eeea14..c1e2bf1e22 100644 --- a/django/contrib/admindocs/templates/admin_doc/model_detail.html +++ b/django/contrib/admindocs/templates/admin_doc/model_detail.html @@ -18,7 +18,7 @@ </div> {% endblock %} -{% block title %}Model: {{ name }}{% endblock %} +{% block title %}{% blocktrans %}Model: {{ name }}{% endblocktrans %}{% endblock %} {% block content %} <div id="content-main"> @@ -32,9 +32,9 @@ <table class="model"> <thead> <tr> - <th>Field</th> - <th>Type</th> - <th>Description</th> + <th>{% trans 'Field' %}</th> + <th>{% trans 'Type' %}</th> + <th>{% trans 'Description' %}</th> </tr> </thead> <tbody> @@ -49,6 +49,6 @@ </table> </div> -<p class="small"><a href="{% url 'django-admindocs-models-index' %}">‹ Back to Models Documentation</a></p> +<p class="small"><a href="{% url 'django-admindocs-models-index' %}">‹ {% trans 'Back to Models Documentation' %}</a></p> </div> {% endblock %} diff --git a/django/contrib/admindocs/templates/admin_doc/model_index.html b/django/contrib/admindocs/templates/admin_doc/model_index.html index 7a8c69953e..d4cde8334f 100644 --- a/django/contrib/admindocs/templates/admin_doc/model_index.html +++ b/django/contrib/admindocs/templates/admin_doc/model_index.html @@ -11,11 +11,11 @@ </div> {% endblock %} -{% block title %}Models{% endblock %} +{% block title %}{% trans 'Models' %}{% endblock %} {% block content %} -<h1>Model documentation</h1> +<h1>{% trans 'Model documentation' %}</h1> {% regroup models by app_label as grouped_models %} @@ -40,7 +40,7 @@ {% block sidebar %} <div id="content-related" class="sidebar"> <div class="module"> -<h2>Model groups</h2> +<h2>{% trans 'Model groups' %}</h2> <ul> {% regroup models by app_label as grouped_models %} {% for group in grouped_models %} diff --git a/django/contrib/admindocs/templates/admin_doc/template_detail.html b/django/contrib/admindocs/templates/admin_doc/template_detail.html index 27fca28b4b..9535724c24 100644 --- a/django/contrib/admindocs/templates/admin_doc/template_detail.html +++ b/django/contrib/admindocs/templates/admin_doc/template_detail.html @@ -10,20 +10,20 @@ </div> {% endblock %} -{% block title %}Template: {{ name }}{% endblock %} +{% block title %}{% blocktrans %}Template: {{ name }}{% endblocktrans %}{% endblock %} {% block content %} -<h1>Template: "{{ name }}"</h1> +<h1>{% blocktrans %}Template: "{{ name }}"{% endblocktrans %}</h1> {% regroup templates|dictsort:"site_id" by site as templates_by_site %} {% for group in templates_by_site %} - <h2>Search path for template "{{ name }}" on {{ group.grouper }}:</h2> + <h2>{% blocktrans with group.grouper as grouper %}Search path for template "{{ name }}" on {{ grouper }}:{% endblocktrans %}</h2> <ol> {% for template in group.list|dictsort:"order" %} - <li><code>{{ template.file }}</code>{% if not template.exists %} <em>(does not exist)</em>{% endif %}</li> + <li><code>{{ template.file }}</code>{% if not template.exists %} <em>{% trans '(does not exist)' %}</em>{% endif %}</li> {% endfor %} </ol> {% endfor %} -<p class="small"><a href="{% url 'django-admindocs-docroot' %}">‹ Back to Documentation</a></p> +<p class="small"><a href="{% url 'django-admindocs-docroot' %}">‹ {% trans 'Back to Documentation' %}</a></p> {% endblock %} diff --git a/django/contrib/admindocs/templates/admin_doc/template_filter_index.html b/django/contrib/admindocs/templates/admin_doc/template_filter_index.html index 687ebbcfec..04aac39105 100644 --- a/django/contrib/admindocs/templates/admin_doc/template_filter_index.html +++ b/django/contrib/admindocs/templates/admin_doc/template_filter_index.html @@ -9,18 +9,18 @@ › {% trans 'Filters' %} </div> {% endblock %} -{% block title %}Template filters{% endblock %} +{% block title %}{% trans 'Template filters' %}{% endblock %} {% block content %} -<h1>Template filter documentation</h1> +<h1>{% trans 'Template filter documentation' %}</h1> <div id="content-main"> {% regroup filters|dictsort:"library" by library as filter_libraries %} {% for library in filter_libraries %} <div class="module"> - <h2>{% firstof library.grouper "Built-in filters" %}</h2> - {% if library.grouper %}<p class="small quiet">To use these filters, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the filter.</p><hr />{% endif %} + <h2>{% firstof library.grouper _("Built-in filters") %}</h2> + {% if library.grouper %}<p class="small quiet">{% blocktrans with code="{"|add:"% load "|add:library.grouper|add:" %"|add:"}" %}To use these filters, put <code>{{ code }}</code> in your template before using the filter.{% endblocktrans %}</p><hr />{% endif %} {% for filter in library.list|dictsort:"name" %} <h3 id="{{ library.grouper|default:"built_in" }}-{{ filter.name }}">{{ filter.name }}</h3> {{ filter.title }} @@ -40,7 +40,7 @@ {% regroup filters|dictsort:"library" by library as filter_libraries %} {% for library in filter_libraries %} <div class="module"> - <h2>{% firstof library.grouper "Built-in filters" %}</h2> + <h2>{% firstof library.grouper _("Built-in filters") %}</h2> <ul> {% for filter in library.list|dictsort:"name" %} <li><a href="#{{ library.grouper|default:"built_in" }}-{{ filter.name }}">{{ filter.name }}</a></li> diff --git a/django/contrib/admindocs/templates/admin_doc/template_tag_index.html b/django/contrib/admindocs/templates/admin_doc/template_tag_index.html index c0fb243a99..a3c6eaadf4 100644 --- a/django/contrib/admindocs/templates/admin_doc/template_tag_index.html +++ b/django/contrib/admindocs/templates/admin_doc/template_tag_index.html @@ -9,18 +9,18 @@ › {% trans 'Tags' %} </div> {% endblock %} -{% block title %}Template tags{% endblock %} +{% block title %}{% trans 'Template tags' %}{% endblock %} {% block content %} -<h1>Template tag documentation</h1> +<h1>{% trans 'Template tag documentation' %}</h1> <div id="content-main"> {% regroup tags|dictsort:"library" by library as tag_libraries %} {% for library in tag_libraries %} <div class="module"> - <h2>{% firstof library.grouper "Built-in tags" %}</h2> - {% if library.grouper %}<p class="small quiet">To use these tags, put <code>{% templatetag openblock %} load {{ library.grouper }} {% templatetag closeblock %}</code> in your template before using the tag.</p><hr />{% endif %} + <h2>{% firstof library.grouper _("Built-in tags") %}</h2> + {% if library.grouper %}<p class="small quiet">{% blocktrans with code="{"|add:"% load "|add:library.grouper|add:" %"|add:"}" %}To use these tags, put <code>{{ code }}</code> in your template before using the tag.{% endblocktrans %}</p><hr />{% endif %} {% for tag in library.list|dictsort:"name" %} <h3 id="{{ library.grouper|default:"built_in" }}-{{ tag.name }}">{{ tag.name }}</h3> <h4>{{ tag.title|striptags }}</h4> @@ -40,7 +40,7 @@ {% regroup tags|dictsort:"library" by library as tag_libraries %} {% for library in tag_libraries %} <div class="module"> - <h2>{% firstof library.grouper "Built-in tags" %}</h2> + <h2>{% firstof library.grouper _("Built-in tags") %}</h2> <ul> {% for tag in library.list|dictsort:"name" %} <li><a href="#{{ library.grouper|default:"built_in" }}-{{ tag.name }}">{{ tag.name }}</a></li> diff --git a/django/contrib/admindocs/templates/admin_doc/view_detail.html b/django/contrib/admindocs/templates/admin_doc/view_detail.html index efe5fed9ed..050e6c800b 100644 --- a/django/contrib/admindocs/templates/admin_doc/view_detail.html +++ b/django/contrib/admindocs/templates/admin_doc/view_detail.html @@ -9,7 +9,7 @@ › {{ name }} </div> {% endblock %} -{% block title %}View: {{ name }}{% endblock %} +{% block title %}{% blocktrans %}View: {{ name }}{% endblocktrans %}{% endblock %} {% block content %} @@ -20,14 +20,14 @@ {{ body }} {% if meta.Context %} -<h3>Context:</h3> +<h3>{% trans 'Context:' %}</h3> <p>{{ meta.Context }}</p> {% endif %} {% if meta.Templates %} -<h3>Templates:</h3> +<h3>{% trans 'Templates:' %}</h3> <p>{{ meta.Templates }}</p> {% endif %} -<p class="small"><a href="{% url 'django-admindocs-views-index' %}">‹ Back to Views Documentation</a></p> +<p class="small"><a href="{% url 'django-admindocs-views-index' %}">‹ {% trans 'Back to Views Documentation' %}</a></p> {% endblock %} diff --git a/django/contrib/admindocs/templates/admin_doc/view_index.html b/django/contrib/admindocs/templates/admin_doc/view_index.html index 86342c6dd4..891eee7eec 100644 --- a/django/contrib/admindocs/templates/admin_doc/view_index.html +++ b/django/contrib/admindocs/templates/admin_doc/view_index.html @@ -9,17 +9,17 @@ › {% trans 'Views' %} </div> {% endblock %} -{% block title %}Views{% endblock %} +{% block title %}{% trans 'Views' %}{% endblock %} {% block content %} -<h1>View documentation</h1> +<h1>{% trans 'View documentation' %}</h1> {% regroup views|dictsort:"site_id" by site as views_by_site %} <div id="content-related" class="sidebar"> <div class="module"> -<h2>Jump to site</h2> +<h2>{% trans 'Jump to site' %}</h2> <ul> {% for site_views in views_by_site %} <li><a href="#site{{ site_views.grouper.id }}">{{ site_views.grouper.name }}</a></li> @@ -32,12 +32,12 @@ {% for site_views in views_by_site %} <div class="module"> -<h2 id="site{{ site_views.grouper.id }}">Views by URL on {{ site_views.grouper.name }}</h2> +<h2 id="site{{ site_views.grouper.id }}">{% blocktrans with site_views.grouper.name as name %}Views by URL on {{ name }}{% endblocktrans %}</h2> {% for view in site_views.list|dictsort:"url" %} {% ifchanged %} <h3><a href="{% url 'django-admindocs-views-detail' view=view.full_name %}">{{ view.url }}</a></h3> -<p class="small quiet">View function: {{ view.full_name }}</p> +<p class="small quiet">{% blocktrans with view.full_name as name %}View function: {{ name }}{% endblocktrans %}</p> <p>{{ view.title }}</p> <hr /> {% endifchanged %} @@ -46,5 +46,3 @@ {% endfor %} </div> {% endblock %} - - diff --git a/django/contrib/admindocs/views.py b/django/contrib/admindocs/views.py index 348727eb21..c03883def7 100644 --- a/django/contrib/admindocs/views.py +++ b/django/contrib/admindocs/views.py @@ -39,7 +39,7 @@ def bookmarklets(request): admin_root = urlresolvers.reverse('admin:index') return render_to_response('admin_doc/bookmarklets.html', { 'root_path': admin_root, - 'admin_url': "%s://%s%s" % (request.is_secure() and 'https' or 'http', request.get_host(), admin_root), + 'admin_url': "%s://%s%s" % ('https' if request.is_secure() else 'http', request.get_host(), admin_root), }, context_instance=RequestContext(request)) @staff_member_required @@ -267,6 +267,7 @@ def model_detail(request, app_label, model_name): return render_to_response('admin_doc/model_detail.html', { 'root_path': urlresolvers.reverse('admin:index'), 'name': '%s.%s' % (opts.app_label, opts.object_name), + # Translators: %s is an object type name 'summary': _("Attributes on %s objects") % opts.object_name, 'description': model.__doc__, 'fields': fields, @@ -286,7 +287,7 @@ def template_detail(request, template): templates.append({ 'file': template_file, 'exists': os.path.exists(template_file), - 'contents': lambda: os.path.exists(template_file) and open(template_file).read() or '', + 'contents': lambda: open(template_file).read() if os.path.exists(template_file) else '', 'site_id': settings_mod.SITE_ID, 'site': site_obj, 'order': list(settings_mod.TEMPLATE_DIRS).index(dir), diff --git a/django/contrib/auth/__init__.py b/django/contrib/auth/__init__.py index ef9066657d..029193d582 100644 --- a/django/contrib/auth/__init__.py +++ b/django/contrib/auth/__init__.py @@ -1,8 +1,11 @@ import re -from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed +from django.conf import settings from django.core.exceptions import ImproperlyConfigured, PermissionDenied from django.utils.module_loading import import_by_path +from django.middleware.csrf import rotate_token + +from .signals import user_logged_in, user_logged_out, user_login_failed SESSION_KEY = '_auth_user_id' BACKEND_SESSION_KEY = '_auth_user_backend' @@ -14,7 +17,6 @@ def load_backend(path): def get_backends(): - from django.conf import settings backends = [] for backend_path in settings.AUTHENTICATION_BACKENDS: backends.append(load_backend(backend_path)) @@ -83,6 +85,7 @@ def login(request, user): request.session[BACKEND_SESSION_KEY] = user.backend if hasattr(request, 'user'): request.user = user + rotate_token(request) user_logged_in.send(sender=user.__class__, request=request, user=user) @@ -106,7 +109,6 @@ def logout(request): def get_user_model(): "Return the User model that is active in this project" - from django.conf import settings from django.db.models import get_model try: @@ -120,12 +122,13 @@ def get_user_model(): def get_user(request): - from django.contrib.auth.models import AnonymousUser + from .models import AnonymousUser try: user_id = request.session[SESSION_KEY] backend_path = request.session[BACKEND_SESSION_KEY] + assert backend_path in settings.AUTHENTICATION_BACKENDS backend = load_backend(backend_path) user = backend.get_user(user_id) or AnonymousUser() - except KeyError: + except (KeyError, AssertionError): user = AnonymousUser() return user diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py index 0e08d8ef31..edf2727b07 100644 --- a/django/contrib/auth/forms.py +++ b/django/contrib/auth/forms.py @@ -237,7 +237,7 @@ class PasswordResetForm(forms.Form): 'uid': int_to_base36(user.pk), 'user': user, 'token': token_generator.make_token(user), - 'protocol': use_https and 'https' or 'http', + 'protocol': 'https' if use_https else 'http', } subject = loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py index 685e50d498..fdf822ff74 100644 --- a/django/contrib/auth/management/__init__.py +++ b/django/contrib/auth/management/__init__.py @@ -80,7 +80,7 @@ def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kw for perm in _get_all_permissions(klass._meta, ctype): searched_perms.append((ctype, perm)) - # Find all the Permissions that have a context_type for a model we're + # Find all the Permissions that have a content_type for a model we're # looking for. We don't need to check for codenames since we already have # a list of the ones we're going to create. all_perms = set(auth_app.Permission.objects.using(db).filter( diff --git a/django/contrib/auth/models.py b/django/contrib/auth/models.py index 5709d25d7f..798cc805a0 100644 --- a/django/contrib/auth/models.py +++ b/django/contrib/auth/models.py @@ -177,7 +177,7 @@ class UserManager(BaseUserManager): now = timezone.now() if not username: raise ValueError('The given username must be set') - email = UserManager.normalize_email(email) + email = self.normalize_email(email) user = self.model(username=username, email=email, is_staff=False, is_active=True, is_superuser=False, last_login=now, date_joined=now, **extra_fields) diff --git a/django/contrib/auth/tests/test_auth_backends.py b/django/contrib/auth/tests/test_auth_backends.py index bb97c54a11..fc5a80e8dd 100644 --- a/django/contrib/auth/tests/test_auth_backends.py +++ b/django/contrib/auth/tests/test_auth_backends.py @@ -2,12 +2,14 @@ from __future__ import unicode_literals from datetime import date from django.conf import settings +from django.contrib.auth.backends import ModelBackend from django.contrib.auth.models import User, Group, Permission, AnonymousUser from django.contrib.auth.tests.utils import skipIfCustomUser from django.contrib.auth.tests.test_custom_user import ExtensionUser, CustomPermissionsUser, CustomUser from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured, PermissionDenied -from django.contrib.auth import authenticate +from django.contrib.auth import authenticate, get_user +from django.http import HttpRequest from django.test import TestCase from django.test.utils import override_settings @@ -402,3 +404,52 @@ class PermissionDeniedBackendTest(TestCase): settings.AUTHENTICATION_BACKENDS) + (backend, )) def test_authenticates(self): self.assertEqual(authenticate(username='test', password='test'), self.user1) + + +class NewModelBackend(ModelBackend): + pass + + +@skipIfCustomUser +class ChangedBackendSettingsTest(TestCase): + """ + Tests for changes in the settings.AUTHENTICATION_BACKENDS + """ + backend = 'django.contrib.auth.tests.test_auth_backends.NewModelBackend' + + TEST_USERNAME = 'test_user' + TEST_PASSWORD = 'test_password' + TEST_EMAIL = 'test@example.com' + + def setUp(self): + User.objects.create_user(self.TEST_USERNAME, + self.TEST_EMAIL, + self.TEST_PASSWORD) + + @override_settings(AUTHENTICATION_BACKENDS=(backend, )) + def test_changed_backend_settings(self): + """ + Tests that removing a backend configured in AUTHENTICATION_BACKENDS + make already logged-in users disconnect. + """ + + # Get a session for the test user + self.assertTrue(self.client.login( + username=self.TEST_USERNAME, + password=self.TEST_PASSWORD) + ) + + # Prepare a request object + request = HttpRequest() + request.session = self.client.session + + # Remove NewModelBackend + with self.settings(AUTHENTICATION_BACKENDS=( + 'django.contrib.auth.backends.ModelBackend',)): + # Get the user from the request + user = get_user(request) + + # Assert that the user retrieval is successful and the user is + # anonymous as the backend is not longer available. + self.assertIsNotNone(user) + self.assertTrue(user.is_anonymous()) diff --git a/django/contrib/auth/tests/test_custom_user.py b/django/contrib/auth/tests/test_custom_user.py index 0d324f0953..a3a159880a 100644 --- a/django/contrib/auth/tests/test_custom_user.py +++ b/django/contrib/auth/tests/test_custom_user.py @@ -21,7 +21,7 @@ class CustomUserManager(BaseUserManager): raise ValueError('Users must have an email address') user = self.model( - email=CustomUserManager.normalize_email(email), + email=self.normalize_email(email), date_of_birth=date_of_birth, ) diff --git a/django/contrib/auth/tests/test_management.py b/django/contrib/auth/tests/test_management.py index 04fd4941ab..fee0a29e7b 100644 --- a/django/contrib/auth/tests/test_management.py +++ b/django/contrib/auth/tests/test_management.py @@ -7,6 +7,7 @@ from django.contrib.auth.management.commands import changepassword from django.contrib.auth.models import User from django.contrib.auth.tests.test_custom_user import CustomUser from django.contrib.auth.tests.utils import skipIfCustomUser +from django.contrib.contenttypes.models import ContentType from django.core.management import call_command from django.core.management.base import CommandError from django.core.management.validation import get_validation_errors @@ -195,6 +196,7 @@ class PermissionDuplicationTestCase(TestCase): def tearDown(self): models.Permission._meta.permissions = self._original_permissions + ContentType.objects.clear_cache() def test_duplicated_permissions(self): """ diff --git a/django/contrib/auth/tests/test_views.py b/django/contrib/auth/tests/test_views.py index 7cbf72327e..ef305ac8f1 100644 --- a/django/contrib/auth/tests/test_views.py +++ b/django/contrib/auth/tests/test_views.py @@ -1,25 +1,31 @@ import itertools import os import re +try: + from urllib.parse import urlparse, ParseResult +except ImportError: # Python 2 + from urlparse import urlparse, ParseResult from django.conf import global_settings, settings from django.contrib.sites.models import Site, RequestSite from django.contrib.auth.models import User from django.core import mail -from django.core.exceptions import SuspiciousOperation from django.core.urlresolvers import reverse, NoReverseMatch -from django.http import QueryDict +from django.http import QueryDict, HttpRequest from django.utils.encoding import force_text from django.utils.html import escape from django.utils.http import urlquote from django.utils._os import upath from django.test import TestCase -from django.test.utils import override_settings +from django.test.utils import override_settings, patch_logger +from django.middleware.csrf import CsrfViewMiddleware +from django.contrib.sessions.middleware import SessionMiddleware from django.contrib.auth import SESSION_KEY, REDIRECT_FIELD_NAME from django.contrib.auth.forms import (AuthenticationForm, PasswordChangeForm, SetPasswordForm, PasswordResetForm) from django.contrib.auth.tests.utils import skipIfCustomUser +from django.contrib.auth.views import login as login_view @override_settings( @@ -46,15 +52,31 @@ class AuthViewsTestCase(TestCase): 'username': 'testclient', 'password': password, }) - self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith(settings.LOGIN_REDIRECT_URL)) self.assertTrue(SESSION_KEY in self.client.session) + return response def assertFormError(self, response, error): """Assert that error is found in response.context['form'] errors""" form_errors = list(itertools.chain(*response.context['form'].errors.values())) self.assertIn(force_text(error), form_errors) + def assertURLEqual(self, url, expected, parse_qs=False): + """ + Given two URLs, make sure all their components (the ones given by + urlparse) are equal, only comparing components that are present in both + URLs. + If `parse_qs` is True, then the querystrings are parsed with QueryDict. + This is useful if you don't want the order of parameters to matter. + Otherwise, the query strings are compared as-is. + """ + fields = ParseResult._fields + + for attr, x, y in zip(fields, urlparse(url), urlparse(expected)): + if parse_qs and attr == 'query': + x, y = QueryDict(x), QueryDict(y) + if x and y and x != y: + self.fail("%r != %r (%s doesn't match)" % (url, expected, attr)) + @skipIfCustomUser class AuthViewNamedURLTests(AuthViewsTestCase): @@ -132,28 +154,32 @@ class PasswordResetTest(AuthViewsTestCase): # produce a meaningful reset URL, we need to be certain that the # HTTP_HOST header isn't poisoned. This is done as a check when get_host() # is invoked, but we check here as a practical consequence. - with self.assertRaises(SuspiciousOperation): - self.client.post('/password_reset/', - {'email': 'staffmember@example.com'}, - HTTP_HOST='www.example:dr.frankenstein@evil.tld' - ) - self.assertEqual(len(mail.outbox), 0) + with patch_logger('django.security.DisallowedHost', 'error') as logger_calls: + response = self.client.post('/password_reset/', + {'email': 'staffmember@example.com'}, + HTTP_HOST='www.example:dr.frankenstein@evil.tld' + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(len(mail.outbox), 0) + self.assertEqual(len(logger_calls), 1) # Skip any 500 handler action (like sending more mail...) @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True) def test_poisoned_http_host_admin_site(self): "Poisoned HTTP_HOST headers can't be used for reset emails on admin views" - with self.assertRaises(SuspiciousOperation): - self.client.post('/admin_password_reset/', - {'email': 'staffmember@example.com'}, - HTTP_HOST='www.example:dr.frankenstein@evil.tld' - ) - self.assertEqual(len(mail.outbox), 0) + with patch_logger('django.security.DisallowedHost', 'error') as logger_calls: + response = self.client.post('/admin_password_reset/', + {'email': 'staffmember@example.com'}, + HTTP_HOST='www.example:dr.frankenstein@evil.tld' + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(len(mail.outbox), 0) + self.assertEqual(len(logger_calls), 1) + def _test_confirm_start(self): # Start by creating the email response = self.client.post('/password_reset/', {'email': 'staffmember@example.com'}) - self.assertEqual(response.status_code, 302) self.assertEqual(len(mail.outbox), 1) return self._read_signup_email(mail.outbox[0]) @@ -205,8 +231,6 @@ class PasswordResetTest(AuthViewsTestCase): url, path = self._test_confirm_start() response = self.client.post(path, {'new_password1': 'anewpassword', 'new_password2': 'anewpassword'}) - # It redirects us to a 'complete' page: - self.assertEqual(response.status_code, 302) # Check the password has been changed u = User.objects.get(email='staffmember@example.com') self.assertTrue(u.check_password("anewpassword")) @@ -221,6 +245,47 @@ class PasswordResetTest(AuthViewsTestCase): 'new_password2': 'x'}) self.assertFormError(response, SetPasswordForm.error_messages['password_mismatch']) + def test_reset_redirect_default(self): + response = self.client.post('/password_reset/', + {'email': 'staffmember@example.com'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_reset/done/') + + def test_reset_custom_redirect(self): + response = self.client.post('/password_reset/custom_redirect/', + {'email': 'staffmember@example.com'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/custom/') + + def test_reset_custom_redirect_named(self): + response = self.client.post('/password_reset/custom_redirect/named/', + {'email': 'staffmember@example.com'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_reset/') + + def test_confirm_redirect_default(self): + url, path = self._test_confirm_start() + response = self.client.post(path, {'new_password1': 'anewpassword', + 'new_password2': 'anewpassword'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/reset/done/') + + def test_confirm_redirect_custom(self): + url, path = self._test_confirm_start() + path = path.replace('/reset/', '/reset/custom/') + response = self.client.post(path, {'new_password1': 'anewpassword', + 'new_password2': 'anewpassword'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/custom/') + + def test_confirm_redirect_custom_named(self): + url, path = self._test_confirm_start() + path = path.replace('/reset/', '/reset/custom/named/') + response = self.client.post(path, {'new_password1': 'anewpassword', + 'new_password2': 'anewpassword'}) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_reset/') + @override_settings(AUTH_USER_MODEL='auth.CustomUser') class CustomUserPasswordResetTest(AuthViewsTestCase): @@ -285,8 +350,6 @@ class ChangePasswordTest(AuthViewsTestCase): 'new_password1': 'password1', 'new_password2': 'password1', }) - self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/password_change/done/')) self.fail_login() self.login(password='password1') @@ -298,20 +361,50 @@ class ChangePasswordTest(AuthViewsTestCase): 'new_password2': 'password1', }) self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/password_change/done/')) + self.assertURLEqual(response.url, '/password_change/done/') + @override_settings(LOGIN_URL='/login/') def test_password_change_done_fails(self): - with self.settings(LOGIN_URL='/login/'): - response = self.client.get('/password_change/done/') - self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/login/?next=/password_change/done/')) + response = self.client.get('/password_change/done/') + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/login/?next=/password_change/done/') + + def test_password_change_redirect_default(self): + self.login() + response = self.client.post('/password_change/', { + 'old_password': 'password', + 'new_password1': 'password1', + 'new_password2': 'password1', + }) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_change/done/') + + def test_password_change_redirect_custom(self): + self.login() + response = self.client.post('/password_change/custom/', { + 'old_password': 'password', + 'new_password1': 'password1', + 'new_password2': 'password1', + }) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/custom/') + + def test_password_change_redirect_custom_named(self): + self.login() + response = self.client.post('/password_change/custom/named/', { + 'old_password': 'password', + 'new_password1': 'password1', + 'new_password2': 'password1', + }) + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_reset/') @skipIfCustomUser class LoginTest(AuthViewsTestCase): def test_current_site_in_context_after_login(self): - response = self.client.get(reverse('django.contrib.auth.views.login')) + response = self.client.get(reverse('login')) self.assertEqual(response.status_code, 200) if Site._meta.installed: site = Site.objects.get_current() @@ -323,7 +416,7 @@ class LoginTest(AuthViewsTestCase): 'Login form is not an AuthenticationForm') def test_security_check(self, password='password'): - login_url = reverse('django.contrib.auth.views.login') + login_url = reverse('login') # Those URLs should not pass the security check for bad_url in ('http://example.com', @@ -374,63 +467,103 @@ class LoginTest(AuthViewsTestCase): # the custom authentication form used by this login asserts # that a request is passed to the form successfully. -@skipIfCustomUser -class LoginURLSettings(AuthViewsTestCase): + def test_login_csrf_rotate(self, password='password'): + """ + Makes sure that a login rotates the currently-used CSRF token. + """ + # Do a GET to establish a CSRF token + # TestClient isn't used here as we're testing middleware, essentially. + req = HttpRequest() + CsrfViewMiddleware().process_view(req, login_view, (), {}) + req.META["CSRF_COOKIE_USED"] = True + resp = login_view(req) + resp2 = CsrfViewMiddleware().process_response(req, resp) + csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, None) + token1 = csrf_cookie.coded_value + + # Prepare the POST request + req = HttpRequest() + req.COOKIES[settings.CSRF_COOKIE_NAME] = token1 + req.method = "POST" + req.POST = {'username': 'testclient', 'password': password, 'csrfmiddlewaretoken': token1} + req.REQUEST = req.POST + + # Use POST request to log in + SessionMiddleware().process_request(req) + CsrfViewMiddleware().process_view(req, login_view, (), {}) + req.META["SERVER_NAME"] = "testserver" # Required to have redirect work in login view + req.META["SERVER_PORT"] = 80 + req.META["CSRF_COOKIE_USED"] = True + resp = login_view(req) + resp2 = CsrfViewMiddleware().process_response(req, resp) + csrf_cookie = resp2.cookies.get(settings.CSRF_COOKIE_NAME, None) + token2 = csrf_cookie.coded_value - def setUp(self): - super(LoginURLSettings, self).setUp() - self.old_LOGIN_URL = settings.LOGIN_URL + # Check the CSRF token switched + self.assertNotEqual(token1, token2) - def tearDown(self): - super(LoginURLSettings, self).tearDown() - settings.LOGIN_URL = self.old_LOGIN_URL - def get_login_required_url(self, login_url): - settings.LOGIN_URL = login_url +@skipIfCustomUser +class LoginURLSettings(AuthViewsTestCase): + """Tests for settings.LOGIN_URL.""" + def assertLoginURLEquals(self, url, parse_qs=False): response = self.client.get('/login_required/') self.assertEqual(response.status_code, 302) - return response.url + self.assertURLEqual(response.url, url, parse_qs=parse_qs) + @override_settings(LOGIN_URL='/login/') def test_standard_login_url(self): - login_url = '/login/' - login_required_url = self.get_login_required_url(login_url) - querystring = QueryDict('', mutable=True) - querystring['next'] = '/login_required/' - self.assertEqual(login_required_url, 'http://testserver%s?%s' % - (login_url, querystring.urlencode('/'))) + self.assertLoginURLEquals('/login/?next=/login_required/') + @override_settings(LOGIN_URL='login') + def test_named_login_url(self): + self.assertLoginURLEquals('/login/?next=/login_required/') + + @override_settings(LOGIN_URL='http://remote.example.com/login') def test_remote_login_url(self): - login_url = 'http://remote.example.com/login' - login_required_url = self.get_login_required_url(login_url) - querystring = QueryDict('', mutable=True) - querystring['next'] = 'http://testserver/login_required/' - self.assertEqual(login_required_url, - '%s?%s' % (login_url, querystring.urlencode('/'))) + quoted_next = urlquote('http://testserver/login_required/') + expected = 'http://remote.example.com/login?next=%s' % quoted_next + self.assertLoginURLEquals(expected) + @override_settings(LOGIN_URL='https:///login/') def test_https_login_url(self): - login_url = 'https:///login/' - login_required_url = self.get_login_required_url(login_url) - querystring = QueryDict('', mutable=True) - querystring['next'] = 'http://testserver/login_required/' - self.assertEqual(login_required_url, - '%s?%s' % (login_url, querystring.urlencode('/'))) + quoted_next = urlquote('http://testserver/login_required/') + expected = 'https:///login/?next=%s' % quoted_next + self.assertLoginURLEquals(expected) + @override_settings(LOGIN_URL='/login/?pretty=1') def test_login_url_with_querystring(self): - login_url = '/login/?pretty=1' - login_required_url = self.get_login_required_url(login_url) - querystring = QueryDict('pretty=1', mutable=True) - querystring['next'] = '/login_required/' - self.assertEqual(login_required_url, 'http://testserver/login/?%s' % - querystring.urlencode('/')) + self.assertLoginURLEquals('/login/?pretty=1&next=/login_required/', parse_qs=True) + @override_settings(LOGIN_URL='http://remote.example.com/login/?next=/default/') def test_remote_login_url_with_next_querystring(self): - login_url = 'http://remote.example.com/login/' - login_required_url = self.get_login_required_url('%s?next=/default/' % - login_url) - querystring = QueryDict('', mutable=True) - querystring['next'] = 'http://testserver/login_required/' - self.assertEqual(login_required_url, '%s?%s' % (login_url, - querystring.urlencode('/'))) + quoted_next = urlquote('http://testserver/login_required/') + expected = 'http://remote.example.com/login/?next=%s' % quoted_next + self.assertLoginURLEquals(expected) + + +@skipIfCustomUser +class LoginRedirectUrlTest(AuthViewsTestCase): + """Tests for settings.LOGIN_REDIRECT_URL.""" + def assertLoginRedirectURLEqual(self, url): + response = self.login() + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, url) + + def test_default(self): + self.assertLoginRedirectURLEqual('/accounts/profile/') + + @override_settings(LOGIN_REDIRECT_URL='/custom/') + def test_custom(self): + self.assertLoginRedirectURLEqual('/custom/') + + @override_settings(LOGIN_REDIRECT_URL='password_reset') + def test_named(self): + self.assertLoginRedirectURLEqual('/password_reset/') + + @override_settings(LOGIN_REDIRECT_URL='http://remote.example.com/welcome/') + def test_remote(self): + self.assertLoginRedirectURLEqual('http://remote.example.com/welcome/') @skipIfCustomUser @@ -457,11 +590,11 @@ class LogoutTest(AuthViewsTestCase): self.login() response = self.client.get('/logout/next_page/') self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/somewhere/')) + self.assertURLEqual(response.url, '/somewhere/') response = self.client.get('/logout/next_page/?next=/login/') self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/login/')) + self.assertURLEqual(response.url, '/login/') self.confirm_logged_out() @@ -470,7 +603,7 @@ class LogoutTest(AuthViewsTestCase): self.login() response = self.client.get('/logout/next_page/') self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/somewhere/')) + self.assertURLEqual(response.url, '/somewhere/') self.confirm_logged_out() def test_logout_with_redirect_argument(self): @@ -478,7 +611,7 @@ class LogoutTest(AuthViewsTestCase): self.login() response = self.client.get('/logout/?next=/login/') self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/login/')) + self.assertURLEqual(response.url, '/login/') self.confirm_logged_out() def test_logout_with_custom_redirect_argument(self): @@ -486,11 +619,19 @@ class LogoutTest(AuthViewsTestCase): self.login() response = self.client.get('/logout/custom_query/?follow=/somewhere/') self.assertEqual(response.status_code, 302) - self.assertTrue(response.url.endswith('/somewhere/')) + self.assertURLEqual(response.url, '/somewhere/') + self.confirm_logged_out() + + def test_logout_with_named_redirect(self): + "Logout resolves names or URLs passed as next_page." + self.login() + response = self.client.get('/logout/next_page/named/') + self.assertEqual(response.status_code, 302) + self.assertURLEqual(response.url, '/password_reset/') self.confirm_logged_out() def test_security_check(self, password='password'): - logout_url = reverse('django.contrib.auth.views.logout') + logout_url = reverse('logout') # Those URLs should not pass the security check for bad_url in ('http://example.com', @@ -541,5 +682,7 @@ class ChangelistTests(AuthViewsTestCase): self.login() # A lookup that tries to filter on password isn't OK - with self.assertRaises(SuspiciousOperation): + with patch_logger('django.security.DisallowedModelAdminLookup', 'error') as logger_calls: response = self.client.get('/admin/auth/user/?password__startswith=sha1$') + self.assertEqual(response.status_code, 400) + self.assertEqual(len(logger_calls), 1) diff --git a/django/contrib/auth/tests/urls.py b/django/contrib/auth/tests/urls.py index 51b05be648..835ff41de7 100644 --- a/django/contrib/auth/tests/urls.py +++ b/django/contrib/auth/tests/urls.py @@ -62,8 +62,19 @@ def custom_request_auth_login(request): urlpatterns = urlpatterns + patterns('', (r'^logout/custom_query/$', 'django.contrib.auth.views.logout', dict(redirect_field_name='follow')), (r'^logout/next_page/$', 'django.contrib.auth.views.logout', dict(next_page='/somewhere/')), + (r'^logout/next_page/named/$', 'django.contrib.auth.views.logout', dict(next_page='password_reset')), (r'^remote_user/$', remote_user_auth_view), (r'^password_reset_from_email/$', 'django.contrib.auth.views.password_reset', dict(from_email='staffmember@example.com')), + (r'^password_reset/custom_redirect/$', 'django.contrib.auth.views.password_reset', dict(post_reset_redirect='/custom/')), + (r'^password_reset/custom_redirect/named/$', 'django.contrib.auth.views.password_reset', dict(post_reset_redirect='password_reset')), + (r'^reset/custom/(?P<uidb36>[0-9A-Za-z]{1,13})-(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$', + 'django.contrib.auth.views.password_reset_confirm', + dict(post_reset_redirect='/custom/')), + (r'^reset/custom/named/(?P<uidb36>[0-9A-Za-z]{1,13})-(?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,20})/$', + 'django.contrib.auth.views.password_reset_confirm', + dict(post_reset_redirect='password_reset')), + (r'^password_change/custom/$', 'django.contrib.auth.views.password_change', dict(post_change_redirect='/custom/')), + (r'^password_change/custom/named/$', 'django.contrib.auth.views.password_change', dict(post_change_redirect='password_reset')), (r'^admin_password_reset/$', 'django.contrib.auth.views.password_reset', dict(is_admin_site=True)), (r'^login_required/$', login_required(password_reset)), (r'^login_required_login_url/$', login_required(password_reset, login_url='/somewhere/')), diff --git a/django/contrib/auth/views.py b/django/contrib/auth/views.py index 8a554b0ad8..fe21683323 100644 --- a/django/contrib/auth/views.py +++ b/django/contrib/auth/views.py @@ -72,6 +72,9 @@ def logout(request, next_page=None, """ auth_logout(request) + if next_page is not None: + next_page = resolve_url(next_page) + if redirect_field_name in request.REQUEST: next_page = request.REQUEST[redirect_field_name] # Security check -- don't allow redirection to a different host. @@ -139,7 +142,9 @@ def password_reset(request, is_admin_site=False, current_app=None, extra_context=None): if post_reset_redirect is None: - post_reset_redirect = reverse('django.contrib.auth.views.password_reset_done') + post_reset_redirect = reverse('password_reset_done') + else: + post_reset_redirect = resolve_url(post_reset_redirect) if request.method == "POST": form = password_reset_form(request.POST) if form.is_valid(): @@ -192,7 +197,9 @@ def password_reset_confirm(request, uidb36=None, token=None, UserModel = get_user_model() assert uidb36 is not None and token is not None # checked by URLconf if post_reset_redirect is None: - post_reset_redirect = reverse('django.contrib.auth.views.password_reset_complete') + post_reset_redirect = reverse('password_reset_complete') + else: + post_reset_redirect = resolve_url(post_reset_redirect) try: uid_int = base36_to_int(uidb36) user = UserModel._default_manager.get(pk=uid_int) @@ -242,7 +249,9 @@ def password_change(request, password_change_form=PasswordChangeForm, current_app=None, extra_context=None): if post_change_redirect is None: - post_change_redirect = reverse('django.contrib.auth.views.password_change_done') + post_change_redirect = reverse('password_change_done') + else: + post_change_redirect = resolve_url(post_change_redirect) if request.method == "POST": form = password_change_form(user=request.user, data=request.POST) if form.is_valid(): diff --git a/django/contrib/comments/locale/en/LC_MESSAGES/django.po b/django/contrib/comments/locale/en/LC_MESSAGES/django.po index 6aca84c3cc..43ca058b6c 100644 --- a/django/contrib/comments/locale/en/LC_MESSAGES/django.po +++ b/django/contrib/comments/locale/en/LC_MESSAGES/django.po @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: Django\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2013-05-02 16:18+0200\n" +"POT-Creation-Date: 2013-05-25 14:19+0200\n" "PO-Revision-Date: 2010-05-13 15:35+0200\n" "Last-Translator: Django team\n" "Language-Team: English <en@li.org>\n" @@ -269,9 +269,11 @@ msgstr "" #: templates/comments/preview.html:11 msgid "Please correct the error below" -msgid_plural "Please correct the errors below" -msgstr[0] "" -msgstr[1] "" +msgstr "" + +#: templates/comments/preview.html:11 +msgid "Please correct the errors below" +msgstr "" #: templates/comments/preview.html:16 msgid "Post your comment" diff --git a/django/contrib/comments/templates/comments/preview.html b/django/contrib/comments/templates/comments/preview.html index 882d0fe714..0e7056795b 100644 --- a/django/contrib/comments/templates/comments/preview.html +++ b/django/contrib/comments/templates/comments/preview.html @@ -8,7 +8,7 @@ <form action="{% comment_form_target %}" method="post">{% csrf_token %} {% if next %}<div><input type="hidden" name="next" value="{{ next }}" /></div>{% endif %} {% if form.errors %} - <h1>{% blocktrans count counter=form.errors|length %}Please correct the error below{% plural %}Please correct the errors below{% endblocktrans %}</h1> + <h1>{% if form.errors|length == 1 %}{% trans "Please correct the error below" %}{% else %}{% trans "Please correct the errors below" %}{% endif %}</h1> {% else %} <h1>{% trans "Preview your comment" %}</h1> <blockquote>{{ comment|linebreaks }}</blockquote> diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index 399d24aa87..26db4ab171 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -35,9 +35,10 @@ class GenericForeignKey(six.with_metaclass(RenameGenericForeignKeyMethods)): fields. """ - def __init__(self, ct_field="content_type", fk_field="object_id"): + def __init__(self, ct_field="content_type", fk_field="object_id", for_concrete_model=True): self.ct_field = ct_field self.fk_field = fk_field + self.for_concrete_model = for_concrete_model def contribute_to_class(self, cls, name): self.name = name @@ -63,7 +64,8 @@ class GenericForeignKey(six.with_metaclass(RenameGenericForeignKeyMethods)): def get_content_type(self, obj=None, id=None, using=None): if obj is not None: - return ContentType.objects.db_manager(obj._state.db).get_for_model(obj) + return ContentType.objects.db_manager(obj._state.db).get_for_model( + obj, for_concrete_model=self.for_concrete_model) elif id: return ContentType.objects.db_manager(using).get_for_id(id) else: @@ -160,6 +162,8 @@ class GenericRelation(ForeignObject): self.object_id_field_name = kwargs.pop("object_id_field", "object_id") self.content_type_field_name = kwargs.pop("content_type_field", "content_type") + self.for_concrete_model = kwargs.pop("for_concrete_model", True) + kwargs['blank'] = True kwargs['editable'] = False kwargs['serialize'] = False @@ -201,7 +205,7 @@ class GenericRelation(ForeignObject): # Save a reference to which model this class is on for future use self.model = cls # Add the descriptor for the relation - setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self)) + setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self, self.for_concrete_model)) def contribute_to_related_class(self, cls, related): pass @@ -216,7 +220,8 @@ class GenericRelation(ForeignObject): """ Returns the content type associated with this field's model. """ - return ContentType.objects.get_for_model(self.model) + return ContentType.objects.get_for_model(self.model, + for_concrete_model=self.for_concrete_model) def get_extra_restriction(self, where_class, alias, remote_alias): field = self.rel.to._meta.get_field_by_name(self.content_type_field_name)[0] @@ -232,7 +237,8 @@ class GenericRelation(ForeignObject): """ return self.rel.to._base_manager.db_manager(using).filter(**{ "%s__pk" % self.content_type_field_name: - ContentType.objects.db_manager(using).get_for_model(self.model).pk, + ContentType.objects.db_manager(using).get_for_model( + self.model, for_concrete_model=self.for_concrete_model).pk, "%s__in" % self.object_id_field_name: [obj.pk for obj in objs] }) @@ -247,8 +253,9 @@ class ReverseGenericRelatedObjectsDescriptor(object): "article.publications", the publications attribute is a ReverseGenericRelatedObjectsDescriptor instance. """ - def __init__(self, field): + def __init__(self, field, for_concrete_model=True): self.field = field + self.for_concrete_model = for_concrete_model def __get__(self, instance, instance_type=None): if instance is None: @@ -261,7 +268,8 @@ class ReverseGenericRelatedObjectsDescriptor(object): RelatedManager = create_generic_related_manager(superclass) qn = connection.ops.quote_name - content_type = ContentType.objects.db_manager(instance._state.db).get_for_model(instance) + content_type = ContentType.objects.db_manager(instance._state.db).get_for_model( + instance, for_concrete_model=self.for_concrete_model) join_cols = self.field.get_joining_columns(reverse_join=True)[0] manager = RelatedManager( @@ -376,7 +384,7 @@ class BaseGenericInlineFormSet(BaseModelFormSet): """ def __init__(self, data=None, files=None, instance=None, save_as_new=None, - prefix=None, queryset=None): + prefix=None, queryset=None, **kwargs): opts = self.model._meta self.instance = instance self.rel_name = '-'.join(( @@ -389,12 +397,14 @@ class BaseGenericInlineFormSet(BaseModelFormSet): if queryset is None: queryset = self.model._default_manager qs = queryset.filter(**{ - self.ct_field.name: ContentType.objects.get_for_model(self.instance), + self.ct_field.name: ContentType.objects.get_for_model( + self.instance, for_concrete_model=self.for_concrete_model), self.ct_fk_field.name: self.instance.pk, }) super(BaseGenericInlineFormSet, self).__init__( queryset=qs, data=data, files=files, - prefix=prefix + prefix=prefix, + **kwargs ) @classmethod @@ -406,7 +416,8 @@ class BaseGenericInlineFormSet(BaseModelFormSet): def save_new(self, form, commit=True): kwargs = { - self.ct_field.get_attname(): ContentType.objects.get_for_model(self.instance).pk, + self.ct_field.get_attname(): ContentType.objects.get_for_model( + self.instance, for_concrete_model=self.for_concrete_model).pk, self.ct_fk_field.get_attname(): self.instance.pk, } new_obj = self.model(**kwargs) @@ -418,7 +429,8 @@ def generic_inlineformset_factory(model, form=ModelForm, fields=None, exclude=None, extra=3, can_order=False, can_delete=True, max_num=None, - formfield_callback=None, validate_max=False): + formfield_callback=None, validate_max=False, + for_concrete_model=True): """ Returns a ``GenericInlineFormSet`` for the given kwargs. @@ -444,6 +456,7 @@ def generic_inlineformset_factory(model, form=ModelForm, validate_max=validate_max) FormSet.ct_field = ct_field FormSet.ct_fk_field = fk_field + FormSet.for_concrete_model = for_concrete_model return FormSet class GenericInlineModelAdmin(InlineModelAdmin): diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py index f0bd109b00..34d54441fe 100644 --- a/django/contrib/contenttypes/models.py +++ b/django/contrib/contenttypes/models.py @@ -118,11 +118,13 @@ class ContentTypeManager(models.Manager): def _add_to_cache(self, using, ct): """Insert a ContentType into the cache.""" - model = ct.model_class() - key = (model._meta.app_label, model._meta.model_name) + # Note it's possible for ContentType objects to be stale; model_class() will return None. + # Hence, there is no reliance on model._meta.app_label here, just using the model fields instead. + key = (ct.app_label, ct.model) self.__class__._cache.setdefault(using, {})[key] = ct self.__class__._cache.setdefault(using, {})[ct.id] = ct + @python_2_unicode_compatible class ContentType(models.Model): name = models.CharField(max_length=100) @@ -153,7 +155,6 @@ class ContentType(models.Model): def model_class(self): "Returns the Python model class for this type of content." - from django.db import models return models.get_model(self.app_label, self.model, only_installed=False) diff --git a/django/contrib/contenttypes/tests.py b/django/contrib/contenttypes/tests.py index 7937873a00..f300294cd6 100644 --- a/django/contrib/contenttypes/tests.py +++ b/django/contrib/contenttypes/tests.py @@ -274,3 +274,10 @@ class ContentTypesTests(TestCase): model = 'OldModel', ) self.assertEqual(six.text_type(ct), 'Old model') + self.assertIsNone(ct.model_class()) + + # Make sure stale ContentTypes can be fetched like any other object. + # Before Django 1.6 this caused a NoneType error in the caching mechanism. + # Instead, just return the ContentType object and let the app detect stale states. + ct_fetched = ContentType.objects.get_for_id(ct.pk) + self.assertIsNone(ct_fetched.model_class()) diff --git a/django/contrib/flatpages/views.py b/django/contrib/flatpages/views.py index 497979e497..20e930f343 100644 --- a/django/contrib/flatpages/views.py +++ b/django/contrib/flatpages/views.py @@ -1,7 +1,6 @@ from django.conf import settings from django.contrib.flatpages.models import FlatPage from django.contrib.sites.models import get_current_site -from django.core.xheaders import populate_xheaders from django.http import Http404, HttpResponse, HttpResponsePermanentRedirect from django.shortcuts import get_object_or_404 from django.template import loader, RequestContext @@ -70,5 +69,4 @@ def render_flatpage(request, f): 'flatpage': f, }) response = HttpResponse(t.render(c)) - populate_xheaders(request, response, FlatPage, f.id) return response diff --git a/django/contrib/formtools/exceptions.py b/django/contrib/formtools/exceptions.py new file mode 100644 index 0000000000..f07ac9f745 --- /dev/null +++ b/django/contrib/formtools/exceptions.py @@ -0,0 +1,6 @@ +from django.core.exceptions import SuspiciousOperation + + +class WizardViewCookieModified(SuspiciousOperation): + """Signature of cookie modified""" + pass diff --git a/django/contrib/formtools/wizard/storage/cookie.py b/django/contrib/formtools/wizard/storage/cookie.py index e80361042f..9bf6503f18 100644 --- a/django/contrib/formtools/wizard/storage/cookie.py +++ b/django/contrib/formtools/wizard/storage/cookie.py @@ -1,8 +1,8 @@ import json -from django.core.exceptions import SuspiciousOperation from django.core.signing import BadSignature +from django.contrib.formtools.exceptions import WizardViewCookieModified from django.contrib.formtools.wizard import storage @@ -21,7 +21,7 @@ class CookieStorage(storage.BaseStorage): except KeyError: data = None except BadSignature: - raise SuspiciousOperation('WizardView cookie manipulated') + raise WizardViewCookieModified('WizardView cookie manipulated') if data is None: return None return json.loads(data, cls=json.JSONDecoder) diff --git a/django/contrib/gis/db/backends/postgis/operations.py b/django/contrib/gis/db/backends/postgis/operations.py index 734f39c752..84dbda3239 100644 --- a/django/contrib/gis/db/backends/postgis/operations.py +++ b/django/contrib/gis/db/backends/postgis/operations.py @@ -11,6 +11,10 @@ from django.core.exceptions import ImproperlyConfigured from django.db.backends.postgresql_psycopg2.base import DatabaseOperations from django.db.utils import DatabaseError from django.utils import six +from django.utils.functional import cached_property + +from .models import GeometryColumns, SpatialRefSys + #### Classes used in constructing PostGIS spatial SQL #### class PostGISOperator(SpatialOperation): @@ -62,6 +66,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): compiler_module = 'django.contrib.gis.db.models.sql.compiler' name = 'postgis' postgis = True + geom_func_prefix = 'ST_' version_regex = re.compile(r'^(?P<major>\d)\.(?P<minor1>\d)\.(?P<minor2>\d+)') valid_aggregates = dict([(k, None) for k in ('Collect', 'Extent', 'Extent3D', 'MakeLine', 'Union')]) @@ -72,45 +77,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): def __init__(self, connection): super(PostGISOperations, self).__init__(connection) - # Trying to get the PostGIS version because the function - # signatures will depend on the version used. The cost - # here is a database query to determine the version, which - # can be mitigated by setting `POSTGIS_VERSION` with a 3-tuple - # comprising user-supplied values for the major, minor, and - # subminor revision of PostGIS. - try: - if hasattr(settings, 'POSTGIS_VERSION'): - vtup = settings.POSTGIS_VERSION - if len(vtup) == 3: - # The user-supplied PostGIS version. - version = vtup - else: - # This was the old documented way, but it's stupid to - # include the string. - version = vtup[1:4] - else: - vtup = self.postgis_version_tuple() - version = vtup[1:] - - # Getting the prefix -- even though we don't officially support - # PostGIS 1.2 anymore, keeping it anyway in case a prefix change - # for something else is necessary. - if version >= (1, 2, 2): - prefix = 'ST_' - else: - prefix = '' - - self.geom_func_prefix = prefix - self.spatial_version = version - except DatabaseError: - raise ImproperlyConfigured( - 'Cannot determine PostGIS version for database "%s". ' - 'GeoDjango requires at least PostGIS version 1.3. ' - 'Was the database created from a spatial database ' - 'template?' % self.connection.settings_dict['NAME'] - ) - # TODO: Raise helpful exceptions as they become known. - + prefix = self.geom_func_prefix # PostGIS-specific operators. The commented descriptions of these # operators come from Section 7.6 of the PostGIS 1.4 documentation. self.geometry_operators = { @@ -188,13 +155,13 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): self.geometry_functions.update(self.distance_functions) # Only PostGIS versions 1.3.4+ have GeoJSON serialization support. - if version < (1, 3, 4): + if self.spatial_version < (1, 3, 4): GEOJSON = False else: GEOJSON = prefix + 'AsGeoJson' # ST_ContainsProperly ST_MakeLine, and ST_GeoHash added in 1.4. - if version >= (1, 4, 0): + if self.spatial_version >= (1, 4, 0): GEOHASH = 'ST_GeoHash' BOUNDINGCIRCLE = 'ST_MinimumBoundingCircle' self.geometry_functions['contains_properly'] = PostGISFunction(prefix, 'ContainsProperly') @@ -202,7 +169,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): GEOHASH, BOUNDINGCIRCLE = False, False # Geography type support added in 1.5. - if version >= (1, 5, 0): + if self.spatial_version >= (1, 5, 0): self.geography = True # Only a subset of the operators and functions are available # for the geography type. @@ -217,7 +184,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): } # Native geometry type support added in PostGIS 2.0. - if version >= (2, 0, 0): + if self.spatial_version >= (2, 0, 0): self.geometry = True # Creating a dictionary lookup of all GIS terms for PostGIS. @@ -260,7 +227,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): self.union = prefix + 'Union' self.unionagg = prefix + 'Union' - if version >= (2, 0, 0): + if self.spatial_version >= (2, 0, 0): self.extent3d = prefix + '3DExtent' self.length3d = prefix + '3DLength' self.perimeter3d = prefix + '3DPerimeter' @@ -269,6 +236,30 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): self.length3d = prefix + 'Length3D' self.perimeter3d = prefix + 'Perimeter3D' + @cached_property + def spatial_version(self): + """Determine the version of the PostGIS library.""" + # Trying to get the PostGIS version because the function + # signatures will depend on the version used. The cost + # here is a database query to determine the version, which + # can be mitigated by setting `POSTGIS_VERSION` with a 3-tuple + # comprising user-supplied values for the major, minor, and + # subminor revision of PostGIS. + if hasattr(settings, 'POSTGIS_VERSION'): + version = settings.POSTGIS_VERSION + else: + try: + vtup = self.postgis_version_tuple() + except DatabaseError: + raise ImproperlyConfigured( + 'Cannot determine PostGIS version for database "%s". ' + 'GeoDjango requires at least PostGIS version 1.3. ' + 'Was the database created from a spatial database ' + 'template?' % self.connection.settings_dict['NAME'] + ) + version = vtup[1:] + return version + def check_aggregate_support(self, aggregate): """ Checks if the given aggregate name is supported (that is, if it's @@ -572,9 +563,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): # Routines for getting the OGC-compliant models. def geometry_columns(self): - from django.contrib.gis.db.backends.postgis.models import GeometryColumns return GeometryColumns def spatial_ref_sys(self): - from django.contrib.gis.db.backends.postgis.models import SpatialRefSys return SpatialRefSys diff --git a/django/contrib/gis/geos/geometry.py b/django/contrib/gis/geos/geometry.py index 01719c21d4..b088ec2dc4 100644 --- a/django/contrib/gis/geos/geometry.py +++ b/django/contrib/gis/geos/geometry.py @@ -384,7 +384,7 @@ class GEOSGeometry(GEOSBase, ListMixin): @property def wkt(self): "Returns the WKT (Well-Known Text) representation of this Geometry." - return wkt_w(self.hasz and 3 or 2).write(self).decode() + return wkt_w(3 if self.hasz else 2).write(self).decode() @property def hex(self): @@ -395,7 +395,7 @@ class GEOSGeometry(GEOSBase, ListMixin): """ # A possible faster, all-python, implementation: # str(self.wkb).encode('hex') - return wkb_w(self.hasz and 3 or 2).write_hex(self) + return wkb_w(3 if self.hasz else 2).write_hex(self) @property def hexewkb(self): @@ -407,7 +407,7 @@ class GEOSGeometry(GEOSBase, ListMixin): if self.hasz and not GEOS_PREPARE: # See: http://trac.osgeo.org/geos/ticket/216 raise GEOSException('Upgrade GEOS to 3.1 to get valid 3D HEXEWKB.') - return ewkb_w(self.hasz and 3 or 2).write_hex(self) + return ewkb_w(3 if self.hasz else 2).write_hex(self) @property def json(self): @@ -427,7 +427,7 @@ class GEOSGeometry(GEOSBase, ListMixin): as a Python buffer. SRID and Z values are not included, use the `ewkb` property instead. """ - return wkb_w(self.hasz and 3 or 2).write(self) + return wkb_w(3 if self.hasz else 2).write(self) @property def ewkb(self): @@ -439,7 +439,7 @@ class GEOSGeometry(GEOSBase, ListMixin): if self.hasz and not GEOS_PREPARE: # See: http://trac.osgeo.org/geos/ticket/216 raise GEOSException('Upgrade GEOS to 3.1 to get valid 3D EWKB.') - return ewkb_w(self.hasz and 3 or 2).write(self) + return ewkb_w(3 if self.hasz else 2).write(self) @property def kml(self): diff --git a/django/contrib/gis/utils/layermapping.py b/django/contrib/gis/utils/layermapping.py index cb56b8bce2..a502aa280d 100644 --- a/django/contrib/gis/utils/layermapping.py +++ b/django/contrib/gis/utils/layermapping.py @@ -546,7 +546,7 @@ class LayerMapping(object): # Attempting to save. m.save(using=self.using) num_saved += 1 - if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m)) + if verbose: stream.write('%s: %s\n' % ('Updated' if is_update else 'Saved', m)) except SystemExit: raise except Exception as msg: diff --git a/django/contrib/humanize/locale/en/LC_MESSAGES/django.po b/django/contrib/humanize/locale/en/LC_MESSAGES/django.po index fc75b677a0..2c3cd0c08d 100644 --- a/django/contrib/humanize/locale/en/LC_MESSAGES/django.po +++ b/django/contrib/humanize/locale/en/LC_MESSAGES/django.po @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: Django\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2013-05-02 16:18+0200\n" +"POT-Creation-Date: 2013-05-18 23:10+0200\n" "PO-Revision-Date: 2010-05-13 15:35+0200\n" "Last-Translator: Django team\n" "Language-Team: English <en@li.org>\n" @@ -237,54 +237,60 @@ msgctxt "naturaltime" msgid "%(delta)s ago" msgstr "" -#: templatetags/humanize.py:194 templatetags/humanize.py:216 +#: templatetags/humanize.py:194 templatetags/humanize.py:219 msgid "now" msgstr "" -#: templatetags/humanize.py:197 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:198 #, python-format msgid "a second ago" -msgid_plural "%(count)s seconds ago" +msgid_plural "%(count)s\\u00a0seconds ago" msgstr[0] "" msgstr[1] "" -#: templatetags/humanize.py:202 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:204 #, python-format msgid "a minute ago" -msgid_plural "%(count)s minutes ago" +msgid_plural "%(count)s\\u00a0minutes ago" msgstr[0] "" msgstr[1] "" -#: templatetags/humanize.py:207 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:210 #, python-format msgid "an hour ago" -msgid_plural "%(count)s hours ago" +msgid_plural "%(count)s\\u00a0hours ago" msgstr[0] "" msgstr[1] "" -#: templatetags/humanize.py:213 +#: templatetags/humanize.py:216 #, python-format msgctxt "naturaltime" msgid "%(delta)s from now" msgstr "" -#: templatetags/humanize.py:219 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:223 #, python-format msgid "a second from now" -msgid_plural "%(count)s seconds from now" +msgid_plural "%(count)s\\u00a0seconds from now" msgstr[0] "" msgstr[1] "" -#: templatetags/humanize.py:224 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:229 #, python-format msgid "a minute from now" -msgid_plural "%(count)s minutes from now" +msgid_plural "%(count)s\\u00a0minutes from now" msgstr[0] "" msgstr[1] "" -#: templatetags/humanize.py:229 +#. Translators: \\u00a0 is non-breaking space +#: templatetags/humanize.py:235 #, python-format msgid "an hour from now" -msgid_plural "%(count)s hours from now" +msgid_plural "%(count)s\\u00a0hours from now" msgstr[0] "" msgstr[1] "" diff --git a/django/contrib/humanize/templatetags/humanize.py b/django/contrib/humanize/templatetags/humanize.py index 21f4c452fa..eaee734f75 100644 --- a/django/contrib/humanize/templatetags/humanize.py +++ b/django/contrib/humanize/templatetags/humanize.py @@ -194,17 +194,20 @@ def naturaltime(value): return _('now') elif delta.seconds < 60: return ungettext( - 'a second ago', '%(count)s seconds ago', delta.seconds + # Translators: \\u00a0 is non-breaking space + 'a second ago', '%(count)s\u00a0seconds ago', delta.seconds ) % {'count': delta.seconds} elif delta.seconds // 60 < 60: count = delta.seconds // 60 return ungettext( - 'a minute ago', '%(count)s minutes ago', count + # Translators: \\u00a0 is non-breaking space + 'a minute ago', '%(count)s\u00a0minutes ago', count ) % {'count': count} else: count = delta.seconds // 60 // 60 return ungettext( - 'an hour ago', '%(count)s hours ago', count + # Translators: \\u00a0 is non-breaking space + 'an hour ago', '%(count)s\u00a0hours ago', count ) % {'count': count} else: delta = value - now @@ -216,15 +219,18 @@ def naturaltime(value): return _('now') elif delta.seconds < 60: return ungettext( - 'a second from now', '%(count)s seconds from now', delta.seconds + # Translators: \\u00a0 is non-breaking space + 'a second from now', '%(count)s\u00a0seconds from now', delta.seconds ) % {'count': delta.seconds} elif delta.seconds // 60 < 60: count = delta.seconds // 60 return ungettext( - 'a minute from now', '%(count)s minutes from now', count + # Translators: \\u00a0 is non-breaking space + 'a minute from now', '%(count)s\u00a0minutes from now', count ) % {'count': count} else: count = delta.seconds // 60 // 60 return ungettext( - 'an hour from now', '%(count)s hours from now', count + # Translators: \\u00a0 is non-breaking space + 'an hour from now', '%(count)s\u00a0hours from now', count ) % {'count': count} diff --git a/django/contrib/humanize/tests.py b/django/contrib/humanize/tests.py index 1e1c8424e6..54a60f8fd6 100644 --- a/django/contrib/humanize/tests.py +++ b/django/contrib/humanize/tests.py @@ -19,6 +19,8 @@ from django.utils.translation import ugettext as _ from django.utils import tzinfo from django.utils.unittest import skipIf +from i18n import TransRealMixin + # Mock out datetime in some tests so they don't fail occasionally when they # run too slow. Use a fixed datetime for datetime.now(). DST change in @@ -36,7 +38,7 @@ class MockDateTime(datetime.datetime): return now.replace(tzinfo=tz) + tz.utcoffset(now) -class HumanizeTests(TestCase): +class HumanizeTests(TransRealMixin, TestCase): def humanize_tester(self, test_list, result_list, method): for test_content, result in zip(test_list, result_list): @@ -195,22 +197,22 @@ class HumanizeTests(TestCase): result_list = [ 'now', 'a second ago', - '30 seconds ago', + '30\xa0seconds ago', 'a minute ago', - '2 minutes ago', + '2\xa0minutes ago', 'an hour ago', - '23 hours ago', - '1 day ago', - '1 year, 4 months ago', + '23\xa0hours ago', + '1\xa0day ago', + '1\xa0year, 4\xa0months ago', 'a second from now', - '30 seconds from now', + '30\xa0seconds from now', 'a minute from now', - '2 minutes from now', + '2\xa0minutes from now', 'an hour from now', - '23 hours from now', - '1 day from now', - '2 days, 6 hours from now', - '1 year, 4 months from now', + '23\xa0hours from now', + '1\xa0day from now', + '2\xa0days, 6\xa0hours from now', + '1\xa0year, 4\xa0months from now', 'now', 'now', ] @@ -218,8 +220,8 @@ class HumanizeTests(TestCase): # date in naive arithmetic is only 2 days and 5 hours after in # aware arithmetic. result_list_with_tz_support = result_list[:] - assert result_list_with_tz_support[-4] == '2 days, 6 hours from now' - result_list_with_tz_support[-4] == '2 days, 5 hours from now' + assert result_list_with_tz_support[-4] == '2\xa0days, 6\xa0hours from now' + result_list_with_tz_support[-4] == '2\xa0days, 5\xa0hours from now' orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime try: diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index f79a264500..759d7ac7ad 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import base64 from datetime import datetime, timedelta +import logging + try: from django.utils.six.moves import cPickle as pickle except ImportError: @@ -14,7 +16,9 @@ from django.utils.crypto import constant_time_compare from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone -from django.utils.encoding import force_bytes +from django.utils.encoding import force_bytes, force_text + +from django.contrib.sessions.exceptions import SuspiciousSession # session_key should not be case sensitive because some backends can store it # on case insensitive file systems. @@ -94,12 +98,16 @@ class SessionBase(object): hash, pickled = encoded_data.split(b':', 1) expected_hash = self._hash(pickled) if not constant_time_compare(hash.decode(), expected_hash): - raise SuspiciousOperation("Session data corrupted") + raise SuspiciousSession("Session data corrupted") else: return pickle.loads(pickled) - except Exception: + except Exception as e: # ValueError, SuspiciousOperation, unpickling exceptions. If any of # these happen, just return an empty dictionary (an empty session). + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) return {} def update(self, dict_): diff --git a/django/contrib/sessions/backends/cached_db.py b/django/contrib/sessions/backends/cached_db.py index 31c6fbfce3..be22c1f97a 100644 --- a/django/contrib/sessions/backends/cached_db.py +++ b/django/contrib/sessions/backends/cached_db.py @@ -2,10 +2,13 @@ Cached, database-backed sessions. """ +import logging + from django.contrib.sessions.backends.db import SessionStore as DBStore from django.core.cache import cache from django.core.exceptions import SuspiciousOperation from django.utils import timezone +from django.utils.encoding import force_text KEY_PREFIX = "django.contrib.sessions.cached_db" @@ -41,7 +44,11 @@ class SessionStore(DBStore): data = self.decode(s.session_data) cache.set(self.cache_key, data, self.get_expiry_age(expiry=s.expire_date)) - except (Session.DoesNotExist, SuspiciousOperation): + except (Session.DoesNotExist, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() data = {} return data diff --git a/django/contrib/sessions/backends/db.py b/django/contrib/sessions/backends/db.py index 30da0b7a10..206fca2700 100644 --- a/django/contrib/sessions/backends/db.py +++ b/django/contrib/sessions/backends/db.py @@ -1,8 +1,10 @@ +import logging + from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation from django.db import IntegrityError, transaction, router from django.utils import timezone - +from django.utils.encoding import force_text class SessionStore(SessionBase): """ @@ -18,7 +20,11 @@ class SessionStore(SessionBase): expire_date__gt=timezone.now() ) return self.decode(s.session_data) - except (Session.DoesNotExist, SuspiciousOperation): + except (Session.DoesNotExist, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() return {} diff --git a/django/contrib/sessions/backends/file.py b/django/contrib/sessions/backends/file.py index 9588680fea..f47aa2d867 100644 --- a/django/contrib/sessions/backends/file.py +++ b/django/contrib/sessions/backends/file.py @@ -1,5 +1,6 @@ import datetime import errno +import logging import os import shutil import tempfile @@ -8,6 +9,9 @@ from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError, VALID_KEY_CHARS from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured from django.utils import timezone +from django.utils.encoding import force_text + +from django.contrib.sessions.exceptions import InvalidSessionKey class SessionStore(SessionBase): """ @@ -48,7 +52,7 @@ class SessionStore(SessionBase): # should always be md5s, so they should never contain directory # components. if not set(session_key).issubset(set(VALID_KEY_CHARS)): - raise SuspiciousOperation( + raise InvalidSessionKey( "Invalid characters in session key") return os.path.join(self.storage_path, self.file_prefix + session_key) @@ -75,7 +79,11 @@ class SessionStore(SessionBase): if file_data: try: session_data = self.decode(file_data) - except (EOFError, SuspiciousOperation): + except (EOFError, SuspiciousOperation) as e: + if isinstance(e, SuspiciousOperation): + logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + logger.warning(force_text(e)) self.create() # Remove expired sessions. @@ -86,7 +94,7 @@ class SessionStore(SessionBase): session_data = {} self.delete() self.create() - except IOError: + except (IOError, SuspiciousOperation): self.create() return session_data diff --git a/django/contrib/sessions/exceptions.py b/django/contrib/sessions/exceptions.py new file mode 100644 index 0000000000..4f4dc6b048 --- /dev/null +++ b/django/contrib/sessions/exceptions.py @@ -0,0 +1,11 @@ +from django.core.exceptions import SuspiciousOperation + + +class InvalidSessionKey(SuspiciousOperation): + """Invalid characters in session key""" + pass + + +class SuspiciousSession(SuspiciousOperation): + """The session may be tampered with""" + pass diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 8bcc505ee6..cd8191a6a4 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -1,3 +1,4 @@ +import base64 from datetime import timedelta import os import shutil @@ -15,14 +16,16 @@ from django.contrib.sessions.models import Session from django.contrib.sessions.middleware import SessionMiddleware from django.core.cache import get_cache from django.core import management -from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation +from django.core.exceptions import ImproperlyConfigured from django.http import HttpResponse from django.test import TestCase, RequestFactory -from django.test.utils import override_settings +from django.test.utils import override_settings, patch_logger from django.utils import six from django.utils import timezone from django.utils import unittest +from django.contrib.sessions.exceptions import InvalidSessionKey + class SessionTestsMixin(object): # This does not inherit from TestCase to avoid any tests being run with this @@ -272,6 +275,15 @@ class SessionTestsMixin(object): encoded = self.session.encode(data) self.assertEqual(self.session.decode(encoded), data) + def test_decode_failure_logged_to_security(self): + bad_encode = base64.b64encode(b'flaskdj:alkdjf') + with patch_logger('django.security.SuspiciousSession', 'warning') as calls: + self.assertEqual({}, self.session.decode(bad_encode)) + # check that the failed decode is logged + self.assertEqual(len(calls), 1) + self.assertTrue('corrupted' in calls[0]) + + def test_actual_expiry(self): # Regression test for #19200 old_session_key = None @@ -403,14 +415,21 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): self.assertRaises(ImproperlyConfigured, self.backend) def test_invalid_key_backslash(self): - # Ensure we don't allow directory-traversal - self.assertRaises(SuspiciousOperation, - self.backend("a\\b\\c").load) + # This key should be refused and a new session should be created + self.assertTrue(self.backend("a\\b\\c").load()) + + def test_invalid_key_backslash(self): + # Ensure we don't allow directory-traversal. + # This is tested directly on _key_to_file, as load() will swallow + # a SuspiciousOperation in the same way as an IOError - by creating + # a new session, making it unclear whether the slashes were detected. + self.assertRaises(InvalidSessionKey, + self.backend()._key_to_file, "a\\b\\c") def test_invalid_key_forwardslash(self): # Ensure we don't allow directory-traversal - self.assertRaises(SuspiciousOperation, - self.backend("a/b/c").load) + self.assertRaises(InvalidSessionKey, + self.backend()._key_to_file, "a/b/c") @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.file") def test_clearsessions_command(self): diff --git a/django/contrib/sitemaps/__init__.py b/django/contrib/sitemaps/__init__.py index 7d03ef19f5..1acae8296c 100644 --- a/django/contrib/sitemaps/__init__.py +++ b/django/contrib/sitemaps/__init__.py @@ -94,7 +94,7 @@ class Sitemap(object): 'location': loc, 'lastmod': self.__get('lastmod', item, None), 'changefreq': self.__get('changefreq', item, None), - 'priority': str(priority is not None and priority or ''), + 'priority': str(priority if priority is not None else ''), } urls.append(url_info) return urls diff --git a/django/contrib/staticfiles/finders.py b/django/contrib/staticfiles/finders.py index 8631570e79..7d266d95a0 100644 --- a/django/contrib/staticfiles/finders.py +++ b/django/contrib/staticfiles/finders.py @@ -245,7 +245,7 @@ def find(path, all=False): if matches: return matches # No match. - return all and [] or None + return [] if all else None def get_finders(): diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py index c620993f33..7c3de80e93 100644 --- a/django/contrib/staticfiles/management/commands/collectstatic.py +++ b/django/contrib/staticfiles/management/commands/collectstatic.py @@ -175,11 +175,9 @@ Type 'yes' to continue, or 'no' to cancel: """ summary = template % { 'modified_count': modified_count, 'identifier': 'static file' + ('' if modified_count == 1 else 's'), - 'action': self.symlink and 'symlinked' or 'copied', - 'destination': (destination_path and " to '%s'" - % destination_path or ''), - 'unmodified': (collected['unmodified'] and ', %s unmodified' - % unmodified_count or ''), + 'action': 'symlinked' if self.symlink else 'copied', + 'destination': (" to '%s'" % destination_path if destination_path else ''), + 'unmodified': (', %s unmodified' % unmodified_count if collected['unmodified'] else ''), 'post_processed': (collected['post_processed'] and ', %s post-processed' % post_processed_count or ''), diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index 53b0270a57..deb98e7714 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -15,6 +15,10 @@ class CacheKeyWarning(DjangoRuntimeWarning): pass +# Stub class to ensure not passing in a `timeout` argument results in +# the default timeout +DEFAULT_TIMEOUT = object() + # Memcached does not accept keys longer than this. MEMCACHE_MAX_KEY_LENGTH = 250 @@ -84,7 +88,7 @@ class BaseCache(object): new_key = self.key_func(key, self.key_prefix, version) return new_key - def add(self, key, value, timeout=None, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): """ Set a value in the cache if the key does not already exist. If timeout is given, that timeout will be used for the key; otherwise @@ -101,7 +105,7 @@ class BaseCache(object): """ raise NotImplementedError - def set(self, key, value, timeout=None, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): """ Set a value in the cache. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -163,7 +167,7 @@ class BaseCache(object): # if a subclass overrides it. return self.has_key(key) - def set_many(self, data, timeout=None, version=None): + def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): """ Set a bunch of values in the cache at once from a dict of key/value pairs. For certain backends (memcached), this is much more efficient diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py index 7749284122..0e59c6d65e 100644 --- a/django/core/cache/backends/db.py +++ b/django/core/cache/backends/db.py @@ -9,7 +9,7 @@ except ImportError: import pickle from django.conf import settings -from django.core.cache.backends.base import BaseCache +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT from django.db import connections, transaction, router, DatabaseError from django.utils import timezone, six from django.utils.encoding import force_bytes @@ -65,6 +65,7 @@ class DatabaseCache(BaseDatabaseCache): if row is None: return default now = timezone.now() + if row[2] < now: db = router.db_for_write(self.cache_model_class) cursor = connections[db].cursor() @@ -74,18 +75,18 @@ class DatabaseCache(BaseDatabaseCache): value = connections[db].ops.process_clob(row[1]) return pickle.loads(base64.b64decode(force_bytes(value))) - def set(self, key, value, timeout=None, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) self._base_set('set', key, value, timeout) - def add(self, key, value, timeout=None, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) return self._base_set('add', key, value, timeout) - def _base_set(self, mode, key, value, timeout=None): - if timeout is None: + def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT): + if timeout == DEFAULT_TIMEOUT: timeout = self.default_timeout db = router.db_for_write(self.cache_model_class) table = connections[db].ops.quote_name(self._table) @@ -95,7 +96,9 @@ class DatabaseCache(BaseDatabaseCache): num = cursor.fetchone()[0] now = timezone.now() now = now.replace(microsecond=0) - if settings.USE_TZ: + if timeout is None: + exp = datetime.max + elif settings.USE_TZ: exp = datetime.utcfromtimestamp(time.time() + timeout) else: exp = datetime.fromtimestamp(time.time() + timeout) diff --git a/django/core/cache/backends/dummy.py b/django/core/cache/backends/dummy.py index 9fe9b3b5f5..7ca6114ee4 100644 --- a/django/core/cache/backends/dummy.py +++ b/django/core/cache/backends/dummy.py @@ -1,12 +1,12 @@ "Dummy cache backend" -from django.core.cache.backends.base import BaseCache +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT class DummyCache(BaseCache): def __init__(self, host, *args, **kwargs): BaseCache.__init__(self, *args, **kwargs) - def add(self, key, value, timeout=None, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) return True @@ -16,7 +16,7 @@ class DummyCache(BaseCache): self.validate_key(key) return default - def set(self, key, value, timeout=None, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) @@ -32,7 +32,7 @@ class DummyCache(BaseCache): self.validate_key(key) return False - def set_many(self, data, timeout=0, version=None): + def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): pass def delete_many(self, keys, version=None): diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index 96194d458f..d19eed4a95 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -9,9 +9,10 @@ try: except ImportError: import pickle -from django.core.cache.backends.base import BaseCache +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT from django.utils.encoding import force_bytes + class FileBasedCache(BaseCache): def __init__(self, dir, params): BaseCache.__init__(self, params) @@ -19,7 +20,7 @@ class FileBasedCache(BaseCache): if not os.path.exists(self._dir): self._createdir() - def add(self, key, value, timeout=None, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): if self.has_key(key, version=version): return False @@ -35,7 +36,7 @@ class FileBasedCache(BaseCache): with open(fname, 'rb') as f: exp = pickle.load(f) now = time.time() - if exp < now: + if exp is not None and exp < now: self._delete(fname) else: return pickle.load(f) @@ -43,14 +44,14 @@ class FileBasedCache(BaseCache): pass return default - def set(self, key, value, timeout=None, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) fname = self._key_to_file(key) dirname = os.path.dirname(fname) - if timeout is None: + if timeout == DEFAULT_TIMEOUT: timeout = self.default_timeout self._cull() @@ -60,8 +61,8 @@ class FileBasedCache(BaseCache): os.makedirs(dirname) with open(fname, 'wb') as f: - now = time.time() - pickle.dump(now + timeout, f, pickle.HIGHEST_PROTOCOL) + expiry = None if timeout is None else time.time() + timeout + pickle.dump(expiry, f, pickle.HIGHEST_PROTOCOL) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) except (IOError, OSError): pass diff --git a/django/core/cache/backends/locmem.py b/django/core/cache/backends/locmem.py index 76667e9609..1fa17052fd 100644 --- a/django/core/cache/backends/locmem.py +++ b/django/core/cache/backends/locmem.py @@ -6,7 +6,7 @@ try: except ImportError: import pickle -from django.core.cache.backends.base import BaseCache +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT from django.utils.synch import RWLock # Global in-memory store of cache data. Keyed by name, to provide @@ -23,7 +23,7 @@ class LocMemCache(BaseCache): self._expire_info = _expire_info.setdefault(name, {}) self._lock = _locks.setdefault(name, RWLock()) - def add(self, key, value, timeout=None, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) with self._lock.writer(): @@ -41,10 +41,8 @@ class LocMemCache(BaseCache): key = self.make_key(key, version=version) self.validate_key(key) with self._lock.reader(): - exp = self._expire_info.get(key) - if exp is None: - return default - elif exp > time.time(): + exp = self._expire_info.get(key, 0) + if exp is None or exp > time.time(): try: pickled = self._cache[key] return pickle.loads(pickled) @@ -58,15 +56,16 @@ class LocMemCache(BaseCache): pass return default - def _set(self, key, value, timeout=None): + def _set(self, key, value, timeout=DEFAULT_TIMEOUT): if len(self._cache) >= self._max_entries: self._cull() - if timeout is None: + if timeout == DEFAULT_TIMEOUT: timeout = self.default_timeout + expiry = None if timeout is None else time.time() + timeout self._cache[key] = value - self._expire_info[key] = time.time() + timeout + self._expire_info[key] = expiry - def set(self, key, value, timeout=None, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self.validate_key(key) with self._lock.writer(): diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py index c942acd52f..64d1c41dc5 100644 --- a/django/core/cache/backends/memcached.py +++ b/django/core/cache/backends/memcached.py @@ -4,7 +4,7 @@ import time import pickle from threading import local -from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError +from django.core.cache.backends.base import BaseCache, DEFAULT_TIMEOUT from django.utils import six from django.utils.encoding import force_str @@ -36,12 +36,22 @@ class BaseMemcachedCache(BaseCache): return self._client - def _get_memcache_timeout(self, timeout): + def _get_memcache_timeout(self, timeout=DEFAULT_TIMEOUT): """ Memcached deals with long (> 30 days) timeouts in a special way. Call this function to obtain a safe value for your timeout. """ - timeout = timeout or self.default_timeout + if timeout == DEFAULT_TIMEOUT: + return self.default_timeout + + if timeout is None: + # Using 0 in memcache sets a non-expiring timeout. + return 0 + elif int(timeout) == 0: + # Other cache backends treat 0 as set-and-expire. To achieve this + # in memcache backends, a negative timeout must be passed. + timeout = -1 + if timeout > 2592000: # 60*60*24*30, 30 days # See http://code.google.com/p/memcached/wiki/FAQ # "You can set expire times up to 30 days in the future. After that @@ -56,7 +66,7 @@ class BaseMemcachedCache(BaseCache): # Python 2 memcache requires the key to be a byte string. return force_str(super(BaseMemcachedCache, self).make_key(key, version)) - def add(self, key, value, timeout=0, version=None): + def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) return self._cache.add(key, value, self._get_memcache_timeout(timeout)) @@ -67,7 +77,7 @@ class BaseMemcachedCache(BaseCache): return default return val - def set(self, key, value, timeout=0, version=None): + def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_key(key, version=version) self._cache.set(key, value, self._get_memcache_timeout(timeout)) @@ -125,7 +135,7 @@ class BaseMemcachedCache(BaseCache): raise ValueError("Key '%s' not found" % key) return val - def set_many(self, data, timeout=0, version=None): + def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): safe_data = {} for key, value in data.items(): key = self.make_key(key, version=version) diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 233af40f88..2c79736e33 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -1,6 +1,7 @@ """ Global Django exception and warning classes. """ +import logging from functools import reduce @@ -9,37 +10,56 @@ class DjangoRuntimeWarning(RuntimeWarning): class ObjectDoesNotExist(Exception): - "The requested object does not exist" + """The requested object does not exist""" silent_variable_failure = True class MultipleObjectsReturned(Exception): - "The query returned multiple objects when only one was expected." + """The query returned multiple objects when only one was expected.""" pass class SuspiciousOperation(Exception): - "The user did something suspicious" + """The user did something suspicious""" + + +class SuspiciousMultipartForm(SuspiciousOperation): + """Suspect MIME request in multipart form data""" + pass + + +class SuspiciousFileOperation(SuspiciousOperation): + """A Suspicious filesystem operation was attempted""" + pass + + +class DisallowedHost(SuspiciousOperation): + """HTTP_HOST header contains invalid value""" + pass + + +class DisallowedRedirect(SuspiciousOperation): + """Redirect to scheme not in allowed list""" pass class PermissionDenied(Exception): - "The user did not have permission to do that" + """The user did not have permission to do that""" pass class ViewDoesNotExist(Exception): - "The requested view does not exist" + """The requested view does not exist""" pass class MiddlewareNotUsed(Exception): - "This middleware is not used in this server configuration" + """This middleware is not used in this server configuration""" pass class ImproperlyConfigured(Exception): - "Django is somehow improperly configured" + """Django is somehow improperly configured""" pass diff --git a/django/core/files/locks.py b/django/core/files/locks.py index d1384329dc..6f0e4b9508 100644 --- a/django/core/files/locks.py +++ b/django/core/files/locks.py @@ -41,7 +41,7 @@ except (ImportError, AttributeError): def fd(f): """Get a filedescriptor from something which could be a file or an fd.""" - return hasattr(f, 'fileno') and f.fileno() or f + return f.fileno() if hasattr(f, 'fileno') else f if system_type == 'nt': def lock(file, flags): diff --git a/django/core/files/move.py b/django/core/files/move.py index 3af02634fe..4519dedf97 100644 --- a/django/core/files/move.py +++ b/django/core/files/move.py @@ -62,7 +62,7 @@ def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_ove with open(old_file_name, 'rb') as old_file: # now open the new file, not forgetting allow_overwrite fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) | - (not allow_overwrite and os.O_EXCL or 0)) + (os.O_EXCL if not allow_overwrite else 0)) try: locks.lock(fd, locks.LOCK_EX) current_chunk = None @@ -77,8 +77,8 @@ def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_ove try: os.remove(old_file_name) except OSError as e: - # Certain operating systems (Cygwin and Windows) - # fail when deleting opened files, ignore it. (For the + # Certain operating systems (Cygwin and Windows) + # fail when deleting opened files, ignore it. (For the # systems where this happens, temporary files will be auto-deleted # on close anyway.) if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13: diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 18d15e1ab6..977b6a68a8 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -8,7 +8,7 @@ import itertools from datetime import datetime from django.conf import settings -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import SuspiciousFileOperation from django.core.files import locks, File from django.core.files.move import file_move_safe from django.utils.encoding import force_text, filepath_to_uri @@ -260,7 +260,7 @@ class FileSystemStorage(Storage): try: path = safe_join(self.location, name) except ValueError: - raise SuspiciousOperation("Attempted access to '%s' denied." % name) + raise SuspiciousFileOperation("Attempted access to '%s' denied." % name) return os.path.normpath(path) def size(self, name): diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py index acc74db6f5..59118656c6 100644 --- a/django/core/handlers/base.py +++ b/django/core/handlers/base.py @@ -8,7 +8,7 @@ from django import http from django.conf import settings from django.core import urlresolvers from django.core import signals -from django.core.exceptions import MiddlewareNotUsed, PermissionDenied +from django.core.exceptions import MiddlewareNotUsed, PermissionDenied, SuspiciousOperation from django.db import connections, transaction from django.utils.encoding import force_text from django.utils.module_loading import import_by_path @@ -66,10 +66,11 @@ class BaseHandler(object): self._request_middleware = request_middleware def make_view_atomic(self, view): - if getattr(view, 'transactions_per_request', True): - for db in connections.all(): - if db.settings_dict['ATOMIC_REQUESTS']: - view = transaction.atomic(using=db.alias)(view) + non_atomic_requests = getattr(view, '_non_atomic_requests', set()) + for db in connections.all(): + if (db.settings_dict['ATOMIC_REQUESTS'] + and db.alias not in non_atomic_requests): + view = transaction.atomic(using=db.alias)(view) return view def get_response(self, request): @@ -169,11 +170,27 @@ class BaseHandler(object): response = self.handle_uncaught_exception(request, resolver, sys.exc_info()) + except SuspiciousOperation as e: + # The request logger receives events for any problematic request + # The security logger receives events for all SuspiciousOperations + security_logger = logging.getLogger('django.security.%s' % + e.__class__.__name__) + security_logger.error(force_text(e)) + + try: + callback, param_dict = resolver.resolve400() + response = callback(request, **param_dict) + except: + signals.got_request_exception.send( + sender=self.__class__, request=request) + response = self.handle_uncaught_exception(request, + resolver, sys.exc_info()) + except SystemExit: # Allow sys.exit() to actually exit. See tickets #1023 and #4701 raise - except: # Handle everything else, including SuspiciousOperation, etc. + except: # Handle everything else. # Get the exception info now, in case another exception is thrown later. signals.got_request_exception.send(sender=self.__class__, request=request) response = self.handle_uncaught_exception(request, resolver, sys.exc_info()) diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py index c348c6c8da..af78d1d269 100644 --- a/django/core/handlers/wsgi.py +++ b/django/core/handlers/wsgi.py @@ -13,66 +13,11 @@ from django.core.urlresolvers import set_script_prefix from django.utils import datastructures from django.utils.encoding import force_str, force_text, iri_to_uri -logger = logging.getLogger('django.request') +# For backwards compatibility -- lots of code uses this in the wild! +from django.http.response import REASON_PHRASES as STATUS_CODE_TEXT +logger = logging.getLogger('django.request') -# See http://www.iana.org/assignments/http-status-codes -STATUS_CODE_TEXT = { - 100: 'CONTINUE', - 101: 'SWITCHING PROTOCOLS', - 102: 'PROCESSING', - 200: 'OK', - 201: 'CREATED', - 202: 'ACCEPTED', - 203: 'NON-AUTHORITATIVE INFORMATION', - 204: 'NO CONTENT', - 205: 'RESET CONTENT', - 206: 'PARTIAL CONTENT', - 207: 'MULTI-STATUS', - 208: 'ALREADY REPORTED', - 226: 'IM USED', - 300: 'MULTIPLE CHOICES', - 301: 'MOVED PERMANENTLY', - 302: 'FOUND', - 303: 'SEE OTHER', - 304: 'NOT MODIFIED', - 305: 'USE PROXY', - 306: 'RESERVED', - 307: 'TEMPORARY REDIRECT', - 400: 'BAD REQUEST', - 401: 'UNAUTHORIZED', - 402: 'PAYMENT REQUIRED', - 403: 'FORBIDDEN', - 404: 'NOT FOUND', - 405: 'METHOD NOT ALLOWED', - 406: 'NOT ACCEPTABLE', - 407: 'PROXY AUTHENTICATION REQUIRED', - 408: 'REQUEST TIMEOUT', - 409: 'CONFLICT', - 410: 'GONE', - 411: 'LENGTH REQUIRED', - 412: 'PRECONDITION FAILED', - 413: 'REQUEST ENTITY TOO LARGE', - 414: 'REQUEST-URI TOO LONG', - 415: 'UNSUPPORTED MEDIA TYPE', - 416: 'REQUESTED RANGE NOT SATISFIABLE', - 417: 'EXPECTATION FAILED', - 418: "I'M A TEAPOT", - 422: 'UNPROCESSABLE ENTITY', - 423: 'LOCKED', - 424: 'FAILED DEPENDENCY', - 426: 'UPGRADE REQUIRED', - 500: 'INTERNAL SERVER ERROR', - 501: 'NOT IMPLEMENTED', - 502: 'BAD GATEWAY', - 503: 'SERVICE UNAVAILABLE', - 504: 'GATEWAY TIMEOUT', - 505: 'HTTP VERSION NOT SUPPORTED', - 506: 'VARIANT ALSO NEGOTIATES', - 507: 'INSUFFICIENT STORAGE', - 508: 'LOOP DETECTED', - 510: 'NOT EXTENDED', -} class LimitedStream(object): ''' @@ -254,11 +199,7 @@ class WSGIHandler(base.BaseHandler): response._handler_class = self.__class__ - try: - status_text = STATUS_CODE_TEXT[response.status_code] - except KeyError: - status_text = 'UNKNOWN STATUS CODE' - status = '%s %s' % (response.status_code, status_text) + status = '%s %s' % (response.status_code, response.reason_phrase) response_headers = [(str(k), str(v)) for k, v in response.items()] for c in response.cookies.values(): response_headers.append((str('Set-Cookie'), str(c.output(header='')))) diff --git a/django/core/management/base.py b/django/core/management/base.py index ba6ad8f4c0..af040288d0 100644 --- a/django/core/management/base.py +++ b/django/core/management/base.py @@ -7,7 +7,6 @@ import os import sys from optparse import make_option, OptionParser -import traceback import django from django.core.exceptions import ImproperlyConfigured @@ -171,7 +170,7 @@ class BaseCommand(object): make_option('--pythonpath', help='A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".'), make_option('--traceback', action='store_true', - help='Print traceback on exception'), + help='Raise on exception'), ) help = '' args = '' @@ -231,7 +230,8 @@ class BaseCommand(object): Set up any environment changes requested (e.g., Python path and Django settings), then run this command. If the command raises a ``CommandError``, intercept it and print it sensibly - to stderr. + to stderr. If the ``--traceback`` option is present or the raised + ``Exception`` is not ``CommandError``, raise it. """ parser = self.create_parser(argv[0], argv[1]) options, args = parser.parse_args(argv[2:]) @@ -239,12 +239,12 @@ class BaseCommand(object): try: self.execute(*args, **options.__dict__) except Exception as e: + if options.traceback or not isinstance(e, CommandError): + raise + # self.stderr is not guaranteed to be set here stderr = getattr(self, 'stderr', OutputWrapper(sys.stderr, self.style.ERROR)) - if options.traceback or not isinstance(e, CommandError): - stderr.write(traceback.format_exc()) - else: - stderr.write('%s: %s' % (e.__class__.__name__, e)) + stderr.write('%s: %s' % (e.__class__.__name__, e)) sys.exit(1) def execute(self, *args, **options): diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py index 4d1bc0403e..d7ce3e93fd 100644 --- a/django/core/management/commands/createcachetable.py +++ b/django/core/management/commands/createcachetable.py @@ -38,7 +38,7 @@ class Command(LabelCommand): qn = connection.ops.quote_name for f in fields: field_output = [qn(f.name), f.db_type(connection=connection)] - field_output.append("%sNULL" % (not f.null and "NOT " or "")) + field_output.append("%sNULL" % ("NOT " if not f.null else "")) if f.primary_key: field_output.append("PRIMARY KEY") elif f.unique: @@ -51,7 +51,7 @@ class Command(LabelCommand): table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): - full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or '')) + full_statement.append(' %s%s' % (line, ',' if i < len(table_output)-1 else '')) full_statement.append(');') with transaction.commit_on_success_unless_managed(): curs = connection.cursor() diff --git a/django/core/management/commands/dumpdata.py b/django/core/management/commands/dumpdata.py index d3650b1eb8..c5eb1b9a9e 100644 --- a/django/core/management/commands/dumpdata.py +++ b/django/core/management/commands/dumpdata.py @@ -21,6 +21,9 @@ class Command(BaseCommand): help='Use natural keys if they are available.'), make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False, help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."), + make_option('--pks', dest='primary_keys', help="Only dump objects with " + "given primary keys. Accepts a comma seperated list of keys. " + "This option will only work when you specify one model."), ) help = ("Output the contents of the database as a fixture of the given " "format (using each model's default manager unless --all is " @@ -37,6 +40,12 @@ class Command(BaseCommand): show_traceback = options.get('traceback') use_natural_keys = options.get('use_natural_keys') use_base_manager = options.get('use_base_manager') + pks = options.get('primary_keys') + + if pks: + primary_keys = pks.split(',') + else: + primary_keys = [] excluded_apps = set() excluded_models = set() @@ -55,8 +64,12 @@ class Command(BaseCommand): raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: + if primary_keys: + raise CommandError("You can only use --pks option with one model") app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps) else: + if len(app_labels) > 1 and primary_keys: + raise CommandError("You can only use --pks option with one model") app_list = SortedDict() for label in app_labels: try: @@ -77,6 +90,8 @@ class Command(BaseCommand): else: app_list[app] = [model] except ValueError: + if primary_keys: + raise CommandError("You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: @@ -107,8 +122,11 @@ class Command(BaseCommand): objects = model._base_manager else: objects = model._default_manager - for obj in objects.using(using).\ - order_by(model._meta.pk.name).iterator(): + + queryset = objects.using(using).order_by(model._meta.pk.name) + if primary_keys: + queryset = queryset.filter(pk__in=primary_keys) + for obj in queryset.iterator(): yield obj try: diff --git a/django/core/management/commands/flush.py b/django/core/management/commands/flush.py index 10066417a1..c56fc1e1b0 100644 --- a/django/core/management/commands/flush.py +++ b/django/core/management/commands/flush.py @@ -20,7 +20,7 @@ class Command(NoArgsCommand): default=DEFAULT_DB_ALIAS, help='Nominates a database to flush. ' 'Defaults to the "default" database.'), make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True, - help='Tells Django not to load any initial data after database synchronization.'), + help='Tells Django not to load any initial data after database synchronization.'), ) help = ('Returns the database to the state it was in immediately after ' 'syncdb was executed. This means that all data will be removed ' diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index c95d11cf60..6856e85e45 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -1,9 +1,11 @@ from __future__ import unicode_literals -import os +import glob import gzip +import os import zipfile from optparse import make_option +import warnings from django.conf import settings from django.core import serializers @@ -11,8 +13,9 @@ from django.core.management.base import BaseCommand, CommandError from django.core.management.color import no_style from django.db import (connections, router, transaction, DEFAULT_DB_ALIAS, IntegrityError, DatabaseError) -from django.db.models import get_apps +from django.db.models import get_app_paths from django.utils.encoding import force_text +from django.utils.functional import cached_property, memoize from django.utils._os import upath from itertools import product @@ -43,9 +46,8 @@ class Command(BaseCommand): if not len(fixture_labels): raise CommandError( - "No database fixture specified. Please provide the path of at " - "least one fixture in the command line." - ) + "No database fixture specified. Please provide the path " + "of at least one fixture in the command line.") self.verbosity = int(options.get('verbosity')) @@ -68,37 +70,18 @@ class Command(BaseCommand): self.fixture_object_count = 0 self.models = set() - class SingleZipReader(zipfile.ZipFile): - def __init__(self, *args, **kwargs): - zipfile.ZipFile.__init__(self, *args, **kwargs) - if settings.DEBUG: - assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file." - def read(self): - return zipfile.ZipFile.read(self, self.namelist()[0]) - - self.compression_types = { + self.serialization_formats = serializers.get_public_serializer_formats() + self.compression_formats = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: - self.compression_types['bz2'] = bz2.BZ2File - - app_module_paths = [] - for app in get_apps(): - if hasattr(app, '__path__'): - # It's a 'models/' subpackage - for path in app.__path__: - app_module_paths.append(upath(path)) - else: - # It's a models.py module - app_module_paths.append(upath(app.__file__)) - - app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] + self.compression_formats['bz2'] = bz2.BZ2File with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: - self.load_label(fixture_label, app_fixtures) + self.load_label(fixture_label) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added @@ -123,122 +106,174 @@ class Command(BaseCommand): if self.verbosity >= 1: if self.fixture_object_count == self.loaded_object_count: - self.stdout.write("Installed %d object(s) from %d fixture(s)" % ( - self.loaded_object_count, self.fixture_count)) + self.stdout.write("Installed %d object(s) from %d fixture(s)" % + (self.loaded_object_count, self.fixture_count)) else: - self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % ( - self.loaded_object_count, self.fixture_object_count, self.fixture_count)) + self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % + (self.loaded_object_count, self.fixture_object_count, self.fixture_count)) - def load_label(self, fixture_label, app_fixtures): + def load_label(self, fixture_label): + """ + Loads fixtures files for a given label. + """ + for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label): + _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file)) + open_method = self.compression_formats[cmp_fmt] + fixture = open_method(fixture_file, 'r') + try: + self.fixture_count += 1 + objects_in_fixture = 0 + loaded_objects_in_fixture = 0 + if self.verbosity >= 2: + self.stdout.write("Installing %s fixture '%s' from %s." % + (ser_fmt, fixture_name, humanize(fixture_dir))) - parts = fixture_label.split('.') + objects = serializers.deserialize(ser_fmt, fixture, + using=self.using, ignorenonexistent=self.ignore) - if len(parts) > 1 and parts[-1] in self.compression_types: - compression_formats = [parts[-1]] - parts = parts[:-1] - else: - compression_formats = self.compression_types.keys() + for obj in objects: + objects_in_fixture += 1 + if router.allow_syncdb(self.using, obj.object.__class__): + loaded_objects_in_fixture += 1 + self.models.add(obj.object.__class__) + try: + obj.save(using=self.using) + except (DatabaseError, IntegrityError) as e: + e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { + 'app_label': obj.object._meta.app_label, + 'object_name': obj.object._meta.object_name, + 'pk': obj.object.pk, + 'error_msg': force_text(e) + },) + raise - if len(parts) == 1: - fixture_name = parts[0] - formats = serializers.get_public_serializer_formats() - else: - fixture_name, format = '.'.join(parts[:-1]), parts[-1] - if format in serializers.get_public_serializer_formats(): - formats = [format] - else: - formats = [] + self.loaded_object_count += loaded_objects_in_fixture + self.fixture_object_count += objects_in_fixture + except Exception as e: + if not isinstance(e, CommandError): + e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),) + raise + finally: + fixture.close() - if formats: - if self.verbosity >= 2: - self.stdout.write("Loading '%s' fixtures..." % fixture_name) - else: + # If the fixture we loaded contains 0 objects, assume that an + # error was encountered during fixture loading. + if objects_in_fixture == 0: + raise CommandError( + "No fixture data found for '%s'. " + "(File format may be invalid.)" % fixture_name) + + def _find_fixtures(self, fixture_label): + """ + Finds fixture files for a given label. + """ + fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) + databases = [self.using, None] + cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] + ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] + + # Check kept for backwards-compatibility; it doesn't look very useful. + if '.' in os.path.basename(fixture_name): raise CommandError( - "Problem installing fixture '%s': %s is not a known serialization format." % - (fixture_name, format)) + "Problem installing fixture '%s': %s is not a known " + "serialization format." % tuple(fixture_name.rsplit('.'))) + + if self.verbosity >= 2: + self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): - fixture_dirs = [fixture_name] + fixture_dirs = [os.path.dirname(fixture_name)] + fixture_name = os.path.basename(fixture_name) else: - fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] + fixture_dirs = self.fixture_dirs + suffixes = ('.'.join(ext for ext in combo if ext) + for combo in product(databases, ser_fmts, cmp_fmts)) + targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes) + + fixture_files = [] for fixture_dir in fixture_dirs: - self.process_dir(fixture_dir, fixture_name, compression_formats, - formats) + if self.verbosity >= 2: + self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) + fixture_files_in_dir = [] + for candidate in glob.iglob(os.path.join(fixture_dir, fixture_name + '*')): + if os.path.basename(candidate) in targets: + # Save the fixture_dir and fixture_name for future error messages. + fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) - def process_dir(self, fixture_dir, fixture_name, compression_formats, - serialization_formats): + if self.verbosity >= 2 and not fixture_files_in_dir: + self.stdout.write("No fixture '%s' in %s." % + (fixture_name, humanize(fixture_dir))) - humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' + # Check kept for backwards-compatibility; it isn't clear why + # duplicates are only allowed in different directories. + if len(fixture_files_in_dir) > 1: + raise CommandError( + "Multiple fixtures named '%s' in %s. Aborting." % + (fixture_name, humanize(fixture_dir))) + fixture_files.extend(fixture_files_in_dir) - if self.verbosity >= 2: - self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) + if fixture_name != 'initial_data' and not fixture_files: + # Warning kept for backwards-compatibility; why not an exception? + warnings.warn("No fixture named '%s' found." % fixture_name) - label_found = False - for combo in product([self.using, None], serialization_formats, compression_formats): - database, format, compression_format = combo - file_name = '.'.join( - p for p in [ - fixture_name, database, format, compression_format - ] - if p - ) + return fixture_files + + _label_to_fixtures_cache = {} + find_fixtures = memoize(_find_fixtures, _label_to_fixtures_cache, 2) + + @cached_property + def fixture_dirs(self): + """ + Return a list of fixture directories. + + The list contains the 'fixtures' subdirectory of each installed + application, if it exists, the directories in FIXTURE_DIRS, and the + current directory. + """ + dirs = [] + for path in get_app_paths(): + d = os.path.join(os.path.dirname(path), 'fixtures') + if os.path.isdir(d): + dirs.append(d) + dirs.extend(list(settings.FIXTURE_DIRS)) + dirs.append('') + dirs = [upath(os.path.abspath(os.path.realpath(d))) for d in dirs] + return dirs + + def parse_name(self, fixture_name): + """ + Splits fixture name in name, serialization format, compression format. + """ + parts = fixture_name.rsplit('.', 2) + + if len(parts) > 1 and parts[-1] in self.compression_formats: + cmp_fmt = parts[-1] + parts = parts[:-1] + else: + cmp_fmt = None + + if len(parts) > 1 and parts[-1] in self.serialization_formats: + ser_fmt = parts[-1] + parts = parts[:-1] + else: + ser_fmt = None + + name = '.'.join(parts) + + return name, ser_fmt, cmp_fmt - if self.verbosity >= 3: - self.stdout.write("Trying %s for %s fixture '%s'..." % \ - (humanize(fixture_dir), file_name, fixture_name)) - full_path = os.path.join(fixture_dir, file_name) - open_method = self.compression_types[compression_format] - try: - fixture = open_method(full_path, 'r') - except IOError: - if self.verbosity >= 2: - self.stdout.write("No %s fixture '%s' in %s." % \ - (format, fixture_name, humanize(fixture_dir))) - else: - try: - if label_found: - raise CommandError("Multiple fixtures named '%s' in %s. Aborting." % - (fixture_name, humanize(fixture_dir))) - self.fixture_count += 1 - objects_in_fixture = 0 - loaded_objects_in_fixture = 0 - if self.verbosity >= 2: - self.stdout.write("Installing %s fixture '%s' from %s." % \ - (format, fixture_name, humanize(fixture_dir))) +class SingleZipReader(zipfile.ZipFile): - objects = serializers.deserialize(format, fixture, using=self.using, ignorenonexistent=self.ignore) + def __init__(self, *args, **kwargs): + zipfile.ZipFile.__init__(self, *args, **kwargs) + if len(self.namelist()) != 1: + raise ValueError("Zip-compressed fixtures must contain one file.") - for obj in objects: - objects_in_fixture += 1 - if router.allow_syncdb(self.using, obj.object.__class__): - loaded_objects_in_fixture += 1 - self.models.add(obj.object.__class__) - try: - obj.save(using=self.using) - except (DatabaseError, IntegrityError) as e: - e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { - 'app_label': obj.object._meta.app_label, - 'object_name': obj.object._meta.object_name, - 'pk': obj.object.pk, - 'error_msg': force_text(e) - },) - raise + def read(self): + return zipfile.ZipFile.read(self, self.namelist()[0]) - self.loaded_object_count += loaded_objects_in_fixture - self.fixture_object_count += objects_in_fixture - label_found = True - except Exception as e: - if not isinstance(e, CommandError): - e.args = ("Problem installing fixture '%s': %s" % (full_path, e),) - raise - finally: - fixture.close() - # If the fixture we loaded contains 0 objects, assume that an - # error was encountered during fixture loading. - if objects_in_fixture == 0: - raise CommandError( - "No fixture data found for '%s'. (File format may be invalid.)" % - (fixture_name)) +def humanize(dirname): + return "'%s'" % dirname if dirname else 'absolute path' diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index bc171176c2..060def5d5a 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -250,18 +250,6 @@ class Command(NoArgsCommand): "if you want to enable i18n for your project or application.") check_programs('xgettext') - # We require gettext version 0.15 or newer. - output, errors, status = popen_wrapper(['xgettext', '--version']) - if status != STATUS_OK: - raise CommandError("Error running xgettext. Note that Django " - "internationalization requires GNU gettext 0.15 or newer.") - match = re.search(r'(?P<major>\d+)\.(?P<minor>\d+)', output) - if match: - xversion = (int(match.group('major')), int(match.group('minor'))) - if xversion < (0, 15): - raise CommandError("Django internationalization requires GNU " - "gettext 0.15 or newer. You are using version %s, please " - "upgrade your gettext toolset." % match.group()) potfile = self.build_pot_file(localedir) @@ -309,10 +297,9 @@ class Command(NoArgsCommand): """ Check if the given path should be ignored or not. """ - for pattern in ignore_patterns: - if fnmatch.fnmatchcase(path, pattern): - return True - return False + filename = os.path.basename(path) + ignore = lambda pattern: fnmatch.fnmatchcase(filename, pattern) + return any(ignore(pattern) for pattern in ignore_patterns) dir_suffix = '%s*' % os.sep norm_patterns = [p[:-len(dir_suffix)] if p.endswith(dir_suffix) else p for p in self.ignore_patterns] diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py index 4fe03216d5..c4a0b78cd4 100644 --- a/django/core/management/commands/runserver.py +++ b/django/core/management/commands/runserver.py @@ -99,7 +99,7 @@ class Command(BaseCommand): "started_at": datetime.now().strftime('%B %d, %Y - %X'), "version": self.get_version(), "settings": settings.SETTINGS_MODULE, - "addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr, + "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, }) diff --git a/django/core/management/commands/syncdb.py b/django/core/management/commands/syncdb.py index 155c3fb67b..51470d7bda 100644 --- a/django/core/management/commands/syncdb.py +++ b/django/core/management/commands/syncdb.py @@ -1,11 +1,12 @@ from optparse import make_option +import itertools import traceback from django.conf import settings from django.core.management import call_command from django.core.management.base import NoArgsCommand from django.core.management.color import no_style -from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal +from django.core.management.sql import custom_sql_for_model, emit_post_sync_signal, emit_pre_sync_signal from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS from django.utils.datastructures import SortedDict from django.utils.importlib import import_module @@ -82,6 +83,9 @@ class Command(NoArgsCommand): for app_name, model_list in all_models ) + create_models = set([x for x in itertools.chain(*manifest.values())]) + emit_pre_sync_signal(create_models, verbosity, interactive, db) + # Create the tables for each model if verbosity >= 1: self.stdout.write("Creating tables ...\n") diff --git a/django/core/management/commands/test.py b/django/core/management/commands/test.py index 2b8e8019eb..d6bed7cdb5 100644 --- a/django/core/management/commands/test.py +++ b/django/core/management/commands/test.py @@ -29,7 +29,7 @@ class Command(BaseCommand): ) help = ('Runs the test suite for the specified applications, or the ' 'entire site if no apps are specified.') - args = '[appname ...]' + args = '[appname|appname.tests.TestCase|appname.tests.TestCase.test_method]...' requires_model_validation = False diff --git a/django/core/management/sql.py b/django/core/management/sql.py index ac60ed470c..42ccafa2c5 100644 --- a/django/core/management/sql.py +++ b/django/core/management/sql.py @@ -133,14 +133,15 @@ def sql_custom(app, style, connection): def sql_indexes(app, style, connection): "Returns a list of the CREATE INDEX SQL statements for all models in the given app." output = [] - for model in models.get_models(app): + for model in models.get_models(app, include_auto_created=True): output.extend(connection.creation.sql_indexes_for_model(model, style)) return output + def sql_destroy_indexes(app, style, connection): "Returns a list of the DROP INDEX SQL statements for all models in the given app." output = [] - for model in models.get_models(app): + for model in models.get_models(app, include_auto_created=True): output.extend(connection.creation.sql_destroy_indexes_for_model(model, style)) return output @@ -191,6 +192,19 @@ def custom_sql_for_model(model, style, connection): return output +def emit_pre_sync_signal(create_models, verbosity, interactive, db): + # Emit the pre_sync signal for every application. + for app in models.get_apps(): + app_name = app.__name__.split('.')[-2] + if verbosity >= 2: + print("Running pre-sync handlers for application %s" % app_name) + models.signals.pre_syncdb.send(sender=app, app=app, + create_models=create_models, + verbosity=verbosity, + interactive=interactive, + db=db) + + def emit_post_sync_signal(created_models, verbosity, interactive, db): # Emit the post_sync signal for every application. for app in models.get_apps(): diff --git a/django/core/management/validation.py b/django/core/management/validation.py index 0f0eade569..a6d6a76985 100644 --- a/django/core/management/validation.py +++ b/django/core/management/validation.py @@ -113,13 +113,15 @@ def get_validation_errors(outfile, app=None): e.add(opts, '"%s": BooleanFields do not accept null values. Use a NullBooleanField instead.' % f.name) if isinstance(f, models.FilePathField) and not (f.allow_files or f.allow_folders): e.add(opts, '"%s": FilePathFields must have either allow_files or allow_folders set to True.' % f.name) + if isinstance(f, models.GenericIPAddressField) and not getattr(f, 'null', False) and getattr(f, 'blank', False): + e.add(opts, '"%s": GenericIPAddressField can not accept blank values if null values are not allowed, as blank values are stored as null.' % f.name) if f.choices: if isinstance(f.choices, six.string_types) or not is_iterable(f.choices): e.add(opts, '"%s": "choices" should be iterable (e.g., a tuple or list).' % f.name) else: for c in f.choices: - if not isinstance(c, (list, tuple)) or len(c) != 2: - e.add(opts, '"%s": "choices" should be a sequence of two-tuples.' % f.name) + if isinstance(c, six.string_types) or not is_iterable(c) or len(c) != 2: + e.add(opts, '"%s": "choices" should be a sequence of two-item iterables (e.g. list of 2 item tuples).' % f.name) if f.db_index not in (None, True, False): e.add(opts, '"%s": "db_index" should be either None, True or False.' % f.name) diff --git a/django/core/paginator.py b/django/core/paginator.py index 9ccff51a34..c8b9377856 100644 --- a/django/core/paginator.py +++ b/django/core/paginator.py @@ -121,7 +121,9 @@ class Page(collections.Sequence): raise TypeError # The object_list is converted to a list so that if it was a QuerySet # it won't be a database hit per __getitem__. - return list(self.object_list)[index] + if not isinstance(self.object_list, list): + self.object_list = list(self.object_list) + return self.object_list[index] def has_next(self): return self.number < self.paginator.num_pages diff --git a/django/core/urlresolvers.py b/django/core/urlresolvers.py index c3d93bb247..d58f2a9fa3 100644 --- a/django/core/urlresolvers.py +++ b/django/core/urlresolvers.py @@ -75,8 +75,7 @@ class Resolver404(Http404): pass class NoReverseMatch(Exception): - # Don't make this raise an error when used in a template. - silent_variable_failure = True + pass def get_callable(lookup_view, can_fail=False): """ @@ -360,6 +359,9 @@ class RegexURLResolver(LocaleRegexProvider): callback = getattr(urls, 'handler%s' % view_type) return get_callable(callback), {} + def resolve400(self): + return self._resolve_special('400') + def resolve403(self): return self._resolve_special('403') diff --git a/django/core/xheaders.py b/django/core/xheaders.py deleted file mode 100644 index 3766628c98..0000000000 --- a/django/core/xheaders.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Pages in Django can are served up with custom HTTP headers containing useful -information about those pages -- namely, the content type and object ID. - -This module contains utility functions for retrieving and doing interesting -things with these special "X-Headers" (so called because the HTTP spec demands -that custom headers are prefixed with "X-"). - -Next time you're at slashdot.org, watch out for X-Fry and X-Bender. :) -""" - -def populate_xheaders(request, response, model, object_id): - """ - Adds the "X-Object-Type" and "X-Object-Id" headers to the given - HttpResponse according to the given model and object_id -- but only if the - given HttpRequest object has an IP address within the INTERNAL_IPS setting - or if the request is from a logged in staff member. - """ - from django.conf import settings - if (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS - or (hasattr(request, 'user') and request.user.is_active - and request.user.is_staff)): - response['X-Object-Type'] = "%s.%s" % (model._meta.app_label, model._meta.model_name) - response['X-Object-Id'] = str(object_id) diff --git a/django/db/__init__.py b/django/db/__init__.py index 08c901ab7b..2421ddeab8 100644 --- a/django/db/__init__.py +++ b/django/db/__init__.py @@ -1,24 +1,19 @@ import warnings -from django.conf import settings from django.core import signals -from django.core.exceptions import ImproperlyConfigured from django.db.utils import (DEFAULT_DB_ALIAS, DataError, OperationalError, IntegrityError, InternalError, ProgrammingError, NotSupportedError, DatabaseError, InterfaceError, Error, load_backend, ConnectionHandler, ConnectionRouter) +from django.utils.functional import cached_property __all__ = ('backend', 'connection', 'connections', 'router', 'DatabaseError', 'IntegrityError', 'DEFAULT_DB_ALIAS') +connections = ConnectionHandler() -if settings.DATABASES and DEFAULT_DB_ALIAS not in settings.DATABASES: - raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS) - -connections = ConnectionHandler(settings.DATABASES) - -router = ConnectionRouter(settings.DATABASE_ROUTERS) +router = ConnectionRouter() # `connection`, `DatabaseError` and `IntegrityError` are convenient aliases # for backend bits. @@ -45,7 +40,28 @@ class DefaultConnectionProxy(object): return delattr(connections[DEFAULT_DB_ALIAS], name) connection = DefaultConnectionProxy() -backend = load_backend(connection.settings_dict['ENGINE']) + +class DefaultBackendProxy(object): + """ + Temporary proxy class used during deprecation period of the `backend` module + variable. + """ + @cached_property + def _backend(self): + warnings.warn("Accessing django.db.backend is deprecated.", + PendingDeprecationWarning, stacklevel=2) + return load_backend(connections[DEFAULT_DB_ALIAS].settings_dict['ENGINE']) + + def __getattr__(self, item): + return getattr(self._backend, item) + + def __setattr__(self, name, value): + return setattr(self._backend, name, value) + + def __delattr__(self, name): + return delattr(self._backend, name) + +backend = DefaultBackendProxy() def close_connection(**kwargs): warnings.warn( diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 0b9e75cbbc..1b8e6ae447 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -390,9 +390,10 @@ class BaseDatabaseWrapper(object): def disable_constraint_checking(self): """ Backends can implement as needed to temporarily disable foreign key - constraint checking. + constraint checking. Should return True if the constraints were + disabled and will need to be reenabled. """ - pass + return False def enable_constraint_checking(self): """ @@ -784,12 +785,12 @@ class BaseDatabaseOperations(object): """ return cursor.fetchone()[0] - def field_cast_sql(self, db_type): + def field_cast_sql(self, db_type, internal_type): """ - Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary - to cast it before using it in a WHERE statement. Note that the - resulting string should contain a '%s' placeholder for the column being - searched against. + Given a column type (e.g. 'BLOB', 'VARCHAR'), and an internal type + (e.g. 'GenericIPAddressField'), returns the SQL necessary to cast it + before using it in a WHERE statement. Note that the resulting string + should contain a '%s' placeholder for the column being searched against. """ return '%s' diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py index 21cea1fef8..98830407fb 100644 --- a/django/db/backends/creation.py +++ b/django/db/backends/creation.py @@ -99,7 +99,7 @@ class BaseDatabaseCreation(object): style.SQL_TABLE(qn(opts.db_table)) + ' ('] for i, line in enumerate(table_output): # Combine and add commas. full_statement.append( - ' %s%s' % (line, i < len(table_output) - 1 and ',' or '')) + ' %s%s' % (line, ',' if i < len(table_output) - 1 else '')) full_statement.append(')') if opts.db_tablespace: tablespace_sql = self.connection.ops.tablespace_sql( diff --git a/django/db/backends/mysql/compiler.py b/django/db/backends/mysql/compiler.py index 50a085212b..4e033e3d93 100644 --- a/django/db/backends/mysql/compiler.py +++ b/django/db/backends/mysql/compiler.py @@ -17,8 +17,7 @@ class SQLCompiler(compiler.SQLCompiler): values.append(value) return row[:index_extra_select] + tuple(values) - def as_subquery_condition(self, alias, columns): - qn = self.quote_name_unless_alias + def as_subquery_condition(self, alias, columns, qn): qn2 = self.connection.ops.quote_name sql, params = self.as_sql() return '(%s) IN (%s)' % (', '.join(['%s.%s' % (qn(alias), qn2(column)) for column in columns]), sql), params diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 66860d3d01..798c735d7b 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -44,6 +44,11 @@ except ImportError as e: from django.core.exceptions import ImproperlyConfigured raise ImproperlyConfigured("Error loading cx_Oracle module: %s" % e) +try: + import pytz +except ImportError: + pytz = None + from django.db import utils from django.db.backends import * from django.db.backends.oracle.client import DatabaseClient @@ -78,6 +83,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_subqueries_in_group_by = False supports_transactions = True supports_timezones = False + has_zoneinfo_database = pytz is not None supports_bitwise_or = False can_defer_constraint_checks = True ignores_nulls_in_unique_constraints = False @@ -243,9 +249,6 @@ WHEN (new.%(col_name)s IS NULL) value = value.date() return value - def datetime_cast_sql(self): - return "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS.FF')" - def deferrable_sql(self): return " DEFERRABLE INITIALLY DEFERRED" @@ -255,7 +258,7 @@ WHEN (new.%(col_name)s IS NULL) def fetch_returned_insert_id(self, cursor): return int(cursor._insert_id_var.getvalue()) - def field_cast_sql(self, db_type): + def field_cast_sql(self, db_type, internal_type): if db_type and db_type.endswith('LOB'): return "DBMS_LOB.SUBSTR(%s)" else: @@ -434,6 +437,17 @@ WHEN (new.%(col_name)s IS NULL) second = '%s-12-31' return [first % value, second % value] + def year_lookup_bounds_for_datetime_field(self, value): + # The default implementation uses datetime objects for the bounds. + # This must be overridden here, to use a formatted date (string) as + # 'second' instead -- cx_Oracle chops the fraction-of-second part + # off of datetime objects, leaving almost an entire second out of + # the year under the default implementation. + bounds = super(DatabaseOperations, self).year_lookup_bounds_for_datetime_field(value) + if settings.USE_TZ: + bounds = [b.astimezone(timezone.utc).replace(tzinfo=None) for b in bounds] + return [b.isoformat(b' ') for b in bounds] + def combine_expression(self, connector, sub_expressions): "Oracle requires special cases for %% and & operators in query expressions" if connector == '%%': diff --git a/django/db/backends/oracle/creation.py b/django/db/backends/oracle/creation.py index c8a436c8cd..7f1192ed8a 100644 --- a/django/db/backends/oracle/creation.py +++ b/django/db/backends/oracle/creation.py @@ -1,5 +1,7 @@ import sys import time + +from django.conf import settings from django.db.backends.creation import BaseDatabaseCreation from django.utils.six.moves import input @@ -112,7 +114,6 @@ class DatabaseCreation(BaseDatabaseCreation): print("Tests cancelled.") sys.exit(1) - from django.db import settings real_settings = settings.DATABASES[self.connection.alias] real_settings['SAVED_USER'] = self.connection.settings_dict['SAVED_USER'] = self.connection.settings_dict['USER'] real_settings['SAVED_PASSWORD'] = self.connection.settings_dict['SAVED_PASSWORD'] = self.connection.settings_dict['PASSWORD'] diff --git a/django/db/backends/oracle/introspection.py b/django/db/backends/oracle/introspection.py index ff56dca5c2..361308a62c 100644 --- a/django/db/backends/oracle/introspection.py +++ b/django/db/backends/oracle/introspection.py @@ -1,4 +1,5 @@ from django.db.backends import BaseDatabaseIntrospection, FieldInfo +from django.utils.encoding import force_text import cx_Oracle import re @@ -48,7 +49,9 @@ class DatabaseIntrospection(BaseDatabaseIntrospection): cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name)) description = [] for desc in cursor.description: - description.append(FieldInfo(*((desc[0].lower(),) + desc[1:]))) + name = force_text(desc[0]) # cx_Oracle always returns a 'str' on both Python 2 and 3 + name = name % {} # cx_Oracle, for some reason, doubles percent signs. + description.append(FieldInfo(*(name.lower(),) + desc[1:])) return description def table_name_converter(self, name): @@ -87,6 +90,18 @@ class DatabaseIntrospection(BaseDatabaseIntrospection): relations[row[0]] = (row[2], row[1].lower()) return relations + def get_key_columns(self, cursor, table_name): + cursor.execute(""" + SELECT ccol.column_name, rcol.table_name AS referenced_table, rcol.column_name AS referenced_column + FROM user_constraints c + JOIN user_cons_columns ccol + ON ccol.constraint_name = c.constraint_name + JOIN user_cons_columns rcol + ON rcol.constraint_name = c.r_constraint_name + WHERE c.table_name = %s AND c.constraint_type = 'R'""" , [table_name.upper()]) + return [tuple(cell.lower() for cell in row) + for row in cursor.fetchall()] + def get_indexes(self, cursor, table_name): sql = """ SELECT LOWER(uic1.column_name) AS column_name, diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py index b17a0c17bb..f06eec5a1d 100644 --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -78,8 +78,8 @@ class DatabaseOperations(BaseDatabaseOperations): return lookup - def field_cast_sql(self, db_type): - if db_type == 'inet': + def field_cast_sql(self, db_type, internal_type): + if internal_type == "GenericIPAddressField" or internal_type == "IPAddressField": return 'HOST(%s)' return '%s' diff --git a/django/db/backends/util.py b/django/db/backends/util.py index 084f4c200b..aa2601277a 100644 --- a/django/db/backends/util.py +++ b/django/db/backends/util.py @@ -83,7 +83,7 @@ class CursorDebugWrapper(CursorWrapper): ############################################### def typecast_date(s): - return s and datetime.date(*map(int, s.split('-'))) or None # returns None if s is null + return datetime.date(*map(int, s.split('-'))) if s else None # returns None if s is null def typecast_time(s): # does NOT store time zone information if not s: return None diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py index 5f17229753..3eac2167d4 100644 --- a/django/db/models/__init__.py +++ b/django/db/models/__init__.py @@ -1,7 +1,7 @@ from functools import wraps from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured -from django.db.models.loading import get_apps, get_app, get_models, get_model, register_models +from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models from django.db.models.query import Q from django.db.models.expressions import F from django.db.models.manager import Manager diff --git a/django/db/models/base.py b/django/db/models/base.py index 556249fa54..5f1c21c255 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -632,15 +632,7 @@ class Model(six.with_metaclass(ModelBase)): base_qs = cls._base_manager.using(using) values = [(f, None, (getattr(self, f.attname) if raw else f.pre_save(self, False))) for f in non_pks] - if not values: - # We can end up here when saving a model in inheritance chain where - # update_fields doesn't target any field in current model. In that - # case we just say the update succeeded. Another case ending up here - # is a model with just PK - in that case check that the PK still - # exists. - updated = update_fields is not None or base_qs.filter(pk=pk_val).exists() - else: - updated = self._do_update(base_qs, using, pk_val, values) + updated = self._do_update(base_qs, using, pk_val, values, update_fields) if force_update and not updated: raise DatabaseError("Forced update did not affect any rows.") if update_fields and not updated: @@ -664,13 +656,21 @@ class Model(six.with_metaclass(ModelBase)): setattr(self, meta.pk.attname, result) return updated - def _do_update(self, base_qs, using, pk_val, values): + def _do_update(self, base_qs, using, pk_val, values, update_fields): """ This method will try to update the model. If the model was updated (in the sense that an update query was done and a matching row was found from the DB) the method will return True. """ - return base_qs.filter(pk=pk_val)._update(values) > 0 + if not values: + # We can end up here when saving a model in inheritance chain where + # update_fields doesn't target any field in current model. In that + # case we just say the update succeeded. Another case ending up here + # is a model with just PK - in that case check that the PK still + # exists. + return update_fields is not None or base_qs.filter(pk=pk_val).exists() + else: + return base_qs.filter(pk=pk_val)._update(values) > 0 def _do_insert(self, manager, using, fields, update_pk, raw): """ diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 86a0711d7c..691eeffb08 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -243,6 +243,8 @@ class Field(object): obj = copy.copy(self) if self.rel: obj.rel = copy.copy(self.rel) + if hasattr(self.rel, 'field') and self.rel.field is self: + obj.rel.field = obj memodict[id(self)] = obj return obj diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 5ef713e5e6..754a97633b 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -11,7 +11,7 @@ from django.db.models.deletion import CASCADE from django.utils.encoding import smart_text from django.utils import six from django.utils.deprecation import RenameMethodsBase -from django.utils.translation import ugettext_lazy as _, string_concat +from django.utils.translation import ugettext_lazy as _ from django.utils.functional import curry, cached_property from django.core import exceptions from django import forms @@ -199,7 +199,9 @@ class SingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjectDescri setattr(rel_obj, self.related.field.get_cache_name(), instance) setattr(instance, self.cache_name, rel_obj) if rel_obj is None: - raise self.related.model.DoesNotExist + raise self.related.model.DoesNotExist("%s has no %s." % ( + instance.__class__.__name__, + self.related.get_accessor_name())) else: return rel_obj @@ -224,8 +226,7 @@ class SingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjectDescri value._state.db = router.db_for_write(value.__class__, instance=instance) elif value._state.db is not None and instance._state.db is not None: if not router.allow_relation(value, instance): - raise ValueError('Cannot assign "%r": instance is on database "%s", value is on database "%s"' % - (value, instance._state.db, value._state.db)) + raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value) related_pk = tuple([getattr(instance, field.attname) for field in self.related.field.foreign_related_fields]) if None in related_pk: @@ -302,7 +303,8 @@ class ReverseSingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjec setattr(rel_obj, self.field.related.get_cache_name(), instance) setattr(instance, self.cache_name, rel_obj) if rel_obj is None and not self.field.null: - raise self.field.rel.to.DoesNotExist + raise self.field.rel.to.DoesNotExist( + "%s has no %s." % (self.field.model.__name__, self.field.name)) else: return rel_obj @@ -323,8 +325,7 @@ class ReverseSingleRelatedObjectDescriptor(six.with_metaclass(RenameRelatedObjec value._state.db = router.db_for_write(value.__class__, instance=instance) elif value._state.db is not None and instance._state.db is not None: if not router.allow_relation(value, instance): - raise ValueError('Cannot assign "%r": instance is on database "%s", value is on database "%s"' % - (value, instance._state.db, value._state.db)) + raise ValueError('Cannot assign "%r": the current database router prevents this relation.' % value) # If we're setting the value of a OneToOneField to None, we need to clear # out the cache on any old related object. Otherwise, deleting the @@ -1379,9 +1380,6 @@ class ManyToManyField(RelatedField): super(ManyToManyField, self).__init__(**kwargs) - msg = _('Hold down "Control", or "Command" on a Mac, to select more than one.') - self.help_text = string_concat(self.help_text, ' ', msg) - def deconstruct(self): name, path, args, kwargs = super(ManyToManyField, self).deconstruct() # Handle the simpler arguments diff --git a/django/db/models/loading.py b/django/db/models/loading.py index 075cae4c61..535df7ce80 100644 --- a/django/db/models/loading.py +++ b/django/db/models/loading.py @@ -152,7 +152,9 @@ class BaseAppCache(object): return self.loaded def get_apps(self): - "Returns a list of all installed modules that contain models." + """ + Returns a list of all installed modules that contain models. + """ self._populate() # Ensure the returned list is always in the same order (with new apps @@ -162,6 +164,23 @@ class BaseAppCache(object): apps.sort() return [elt[1] for elt in apps] + def get_app_paths(self): + """ + Returns a list of paths to all installed apps. + + Useful for discovering files at conventional locations inside apps + (static files, templates, etc.) + """ + self._populate() + + app_paths = [] + for app in self.get_apps(): + if hasattr(app, '__path__'): # models/__init__.py package + app_paths.extend([upath(path) for path in app.__path__]) + else: # models.py module + app_paths.append(upath(app.__file__)) + return app_paths + def get_app(self, app_label, emptyOK=False): """ Returns the module containing the models for the given app_label. If @@ -302,6 +321,7 @@ cache = AppCache() # These methods were always module level, so are kept that way for backwards # compatibility. get_apps = cache.get_apps +get_app_paths = cache.get_app_paths get_app = cache.get_app get_app_errors = cache.get_app_errors get_models = cache.get_models diff --git a/django/db/models/manager.py b/django/db/models/manager.py index 43a8264f11..a1aa79f809 100644 --- a/django/db/models/manager.py +++ b/django/db/models/manager.py @@ -186,6 +186,12 @@ class Manager(six.with_metaclass(RenameManagerMethods)): def latest(self, *args, **kwargs): return self.get_queryset().latest(*args, **kwargs) + def first(self): + return self.get_queryset().first() + + def last(self): + return self.get_queryset().last() + def order_by(self, *args, **kwargs): return self.get_queryset().order_by(*args, **kwargs) diff --git a/django/db/models/query.py b/django/db/models/query.py index d3763d3934..b0ce25f5b5 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -9,7 +9,7 @@ import warnings from django.conf import settings from django.core import exceptions -from django.db import connections, router, transaction, IntegrityError +from django.db import connections, router, transaction, DatabaseError from django.db.models.constants import LOOKUP_SEP from django.db.models.fields import AutoField from django.db.models.query_utils import (Q, select_related_descend, @@ -20,11 +20,6 @@ from django.utils.functional import partition from django.utils import six from django.utils import timezone -# Used to control how many objects are worked with at once in some cases (e.g. -# when deleting objects). -CHUNK_SIZE = 100 -ITER_CHUNK_SIZE = CHUNK_SIZE - # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 @@ -41,7 +36,6 @@ class QuerySet(object): self._db = using self.query = query or sql.Query(self.model) self._result_cache = None - self._iter = None self._sticky_filter = False self._for_write = False self._prefetch_related_lookups = [] @@ -57,8 +51,8 @@ class QuerySet(object): Deep copy of a QuerySet doesn't populate the cache """ obj = self.__class__() - for k,v in self.__dict__.items(): - if k in ('_iter','_result_cache'): + for k, v in self.__dict__.items(): + if k == '_result_cache': obj.__dict__[k] = None else: obj.__dict__[k] = copy.deepcopy(v, memo) @@ -69,10 +63,8 @@ class QuerySet(object): Allows the QuerySet to be pickled. """ # Force the cache to be fully populated. - len(self) - + self._fetch_all() obj_dict = self.__dict__.copy() - obj_dict['_iter'] = None return obj_dict def __repr__(self): @@ -82,95 +74,31 @@ class QuerySet(object): return repr(data) def __len__(self): - # Since __len__ is called quite frequently (for example, as part of - # list(qs), we make some effort here to be as efficient as possible - # whilst not messing up any existing iterators against the QuerySet. - if self._result_cache is None: - if self._iter: - self._result_cache = list(self._iter) - else: - self._result_cache = list(self.iterator()) - elif self._iter: - self._result_cache.extend(self._iter) - if self._prefetch_related_lookups and not self._prefetch_done: - self._prefetch_related_objects() + self._fetch_all() return len(self._result_cache) def __iter__(self): - if self._prefetch_related_lookups and not self._prefetch_done: - # We need all the results in order to be able to do the prefetch - # in one go. To minimize code duplication, we use the __len__ - # code path which also forces this, and also does the prefetch - len(self) - - if self._result_cache is None: - self._iter = self.iterator() - self._result_cache = [] - if self._iter: - return self._result_iter() - # Python's list iterator is better than our version when we're just - # iterating over the cache. + """ + The queryset iterator protocol uses three nested iterators in the + default case: + 1. sql.compiler:execute_sql() + - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE) + using cursor.fetchmany(). This part is responsible for + doing some column masking, and returning the rows in chunks. + 2. sql/compiler.results_iter() + - Returns one row at time. At this point the rows are still just + tuples. In some cases the return values are converted to + Python values at this location (see resolve_columns(), + resolve_aggregate()). + 3. self.iterator() + - Responsible for turning the rows into model objects. + """ + self._fetch_all() return iter(self._result_cache) - def _result_iter(self): - pos = 0 - while 1: - upper = len(self._result_cache) - while pos < upper: - yield self._result_cache[pos] - pos = pos + 1 - if not self._iter: - raise StopIteration - if len(self._result_cache) <= pos: - self._fill_cache() - - def __bool__(self): - if self._prefetch_related_lookups and not self._prefetch_done: - # We need all the results in order to be able to do the prefetch - # in one go. To minimize code duplication, we use the __len__ - # code path which also forces this, and also does the prefetch - len(self) - - if self._result_cache is not None: - return bool(self._result_cache) - try: - next(iter(self)) - except StopIteration: - return False - return True - - def __nonzero__(self): # Python 2 compatibility - return type(self).__bool__(self) - - def __contains__(self, val): - # The 'in' operator works without this method, due to __iter__. This - # implementation exists only to shortcut the creation of Model - # instances, by bailing out early if we find a matching element. - pos = 0 - if self._result_cache is not None: - if val in self._result_cache: - return True - elif self._iter is None: - # iterator is exhausted, so we have our answer - return False - # remember not to check these again: - pos = len(self._result_cache) - else: - # We need to start filling the result cache out. The following - # ensures that self._iter is not None and self._result_cache is not - # None - it = iter(self) - - # Carry on, one result at a time. - while True: - if len(self._result_cache) <= pos: - self._fill_cache(num=1) - if self._iter is None: - # we ran out of items - return False - if self._result_cache[pos] == val: - return True - pos += 1 + def __nonzero__(self): + self._fetch_all() + return bool(self._result_cache) def __getitem__(self, k): """ @@ -184,19 +112,6 @@ class QuerySet(object): "Negative indexing is not supported." if self._result_cache is not None: - if self._iter is not None: - # The result cache has only been partially populated, so we may - # need to fill it out a bit more. - if isinstance(k, slice): - if k.stop is not None: - # Some people insist on passing in strings here. - bound = int(k.stop) - else: - bound = None - else: - bound = k + 1 - if len(self._result_cache) < bound: - self._fill_cache(bound - len(self._result_cache)) return self._result_cache[k] if isinstance(k, slice): @@ -210,7 +125,7 @@ class QuerySet(object): else: stop = None qs.query.set_limits(start, stop) - return k.step and list(qs)[::k.step] or qs + return list(qs)[::k.step] if k.step else qs qs = self._clone() qs.query.set_limits(k, k + 1) @@ -370,7 +285,7 @@ class QuerySet(object): If the QuerySet is already fully cached this simply returns the length of the cached results set to avoid multiple SELECT COUNT(*) calls. """ - if self._result_cache is not None and not self._iter: + if self._result_cache is not None: return len(self._result_cache) return self.query.get_count(using=self.db) @@ -388,13 +303,11 @@ class QuerySet(object): return clone._result_cache[0] if not num: raise self.model.DoesNotExist( - "%s matching query does not exist. " - "Lookup parameters were %s" % - (self.model._meta.object_name, kwargs)) + "%s matching query does not exist." % + self.model._meta.object_name) raise self.model.MultipleObjectsReturned( - "get() returned more than one %s -- it returned %s! " - "Lookup parameters were %s" % - (self.model._meta.object_name, num, kwargs)) + "get() returned more than one %s -- it returned %s!" % + (self.model._meta.object_name, num)) def create(self, **kwargs): """ @@ -450,8 +363,6 @@ class QuerySet(object): Returns a tuple of (object, created), where created is a boolean specifying whether an object was created. """ - assert kwargs, \ - 'get_or_create() must be passed at least one keyword argument' defaults = kwargs.pop('defaults', {}) lookup = kwargs.copy() for f in self.model._meta.fields: @@ -469,13 +380,13 @@ class QuerySet(object): obj.save(force_insert=True, using=self.db) transaction.savepoint_commit(sid, using=self.db) return obj, True - except IntegrityError: + except DatabaseError: transaction.savepoint_rollback(sid, using=self.db) exc_info = sys.exc_info() try: return self.get(**lookup), False except self.model.DoesNotExist: - # Re-raise the IntegrityError with its original traceback. + # Re-raise the DatabaseError with its original traceback. six.reraise(*exc_info) def _earliest_or_latest(self, field_name=None, direction="-"): @@ -500,6 +411,26 @@ class QuerySet(object): def latest(self, field_name=None): return self._earliest_or_latest(field_name=field_name, direction="-") + def first(self): + """ + Returns the first object of a query, returns None if no match is found. + """ + qs = self if self.ordered else self.order_by('pk') + try: + return qs[0] + except IndexError: + return None + + def last(self): + """ + Returns the last object of a query, returns None if no match is found. + """ + qs = self.reverse() if self.ordered else self.order_by('-pk') + try: + return qs[0] + except IndexError: + return None + def in_bulk(self, id_list): """ Returns a dictionary mapping each of the given IDs to the object with @@ -714,6 +645,8 @@ class QuerySet(object): If fields are specified, they must be ForeignKey fields and only those related objects are included in the selection. + + If select_related(None) is called, the list is cleared. """ if 'depth' in kwargs: warnings.warn('The "depth" keyword argument has been deprecated.\n' @@ -723,7 +656,9 @@ class QuerySet(object): raise TypeError('Unexpected keyword arguments to select_related: %s' % (list(kwargs),)) obj = self._clone() - if fields: + if fields == (None,): + obj.query.select_related = False + elif fields: if depth: raise TypeError('Cannot pass both "depth" and fields to select_related()') obj.query.add_select_related(fields) @@ -915,17 +850,11 @@ class QuerySet(object): c._setup_query() return c - def _fill_cache(self, num=None): - """ - Fills the result cache with 'num' more entries (or until the results - iterator is exhausted). - """ - if self._iter: - try: - for i in range(num or ITER_CHUNK_SIZE): - self._result_cache.append(next(self._iter)) - except StopIteration: - self._iter = None + def _fetch_all(self): + if self._result_cache is None: + self._result_cache = list(self.iterator()) + if self._prefetch_related_lookups and not self._prefetch_done: + self._prefetch_related_objects() def _next_is_sticky(self): """ @@ -1618,8 +1547,18 @@ def prefetch_related_objects(result_cache, related_lookups): if len(obj_list) == 0: break + current_lookup = LOOKUP_SEP.join(attrs[0:level+1]) + if current_lookup in done_queries: + # Skip any prefetching, and any object preparation + obj_list = done_queries[current_lookup] + continue + + # Prepare objects: good_objects = True for obj in obj_list: + # Since prefetching can re-use instances, it is possible to have + # the same instance multiple times in obj_list, so obj might + # already be prepared. if not hasattr(obj, '_prefetched_objects_cache'): try: obj._prefetched_objects_cache = {} @@ -1630,9 +1569,6 @@ def prefetch_related_objects(result_cache, related_lookups): # now. good_objects = False break - else: - # We already did this list - break if not good_objects: break @@ -1657,23 +1593,18 @@ def prefetch_related_objects(result_cache, related_lookups): "prefetch_related()." % lookup) if prefetcher is not None and not is_fetched: - # Check we didn't do this already - current_lookup = LOOKUP_SEP.join(attrs[0:level+1]) - if current_lookup in done_queries: - obj_list = done_queries[current_lookup] - else: - obj_list, additional_prl = prefetch_one_level(obj_list, prefetcher, attr) - # We need to ensure we don't keep adding lookups from the - # same relationships to stop infinite recursion. So, if we - # are already on an automatically added lookup, don't add - # the new lookups from relationships we've seen already. - if not (lookup in auto_lookups and - descriptor in followed_descriptors): - for f in additional_prl: - new_prl = LOOKUP_SEP.join([current_lookup, f]) - auto_lookups.append(new_prl) - done_queries[current_lookup] = obj_list - followed_descriptors.add(descriptor) + obj_list, additional_prl = prefetch_one_level(obj_list, prefetcher, attr) + # We need to ensure we don't keep adding lookups from the + # same relationships to stop infinite recursion. So, if we + # are already on an automatically added lookup, don't add + # the new lookups from relationships we've seen already. + if not (lookup in auto_lookups and + descriptor in followed_descriptors): + for f in additional_prl: + new_prl = LOOKUP_SEP.join([current_lookup, f]) + auto_lookups.append(new_prl) + done_queries[current_lookup] = obj_list + followed_descriptors.add(descriptor) else: # Either a singly related object that has already been fetched # (e.g. via select_related), or hopefully some other property diff --git a/django/db/models/signals.py b/django/db/models/signals.py index 09f93d0f77..3e321893c1 100644 --- a/django/db/models/signals.py +++ b/django/db/models/signals.py @@ -12,6 +12,7 @@ post_save = Signal(providing_args=["instance", "raw", "created", "using", "updat pre_delete = Signal(providing_args=["instance", "using"], use_caching=True) post_delete = Signal(providing_args=["instance", "using"], use_caching=True) +pre_syncdb = Signal(providing_args=["app", "create_models", "verbosity", "interactive", "db"]) post_syncdb = Signal(providing_args=["class", "app", "created_models", "verbosity", "interactive", "db"], use_caching=True) m2m_changed = Signal(providing_args=["action", "instance", "reverse", "model", "pk_set", "using"], use_caching=True) diff --git a/django/db/models/sql/aggregates.py b/django/db/models/sql/aggregates.py index 23b79923d1..2bd2b2f76f 100644 --- a/django/db/models/sql/aggregates.py +++ b/django/db/models/sql/aggregates.py @@ -99,7 +99,7 @@ class Count(Aggregate): sql_template = '%(function)s(%(distinct)s%(field)s)' def __init__(self, col, distinct=False, **extra): - super(Count, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra) + super(Count, self).__init__(col, distinct='DISTINCT ' if distinct else '', **extra) class Max(Aggregate): sql_function = 'MAX' diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index bbe310c8c3..0bfd1b38d3 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -729,7 +729,8 @@ class SQLCompiler(object): row = self.resolve_columns(row, fields) if has_aggregate_select: - aggregate_start = len(self.query.extra_select) + len(self.query.select) + loaded_fields = self.query.get_loaded_field_names().get(self.query.model, set()) or self.query.select + aggregate_start = len(self.query.extra_select) + len(loaded_fields) aggregate_end = aggregate_start + len(self.query.aggregate_select) row = tuple(row[:aggregate_start]) + tuple([ self.query.resolve_aggregate(value, aggregate, self.connection) @@ -786,8 +787,7 @@ class SQLCompiler(object): return list(result) return result - def as_subquery_condition(self, alias, columns): - qn = self.quote_name_unless_alias + def as_subquery_condition(self, alias, columns, qn): qn2 = self.connection.ops.quote_name if len(columns) == 1: sql, params = self.as_sql() diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 0a4152587d..154b6bd204 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -1422,7 +1422,9 @@ class Query(object): query.clear_ordering(True) # Try to have as simple as possible subquery -> trim leading joins from # the subquery. - trimmed_joins = query.trim_start(names_with_path) + trimmed_prefix, contains_louter = query.trim_start(names_with_path) + query.remove_inherited_models() + # Add extra check to make sure the selected field will not be null # since we are adding a IN <subquery> clause. This prevents the # database from tripping over IN (...,NULL,...) selects and returning @@ -1431,38 +1433,20 @@ class Query(object): alias, col = query.select[0].col query.where.add((Constraint(alias, col, query.select[0].field), 'isnull', False), AND) - # Still make sure that the trimmed parts in the inner query and - # trimmed prefix are in sync. So, use the trimmed_joins to make sure - # as many path elements are in the prefix as there were trimmed joins. - # In addition, convert the path elements back to names so that - # add_filter() can handle them. - trimmed_prefix = [] - paths_in_prefix = trimmed_joins - for name, path in names_with_path: - if paths_in_prefix - len(path) < 0: - break - trimmed_prefix.append(name) - paths_in_prefix -= len(path) - join_field = path[paths_in_prefix].join_field - # TODO: This should be made properly multicolumn - # join aware. It is likely better to not use build_filter - # at all, instead construct joins up to the correct point, - # then construct the needed equality constraint manually, - # or maybe using SubqueryConstraint would work, too. - # The foreign_related_fields attribute is right here, we - # don't ever split joins for direct case. - trimmed_prefix.append( - join_field.field.foreign_related_fields[0].name) - trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix) condition = self.build_filter( ('%s__in' % trimmed_prefix, query), current_negated=True, branch_negated=True, can_reuse=can_reuse) - # Intentionally leave the other alias as blank, if the condition - # refers it, things will break here. - extra_restriction = join_field.get_extra_restriction( - self.where_class, None, [t for t in query.tables if query.alias_refcount[t]][0]) - if extra_restriction: - query.where.add(extra_restriction, 'AND') + if contains_louter: + or_null_condition = self.build_filter( + ('%s__isnull' % trimmed_prefix, True), + current_negated=True, branch_negated=True, can_reuse=can_reuse) + condition.add(or_null_condition, OR) + # Note that the end result will be: + # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL. + # This might look crazy but due to how IN works, this seems to be + # correct. If the IS NOT NULL check is removed then outercol NOT + # IN will return UNKNOWN. If the IS NULL check is removed, then if + # outercol IS NULL we will not match the row. return condition def set_empty(self): @@ -1821,35 +1805,58 @@ class Query(object): def trim_start(self, names_with_path): """ Trims joins from the start of the join path. The candidates for trim - are the PathInfos in names_with_path structure. Outer joins are not - eligible for removal. Also sets the select column so the start - matches the join. + are the PathInfos in names_with_path structure that are m2m joins. + + Also sets the select column so the start matches the join. + + This method is meant to be used for generating the subquery joins & + cols in split_exclude(). - This method is mostly useful for generating the subquery joins & col - in "WHERE somecol IN (subquery)". This construct is needed by - split_exclude(). + Returns a lookup usable for doing outerq.filter(lookup=self). Returns + also if the joins in the prefix contain a LEFT OUTER join. _""" all_paths = [] for _, paths in names_with_path: all_paths.extend(paths) - direct_join = True + contains_louter = False for pos, path in enumerate(all_paths): + if path.m2m: + break if self.alias_map[self.tables[pos + 1]].join_type == self.LOUTER: - direct_join = False - pos -= 1 + contains_louter = True + self.unref_alias(self.tables[pos]) + # The path.join_field is a Rel, lets get the other side's field + join_field = path.join_field.field + # Build the filter prefix. + trimmed_prefix = [] + paths_in_prefix = pos + for name, path in names_with_path: + if paths_in_prefix - len(path) < 0: break + trimmed_prefix.append(name) + paths_in_prefix -= len(path) + trimmed_prefix.append( + join_field.foreign_related_fields[0].name) + trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix) + # Lets still see if we can trim the first join from the inner query + # (that is, self). We can't do this for LEFT JOINs because we would + # miss those rows that have nothing on the outer side. + if self.alias_map[self.tables[pos + 1]].join_type != self.LOUTER: + select_fields = [r[0] for r in join_field.related_fields] + select_alias = self.tables[pos + 1] self.unref_alias(self.tables[pos]) - if path.direct: - direct_join = not direct_join - join_side = 0 if direct_join else 1 - select_alias = self.tables[pos + 1] - join_field = path.join_field - if hasattr(join_field, 'field'): - join_field = join_field.field - select_fields = [r[join_side] for r in join_field.related_fields] + extra_restriction = join_field.get_extra_restriction( + self.where_class, None, self.tables[pos + 1]) + if extra_restriction: + self.where.add(extra_restriction, AND) + else: + # TODO: It might be possible to trim more joins from the start of the + # inner query if it happens to have a longer join chain containing the + # values in select_fields. Lets punt this one for now. + select_fields = [r[1] for r in join_field.related_fields] + select_alias = self.tables[pos] self.select = [SelectInfo((select_alias, f.column), f) for f in select_fields] - self.remove_inherited_models() - return pos + return trimmed_prefix, contains_louter def is_nullable(self, field): """ diff --git a/django/db/models/sql/where.py b/django/db/models/sql/where.py index 029226383d..2a342d417a 100644 --- a/django/db/models/sql/where.py +++ b/django/db/models/sql/where.py @@ -174,6 +174,8 @@ class WhereNode(tree.Node): it. """ lvalue, lookup_type, value_annotation, params_or_value = child + field_internal_type = lvalue.field.get_internal_type() if lvalue.field else None + if isinstance(lvalue, Constraint): try: lvalue, params = lvalue.process(lookup_type, params_or_value, connection) @@ -187,7 +189,7 @@ class WhereNode(tree.Node): if isinstance(lvalue, tuple): # A direct database column lookup. - field_sql, field_params = self.sql_for_columns(lvalue, qn, connection), [] + field_sql, field_params = self.sql_for_columns(lvalue, qn, connection, field_internal_type), [] else: # A smart object with an as_sql() method. field_sql, field_params = lvalue.as_sql(qn, connection) @@ -257,7 +259,7 @@ class WhereNode(tree.Node): raise TypeError('Invalid lookup_type: %r' % lookup_type) - def sql_for_columns(self, data, qn, connection): + def sql_for_columns(self, data, qn, connection, internal_type=None): """ Returns the SQL fragment used for the left-hand side of a column constraint (for example, the "T1.foo" portion in the clause @@ -268,7 +270,7 @@ class WhereNode(tree.Node): lhs = '%s.%s' % (qn(table_alias), qn(name)) else: lhs = qn(name) - return connection.ops.field_cast_sql(db_type) % lhs + return connection.ops.field_cast_sql(db_type, internal_type) % lhs def relabel_aliases(self, change_map): """ @@ -397,13 +399,21 @@ class SubqueryConstraint(object): if hasattr(query, 'values'): if query._db and connection.alias != query._db: raise ValueError("Can't do subqueries with queries on different DBs.") - query = query.values(*self.targets).query + # Do not override already existing values. + if not hasattr(query, 'field_names'): + query = query.values(*self.targets) + else: + query = query._clone() + query = query.query query.clear_ordering(True) query_compiler = query.get_compiler(connection=connection) - return query_compiler.as_subquery_condition(self.alias, self.columns) + return query_compiler.as_subquery_condition(self.alias, self.columns, qn) + + def relabel_aliases(self, change_map): + self.alias = change_map.get(self.alias, self.alias) - def relabeled_clone(self, relabels): + def clone(self): return self.__class__( - relabels.get(self.alias, self.alias), - self.columns, self.query_object) + self.alias, self.columns, self.targets, + self.query_object) diff --git a/django/db/transaction.py b/django/db/transaction.py index 48e7f900dd..f770f2efa7 100644 --- a/django/db/transaction.py +++ b/django/db/transaction.py @@ -333,6 +333,23 @@ def atomic(using=None, savepoint=True): return Atomic(using, savepoint) +def _non_atomic_requests(view, using): + try: + view._non_atomic_requests.add(using) + except AttributeError: + view._non_atomic_requests = set([using]) + return view + + +def non_atomic_requests(using=None): + if callable(using): + return _non_atomic_requests(using, DEFAULT_DB_ALIAS) + else: + if using is None: + using = DEFAULT_DB_ALIAS + return lambda view: _non_atomic_requests(view, using) + + ############################################ # Deprecated decorators / context managers # ############################################ diff --git a/django/db/utils.py b/django/db/utils.py index e84060f9b3..bd7e10d24c 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -6,6 +6,7 @@ import warnings from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from django.utils.functional import cached_property from django.utils.importlib import import_module from django.utils.module_loading import import_by_path from django.utils._os import upath @@ -90,8 +91,7 @@ class DatabaseErrorWrapper(object): except AttributeError: args = (exc_value,) dj_exc_value = dj_exc_type(*args) - if six.PY3: - dj_exc_value.__cause__ = exc_value + dj_exc_value.__cause__ = exc_value # Only set the 'errors_occurred' flag for errors that may make # the connection unusable. if dj_exc_type not in (DataError, IntegrityError): @@ -138,16 +138,27 @@ class ConnectionDoesNotExist(Exception): class ConnectionHandler(object): - def __init__(self, databases): - if not databases: - self.databases = { + def __init__(self, databases=None): + """ + databases is an optional dictionary of database definitions (structured + like settings.DATABASES). + """ + self._databases = databases + self._connections = local() + + @cached_property + def databases(self): + if self._databases is None: + self._databases = settings.DATABASES + if self._databases == {}: + self._databases = { DEFAULT_DB_ALIAS: { 'ENGINE': 'django.db.backends.dummy', }, } - else: - self.databases = databases - self._connections = local() + if DEFAULT_DB_ALIAS not in self._databases: + raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS) + return self._databases def ensure_defaults(self, alias): """ @@ -202,14 +213,24 @@ class ConnectionHandler(object): class ConnectionRouter(object): - def __init__(self, routers): - self.routers = [] - for r in routers: + def __init__(self, routers=None): + """ + If routers is not specified, will default to settings.DATABASE_ROUTERS. + """ + self._routers = routers + + @cached_property + def routers(self): + if self._routers is None: + self._routers = settings.DATABASE_ROUTERS + routers = [] + for r in self._routers: if isinstance(r, six.string_types): router = import_by_path(r)() else: router = r - self.routers.append(router) + routers.append(router) + return routers def _router_func(action): def _route_db(self, model, **hints): diff --git a/django/forms/fields.py b/django/forms/fields.py index 4ce57d34a3..ac68b9f1fc 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -198,14 +198,15 @@ class Field(object): result.validators = self.validators[:] return result + class CharField(Field): def __init__(self, max_length=None, min_length=None, *args, **kwargs): self.max_length, self.min_length = max_length, min_length super(CharField, self).__init__(*args, **kwargs) if min_length is not None: - self.validators.append(validators.MinLengthValidator(min_length)) + self.validators.append(validators.MinLengthValidator(int(min_length))) if max_length is not None: - self.validators.append(validators.MaxLengthValidator(max_length)) + self.validators.append(validators.MaxLengthValidator(int(max_length))) def to_python(self, value): "Returns a Unicode object." @@ -220,6 +221,7 @@ class CharField(Field): attrs.update({'maxlength': str(self.max_length)}) return attrs + class IntegerField(Field): default_error_messages = { 'invalid': _('Enter a whole number.'), @@ -444,6 +446,7 @@ class TimeField(BaseTemporalField): def strptime(self, value, format): return datetime.datetime.strptime(force_str(value), format).time() + class DateTimeField(BaseTemporalField): widget = DateTimeInput input_formats = formats.get_format_lazy('DATETIME_INPUT_FORMATS') @@ -482,6 +485,7 @@ class DateTimeField(BaseTemporalField): def strptime(self, value, format): return datetime.datetime.strptime(force_str(value), format) + class RegexField(CharField): def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs): """ @@ -511,6 +515,7 @@ class RegexField(CharField): regex = property(_get_regex, _set_regex) + class EmailField(CharField): widget = EmailInput default_validators = [validators.validate_email] @@ -519,6 +524,7 @@ class EmailField(CharField): value = self.to_python(value).strip() return super(EmailField, self).clean(value) + class FileField(Field): widget = ClearableFileInput default_error_messages = { @@ -626,6 +632,7 @@ class ImageField(FileField): f.seek(0) return f + class URLField(CharField): widget = URLInput default_error_messages = { @@ -670,6 +677,10 @@ class URLField(CharField): value = urlunsplit(url_fields) return value + def clean(self, value): + value = self.to_python(value).strip() + return super(URLField, self).clean(value) + class BooleanField(Field): widget = CheckboxInput @@ -788,6 +799,7 @@ class ChoiceField(Field): return True return False + class TypedChoiceField(ChoiceField): def __init__(self, *args, **kwargs): self.coerce = kwargs.pop('coerce', lambda val: val) @@ -899,6 +911,7 @@ class ComboField(Field): value = field.clean(value) return value + class MultiValueField(Field): """ A Field that aggregates the logic of multiple Fields. @@ -1043,6 +1056,7 @@ class FilePathField(ChoiceField): self.widget.choices = self.choices + class SplitDateTimeField(MultiValueField): widget = SplitDateTimeWidget hidden_widget = SplitHiddenDateTimeWidget @@ -1105,3 +1119,7 @@ class GenericIPAddressField(CharField): class SlugField(CharField): default_validators = [validators.validate_slug] + + def clean(self, value): + value = self.to_python(value).strip() + return super(SlugField, self).clean(value) diff --git a/django/forms/forms.py b/django/forms/forms.py index b231de421a..0c598ac775 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -134,7 +134,7 @@ class BaseForm(object): Subclasses may wish to override. """ - return self.prefix and ('%s-%s' % (self.prefix, field_name)) or field_name + return '%s-%s' % (self.prefix, field_name) if self.prefix else field_name def add_initial_prefix(self, field_name): """ @@ -342,6 +342,8 @@ class BaseForm(object): data_value = field.widget.value_from_datadict(self.data, self.files, prefixed_name) if not field.show_hidden_initial: initial_value = self.initial.get(name, field.initial) + if callable(initial_value): + initial_value = initial_value() else: initial_prefixed_name = self.add_initial_prefix(name) hidden_widget = field.hidden_widget() @@ -523,10 +525,11 @@ class BoundField(object): widget = self.field.widget id_ = widget.attrs.get('id') or self.auto_id if id_: + id_for_label = widget.id_for_label(id_) + if id_for_label: + attrs = dict(attrs or {}, **{'for': id_for_label}) attrs = flatatt(attrs) if attrs else '' - contents = format_html('<label for="{0}"{1}>{2}</label>', - widget.id_for_label(id_), attrs, contents - ) + contents = format_html('<label{0}>{1}</label>', attrs, contents) else: contents = conditional_escape(contents) return mark_safe(contents) diff --git a/django/forms/formsets.py b/django/forms/formsets.py index d421770093..fd98c43405 100644 --- a/django/forms/formsets.py +++ b/django/forms/formsets.py @@ -250,9 +250,9 @@ class BaseFormSet(object): form -- i.e., from formset.clean(). Returns an empty ErrorList if there are none. """ - if self._non_form_errors is not None: - return self._non_form_errors - return self.error_class() + if self._non_form_errors is None: + self.full_clean() + return self._non_form_errors @property def errors(self): @@ -291,16 +291,20 @@ class BaseFormSet(object): def full_clean(self): """ - Cleans all of self.data and populates self._errors. + Cleans all of self.data and populates self._errors and + self._non_form_errors. """ self._errors = [] + self._non_form_errors = self.error_class() + if not self.is_bound: # Stop further processing. return for i in range(0, self.total_form_count()): form = self.forms[i] self._errors.append(form.errors) try: - if (self.validate_max and self.total_form_count() > self.max_num) or \ + if (self.validate_max and + self.total_form_count() - len(self.deleted_forms) > self.max_num) or \ self.management_form.cleaned_data[TOTAL_FORM_COUNT] > self.absolute_max: raise ValidationError(ungettext( "Please submit %d or fewer forms.", diff --git a/django/forms/models.py b/django/forms/models.py index af5cda8faf..65434a6f6e 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -13,12 +13,12 @@ from django.forms.forms import BaseForm, get_declared_fields from django.forms.formsets import BaseFormSet, formset_factory from django.forms.util import ErrorList from django.forms.widgets import (SelectMultiple, HiddenInput, - MultipleHiddenInput, media_property) + MultipleHiddenInput, media_property, CheckboxSelectMultiple) from django.utils.encoding import smart_text, force_text from django.utils.datastructures import SortedDict from django.utils import six from django.utils.text import get_text_list, capfirst -from django.utils.translation import ugettext_lazy as _, ugettext +from django.utils.translation import ugettext_lazy as _, ugettext, string_concat __all__ = ( @@ -85,6 +85,8 @@ def save_instance(form, instance, fields=None, fail_message='saved', for f in opts.many_to_many: if fields and f.name not in fields: continue + if exclude and f.name in exclude: + continue if f.name in cleaned_data: f.save_form_data(instance, cleaned_data[f.name]) if commit: @@ -136,7 +138,7 @@ def model_to_dict(instance, fields=None, exclude=None): data[f.name] = f.value_from_object(instance) return data -def fields_for_model(model, fields=None, exclude=None, widgets=None, formfield_callback=None): +def fields_for_model(model, fields=None, exclude=None, widgets=None, formfield_callback=None, localized_fields=None): """ Returns a ``SortedDict`` containing form fields for the given model. @@ -162,10 +164,12 @@ def fields_for_model(model, fields=None, exclude=None, widgets=None, formfield_c continue if exclude and f.name in exclude: continue + + kwargs = {} if widgets and f.name in widgets: - kwargs = {'widget': widgets[f.name]} - else: - kwargs = {} + kwargs['widget'] = widgets[f.name] + if localized_fields == ALL_FIELDS or (localized_fields and f.name in localized_fields): + kwargs['localize'] = True if formfield_callback is None: formfield = f.formfield(**kwargs) @@ -192,6 +196,7 @@ class ModelFormOptions(object): self.fields = getattr(options, 'fields', None) self.exclude = getattr(options, 'exclude', None) self.widgets = getattr(options, 'widgets', None) + self.localized_fields = getattr(options, 'localized_fields', None) class ModelFormMetaclass(type): @@ -215,7 +220,7 @@ class ModelFormMetaclass(type): # We check if a string was passed to `fields` or `exclude`, # which is likely to be a mistake where the user typed ('foo') instead # of ('foo',) - for opt in ['fields', 'exclude']: + for opt in ['fields', 'exclude', 'localized_fields']: value = getattr(opts, opt) if isinstance(value, six.string_types) and value != ALL_FIELDS: msg = ("%(model)s.Meta.%(opt)s cannot be a string. " @@ -235,15 +240,16 @@ class ModelFormMetaclass(type): warnings.warn("Creating a ModelForm without either the 'fields' attribute " "or the 'exclude' attribute is deprecated - form %s " "needs updating" % name, - PendingDeprecationWarning) + PendingDeprecationWarning, stacklevel=2) if opts.fields == ALL_FIELDS: # sentinel for fields_for_model to indicate "get the list of # fields from the model" opts.fields = None - fields = fields_for_model(opts.model, opts.fields, - opts.exclude, opts.widgets, formfield_callback) + fields = fields_for_model(opts.model, opts.fields, opts.exclude, + opts.widgets, formfield_callback, opts.localized_fields) + # make sure opts.fields doesn't specify an invalid field none_model_fields = [k for k, v in six.iteritems(fields) if not v] missing_fields = set(none_model_fields) - \ @@ -401,7 +407,8 @@ class BaseModelForm(BaseForm): else: fail_message = 'changed' return save_instance(self, self.instance, self._meta.fields, - fail_message, commit, construct=False) + fail_message, commit, self._meta.exclude, + construct=False) save.alters_data = True @@ -409,7 +416,7 @@ class ModelForm(six.with_metaclass(ModelFormMetaclass, BaseModelForm)): pass def modelform_factory(model, form=ModelForm, fields=None, exclude=None, - formfield_callback=None, widgets=None): + formfield_callback=None, widgets=None, localized_fields=None): """ Returns a ModelForm containing form fields for the given model. @@ -423,6 +430,8 @@ def modelform_factory(model, form=ModelForm, fields=None, exclude=None, ``widgets`` is a dictionary of model field names mapped to a widget. + ``localized_fields`` is a list of names of fields which should be localized. + ``formfield_callback`` is a callable that takes a model field and returns a form field. """ @@ -438,6 +447,8 @@ def modelform_factory(model, form=ModelForm, fields=None, exclude=None, attrs['exclude'] = exclude if widgets is not None: attrs['widgets'] = widgets + if localized_fields is not None: + attrs['localized_fields'] = localized_fields # If parent form class already has an inner Meta, the Meta we're # creating needs to inherit from the parent's inner meta. @@ -726,8 +737,8 @@ class BaseModelFormSet(BaseFormSet): def modelformset_factory(model, form=ModelForm, formfield_callback=None, formset=BaseModelFormSet, extra=1, can_delete=False, - can_order=False, max_num=None, fields=None, - exclude=None, widgets=None, validate_max=False): + can_order=False, max_num=None, fields=None, exclude=None, + widgets=None, validate_max=False, localized_fields=None): """ Returns a FormSet class for the given Django model class. """ @@ -748,7 +759,7 @@ def modelformset_factory(model, form=ModelForm, formfield_callback=None, form = modelform_factory(model, form=form, fields=fields, exclude=exclude, formfield_callback=formfield_callback, - widgets=widgets) + widgets=widgets, localized_fields=localized_fields) FormSet = formset_factory(form, formset, extra=extra, max_num=max_num, can_order=can_order, can_delete=can_delete, validate_max=validate_max) @@ -885,9 +896,9 @@ def _get_foreign_key(parent_model, model, fk_name=None, can_fail=False): def inlineformset_factory(parent_model, model, form=ModelForm, formset=BaseInlineFormSet, fk_name=None, - fields=None, exclude=None, - extra=3, can_order=False, can_delete=True, max_num=None, - formfield_callback=None, widgets=None, validate_max=False): + fields=None, exclude=None, extra=3, can_order=False, + can_delete=True, max_num=None, formfield_callback=None, + widgets=None, validate_max=False, localized_fields=None): """ Returns an ``InlineFormSet`` for the given kwargs. @@ -910,6 +921,7 @@ def inlineformset_factory(parent_model, model, form=ModelForm, 'max_num': max_num, 'widgets': widgets, 'validate_max': validate_max, + 'localized_fields': localized_fields, } FormSet = modelformset_factory(model, **kwargs) FormSet.fk = fk @@ -1095,6 +1107,10 @@ class ModelMultipleChoiceField(ModelChoiceField): super(ModelMultipleChoiceField, self).__init__(queryset, None, cache_choices, required, widget, label, initial, help_text, *args, **kwargs) + # Remove this in Django 1.8 + if isinstance(self.widget, SelectMultiple) and not isinstance(self.widget, CheckboxSelectMultiple): + msg = _('Hold down "Control", or "Command" on a Mac, to select more than one.') + self.help_text = string_concat(self.help_text, ' ', msg) def clean(self, value): if self.required and not value: diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 0e999f2ded..eeb435fa57 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -11,7 +11,7 @@ import cgi import sys from django.conf import settings -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import SuspiciousMultipartForm from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_text from django.utils import six @@ -48,9 +48,9 @@ class MultiPartParser(object): The standard ``META`` dictionary in Django request objects. :input_data: The raw post data, as a file-like object. - :upload_handler: - An UploadHandler instance that performs operations on the uploaded - data. + :upload_handlers: + A list of UploadHandler instances that perform operations on the uploaded + data. :encoding: The encoding with which to treat the incoming data. """ @@ -113,14 +113,15 @@ class MultiPartParser(object): if self._content_length == 0: return QueryDict('', encoding=self._encoding), MultiValueDict() - # See if the handler will want to take care of the parsing. - # This allows overriding everything if somebody wants it. + # See if any of the handlers take care of the parsing. + # This allows overriding everything if need be. for handler in handlers: result = handler.handle_raw_input(self._input_data, self._meta, self._content_length, self._boundary, encoding) + #Check to see if it was handled if result is not None: return result[0], result[1] @@ -369,7 +370,7 @@ class LazyStream(six.Iterator): if current_number == num_bytes]) if number_equal > 40: - raise SuspiciousOperation( + raise SuspiciousMultipartForm( "The multipart parser got stuck, which shouldn't happen with" " normal uploaded files. Check for malicious upload activity;" " if there is none, report this to the Django developers." diff --git a/django/http/request.py b/django/http/request.py index 749b9f2561..37aa1a355a 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -14,7 +14,7 @@ except ImportError: from django.conf import settings from django.core import signing -from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured +from django.core.exceptions import DisallowedHost, ImproperlyConfigured from django.core.files import uploadhandler from django.http.multipartparser import MultiPartParser from django.utils import six @@ -72,7 +72,7 @@ class HttpRequest(object): msg = "Invalid HTTP_HOST header: %r." % host if domain: msg += "You may need to add %r to ALLOWED_HOSTS." % domain - raise SuspiciousOperation(msg) + raise DisallowedHost(msg) def get_full_path(self): # RFC 3986 requires query string arguments to be in the ASCII range. @@ -238,11 +238,17 @@ class HttpRequest(object): def read(self, *args, **kwargs): self._read_started = True - return self._stream.read(*args, **kwargs) + try: + return self._stream.read(*args, **kwargs) + except IOError as e: + six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2]) def readline(self, *args, **kwargs): self._read_started = True - return self._stream.readline(*args, **kwargs) + try: + return self._stream.readline(*args, **kwargs) + except IOError as e: + six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2]) def xreadlines(self): while True: diff --git a/django/http/response.py b/django/http/response.py index 88ac8848c2..9aa49b1d5f 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -12,7 +12,7 @@ except ImportError: from django.conf import settings from django.core import signals from django.core import signing -from django.core.exceptions import SuspiciousOperation +from django.core.exceptions import DisallowedRedirect from django.http.cookie import SimpleCookie from django.utils import six, timezone from django.utils.encoding import force_bytes, iri_to_uri @@ -20,6 +20,65 @@ from django.utils.http import cookie_date from django.utils.six.moves import map +# See http://www.iana.org/assignments/http-status-codes +REASON_PHRASES = { + 100: 'CONTINUE', + 101: 'SWITCHING PROTOCOLS', + 102: 'PROCESSING', + 200: 'OK', + 201: 'CREATED', + 202: 'ACCEPTED', + 203: 'NON-AUTHORITATIVE INFORMATION', + 204: 'NO CONTENT', + 205: 'RESET CONTENT', + 206: 'PARTIAL CONTENT', + 207: 'MULTI-STATUS', + 208: 'ALREADY REPORTED', + 226: 'IM USED', + 300: 'MULTIPLE CHOICES', + 301: 'MOVED PERMANENTLY', + 302: 'FOUND', + 303: 'SEE OTHER', + 304: 'NOT MODIFIED', + 305: 'USE PROXY', + 306: 'RESERVED', + 307: 'TEMPORARY REDIRECT', + 400: 'BAD REQUEST', + 401: 'UNAUTHORIZED', + 402: 'PAYMENT REQUIRED', + 403: 'FORBIDDEN', + 404: 'NOT FOUND', + 405: 'METHOD NOT ALLOWED', + 406: 'NOT ACCEPTABLE', + 407: 'PROXY AUTHENTICATION REQUIRED', + 408: 'REQUEST TIMEOUT', + 409: 'CONFLICT', + 410: 'GONE', + 411: 'LENGTH REQUIRED', + 412: 'PRECONDITION FAILED', + 413: 'REQUEST ENTITY TOO LARGE', + 414: 'REQUEST-URI TOO LONG', + 415: 'UNSUPPORTED MEDIA TYPE', + 416: 'REQUESTED RANGE NOT SATISFIABLE', + 417: 'EXPECTATION FAILED', + 418: "I'M A TEAPOT", + 422: 'UNPROCESSABLE ENTITY', + 423: 'LOCKED', + 424: 'FAILED DEPENDENCY', + 426: 'UPGRADE REQUIRED', + 500: 'INTERNAL SERVER ERROR', + 501: 'NOT IMPLEMENTED', + 502: 'BAD GATEWAY', + 503: 'SERVICE UNAVAILABLE', + 504: 'GATEWAY TIMEOUT', + 505: 'HTTP VERSION NOT SUPPORTED', + 506: 'VARIANT ALSO NEGOTIATES', + 507: 'INSUFFICIENT STORAGE', + 508: 'LOOP DETECTED', + 510: 'NOT EXTENDED', +} + + class BadHeaderError(ValueError): pass @@ -33,8 +92,9 @@ class HttpResponseBase(six.Iterator): """ status_code = 200 + reason_phrase = None # Use default reason phrase for status code. - def __init__(self, content_type=None, status=None, mimetype=None): + def __init__(self, content_type=None, status=None, reason=None, mimetype=None): # _headers is a mapping of the lower-case name to the original case of # the header (required for working with legacy systems) and the header # value. Both the name of the header and its value are ASCII strings. @@ -53,9 +113,13 @@ class HttpResponseBase(six.Iterator): content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, self._charset) self.cookies = SimpleCookie() - if status: + if status is not None: self.status_code = status - + if reason is not None: + self.reason_phrase = reason + elif self.reason_phrase is None: + self.reason_phrase = REASON_PHRASES.get(self.status_code, + 'UNKNOWN STATUS CODE') self['Content-Type'] = content_type def serialize_headers(self): @@ -388,7 +452,7 @@ class HttpResponseRedirectBase(HttpResponse): def __init__(self, redirect_to, *args, **kwargs): parsed = urlparse(redirect_to) if parsed.scheme and parsed.scheme not in self.allowed_schemes: - raise SuspiciousOperation("Unsafe redirect to URL with protocol '%s'" % parsed.scheme) + raise DisallowedRedirect("Unsafe redirect to URL with protocol '%s'" % parsed.scheme) super(HttpResponseRedirectBase, self).__init__(*args, **kwargs) self['Location'] = iri_to_uri(redirect_to) diff --git a/django/http/utils.py b/django/http/utils.py index fcb3fecb6c..e13dc4cbb6 100644 --- a/django/http/utils.py +++ b/django/http/utils.py @@ -31,13 +31,13 @@ def conditional_content_removal(request, response): if response.streaming: response.streaming_content = [] else: - response.content = '' + response.content = b'' response['Content-Length'] = '0' if request.method == 'HEAD': if response.streaming: response.streaming_content = [] else: - response.content = '' + response.content = b'' return response diff --git a/django/middleware/cache.py b/django/middleware/cache.py index 83860e15f3..e13a8c3918 100644 --- a/django/middleware/cache.py +++ b/django/middleware/cache.py @@ -29,11 +29,6 @@ More details about how the caching works: of the response's "Cache-Control" header, falling back to the CACHE_MIDDLEWARE_SECONDS setting if the section was not found. -* If CACHE_MIDDLEWARE_ANONYMOUS_ONLY is set to True, only anonymous requests - (i.e., those not made by a logged-in user) will be cached. This is a simple - and effective way of avoiding the caching of the Django admin (and any other - user-specific content). - * This middleware expects that a HEAD request is answered with the same response headers exactly like the corresponding GET request. @@ -48,6 +43,8 @@ More details about how the caching works: """ +import warnings + from django.conf import settings from django.core.cache import get_cache, DEFAULT_CACHE_ALIAS from django.utils.cache import get_cache_key, learn_cache_key, patch_response_headers, get_max_age @@ -200,5 +197,9 @@ class CacheMiddleware(UpdateCacheMiddleware, FetchFromCacheMiddleware): else: self.cache_anonymous_only = cache_anonymous_only + if self.cache_anonymous_only: + msg = "CACHE_MIDDLEWARE_ANONYMOUS_ONLY has been deprecated and will be removed in Django 1.8." + warnings.warn(msg, PendingDeprecationWarning, stacklevel=1) + self.cache = get_cache(self.cache_alias, **cache_kwargs) self.cache_timeout = self.cache.default_timeout diff --git a/django/middleware/common.py b/django/middleware/common.py index 92f8cb3992..2c76c47756 100644 --- a/django/middleware/common.py +++ b/django/middleware/common.py @@ -7,6 +7,7 @@ from django.conf import settings from django.core.mail import mail_managers from django.core import urlresolvers from django import http +from django.utils.encoding import force_text from django.utils.http import urlquote from django.utils import six @@ -84,7 +85,7 @@ class CommonMiddleware(object): return if new_url[0]: newurl = "%s://%s%s" % ( - request.is_secure() and 'https' or 'http', + 'https' if request.is_secure() else 'http', new_url[0], urlquote(new_url[1])) else: newurl = urlquote(new_url[1]) @@ -140,16 +141,18 @@ class BrokenLinkEmailsMiddleware(object): if response.status_code == 404 and not settings.DEBUG: domain = request.get_host() path = request.get_full_path() - referer = request.META.get('HTTP_REFERER', '') - is_internal = self.is_internal_request(domain, referer) - is_not_search_engine = '?' not in referer - is_ignorable = self.is_ignorable_404(path) - if referer and (is_internal or is_not_search_engine) and not is_ignorable: + referer = force_text(request.META.get('HTTP_REFERER', ''), errors='replace') + + if not self.is_ignorable_request(request, path, domain, referer): ua = request.META.get('HTTP_USER_AGENT', '<none>') ip = request.META.get('REMOTE_ADDR', '<none>') mail_managers( - "Broken %slink on %s" % (('INTERNAL ' if is_internal else ''), domain), - "Referrer: %s\nRequested URL: %s\nUser agent: %s\nIP address: %s\n" % (referer, path, ua, ip), + "Broken %slink on %s" % ( + ('INTERNAL ' if self.is_internal_request(domain, referer) else ''), + domain + ), + "Referrer: %s\nRequested URL: %s\nUser agent: %s\n" + "IP address: %s\n" % (referer, path, ua, ip), fail_silently=True) return response @@ -158,10 +161,14 @@ class BrokenLinkEmailsMiddleware(object): Returns True if the referring URL is the same domain as the current request. """ # Different subdomains are treated as different domains. - return re.match("^https?://%s/" % re.escape(domain), referer) + return bool(re.match("^https?://%s/" % re.escape(domain), referer)) - def is_ignorable_404(self, uri): + def is_ignorable_request(self, request, uri, domain, referer): """ - Returns True if a 404 at the given URL *shouldn't* notify the site managers. + Returns True if the given request *shouldn't* notify the site managers. """ + # '?' in referer is identified as search engine source + if (not referer or + (not self.is_internal_request(domain, referer) and '?' in referer)): + return True return any(pattern.search(uri) for pattern in settings.IGNORABLE_404_URLS) diff --git a/django/middleware/csrf.py b/django/middleware/csrf.py index 423034478b..1b5732fbbf 100644 --- a/django/middleware/csrf.py +++ b/django/middleware/csrf.py @@ -53,6 +53,14 @@ def get_token(request): return request.META.get("CSRF_COOKIE", None) +def rotate_token(request): + """ + Changes the CSRF token in use for a request - should be done on login + for security purposes. + """ + request.META["CSRF_COOKIE"] = _get_new_csrf_key() + + def _sanitize_token(token): # Allow only alphanum if len(token) > CSRF_KEY_LENGTH: @@ -83,6 +91,13 @@ class CsrfViewMiddleware(object): return None def _reject(self, request, reason): + logger.warning('Forbidden (%s): %s', + reason, request.path, + extra={ + 'status_code': 403, + 'request': request, + } + ) return _get_failure_view()(request, reason=reason) def process_view(self, request, callback, callback_args, callback_kwargs): @@ -134,38 +149,18 @@ class CsrfViewMiddleware(object): # we can use strict Referer checking. referer = request.META.get('HTTP_REFERER') if referer is None: - logger.warning('Forbidden (%s): %s', - REASON_NO_REFERER, request.path, - extra={ - 'status_code': 403, - 'request': request, - } - ) return self._reject(request, REASON_NO_REFERER) # Note that request.get_host() includes the port. good_referer = 'https://%s/' % request.get_host() if not same_origin(referer, good_referer): reason = REASON_BAD_REFERER % (referer, good_referer) - logger.warning('Forbidden (%s): %s', reason, request.path, - extra={ - 'status_code': 403, - 'request': request, - } - ) return self._reject(request, reason) if csrf_token is None: # No CSRF cookie. For POST requests, we insist on a CSRF cookie, # and in this way we can avoid all CSRF attacks, including login # CSRF. - logger.warning('Forbidden (%s): %s', - REASON_NO_CSRF_COOKIE, request.path, - extra={ - 'status_code': 403, - 'request': request, - } - ) return self._reject(request, REASON_NO_CSRF_COOKIE) # Check non-cookie token for match. @@ -179,13 +174,6 @@ class CsrfViewMiddleware(object): request_csrf_token = request.META.get('HTTP_X_CSRFTOKEN', '') if not constant_time_compare(request_csrf_token, csrf_token): - logger.warning('Forbidden (%s): %s', - REASON_BAD_TOKEN, request.path, - extra={ - 'status_code': 403, - 'request': request, - } - ) return self._reject(request, REASON_BAD_TOKEN) return self._accept(request) diff --git a/django/middleware/doc.py b/django/middleware/doc.py index ee3fe2cb2f..1af7b6150a 100644 --- a/django/middleware/doc.py +++ b/django/middleware/doc.py @@ -1,23 +1,6 @@ -from django.conf import settings -from django import http +"""XViewMiddleware has been moved to django.contrib.admindocs.middleware.""" -class XViewMiddleware(object): - """ - Adds an X-View header to internal HEAD requests -- used by the documentation system. - """ - def process_view(self, request, view_func, view_args, view_kwargs): - """ - If the request method is HEAD and either the IP is internal or the - user is a logged-in staff member, quickly return with an x-header - indicating the view function. This is used by the documentation module - to lookup the view function for an arbitrary page. - """ - assert hasattr(request, 'user'), ( - "The XView middleware requires authentication middleware to be " - "installed. Edit your MIDDLEWARE_CLASSES setting to insert " - "'django.contrib.auth.middleware.AuthenticationMiddleware'.") - if request.method == 'HEAD' and (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or - (request.user.is_active and request.user.is_staff)): - response = http.HttpResponse() - response['X-View'] = "%s.%s" % (view_func.__module__, view_func.__name__) - return response +import warnings +warnings.warn(__doc__, PendingDeprecationWarning, stacklevel=2) + +from django.contrib.admindocs.middleware import XViewMiddleware diff --git a/django/middleware/locale.py b/django/middleware/locale.py index 9b2ef8ff32..4e0a4753ce 100644 --- a/django/middleware/locale.py +++ b/django/middleware/locale.py @@ -6,6 +6,7 @@ from django.core.urlresolvers import (is_valid_path, get_resolver, from django.http import HttpResponseRedirect from django.utils.cache import patch_vary_headers from django.utils import translation +from django.utils.datastructures import SortedDict class LocaleMiddleware(object): @@ -18,7 +19,7 @@ class LocaleMiddleware(object): """ def __init__(self): - self._supported_languages = dict(settings.LANGUAGES) + self._supported_languages = SortedDict(settings.LANGUAGES) self._is_language_prefix_patterns_used = False for url_pattern in get_resolver(None).url_patterns: if isinstance(url_pattern, LocaleRegexURLResolver): @@ -48,10 +49,14 @@ class LocaleMiddleware(object): if path_valid: language_url = "%s://%s/%s%s" % ( - request.is_secure() and 'https' or 'http', + 'https' if request.is_secure() else 'http', request.get_host(), language, request.get_full_path()) return HttpResponseRedirect(language_url) + # Store language back into session if it is not present + if hasattr(request, 'session'): + request.session.setdefault('django_language', language) + if not (self.is_language_prefix_patterns_used() and language_from_path): patch_vary_headers(response, ('Accept-Language',)) diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index 88526e5a20..4201cfeb67 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -14,7 +14,7 @@ from django.utils import formats from django.utils.dateformat import format, time_format from django.utils.encoding import force_text, iri_to_uri from django.utils.html import (conditional_escape, escapejs, fix_ampersands, - escape, urlize as urlize_impl, linebreaks, strip_tags) + escape, urlize as urlize_impl, linebreaks, strip_tags, avoid_wrapping) from django.utils.http import urlquote from django.utils.text import Truncator, wrap, phone2numeric from django.utils.safestring import mark_safe, SafeData, mark_for_escaping @@ -810,7 +810,8 @@ def filesizeformat(bytes): try: bytes = float(bytes) except (TypeError,ValueError,UnicodeDecodeError): - return ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0} + value = ungettext("%(size)d byte", "%(size)d bytes", 0) % {'size': 0} + return avoid_wrapping(value) filesize_number_format = lambda value: formats.number_format(round(value, 1), 1) @@ -821,16 +822,19 @@ def filesizeformat(bytes): PB = 1<<50 if bytes < KB: - return ungettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes} - if bytes < MB: - return ugettext("%s KB") % filesize_number_format(bytes / KB) - if bytes < GB: - return ugettext("%s MB") % filesize_number_format(bytes / MB) - if bytes < TB: - return ugettext("%s GB") % filesize_number_format(bytes / GB) - if bytes < PB: - return ugettext("%s TB") % filesize_number_format(bytes / TB) - return ugettext("%s PB") % filesize_number_format(bytes / PB) + value = ungettext("%(size)d byte", "%(size)d bytes", bytes) % {'size': bytes} + elif bytes < MB: + value = ugettext("%s KB") % filesize_number_format(bytes / KB) + elif bytes < GB: + value = ugettext("%s MB") % filesize_number_format(bytes / MB) + elif bytes < TB: + value = ugettext("%s GB") % filesize_number_format(bytes / GB) + elif bytes < PB: + value = ugettext("%s TB") % filesize_number_format(bytes / TB) + else: + value = ugettext("%s PB") % filesize_number_format(bytes / PB) + + return avoid_wrapping(value) @register.filter(is_safe=False) def pluralize(value, arg='s'): diff --git a/django/test/client.py b/django/test/client.py index 46f55d7cdc..2ed0df8fea 100644 --- a/django/test/client.py +++ b/django/test/client.py @@ -13,7 +13,7 @@ except ImportError: # Python 2 from urlparse import urlparse, urlsplit from django.conf import settings -from django.contrib.auth import authenticate, login +from django.contrib.auth import authenticate, login, logout, get_user_model from django.core.handlers.base import BaseHandler from django.core.handlers.wsgi import WSGIRequest from django.core.signals import (request_started, request_finished, @@ -571,11 +571,17 @@ class Client(RequestFactory): Causes the authenticated user to be logged out. """ - session = import_module(settings.SESSION_ENGINE).SessionStore() - session_cookie = self.cookies.get(settings.SESSION_COOKIE_NAME) - if session_cookie: - session.delete(session_key=session_cookie.value) - self.cookies = SimpleCookie() + request = HttpRequest() + engine = import_module(settings.SESSION_ENGINE) + UserModel = get_user_model() + if self.session: + request.session = self.session + uid = self.session.get("_auth_user_id") + if uid: + request.user = UserModel._default_manager.get(pk=uid) + else: + request.session = engine.SessionStore() + logout(request) def _handle_redirects(self, response, **extra): "Follows any redirects by requesting responses from the server using GET." diff --git a/django/test/simple.py b/django/test/simple.py index 5117c6452f..f28b8a2830 100644 --- a/django/test/simple.py +++ b/django/test/simple.py @@ -3,14 +3,15 @@ This module is pending deprecation as of Django 1.6 and will be removed in version 1.8. """ - +import json +import re import unittest as real_unittest import warnings from django.db.models import get_app, get_apps from django.test import _doctest as doctest from django.test import runner -from django.test.testcases import OutputChecker, DocTestRunner +from django.test.utils import compare_xml, strip_quotes from django.utils import unittest from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule @@ -25,6 +26,71 @@ warnings.warn( # The module name for tests outside models.py TEST_MODULE = 'tests' + +normalize_long_ints = lambda s: re.sub(r'(?<![\w])(\d+)L(?![\w])', '\\1', s) +normalize_decimals = lambda s: re.sub(r"Decimal\('(\d+(\.\d*)?)'\)", + lambda m: "Decimal(\"%s\")" % m.groups()[0], s) + + +class OutputChecker(doctest.OutputChecker): + def check_output(self, want, got, optionflags): + """ + The entry method for doctest output checking. Defers to a sequence of + child checkers + """ + checks = (self.check_output_default, + self.check_output_numeric, + self.check_output_xml, + self.check_output_json) + for check in checks: + if check(want, got, optionflags): + return True + return False + + def check_output_default(self, want, got, optionflags): + """ + The default comparator provided by doctest - not perfect, but good for + most purposes + """ + return doctest.OutputChecker.check_output(self, want, got, optionflags) + + def check_output_numeric(self, want, got, optionflags): + """Doctest does an exact string comparison of output, which means that + some numerically equivalent values aren't equal. This check normalizes + * long integers (22L) so that they equal normal integers. (22) + * Decimals so that they are comparable, regardless of the change + made to __repr__ in Python 2.6. + """ + return doctest.OutputChecker.check_output(self, + normalize_decimals(normalize_long_ints(want)), + normalize_decimals(normalize_long_ints(got)), + optionflags) + + def check_output_xml(self, want, got, optionsflags): + try: + return compare_xml(want, got) + except Exception: + return False + + def check_output_json(self, want, got, optionsflags): + """ + Tries to compare want and got as if they were JSON-encoded data + """ + want, got = strip_quotes(want, got) + try: + want_json = json.loads(want) + got_json = json.loads(got) + except Exception: + return False + return want_json == got_json + + +class DocTestRunner(doctest.DocTestRunner): + def __init__(self, *args, **kwargs): + doctest.DocTestRunner.__init__(self, *args, **kwargs) + self.optionflags = doctest.ELLIPSIS + + doctestOutputChecker = OutputChecker() diff --git a/django/test/testcases.py b/django/test/testcases.py index 6fe6b9c397..08c03154b1 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -30,12 +30,11 @@ from django.core.urlresolvers import clear_url_caches, set_urlconf from django.db import connection, connections, DEFAULT_DB_ALIAS, transaction from django.forms.fields import CharField from django.http import QueryDict -from django.test import _doctest as doctest from django.test.client import Client from django.test.html import HTMLParseError, parse_html from django.test.signals import template_rendered from django.test.utils import (CaptureQueriesContext, ContextList, - override_settings, compare_xml, strip_quotes) + override_settings, compare_xml) from django.utils import six, unittest as ut2 from django.utils.encoding import force_text from django.utils.unittest import skipIf # Imported here for backward compatibility @@ -43,15 +42,10 @@ from django.utils.unittest.util import safe_repr from django.views.static import serve -__all__ = ('DocTestRunner', 'OutputChecker', 'TestCase', 'TransactionTestCase', +__all__ = ('TestCase', 'TransactionTestCase', 'SimpleTestCase', 'skipIfDBFeature', 'skipUnlessDBFeature') -normalize_long_ints = lambda s: re.sub(r'(?<![\w])(\d+)L(?![\w])', '\\1', s) -normalize_decimals = lambda s: re.sub(r"Decimal\('(\d+(\.\d*)?)'\)", - lambda m: "Decimal(\"%s\")" % m.groups()[0], s) - - def to_list(value): """ Puts value into a list if it's not already one. @@ -96,75 +90,6 @@ def assert_and_parse_html(self, html, user_msg, msg): return dom -class OutputChecker(doctest.OutputChecker): - def __init__(self): - warnings.warn( - "The django.test.testcases.OutputChecker class is deprecated; " - "use the doctest module from the Python standard library instead.", - PendingDeprecationWarning) - - def check_output(self, want, got, optionflags): - """ - The entry method for doctest output checking. Defers to a sequence of - child checkers - """ - checks = (self.check_output_default, - self.check_output_numeric, - self.check_output_xml, - self.check_output_json) - for check in checks: - if check(want, got, optionflags): - return True - return False - - def check_output_default(self, want, got, optionflags): - """ - The default comparator provided by doctest - not perfect, but good for - most purposes - """ - return doctest.OutputChecker.check_output(self, want, got, optionflags) - - def check_output_numeric(self, want, got, optionflags): - """Doctest does an exact string comparison of output, which means that - some numerically equivalent values aren't equal. This check normalizes - * long integers (22L) so that they equal normal integers. (22) - * Decimals so that they are comparable, regardless of the change - made to __repr__ in Python 2.6. - """ - return doctest.OutputChecker.check_output(self, - normalize_decimals(normalize_long_ints(want)), - normalize_decimals(normalize_long_ints(got)), - optionflags) - - def check_output_xml(self, want, got, optionsflags): - try: - return compare_xml(want, got) - except Exception: - return False - - def check_output_json(self, want, got, optionsflags): - """ - Tries to compare want and got as if they were JSON-encoded data - """ - want, got = strip_quotes(want, got) - try: - want_json = json.loads(want) - got_json = json.loads(got) - except Exception: - return False - return want_json == got_json - - -class DocTestRunner(doctest.DocTestRunner): - def __init__(self, *args, **kwargs): - warnings.warn( - "The django.test.testcases.DocTestRunner class is deprecated; " - "use the doctest module from the Python standard library instead.", - PendingDeprecationWarning) - doctest.DocTestRunner.__init__(self, *args, **kwargs) - self.optionflags = doctest.ELLIPSIS - - class _AssertNumQueriesContext(CaptureQueriesContext): def __init__(self, test_case, num, connection): self.test_case = test_case @@ -231,6 +156,10 @@ class _AssertTemplateNotUsedContext(_AssertTemplateUsedContext): class SimpleTestCase(ut2.TestCase): + # The class we'll use for the test client self.client. + # Can be overridden in derived classes. + client_class = Client + _warn_txt = ("save_warnings_state/restore_warnings_state " "django.test.*TestCase methods are deprecated. Use Python's " "warnings.catch_warnings context manager instead.") @@ -264,10 +193,31 @@ class SimpleTestCase(ut2.TestCase): return def _pre_setup(self): - pass + """Performs any pre-test setup. This includes: + + * If the Test Case class has a 'urls' member, replace the + ROOT_URLCONF with it. + * Clearing the mail test outbox. + """ + self.client = self.client_class() + self._urlconf_setup() + mail.outbox = [] + + def _urlconf_setup(self): + set_urlconf(None) + if hasattr(self, 'urls'): + self._old_root_urlconf = settings.ROOT_URLCONF + settings.ROOT_URLCONF = self.urls + clear_url_caches() def _post_teardown(self): - pass + self._urlconf_teardown() + + def _urlconf_teardown(self): + set_urlconf(None) + if hasattr(self, '_old_root_urlconf'): + settings.ROOT_URLCONF = self._old_root_urlconf + clear_url_caches() def save_warnings_state(self): """ @@ -291,258 +241,6 @@ class SimpleTestCase(ut2.TestCase): """ return override_settings(**kwargs) - def assertRaisesMessage(self, expected_exception, expected_message, - callable_obj=None, *args, **kwargs): - """ - Asserts that the message in a raised exception matches the passed - value. - - Args: - expected_exception: Exception class expected to be raised. - expected_message: expected error message string value. - callable_obj: Function to be called. - args: Extra args. - kwargs: Extra kwargs. - """ - return six.assertRaisesRegex(self, expected_exception, - re.escape(expected_message), callable_obj, *args, **kwargs) - - def assertFieldOutput(self, fieldclass, valid, invalid, field_args=None, - field_kwargs=None, empty_value=''): - """ - Asserts that a form field behaves correctly with various inputs. - - Args: - fieldclass: the class of the field to be tested. - valid: a dictionary mapping valid inputs to their expected - cleaned values. - invalid: a dictionary mapping invalid inputs to one or more - raised error messages. - field_args: the args passed to instantiate the field - field_kwargs: the kwargs passed to instantiate the field - empty_value: the expected clean output for inputs in empty_values - - """ - if field_args is None: - field_args = [] - if field_kwargs is None: - field_kwargs = {} - required = fieldclass(*field_args, **field_kwargs) - optional = fieldclass(*field_args, - **dict(field_kwargs, required=False)) - # test valid inputs - for input, output in valid.items(): - self.assertEqual(required.clean(input), output) - self.assertEqual(optional.clean(input), output) - # test invalid inputs - for input, errors in invalid.items(): - with self.assertRaises(ValidationError) as context_manager: - required.clean(input) - self.assertEqual(context_manager.exception.messages, errors) - - with self.assertRaises(ValidationError) as context_manager: - optional.clean(input) - self.assertEqual(context_manager.exception.messages, errors) - # test required inputs - error_required = [force_text(required.error_messages['required'])] - for e in required.empty_values: - with self.assertRaises(ValidationError) as context_manager: - required.clean(e) - self.assertEqual(context_manager.exception.messages, - error_required) - self.assertEqual(optional.clean(e), empty_value) - # test that max_length and min_length are always accepted - if issubclass(fieldclass, CharField): - field_kwargs.update({'min_length':2, 'max_length':20}) - self.assertTrue(isinstance(fieldclass(*field_args, **field_kwargs), - fieldclass)) - - def assertHTMLEqual(self, html1, html2, msg=None): - """ - Asserts that two HTML snippets are semantically the same. - Whitespace in most cases is ignored, and attribute ordering is not - significant. The passed-in arguments must be valid HTML. - """ - dom1 = assert_and_parse_html(self, html1, msg, - 'First argument is not valid HTML:') - dom2 = assert_and_parse_html(self, html2, msg, - 'Second argument is not valid HTML:') - - if dom1 != dom2: - standardMsg = '%s != %s' % ( - safe_repr(dom1, True), safe_repr(dom2, True)) - diff = ('\n' + '\n'.join(difflib.ndiff( - six.text_type(dom1).splitlines(), - six.text_type(dom2).splitlines()))) - standardMsg = self._truncateMessage(standardMsg, diff) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertHTMLNotEqual(self, html1, html2, msg=None): - """Asserts that two HTML snippets are not semantically equivalent.""" - dom1 = assert_and_parse_html(self, html1, msg, - 'First argument is not valid HTML:') - dom2 = assert_and_parse_html(self, html2, msg, - 'Second argument is not valid HTML:') - - if dom1 == dom2: - standardMsg = '%s == %s' % ( - safe_repr(dom1, True), safe_repr(dom2, True)) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertInHTML(self, needle, haystack, count = None, msg_prefix=''): - needle = assert_and_parse_html(self, needle, None, - 'First argument is not valid HTML:') - haystack = assert_and_parse_html(self, haystack, None, - 'Second argument is not valid HTML:') - real_count = haystack.count(needle) - if count is not None: - self.assertEqual(real_count, count, - msg_prefix + "Found %d instances of '%s' in response" - " (expected %d)" % (real_count, needle, count)) - else: - self.assertTrue(real_count != 0, - msg_prefix + "Couldn't find '%s' in response" % needle) - - def assertJSONEqual(self, raw, expected_data, msg=None): - try: - data = json.loads(raw) - except ValueError: - self.fail("First argument is not valid JSON: %r" % raw) - if isinstance(expected_data, six.string_types): - try: - expected_data = json.loads(expected_data) - except ValueError: - self.fail("Second argument is not valid JSON: %r" % expected_data) - self.assertEqual(data, expected_data, msg=msg) - - def assertXMLEqual(self, xml1, xml2, msg=None): - """ - Asserts that two XML snippets are semantically the same. - Whitespace in most cases is ignored, and attribute ordering is not - significant. The passed-in arguments must be valid XML. - """ - try: - result = compare_xml(xml1, xml2) - except Exception as e: - standardMsg = 'First or second argument is not valid XML\n%s' % e - self.fail(self._formatMessage(msg, standardMsg)) - else: - if not result: - standardMsg = '%s != %s' % (safe_repr(xml1, True), safe_repr(xml2, True)) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertXMLNotEqual(self, xml1, xml2, msg=None): - """ - Asserts that two XML snippets are not semantically equivalent. - Whitespace in most cases is ignored, and attribute ordering is not - significant. The passed-in arguments must be valid XML. - """ - try: - result = compare_xml(xml1, xml2) - except Exception as e: - standardMsg = 'First or second argument is not valid XML\n%s' % e - self.fail(self._formatMessage(msg, standardMsg)) - else: - if result: - standardMsg = '%s == %s' % (safe_repr(xml1, True), safe_repr(xml2, True)) - self.fail(self._formatMessage(msg, standardMsg)) - - -class TransactionTestCase(SimpleTestCase): - - # The class we'll use for the test client self.client. - # Can be overridden in derived classes. - client_class = Client - - # Subclasses can ask for resetting of auto increment sequence before each - # test case - reset_sequences = False - - def _pre_setup(self): - """Performs any pre-test setup. This includes: - - * Flushing the database. - * If the Test Case class has a 'fixtures' member, installing the - named fixtures. - * If the Test Case class has a 'urls' member, replace the - ROOT_URLCONF with it. - * Clearing the mail test outbox. - """ - self.client = self.client_class() - self._fixture_setup() - self._urlconf_setup() - mail.outbox = [] - - def _databases_names(self, include_mirrors=True): - # If the test case has a multi_db=True flag, act on all databases, - # including mirrors or not. Otherwise, just on the default DB. - if getattr(self, 'multi_db', False): - return [alias for alias in connections - if include_mirrors or not connections[alias].settings_dict['TEST_MIRROR']] - else: - return [DEFAULT_DB_ALIAS] - - def _reset_sequences(self, db_name): - conn = connections[db_name] - if conn.features.supports_sequence_reset: - sql_list = \ - conn.ops.sequence_reset_by_name_sql(no_style(), - conn.introspection.sequence_list()) - if sql_list: - with transaction.commit_on_success_unless_managed(using=db_name): - cursor = conn.cursor() - for sql in sql_list: - cursor.execute(sql) - - def _fixture_setup(self): - for db_name in self._databases_names(include_mirrors=False): - # Reset sequences - if self.reset_sequences: - self._reset_sequences(db_name) - - if hasattr(self, 'fixtures'): - # We have to use this slightly awkward syntax due to the fact - # that we're using *args and **kwargs together. - call_command('loaddata', *self.fixtures, - **{'verbosity': 0, 'database': db_name, 'skip_validation': True}) - - def _urlconf_setup(self): - set_urlconf(None) - if hasattr(self, 'urls'): - self._old_root_urlconf = settings.ROOT_URLCONF - settings.ROOT_URLCONF = self.urls - clear_url_caches() - - def _post_teardown(self): - """ Performs any post-test things. This includes: - - * Putting back the original ROOT_URLCONF if it was changed. - * Force closing the connection, so that the next test gets - a clean cursor. - """ - self._fixture_teardown() - self._urlconf_teardown() - # Some DB cursors include SQL statements as part of cursor - # creation. If you have a test that does rollback, the effect - # of these statements is lost, which can effect the operation - # of tests (e.g., losing a timezone setting causing objects to - # be created with the wrong time). - # To make sure this doesn't happen, get a clean connection at the - # start of every test. - for conn in connections.all(): - conn.close() - - def _fixture_teardown(self): - for db in self._databases_names(include_mirrors=False): - call_command('flush', verbosity=0, interactive=False, database=db, - skip_validation=True, reset_sequences=False) - - def _urlconf_teardown(self): - set_urlconf(None) - if hasattr(self, '_old_root_urlconf'): - settings.ROOT_URLCONF = self._old_root_urlconf - clear_url_caches() - def assertRedirects(self, response, expected_url, status_code=302, target_status_code=200, host=None, msg_prefix=''): """Asserts that a response redirected to a specific URL, and that the @@ -736,6 +434,83 @@ class TransactionTestCase(SimpleTestCase): self.fail(msg_prefix + "The form '%s' was not used to render the" " response" % form) + def assertFormsetError(self, response, formset, form_index, field, errors, + msg_prefix=''): + """ + Asserts that a formset used to render the response has a specific error. + + For field errors, specify the ``form_index`` and the ``field``. + For non-field errors, specify the ``form_index`` and the ``field`` as + None. + For non-form errors, specify ``form_index`` as None and the ``field`` + as None. + """ + # Add punctuation to msg_prefix + if msg_prefix: + msg_prefix += ": " + + # Put context(s) into a list to simplify processing. + contexts = to_list(response.context) + if not contexts: + self.fail(msg_prefix + 'Response did not use any contexts to ' + 'render the response') + + # Put error(s) into a list to simplify processing. + errors = to_list(errors) + + # Search all contexts for the error. + found_formset = False + for i, context in enumerate(contexts): + if formset not in context: + continue + found_formset = True + for err in errors: + if field is not None: + if field in context[formset].forms[form_index].errors: + field_errors = context[formset].forms[form_index].errors[field] + self.assertTrue(err in field_errors, + msg_prefix + "The field '%s' on formset '%s', " + "form %d in context %d does not contain the " + "error '%s' (actual errors: %s)" % + (field, formset, form_index, i, err, + repr(field_errors))) + elif field in context[formset].forms[form_index].fields: + self.fail(msg_prefix + "The field '%s' " + "on formset '%s', form %d in " + "context %d contains no errors" % + (field, formset, form_index, i)) + else: + self.fail(msg_prefix + "The formset '%s', form %d in " + "context %d does not contain the field '%s'" % + (formset, form_index, i, field)) + elif form_index is not None: + non_field_errors = context[formset].forms[form_index].non_field_errors() + self.assertFalse(len(non_field_errors) == 0, + msg_prefix + "The formset '%s', form %d in " + "context %d does not contain any non-field " + "errors." % (formset, form_index, i)) + self.assertTrue(err in non_field_errors, + msg_prefix + "The formset '%s', form %d " + "in context %d does not contain the " + "non-field error '%s' " + "(actual errors: %s)" % + (formset, form_index, i, err, + repr(non_field_errors))) + else: + non_form_errors = context[formset].non_form_errors() + self.assertFalse(len(non_form_errors) == 0, + msg_prefix + "The formset '%s' in " + "context %d does not contain any " + "non-form errors." % (formset, i)) + self.assertTrue(err in non_form_errors, + msg_prefix + "The formset '%s' in context " + "%d does not contain the " + "non-form error '%s' (actual errors: %s)" % + (formset, i, err, repr(non_form_errors))) + if not found_formset: + self.fail(msg_prefix + "The formset '%s' was not used to render " + "the response" % formset) + def assertTemplateUsed(self, response=None, template_name=None, msg_prefix=''): """ Asserts that the template with the provided name was used in rendering @@ -787,6 +562,236 @@ class TransactionTestCase(SimpleTestCase): msg_prefix + "Template '%s' was used unexpectedly in rendering" " the response" % template_name) + def assertRaisesMessage(self, expected_exception, expected_message, + callable_obj=None, *args, **kwargs): + """ + Asserts that the message in a raised exception matches the passed + value. + + Args: + expected_exception: Exception class expected to be raised. + expected_message: expected error message string value. + callable_obj: Function to be called. + args: Extra args. + kwargs: Extra kwargs. + """ + return six.assertRaisesRegex(self, expected_exception, + re.escape(expected_message), callable_obj, *args, **kwargs) + + def assertFieldOutput(self, fieldclass, valid, invalid, field_args=None, + field_kwargs=None, empty_value=''): + """ + Asserts that a form field behaves correctly with various inputs. + + Args: + fieldclass: the class of the field to be tested. + valid: a dictionary mapping valid inputs to their expected + cleaned values. + invalid: a dictionary mapping invalid inputs to one or more + raised error messages. + field_args: the args passed to instantiate the field + field_kwargs: the kwargs passed to instantiate the field + empty_value: the expected clean output for inputs in empty_values + + """ + if field_args is None: + field_args = [] + if field_kwargs is None: + field_kwargs = {} + required = fieldclass(*field_args, **field_kwargs) + optional = fieldclass(*field_args, + **dict(field_kwargs, required=False)) + # test valid inputs + for input, output in valid.items(): + self.assertEqual(required.clean(input), output) + self.assertEqual(optional.clean(input), output) + # test invalid inputs + for input, errors in invalid.items(): + with self.assertRaises(ValidationError) as context_manager: + required.clean(input) + self.assertEqual(context_manager.exception.messages, errors) + + with self.assertRaises(ValidationError) as context_manager: + optional.clean(input) + self.assertEqual(context_manager.exception.messages, errors) + # test required inputs + error_required = [force_text(required.error_messages['required'])] + for e in required.empty_values: + with self.assertRaises(ValidationError) as context_manager: + required.clean(e) + self.assertEqual(context_manager.exception.messages, + error_required) + self.assertEqual(optional.clean(e), empty_value) + # test that max_length and min_length are always accepted + if issubclass(fieldclass, CharField): + field_kwargs.update({'min_length':2, 'max_length':20}) + self.assertTrue(isinstance(fieldclass(*field_args, **field_kwargs), + fieldclass)) + + def assertHTMLEqual(self, html1, html2, msg=None): + """ + Asserts that two HTML snippets are semantically the same. + Whitespace in most cases is ignored, and attribute ordering is not + significant. The passed-in arguments must be valid HTML. + """ + dom1 = assert_and_parse_html(self, html1, msg, + 'First argument is not valid HTML:') + dom2 = assert_and_parse_html(self, html2, msg, + 'Second argument is not valid HTML:') + + if dom1 != dom2: + standardMsg = '%s != %s' % ( + safe_repr(dom1, True), safe_repr(dom2, True)) + diff = ('\n' + '\n'.join(difflib.ndiff( + six.text_type(dom1).splitlines(), + six.text_type(dom2).splitlines()))) + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHTMLNotEqual(self, html1, html2, msg=None): + """Asserts that two HTML snippets are not semantically equivalent.""" + dom1 = assert_and_parse_html(self, html1, msg, + 'First argument is not valid HTML:') + dom2 = assert_and_parse_html(self, html2, msg, + 'Second argument is not valid HTML:') + + if dom1 == dom2: + standardMsg = '%s == %s' % ( + safe_repr(dom1, True), safe_repr(dom2, True)) + self.fail(self._formatMessage(msg, standardMsg)) + + def assertInHTML(self, needle, haystack, count=None, msg_prefix=''): + needle = assert_and_parse_html(self, needle, None, + 'First argument is not valid HTML:') + haystack = assert_and_parse_html(self, haystack, None, + 'Second argument is not valid HTML:') + real_count = haystack.count(needle) + if count is not None: + self.assertEqual(real_count, count, + msg_prefix + "Found %d instances of '%s' in response" + " (expected %d)" % (real_count, needle, count)) + else: + self.assertTrue(real_count != 0, + msg_prefix + "Couldn't find '%s' in response" % needle) + + def assertJSONEqual(self, raw, expected_data, msg=None): + try: + data = json.loads(raw) + except ValueError: + self.fail("First argument is not valid JSON: %r" % raw) + if isinstance(expected_data, six.string_types): + try: + expected_data = json.loads(expected_data) + except ValueError: + self.fail("Second argument is not valid JSON: %r" % expected_data) + self.assertEqual(data, expected_data, msg=msg) + + def assertXMLEqual(self, xml1, xml2, msg=None): + """ + Asserts that two XML snippets are semantically the same. + Whitespace in most cases is ignored, and attribute ordering is not + significant. The passed-in arguments must be valid XML. + """ + try: + result = compare_xml(xml1, xml2) + except Exception as e: + standardMsg = 'First or second argument is not valid XML\n%s' % e + self.fail(self._formatMessage(msg, standardMsg)) + else: + if not result: + standardMsg = '%s != %s' % (safe_repr(xml1, True), safe_repr(xml2, True)) + self.fail(self._formatMessage(msg, standardMsg)) + + def assertXMLNotEqual(self, xml1, xml2, msg=None): + """ + Asserts that two XML snippets are not semantically equivalent. + Whitespace in most cases is ignored, and attribute ordering is not + significant. The passed-in arguments must be valid XML. + """ + try: + result = compare_xml(xml1, xml2) + except Exception as e: + standardMsg = 'First or second argument is not valid XML\n%s' % e + self.fail(self._formatMessage(msg, standardMsg)) + else: + if result: + standardMsg = '%s == %s' % (safe_repr(xml1, True), safe_repr(xml2, True)) + self.fail(self._formatMessage(msg, standardMsg)) + + +class TransactionTestCase(SimpleTestCase): + + # Subclasses can ask for resetting of auto increment sequence before each + # test case + reset_sequences = False + + def _pre_setup(self): + """Performs any pre-test setup. This includes: + + * Flushing the database. + * If the Test Case class has a 'fixtures' member, installing the + named fixtures. + """ + super(TransactionTestCase, self)._pre_setup() + self._fixture_setup() + + def _databases_names(self, include_mirrors=True): + # If the test case has a multi_db=True flag, act on all databases, + # including mirrors or not. Otherwise, just on the default DB. + if getattr(self, 'multi_db', False): + return [alias for alias in connections + if include_mirrors or not connections[alias].settings_dict['TEST_MIRROR']] + else: + return [DEFAULT_DB_ALIAS] + + def _reset_sequences(self, db_name): + conn = connections[db_name] + if conn.features.supports_sequence_reset: + sql_list = \ + conn.ops.sequence_reset_by_name_sql(no_style(), + conn.introspection.sequence_list()) + if sql_list: + with transaction.commit_on_success_unless_managed(using=db_name): + cursor = conn.cursor() + for sql in sql_list: + cursor.execute(sql) + + def _fixture_setup(self): + for db_name in self._databases_names(include_mirrors=False): + # Reset sequences + if self.reset_sequences: + self._reset_sequences(db_name) + + if hasattr(self, 'fixtures'): + # We have to use this slightly awkward syntax due to the fact + # that we're using *args and **kwargs together. + call_command('loaddata', *self.fixtures, + **{'verbosity': 0, 'database': db_name, 'skip_validation': True}) + + def _post_teardown(self): + """Performs any post-test things. This includes: + + * Putting back the original ROOT_URLCONF if it was changed. + * Force closing the connection, so that the next test gets + a clean cursor. + """ + self._fixture_teardown() + super(TransactionTestCase, self)._post_teardown() + # Some DB cursors include SQL statements as part of cursor + # creation. If you have a test that does rollback, the effect + # of these statements is lost, which can effect the operation + # of tests (e.g., losing a timezone setting causing objects to + # be created with the wrong time). + # To make sure this doesn't happen, get a clean connection at the + # start of every test. + for conn in connections.all(): + conn.close() + + def _fixture_teardown(self): + for db_name in self._databases_names(include_mirrors=False): + call_command('flush', verbosity=0, interactive=False, database=db_name, + skip_validation=True, reset_sequences=False) + def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True): items = six.moves.map(transform, qs) if not ordered: @@ -841,15 +846,19 @@ class TestCase(TransactionTestCase): # Remove this when the legacy transaction management goes away. disable_transaction_methods() - for db in self._databases_names(include_mirrors=False): + for db_name in self._databases_names(include_mirrors=False): if hasattr(self, 'fixtures'): - call_command('loaddata', *self.fixtures, - **{ - 'verbosity': 0, - 'commit': False, - 'database': db, - 'skip_validation': True, - }) + try: + call_command('loaddata', *self.fixtures, + **{ + 'verbosity': 0, + 'commit': False, + 'database': db_name, + 'skip_validation': True, + }) + except Exception: + self._fixture_teardown() + raise def _fixture_teardown(self): if not connections_support_transactions(): diff --git a/django/test/utils.py b/django/test/utils.py index 92cef59f72..be586c75a6 100644 --- a/django/test/utils.py +++ b/django/test/utils.py @@ -1,4 +1,7 @@ +from contextlib import contextmanager +import logging import re +import sys import warnings from functools import wraps from xml.dom.minidom import parseString, Node @@ -57,6 +60,16 @@ class ContextList(list): return False return True + def keys(self): + """ + Flattened keys of subcontexts. + """ + keys = set() + for subcontext in self: + for dict in subcontext: + keys |= set(dict.keys()) + return keys + def instrumented_test_render(self, context): """ @@ -380,3 +393,41 @@ class CaptureQueriesContext(object): if exc_type is not None: return self.final_queries = len(self.connection.queries) + + +class IgnoreDeprecationWarningsMixin(object): + + warning_class = DeprecationWarning + + def setUp(self): + super(IgnoreDeprecationWarningsMixin, self).setUp() + self.catch_warnings = warnings.catch_warnings() + self.catch_warnings.__enter__() + warnings.filterwarnings("ignore", category=self.warning_class) + + def tearDown(self): + self.catch_warnings.__exit__(*sys.exc_info()) + super(IgnoreDeprecationWarningsMixin, self).tearDown() + + +class IgnorePendingDeprecationWarningsMixin(IgnoreDeprecationWarningsMixin): + + warning_class = PendingDeprecationWarning + + +@contextmanager +def patch_logger(logger_name, log_level): + """ + Context manager that takes a named logger and the logging level + and provides a simple mock-like list of messages received + """ + calls = [] + def replacement(msg): + calls.append(msg) + logger = logging.getLogger(logger_name) + orig = getattr(logger, log_level) + setattr(logger, log_level, replacement) + try: + yield calls + finally: + setattr(logger, log_level, orig) diff --git a/django/utils/_os.py b/django/utils/_os.py index 6c1cd17a83..607e02c94d 100644 --- a/django/utils/_os.py +++ b/django/utils/_os.py @@ -38,7 +38,7 @@ def upath(path): """ Always return a unicode path. """ - if not six.PY3: + if not six.PY3 and not isinstance(path, six.text_type): return path.decode(fs_encoding) return path diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 5d0f381ffa..15db972560 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -28,10 +28,6 @@ from django.utils import six from django.utils.six.moves import xrange -_trans_5c = bytearray([(x ^ 0x5C) for x in xrange(256)]) -_trans_36 = bytearray([(x ^ 0x36) for x in xrange(256)]) - - def salted_hmac(key_salt, value, secret=None): """ Returns the HMAC-SHA1 of 'value', using a key generated from key_salt and a @@ -130,9 +126,9 @@ def _fast_hmac(key, msg, digest): if len(key) > dig1.block_size: key = digest(key).digest() key += b'\x00' * (dig1.block_size - len(key)) - dig1.update(key.translate(_trans_36)) + dig1.update(key.translate(hmac.trans_36)) dig1.update(msg) - dig2.update(key.translate(_trans_5c)) + dig2.update(key.translate(hmac.trans_5C)) dig2.update(dig1.digest()) return dig2 diff --git a/django/utils/functional.py b/django/utils/functional.py index cab74886d3..0606c775ef 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -4,6 +4,7 @@ from functools import wraps import sys from django.utils import six +from django.utils.six.moves import copyreg # You can't trivially replace this with `functools.partial` because this binds @@ -328,15 +329,23 @@ class SimpleLazyObject(LazyObject): self._setup() return self._wrapped.__dict__ - # Python 3.3 will call __reduce__ when pickling; these methods are needed - # to serialize and deserialize correctly. They are not called in earlier - # versions of Python. + # Python 3.3 will call __reduce__ when pickling; this method is needed + # to serialize and deserialize correctly. @classmethod def __newobj__(cls, *args): return cls.__new__(cls, *args) - def __reduce__(self): - return (self.__newobj__, (self.__class__,), self.__getstate__()) + def __reduce_ex__(self, proto): + if proto >= 2: + # On Py3, since the default protocol is 3, pickle uses the + # ``__newobj__`` method (& more efficient opcodes) for writing. + return (self.__newobj__, (self.__class__,), self.__getstate__()) + else: + # On Py2, the default protocol is 0 (for back-compat) & the above + # code fails miserably (see regression test). Instead, we return + # exactly what's returned if there's no ``__reduce__`` method at + # all. + return (copyreg._reconstructor, (self.__class__, object, None), self.__getstate__()) # Return a meaningful representation of the lazy object for debugging # without evaluating the wrapped object. diff --git a/django/utils/html.py b/django/utils/html.py index 8b28d97d13..0d28c77a61 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -16,6 +16,9 @@ from django.utils.functional import allow_lazy from django.utils import six from django.utils.text import normalize_newlines +from .html_parser import HTMLParser, HTMLParseError + + # Configuration for urlize() function. TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)'] WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('<', '>')] @@ -33,7 +36,6 @@ link_target_attribute_re = re.compile(r'(<a [^>]*?)target=[^\s>]+') html_gunk_re = re.compile(r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE) hard_coded_bullets_re = re.compile(r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL) trailing_empty_content_re = re.compile(r'(?:<p>(?: |\s|<br \/>)*?</p>\s*)+\Z') -strip_tags_re = re.compile(r'</?\S([^=>]*=(\s*"[^"]*"|\s*\'[^\']*\'|\S*)|[^>])*?>', re.IGNORECASE) def escape(text): @@ -116,9 +118,31 @@ def linebreaks(value, autoescape=False): return '\n\n'.join(paras) linebreaks = allow_lazy(linebreaks, six.text_type) + +class MLStripper(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self.reset() + self.fed = [] + def handle_data(self, d): + self.fed.append(d) + def handle_entityref(self, name): + self.fed.append('&%s;' % name) + def handle_charref(self, name): + self.fed.append('&#%s;' % name) + def get_data(self): + return ''.join(self.fed) + def strip_tags(value): """Returns the given HTML with all tags stripped.""" - return strip_tags_re.sub('', force_text(value)) + s = MLStripper() + try: + s.feed(value) + s.close() + except HTMLParseError: + return value + else: + return s.get_data() strip_tags = allow_lazy(strip_tags) def remove_tags(html, tags): @@ -281,3 +305,10 @@ def clean_html(text): text = trailing_empty_content_re.sub('', text) return text clean_html = allow_lazy(clean_html, six.text_type) + +def avoid_wrapping(value): + """ + Avoid text wrapping in the middle of a phrase by adding non-breaking + spaces where there previously were normal spaces. + """ + return value.replace(" ", "\xa0") diff --git a/django/utils/http.py b/django/utils/http.py index 15fac6bfca..f4911b4ec0 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -71,7 +71,7 @@ urlunquote_plus = allow_lazy(urlunquote_plus, six.text_type) def urlencode(query, doseq=0): """ A version of Python's urllib.urlencode() function that can operate on - unicode strings. The parameters are first case to UTF-8 encoded strings and + unicode strings. The parameters are first cast to UTF-8 encoded strings and then encoded as per normal. """ if isinstance(query, MultiValueDict): @@ -226,7 +226,10 @@ def same_origin(url1, url2): Checks if two URLs are 'same-origin' """ p1, p2 = urllib_parse.urlparse(url1), urllib_parse.urlparse(url2) - return (p1.scheme, p1.hostname, p1.port) == (p2.scheme, p2.hostname, p2.port) + try: + return (p1.scheme, p1.hostname, p1.port) == (p2.scheme, p2.hostname, p2.port) + except ValueError: + return False def is_safe_url(url, host=None): """ diff --git a/django/utils/image.py b/django/utils/image.py index 54c11adfee..d251ab9d0b 100644 --- a/django/utils/image.py +++ b/django/utils/image.py @@ -124,7 +124,7 @@ def _detect_image_library(): import _imaging as PIL_imaging except ImportError as err: raise ImproperlyConfigured( - _("The '_imaging' module for the PIL could not be " + + _("The '_imaging' module for the PIL could not be " "imported: %s" % err) ) diff --git a/django/utils/ipv6.py b/django/utils/ipv6.py index 8881574eaa..eaacfb4623 100644 --- a/django/utils/ipv6.py +++ b/django/utils/ipv6.py @@ -138,8 +138,7 @@ def _unpack_ipv4(ip_str): if not ip_str.lower().startswith('0000:0000:0000:0000:0000:ffff:'): return None - hextets = ip_str.split(':') - return hextets[-1] + return ip_str.rsplit(':', 1)[1] def is_valid_ipv6_address(ip_str): """ diff --git a/django/utils/log.py b/django/utils/log.py index a9b62caae1..6734a7261e 100644 --- a/django/utils/log.py +++ b/django/utils/log.py @@ -63,6 +63,11 @@ DEFAULT_LOGGING = { 'level': 'ERROR', 'propagate': False, }, + 'django.security': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': False, + }, 'py.warnings': { 'handlers': ['console'], }, @@ -87,8 +92,8 @@ class AdminEmailHandler(logging.Handler): request = record.request subject = '%s (%s IP): %s' % ( record.levelname, - (request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS - and 'internal' or 'EXTERNAL'), + ('internal' if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS + else 'EXTERNAL'), record.getMessage() ) filter = get_exception_reporter_filter(request) diff --git a/django/utils/safestring.py b/django/utils/safestring.py index 07e0bf4cea..3774012d32 100644 --- a/django/utils/safestring.py +++ b/django/utils/safestring.py @@ -4,7 +4,7 @@ without further escaping in HTML. Marking something as a "safe string" means that the producer of the string has already turned characters that should not be interpreted by the HTML engine (e.g. '<') into the appropriate entities. """ -from django.utils.functional import curry, Promise +from django.utils.functional import curry, Promise, allow_lazy from django.utils import six class EscapeData(object): @@ -14,13 +14,13 @@ class EscapeBytes(bytes, EscapeData): """ A byte string that should be HTML-escaped when output. """ - pass + __new__ = allow_lazy(bytes.__new__, bytes) class EscapeText(six.text_type, EscapeData): """ A unicode string object that should be HTML-escaped when output. """ - pass + __new__ = allow_lazy(six.text_type.__new__, six.text_type) if six.PY3: EscapeString = EscapeText @@ -37,6 +37,8 @@ class SafeBytes(bytes, SafeData): A bytes subclass that has been specifically marked as "safe" (requires no further escaping) for HTML output purposes. """ + __new__ = allow_lazy(bytes.__new__, bytes) + def __add__(self, rhs): """ Concatenating a safe byte string with another safe byte string or safe @@ -69,6 +71,8 @@ class SafeText(six.text_type, SafeData): A unicode (Python 2) / str (Python 3) subclass that has been specifically marked as "safe" for HTML output purposes. """ + __new__ = allow_lazy(six.text_type.__new__, six.text_type) + def __add__(self, rhs): """ Concatenating a safe unicode string with another safe byte string or diff --git a/django/utils/timesince.py b/django/utils/timesince.py index d70ab2ffe1..46c387f262 100644 --- a/django/utils/timesince.py +++ b/django/utils/timesince.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime +from django.utils.html import avoid_wrapping from django.utils.timezone import is_aware, utc from django.utils.translation import ugettext, ungettext_lazy @@ -40,18 +41,18 @@ def timesince(d, now=None, reversed=False): since = delta.days * 24 * 60 * 60 + delta.seconds if since <= 0: # d is in the future compared to now, stop processing. - return ugettext('0 minutes') + return avoid_wrapping(ugettext('0 minutes')) for i, (seconds, name) in enumerate(chunks): count = since // seconds if count != 0: break - result = name % count + result = avoid_wrapping(name % count) if i + 1 < len(chunks): # Now get the second item seconds2, name2 = chunks[i + 1] count2 = (since - (seconds * count)) // seconds2 if count2 != 0: - result += ugettext(', ') + name2 % count2 + result += ugettext(', ') + avoid_wrapping(name2 % count2) return result def timeuntil(d, now=None): diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 07353c35ee..195badfc00 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -10,7 +10,9 @@ from threading import local import warnings from django.utils.importlib import import_module +from django.utils.datastructures import SortedDict from django.utils.encoding import force_str, force_text +from django.utils.functional import memoize from django.utils._os import upath from django.utils.safestring import mark_safe, SafeData from django.utils import six @@ -29,6 +31,7 @@ _default = None # This is a cache for normalized accept-header languages to prevent multiple # file lookups when checking the same locale on repeated requests. _accepted = {} +_checked_languages = {} # magic gettext number to separate context from message CONTEXT_SEPARATOR = "\x04" @@ -77,7 +80,6 @@ class DjangoTranslation(gettext_module.GNUTranslations): def __init__(self, *args, **kw): gettext_module.GNUTranslations.__init__(self, *args, **kw) self.set_output_charset('utf-8') - self.django_output_charset = 'utf-8' self.__language = '??' def merge(self, other): @@ -140,7 +142,7 @@ def translation(language): # doesn't affect en-gb), even though they will both use the core "en" # translation. So we have to subvert Python's internal gettext caching. base_lang = lambda x: x.split('-', 1)[0] - if base_lang(lang) in [base_lang(trans) for trans in _translations]: + if base_lang(lang) in [base_lang(trans) for trans in list(_translations)]: res._info = res._info.copy() res._catalog = res._catalog.copy() @@ -355,34 +357,54 @@ def check_for_language(lang_code): if gettext_module.find('django', path, [to_locale(lang_code)]) is not None: return True return False +check_for_language = memoize(check_for_language, _checked_languages, 1) -def get_supported_language_variant(lang_code, supported=None): +def get_supported_language_variant(lang_code, supported=None, strict=False): """ Returns the language-code that's listed in supported languages, possibly selecting a more generic variant. Raises LookupError if nothing found. + + If `strict` is False (the default), the function will look for an alternative + country-specific variant when the currently checked is not found. """ if supported is None: from django.conf import settings - supported = dict(settings.LANGUAGES) - if lang_code and lang_code not in supported: - lang_code = lang_code.split('-')[0] # e.g. if fr-ca is not supported fallback to fr - if lang_code and lang_code in supported and check_for_language(lang_code): - return lang_code + supported = SortedDict(settings.LANGUAGES) + if lang_code: + # if fr-CA is not supported, try fr-ca; if that fails, fallback to fr. + generic_lang_code = lang_code.split('-')[0] + variants = (lang_code, lang_code.lower(), generic_lang_code, + generic_lang_code.lower()) + for code in variants: + if code in supported and check_for_language(code): + return code + if not strict: + # if fr-fr is not supported, try fr-ca. + for supported_code in supported: + if supported_code.startswith((generic_lang_code + '-', + generic_lang_code.lower() + '-')): + return supported_code raise LookupError(lang_code) -def get_language_from_path(path, supported=None): +def get_language_from_path(path, supported=None, strict=False): """ Returns the language-code if there is a valid language-code found in the `path`. + + If `strict` is False (the default), the function will look for an alternative + country-specific variant when the currently checked is not found. """ if supported is None: from django.conf import settings - supported = dict(settings.LANGUAGES) + supported = SortedDict(settings.LANGUAGES) regex_match = language_code_prefix_re.match(path) - if regex_match: - lang_code = regex_match.group(1) - if lang_code in supported and check_for_language(lang_code): - return lang_code + if not regex_match: + return None + lang_code = regex_match.group(1) + try: + return get_supported_language_variant(lang_code, supported, strict=strict) + except LookupError: + return None def get_language_from_request(request, check_path=False): """ @@ -396,7 +418,7 @@ def get_language_from_request(request, check_path=False): """ global _accepted from django.conf import settings - supported = dict(settings.LANGUAGES) + supported = SortedDict(settings.LANGUAGES) if check_path: lang_code = get_language_from_path(request.path_info, supported) @@ -420,11 +442,6 @@ def get_language_from_request(request, check_path=False): if accept_lang == '*': break - # We have a very restricted form for our language files (no encoding - # specifier, since they all must be UTF-8 and only one possible - # language each time. So we avoid the overhead of gettext.find() and - # work out the MO file manually. - # 'normalized' is the root name of the locale in POSIX format (which is # the format used for the directories holding the MO files). normalized = locale.locale_alias.get(to_locale(accept_lang, True)) @@ -438,14 +455,13 @@ def get_language_from_request(request, check_path=False): # need to check again. return _accepted[normalized] - for lang, dirname in ((accept_lang, normalized), - (accept_lang.split('-')[0], normalized.split('_')[0])): - if lang.lower() not in supported: - continue - for path in all_locale_paths(): - if os.path.exists(os.path.join(path, dirname, 'LC_MESSAGES', 'django.mo')): - _accepted[normalized] = lang - return lang + try: + accept_lang = get_supported_language_variant(accept_lang, supported) + except LookupError: + continue + else: + _accepted[normalized] = accept_lang + return accept_lang try: return get_supported_language_variant(settings.LANGUAGE_CODE, supported) diff --git a/django/views/debug.py b/django/views/debug.py index 9b95b524d2..0458580221 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -218,6 +218,15 @@ class ExceptionReporter(object): self.exc_value = Exception('Deprecated String Exception: %r' % self.exc_type) self.exc_type = type(self.exc_value) + def format_path_status(self, path): + if not os.path.exists(path): + return "File does not exist" + if not os.path.isfile(path): + return "Not a file" + if not os.access(path, os.R_OK): + return "File is not readable" + return "File exists" + def get_traceback_data(self): "Return a Context instance containing traceback information." @@ -230,8 +239,10 @@ class ExceptionReporter(object): source_list_func = loader.get_template_sources # NOTE: This assumes exc_value is the name of the template that # the loader attempted to load. - template_list = [{'name': t, 'exists': os.path.exists(t)} \ - for t in source_list_func(str(self.exc_value))] + template_list = [{ + 'name': t, + 'status': self.format_path_status(t), + } for t in source_list_func(str(self.exc_value))] except AttributeError: template_list = [] loader_name = loader.__module__ + '.' + loader.__class__.__name__ @@ -347,7 +358,7 @@ class ExceptionReporter(object): if source is None: try: with open(filename, 'rb') as fp: - source = fp.readlines() + source = fp.read().splitlines() except (OSError, IOError): pass if source is None: @@ -370,9 +381,9 @@ class ExceptionReporter(object): lower_bound = max(0, lineno - context_lines) upper_bound = lineno + context_lines - pre_context = [line.strip('\n') for line in source[lower_bound:lineno]] - context_line = source[lineno].strip('\n') - post_context = [line.strip('\n') for line in source[lineno+1:upper_bound]] + pre_context = source[lower_bound:lineno] + context_line = source[lineno] + post_context = source[lineno+1:upper_bound] return lower_bound, pre_context, context_line, post_context @@ -394,7 +405,7 @@ class ExceptionReporter(object): if pre_context_lineno is not None: frames.append({ 'tb': tb, - 'type': module_name.startswith('django.') and 'django' or 'user', + 'type': 'django' if module_name.startswith('django.') else 'user', 'filename': filename, 'function': function, 'lineno': lineno + 1, @@ -584,7 +595,7 @@ TECHNICAL_500_TEMPLATE = """ <body> <div id="summary"> <h1>{% if exception_type %}{{ exception_type }}{% else %}Report{% endif %}{% if request %} at {{ request.path_info|escape }}{% endif %}</h1> - <pre class="exception_value">{% if exception_value %}{{ exception_value|force_escape }}{% else %}No exception supplied{% endif %}</pre> + <pre class="exception_value">{% if exception_value %}{{ exception_value|force_escape }}{% else %}No exception message supplied{% endif %}</pre> <table class="meta"> {% if request %} <tr> @@ -650,7 +661,9 @@ TECHNICAL_500_TEMPLATE = """ <ul> {% for loader in loader_debug_info %} <li>Using loader <code>{{ loader.loader }}</code>: - <ul>{% for t in loader.templates %}<li><code>{{ t.name }}</code> (File {% if t.exists %}exists{% else %}does not exist{% endif %})</li>{% endfor %}</ul> + <ul> + {% for t in loader.templates %}<li><code>{{ t.name }}</code> ({{ t.status }})</li>{% endfor %} + </ul> </li> {% endfor %} </ul> @@ -753,7 +766,7 @@ Installed Middleware: {% if template_does_not_exist %}Template Loader Error: {% if loader_debug_info %}Django tried loading these templates, in this order: {% for loader in loader_debug_info %}Using loader {{ loader.loader }}: -{% for t in loader.templates %}{{ t.name }} (File {% if t.exists %}exists{% else %}does not exist{% endif %}) +{% for t in loader.templates %}{{ t.name }} ({{ t.status }}) {% endfor %}{% endfor %} {% else %}Django couldn't find any templates because your TEMPLATE_LOADERS setting is empty! {% endif %} @@ -927,7 +940,7 @@ Exception Value: {{ exception_value|force_escape }} """ TECHNICAL_500_TEXT_TEMPLATE = """{% load firstof from future %}{% firstof exception_type 'Report' %}{% if request %} at {{ request.path_info }}{% endif %} -{% firstof exception_value 'No exception supplied' %} +{% firstof exception_value 'No exception message supplied' %} {% if request %} Request Method: {{ request.META.REQUEST_METHOD }} Request URL: {{ request.build_absolute_uri }}{% endif %} @@ -943,7 +956,7 @@ Installed Middleware: {% if template_does_not_exist %}Template loader Error: {% if loader_debug_info %}Django tried loading these templates, in this order: {% for loader in loader_debug_info %}Using loader {{ loader.loader }}: -{% for t in loader.templates %}{{ t.name }} (File {% if t.exists %}exists{% else %}does not exist{% endif %}) +{% for t in loader.templates %}{{ t.name }} ({{ t.status }}) {% endfor %}{% endfor %} {% else %}Django couldn't find any templates because your TEMPLATE_LOADERS setting is empty! {% endif %} diff --git a/django/views/decorators/csrf.py b/django/views/decorators/csrf.py index 7a7eb6bba6..a6bd7d8526 100644 --- a/django/views/decorators/csrf.py +++ b/django/views/decorators/csrf.py @@ -15,7 +15,7 @@ using the decorator multiple times, is harmless and efficient. class _EnsureCsrfToken(CsrfViewMiddleware): # We need this to behave just like the CsrfViewMiddleware, but not reject - # requests. + # requests or log warnings. def _reject(self, request, reason): return None diff --git a/django/views/defaults.py b/django/views/defaults.py index 89228c50c9..c8a62fc753 100644 --- a/django/views/defaults.py +++ b/django/views/defaults.py @@ -43,6 +43,21 @@ def server_error(request, template_name='500.html'): return http.HttpResponseServerError(template.render(Context({}))) +@requires_csrf_token +def bad_request(request, template_name='400.html'): + """ + 400 error handler. + + Templates: :template:`400.html` + Context: None + """ + try: + template = loader.get_template(template_name) + except TemplateDoesNotExist: + return http.HttpResponseBadRequest('<h1>Bad Request (400)</h1>') + return http.HttpResponseBadRequest(template.render(Context({}))) + + # This can be called when CsrfViewMiddleware.process_view has not run, # therefore need @requires_csrf_token in case the template needs # {% csrf_token %}. diff --git a/django/views/generic/base.py b/django/views/generic/base.py index d50d6bbc55..286a18d0f2 100644 --- a/django/views/generic/base.py +++ b/django/views/generic/base.py @@ -30,7 +30,7 @@ class View(object): dispatch-by-method and simple sanity checking. """ - http_method_names = ['get', 'post', 'put', 'delete', 'head', 'options', 'trace'] + http_method_names = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options', 'trace'] def __init__(self, **kwargs): """ @@ -206,3 +206,6 @@ class RedirectView(View): def put(self, request, *args, **kwargs): return self.get(request, *args, **kwargs) + + def patch(self, request, *args, **kwargs): + return self.get(request, *args, **kwargs) diff --git a/django/views/generic/detail.py b/django/views/generic/detail.py index 58302bbe23..23000641b4 100644 --- a/django/views/generic/detail.py +++ b/django/views/generic/detail.py @@ -93,9 +93,11 @@ class SingleObjectMixin(ContextMixin): Insert the single object into the context dict. """ context = {} - context_object_name = self.get_context_object_name(self.object) - if context_object_name: - context[context_object_name] = self.object + if self.object: + context['object'] = self.object + context_object_name = self.get_context_object_name(self.object) + if context_object_name: + context[context_object_name] = self.object context.update(kwargs) return super(SingleObjectMixin, self).get_context_data(**context) @@ -122,7 +124,7 @@ class SingleObjectTemplateResponseMixin(TemplateResponseMixin): * the value of ``template_name`` on the view (if provided) * the contents of the ``template_name_field`` field on the object instance that the view is operating upon (if available) - * ``<app_label>/<object_name><template_name_suffix>.html`` + * ``<app_label>/<object_name><template_name_suffix>.html`` """ try: names = super(SingleObjectTemplateResponseMixin, self).get_template_names() diff --git a/django/views/generic/edit.py b/django/views/generic/edit.py index e2cc741ffb..cf87aeed27 100644 --- a/django/views/generic/edit.py +++ b/django/views/generic/edit.py @@ -136,20 +136,6 @@ class ModelFormMixin(FormMixin, SingleObjectMixin): self.object = form.save() return super(ModelFormMixin, self).form_valid(form) - def get_context_data(self, **kwargs): - """ - If an object has been supplied, inject it into the context with the - supplied context_object_name name. - """ - context = {} - if self.object: - context['object'] = self.object - context_object_name = self.get_context_object_name(self.object) - if context_object_name: - context[context_object_name] = self.object - context.update(kwargs) - return super(ModelFormMixin, self).get_context_data(**context) - class ProcessFormView(View): """ diff --git a/django/views/generic/list.py b/django/views/generic/list.py index 08c4bbcda0..1aff3454f4 100644 --- a/django/views/generic/list.py +++ b/django/views/generic/list.py @@ -105,7 +105,7 @@ class MultipleObjectMixin(ContextMixin): """ Get the context for this view. """ - queryset = kwargs.pop('object_list') + queryset = kwargs.pop('object_list', self.object_list) page_size = self.get_paginate_by(queryset) context_object_name = self.get_context_object_name(queryset) if page_size: @@ -149,7 +149,7 @@ class BaseListView(MultipleObjectMixin, View): if is_empty: raise Http404(_("Empty list and '%(class_name)s.allow_empty' is False.") % {'class_name': self.__class__.__name__}) - context = self.get_context_data(object_list=self.object_list) + context = self.get_context_data() return self.render_to_response(context) diff --git a/django/views/i18n.py b/django/views/i18n.py index 37ec10b552..71ac005855 100644 --- a/django/views/i18n.py +++ b/django/views/i18n.py @@ -184,38 +184,8 @@ def render_javascript_catalog(catalog=None, plural=None): return http.HttpResponse(template.render(context), 'text/javascript') -def null_javascript_catalog(request, domain=None, packages=None): - """ - Returns "identity" versions of the JavaScript i18n functions -- i.e., - versions that don't actually do anything. - """ - return render_javascript_catalog() - - -def javascript_catalog(request, domain='djangojs', packages=None): - """ - Returns the selected language catalog as a javascript library. - - Receives the list of packages to check for translations in the - packages parameter either from an infodict or as a +-delimited - string from the request. Default is 'django.conf'. - - Additionally you can override the gettext domain for this view, - but usually you don't want to do that, as JavaScript messages - go to the djangojs domain. But this might be needed if you - deliver your JavaScript source from Django templates. - """ +def get_javascript_catalog(locale, domain, packages): default_locale = to_locale(settings.LANGUAGE_CODE) - locale = to_locale(get_language()) - - if request.GET and 'language' in request.GET: - if check_for_language(request.GET['language']): - locale = to_locale(request.GET['language']) - - if packages is None: - packages = ['django.conf'] - if isinstance(packages, six.string_types): - packages = packages.split('+') packages = [p for p in packages if p == 'django.conf' or p in settings.INSTALLED_APPS] t = {} paths = [] @@ -296,4 +266,40 @@ def javascript_catalog(request, domain='djangojs', packages=None): for k, v in pdict.items(): catalog[k] = [v.get(i, '') for i in range(maxcnts[msgid] + 1)] + return catalog, plural + + +def null_javascript_catalog(request, domain=None, packages=None): + """ + Returns "identity" versions of the JavaScript i18n functions -- i.e., + versions that don't actually do anything. + """ + return render_javascript_catalog() + + +def javascript_catalog(request, domain='djangojs', packages=None): + """ + Returns the selected language catalog as a javascript library. + + Receives the list of packages to check for translations in the + packages parameter either from an infodict or as a +-delimited + string from the request. Default is 'django.conf'. + + Additionally you can override the gettext domain for this view, + but usually you don't want to do that, as JavaScript messages + go to the djangojs domain. But this might be needed if you + deliver your JavaScript source from Django templates. + """ + locale = to_locale(get_language()) + + if request.GET and 'language' in request.GET: + if check_for_language(request.GET['language']): + locale = to_locale(request.GET['language']) + + if packages is None: + packages = ['django.conf'] + if isinstance(packages, six.string_types): + packages = packages.split('+') + + catalog, plural = get_javascript_catalog(locale, domain, packages) return render_javascript_catalog(catalog, plural) |