boulder-oracle-sprint: Merged to trunk [4455].

git-svn-id: http://code.djangoproject.com/svn/django/branches/boulder-oracle-sprint@4456 bcc190cf-cafb-0310-a4f2-bffc1f526a37

28 files changed, 744 insertions, 245 deletions

diff --git a/django/contrib/admin/templates/admin/auth/user/change_password.html b/django/contrib/admin/templates/admin/auth/user/change_password.html
index 80990faa24..3d359ecf8f 100644
--- a/django/contrib/admin/templates/admin/auth/user/change_password.html
+++ b/django/contrib/admin/templates/admin/auth/user/change_password.html
@@ -25,7 +25,7 @@
     </p>
 {% endif %}
 
-<p>{% blocktrans with original.username|escape as username %}Enter a new username and password for the user <strong>{{ username }}</strong>.{% endblocktrans %}</p>
+<p>{% blocktrans with original.username|escape as username %}Enter a new password for the user <strong>{{ username }}</strong>.{% endblocktrans %}</p>
 
 <fieldset class="module aligned">
 
diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py
index 832b3562cd..3c0c6f0ac2 100644
--- a/django/contrib/admin/templatetags/admin_list.py
+++ b/django/contrib/admin/templatetags/admin_list.py
@@ -101,6 +101,10 @@ def result_headers(cl):
                        "url": cl.get_query_string({ORDER_VAR: i, ORDER_TYPE_VAR: new_order_type}),
                        "class_attrib": (th_classes and ' class="%s"' % ' '.join(th_classes) or '')}
 
+def _boolean_icon(field_val):
+    BOOLEAN_MAPPING = {True: 'yes', False: 'no', None: 'unknown'}
+    return '<img src="%simg/admin/icon-%s.gif" alt="%s" />' % (settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], field_val)
+
 def items_for_result(cl, result):
     first = True
     pk = cl.lookup_opts.pk.attname
@@ -114,9 +118,14 @@ def items_for_result(cl, result):
             try:
                 attr = getattr(result, field_name)
                 allow_tags = getattr(attr, 'allow_tags', False)
+                boolean = getattr(attr, 'boolean', False)
                 if callable(attr):
                     attr = attr()
-                result_repr = str(attr)
+                if boolean:
+                    allow_tags = True
+                    result_repr = _boolean_icon(attr)
+                else:
+                    result_repr = str(attr)
             except (AttributeError, ObjectDoesNotExist):
                 result_repr = EMPTY_CHANGELIST_VALUE
             else:
@@ -147,8 +156,7 @@ def items_for_result(cl, result):
                 row_class = ' class="nowrap"'
             # Booleans are special: We use images.
             elif isinstance(f, models.BooleanField) or isinstance(f, models.NullBooleanField):
-                BOOLEAN_MAPPING = {True: 'yes', False: 'no', None: 'unknown'}
-                result_repr = '<img src="%simg/admin/icon-%s.gif" alt="%s" />' % (settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], field_val)
+                result_repr = _boolean_icon(field_val)
             # FloatFields are special: Zero-pad the decimals.
             elif isinstance(f, models.FloatField):
                 if field_val is not None:
diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py
index a95748a9a1..3384134cb2 100644
--- a/django/contrib/contenttypes/models.py
+++ b/django/contrib/contenttypes/models.py
@@ -1,6 +1,7 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
+CONTENT_TYPE_CACHE = {}
 class ContentTypeManager(models.Manager):
     def get_for_model(self, model):
         """
@@ -8,10 +9,15 @@ class ContentTypeManager(models.Manager):
         ContentType if necessary.
         """
         opts = model._meta
-        # The str() is needed around opts.verbose_name because it's a
-        # django.utils.functional.__proxy__ object.
-        ct, created = self.model._default_manager.get_or_create(app_label=opts.app_label,
-            model=opts.object_name.lower(), defaults={'name': str(opts.verbose_name)})
+        key = (opts.app_label, opts.object_name.lower())
+        try:
+            ct = CONTENT_TYPE_CACHE[key]
+        except KeyError:
+            # The str() is needed around opts.verbose_name because it's a
+            # django.utils.functional.__proxy__ object.
+            ct, created = self.model._default_manager.get_or_create(app_label=key[0],
+                model=key[1], defaults={'name': str(opts.verbose_name)})
+            CONTENT_TYPE_CACHE[key] = ct
         return ct
 
 class ContentType(models.Model):
diff --git a/django/contrib/sessions/middleware.py b/django/contrib/sessions/middleware.py
index 2337ad8a61..728caa7e19 100644
--- a/django/contrib/sessions/middleware.py
+++ b/django/contrib/sessions/middleware.py
@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.contrib.sessions.models import Session
+from django.core.exceptions import SuspiciousOperation
 from django.utils.cache import patch_vary_headers
 import datetime
 
@@ -55,7 +56,7 @@ class SessionWrapper(object):
                     s = Session.objects.get(session_key=self.session_key,
                         expire_date__gt=datetime.datetime.now())
                     self._session_cache = s.get_decoded()
-                except Session.DoesNotExist:
+                except (Session.DoesNotExist, SuspiciousOperation):
                     self._session_cache = {}
                     # Set the session_key to None to force creation of a new
                     # key, for extra security.
diff --git a/django/core/cache/backends/dummy.py b/django/core/cache/backends/dummy.py
index c68f33616c..4c64161538 100644
--- a/django/core/cache/backends/dummy.py
+++ b/django/core/cache/backends/dummy.py
@@ -6,8 +6,8 @@ class CacheClass(BaseCache):
     def __init__(self, *args, **kwargs):
         pass
 
-    def get(self, *args, **kwargs):
-        pass
+    def get(self, key, default=None):
+        return default
 
     def set(self, *args, **kwargs):
         pass
@@ -16,7 +16,7 @@ class CacheClass(BaseCache):
         pass
 
     def get_many(self, *args, **kwargs):
-        pass
+        return {}
 
     def has_key(self, *args, **kwargs):
         return False
diff --git a/django/core/management.py b/django/core/management.py
index c0b3d3a15a..10d2532263 100644
--- a/django/core/management.py
+++ b/django/core/management.py
@@ -25,7 +25,7 @@ APP_ARGS = '[appname ...]'
 # which has been installed.
 PROJECT_TEMPLATE_DIR = os.path.join(django.__path__[0], 'conf', '%s_template')
 
-INVALID_PROJECT_NAMES = ('django', 'test')
+INVALID_PROJECT_NAMES = ('django', 'site', 'test')
 
 # Set up the terminal color scheme.
 class dummy: pass
@@ -735,7 +735,7 @@ def startproject(project_name, directory):
     "Creates a Django project for the given project_name in the given directory."
     from random import choice
     if project_name in INVALID_PROJECT_NAMES:
-        sys.stderr.write(style.ERROR("Error: %r isn't a valid project name. Please try another.\n" % project_name))
+        sys.stderr.write(style.ERROR("Error: '%r' conflicts with the name of an existing Python module and cannot be used as a project name. Please try another name.\n" % project_name))
         sys.exit(1)
     _start_helper('project', project_name, directory)
     # Create a random SECRET_KEY hash, and put it in the main settings.
diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py
index 859816c226..1e1e6f4bec 100644
--- a/django/core/serializers/python.py
+++ b/django/core/serializers/python.py
@@ -57,7 +57,7 @@ def Deserializer(object_list, **options):
     for d in object_list:
         # Look up the model and starting build a dict of data for it.
         Model = _get_model(d["model"])
-        data = {Model._meta.pk.name : d["pk"]}
+        data = {Model._meta.pk.attname : d["pk"]}
         m2m_data = {}
         
         # Handle each field
diff --git a/django/db/backends/ado_mssql/creation.py b/django/db/backends/ado_mssql/creation.py
index 4d85d27ea5..5158ba02f9 100644
--- a/django/db/backends/ado_mssql/creation.py
+++ b/django/db/backends/ado_mssql/creation.py
@@ -21,6 +21,5 @@ DATA_TYPES = {
     'SmallIntegerField': 'smallint',
     'TextField':         'text',
     'TimeField':         'time',
-    'URLField':          'varchar(200)',
     'USStateField':      'varchar(2)',
 }
diff --git a/django/db/backends/mysql/creation.py b/django/db/backends/mysql/creation.py
index 116b490124..22ed901653 100644
--- a/django/db/backends/mysql/creation.py
+++ b/django/db/backends/mysql/creation.py
@@ -25,6 +25,5 @@ DATA_TYPES = {
     'SmallIntegerField': 'smallint',
     'TextField':         'longtext',
     'TimeField':         'time',
-    'URLField':          'varchar(200)',
     'USStateField':      'varchar(2)',
 }
diff --git a/django/db/backends/postgresql/creation.py b/django/db/backends/postgresql/creation.py
index 65a804ec40..6c130f368e 100644
--- a/django/db/backends/postgresql/creation.py
+++ b/django/db/backends/postgresql/creation.py
@@ -25,6 +25,5 @@ DATA_TYPES = {
     'SmallIntegerField': 'smallint',
     'TextField':         'text',
     'TimeField':         'time',
-    'URLField':          'varchar(200)',
     'USStateField':      'varchar(2)',
 }
diff --git a/django/db/backends/sqlite3/creation.py b/django/db/backends/sqlite3/creation.py
index e845179e64..77f570b2e8 100644
--- a/django/db/backends/sqlite3/creation.py
+++ b/django/db/backends/sqlite3/creation.py
@@ -24,6 +24,5 @@ DATA_TYPES = {
     'SmallIntegerField':            'smallint',
     'TextField':                    'text',
     'TimeField':                    'time',
-    'URLField':                     'varchar(200)',
     'USStateField':                 'varchar(2)',
 }
diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
index 2471514e0e..e5c9ff2fe2 100644
--- a/django/db/models/fields/__init__.py
+++ b/django/db/models/fields/__init__.py
@@ -337,11 +337,11 @@ class Field(object):
             return self._choices
     choices = property(_get_choices)
 
-    def formfield(self, initial=None):
+    def formfield(self, **kwargs):
         "Returns a django.newforms.Field instance for this database Field."
-        from django.newforms import CharField
-        # TODO: This is just a temporary default during development.
-        return forms.CharField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.CharField(**defaults)
 
     def value_from_object(self, obj):
         "Returns the value of this field in the given model instance."
@@ -383,7 +383,7 @@ class AutoField(Field):
         super(AutoField, self).contribute_to_class(cls, name)
         cls._meta.has_auto_field = True
 
-    def formfield(self, initial=None):
+    def formfield(self, **kwargs):
         return None
 
 class BooleanField(Field):
@@ -400,8 +400,10 @@ class BooleanField(Field):
     def get_manipulator_field_objs(self):
         return [oldforms.CheckboxField]
 
-    def formfield(self, initial=None):
-        return forms.BooleanField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.BooleanField(**defaults)
 
 class CharField(Field):
     def get_manipulator_field_objs(self):
@@ -417,8 +419,10 @@ class CharField(Field):
                 raise validators.ValidationError, gettext_lazy("This field cannot be null.")
         return str(value)
 
-    def formfield(self, initial=None):
-        return forms.CharField(max_length=self.maxlength, required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'max_length': self.maxlength, 'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.CharField(**defaults)
 
 # TODO: Maybe move this into contrib, because it's specialized.
 class CommaSeparatedIntegerField(CharField):
@@ -497,8 +501,10 @@ class DateField(Field):
         val = self._get_val_from_obj(obj)
         return {self.attname: (val is not None and val.strftime("%Y-%m-%d") or '')}
 
-    def formfield(self, initial=None):
-        return forms.DateField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.DateField(**defaults)
 
 class DateTimeField(DateField):
     def to_python(self, value):
@@ -569,8 +575,10 @@ class DateTimeField(DateField):
             return {date_field: (val is not None and val.strftime("%Y-%m-%d") or ''),
                     time_field: (val is not None and val.strftime("%H:%M:%S") or '')}
 
-    def formfield(self, initial=None):
-        return forms.DateTimeField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.DateTimeField(**defaults)
 
 class EmailField(CharField):
     def __init__(self, *args, **kwargs):
@@ -586,8 +594,10 @@ class EmailField(CharField):
     def validate(self, field_data, all_data):
         validators.isValidEmail(field_data, all_data)
 
-    def formfield(self, initial=None):
-        return forms.EmailField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.EmailField(**defaults)
 
 class FileField(Field):
     def __init__(self, verbose_name=None, name=None, upload_to='', **kwargs):
@@ -721,8 +731,10 @@ class IntegerField(Field):
     def get_manipulator_field_objs(self):
         return [oldforms.IntegerField]
 
-    def formfield(self, initial=None):
-        return forms.IntegerField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.IntegerField(**defaults)
 
 class IPAddressField(Field):
     def __init__(self, *args, **kwargs):
@@ -778,6 +790,11 @@ class TextField(Field):
     def get_manipulator_field_objs(self):
         return [oldforms.LargeTextField]
 
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'widget': forms.Textarea, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.CharField(**defaults)
+
 class TimeField(Field):
     empty_strings_allowed = False
     def __init__(self, verbose_name=None, name=None, auto_now=False, auto_now_add=False, **kwargs):
@@ -824,21 +841,29 @@ class TimeField(Field):
         val = self._get_val_from_obj(obj)
         return {self.attname: (val is not None and val.strftime("%H:%M:%S") or '')}
 
-    def formfield(self, initial=None):
-        return forms.TimeField(required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.TimeField(**defaults)
 
-class URLField(Field):
+class URLField(CharField):
     def __init__(self, verbose_name=None, name=None, verify_exists=True, **kwargs):
+        kwargs['maxlength'] = kwargs.get('maxlength', 200)
         if verify_exists:
             kwargs.setdefault('validator_list', []).append(validators.isExistingURL)
         self.verify_exists = verify_exists
-        Field.__init__(self, verbose_name, name, **kwargs)
+        CharField.__init__(self, verbose_name, name, **kwargs)
 
     def get_manipulator_field_objs(self):
         return [oldforms.URLField]
 
-    def formfield(self, initial=None):
-        return forms.URLField(required=not self.blank, verify_exists=self.verify_exists, label=capfirst(self.verbose_name), initial=initial)
+    def get_internal_type(self):
+        return "CharField"
+
+    def formfield(self, **kwargs):
+        defaults = {'required': not self.blank, 'verify_exists': self.verify_exists, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.URLField(**defaults)
 
 class USStateField(Field):
     def get_manipulator_field_objs(self):
diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py
index e51c779550..b517747735 100644
--- a/django/db/models/fields/related.py
+++ b/django/db/models/fields/related.py
@@ -316,18 +316,20 @@ def create_many_related_manager(superclass):
             # join_table: name of the m2m link table
             # source_col_name: the PK colname in join_table for the source object
             # target_col_name: the PK colname in join_table for the target object
-            # *objs - objects to add
+            # *objs - objects to add. Either object instances, or primary keys of object instances.
             from django.db import connection
 
             # If there aren't any objects, there is nothing to do.
             if objs:
                 # Check that all the objects are of the right type
+                new_ids = set()
                 for obj in objs:
-                    if not isinstance(obj, self.model):
-                        raise ValueError, "objects to add() must be %s instances" % self.model._meta.object_name
+                    if isinstance(obj, self.model):
+                        new_ids.add(obj._get_pk_val())
+                    else:
+                        new_ids.add(obj)
                 # Add the newly created or already existing objects to the join table.
                 # First find out which items are already added, to avoid adding them twice
-                new_ids = set([obj._get_pk_val() for obj in objs])
                 cursor = connection.cursor()
                 cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \
                     (target_col_name, self.join_table, source_col_name,
@@ -354,11 +356,13 @@ def create_many_related_manager(superclass):
             # If there aren't any objects, there is nothing to do.
             if objs:
                 # Check that all the objects are of the right type
+                old_ids = set()
                 for obj in objs:
-                    if not isinstance(obj, self.model):
-                        raise ValueError, "objects to remove() must be %s instances" % self.model._meta.object_name
+                    if isinstance(obj, self.model):
+                        old_ids.add(obj._get_pk_val())
+                    else:
+                        old_ids.add(obj)
                 # Remove the specified objects from the join table
-                old_ids = set([obj._get_pk_val() for obj in objs])
                 cursor = connection.cursor()
                 cursor.execute("DELETE FROM %s WHERE %s = %%s AND %s IN (%s)" % \
                     (self.join_table, source_col_name,
@@ -548,8 +552,10 @@ class ForeignKey(RelatedField, Field):
     def contribute_to_related_class(self, cls, related):
         setattr(cls, related.get_accessor_name(), ForeignRelatedObjectsDescriptor(related))
 
-    def formfield(self, initial=None):
-        return forms.ChoiceField(choices=self.get_choices_default(), required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'choices': self.get_choices_default(), 'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.ChoiceField(**defaults)
 
 class OneToOneField(RelatedField, IntegerField):
     def __init__(self, to, to_field=None, **kwargs):
@@ -612,8 +618,10 @@ class OneToOneField(RelatedField, IntegerField):
         if not cls._meta.one_to_one_field:
             cls._meta.one_to_one_field = self
 
-    def formfield(self, initial=None):
-        return forms.ChoiceField(choices=self.get_choices_default(), required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+    def formfield(self, **kwargs):
+        defaults = {'choices': self.get_choices_default(), 'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.ChoiceField(**kwargs)
 
 class ManyToManyField(RelatedField, Field):
     def __init__(self, to, **kwargs):
@@ -625,6 +633,7 @@ class ManyToManyField(RelatedField, Field):
             limit_choices_to=kwargs.pop('limit_choices_to', None),
             raw_id_admin=kwargs.pop('raw_id_admin', False),
             symmetrical=kwargs.pop('symmetrical', True))
+        self.db_table = kwargs.pop('db_table', None)
         if kwargs["rel"].raw_id_admin:
             kwargs.setdefault("validator_list", []).append(self.isValidIDList)
         Field.__init__(self, **kwargs)
@@ -647,10 +656,10 @@ class ManyToManyField(RelatedField, Field):
 
     def _get_m2m_db_table(self, opts):
         "Function that can be curried to provide the m2m table name for this relation"
-        from django.db import backend
-        from django.db.backends.util import truncate_name
-        name = '%s_%s' % (opts.db_table, self.name)
-        return truncate_name(name, backend.get_max_name_length())
+        if self.db_table:
+            return self.db_table
+        else:
+            return '%s_%s' % (opts.db_table, self.name)
 
     def _get_m2m_column_name(self, related):
         "Function that can be curried to provide the source column name for the m2m table"
@@ -728,12 +737,14 @@ class ManyToManyField(RelatedField, Field):
         "Returns the value of this field in the given model instance."
         return getattr(obj, self.attname).all()
 
-    def formfield(self, initial=None):
+    def formfield(self, **kwargs):
         # If initial is passed in, it's a list of related objects, but the
         # MultipleChoiceField takes a list of IDs.
-        if initial is not None:
-            initial = [i._get_pk_val() for i in initial]
-        return forms.MultipleChoiceField(choices=self.get_choices_default(), required=not self.blank, label=capfirst(self.verbose_name), initial=initial)
+        if kwargs.get('initial') is not None:
+            kwargs['initial'] = [i._get_pk_val() for i in kwargs['initial']]
+        defaults = {'choices': self.get_choices_default(), 'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text}
+        defaults.update(kwargs)
+        return forms.MultipleChoiceField(**defaults)
 
 class ManyToOneRel(object):
     def __init__(self, to, field_name, num_in_admin=3, min_num_in_admin=None,
diff --git a/django/db/models/manager.py b/django/db/models/manager.py
index 6005874516..b60eed262a 100644
--- a/django/db/models/manager.py
+++ b/django/db/models/manager.py
@@ -1,4 +1,4 @@
-from django.db.models.query import QuerySet
+from django.db.models.query import QuerySet, EmptyQuerySet
 from django.dispatch import dispatcher
 from django.db.models import signals
 from django.db.models.fields import FieldDoesNotExist
@@ -41,12 +41,18 @@ class Manager(object):
     #######################
     # PROXIES TO QUERYSET #
     #######################
+    
+    def get_empty_query_set(self):
+        return EmptyQuerySet(self.model)
 
     def get_query_set(self):
         """Returns a new QuerySet object.  Subclasses can override this method
         to easily customise the behaviour of the Manager.
         """
         return QuerySet(self.model)
+    
+    def none(self):
+        return self.get_empty_query_set()
 
     def all(self):
         return self.get_query_set()
diff --git a/django/db/models/query.py b/django/db/models/query.py
index 231c5c12b0..a124da640b 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -1,5 +1,6 @@
 from django.db import backend, connection, get_query_module, transaction
 from django.db.models.fields import DateField, FieldDoesNotExist
+from django.db.models.fields.generic import GenericRelation
 from django.db.models import signals
 from django.dispatch import dispatcher
 from django.utils.datastructures import SortedDict
@@ -25,6 +26,9 @@ QUERY_TERMS = (
 # Larger values are slightly faster at the expense of more storage space.
 GET_ITERATOR_CHUNK_SIZE = 100
 
+class EmptyResultSet(Exception):
+    pass
+
 ####################
 # HELPER FUNCTIONS #
 ####################
@@ -169,7 +173,11 @@ class _QuerySet(object):
 
         cursor = connection.cursor()
 
-        select, sql, params, full_query = self._get_sql_clause()
+        try:
+            select, sql, params, full_query = self._get_sql_clause()
+        except EmptyResultSet:
+            raise StopIteration
+
         cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
 
         fill_cache = self._select_related
@@ -194,7 +202,12 @@ class _QuerySet(object):
         counter._offset = None
         counter._limit = None
         counter._select_related = False
-        select, sql, params, full_query = counter._get_sql_clause()
+
+        try:
+            select, sql, params, full_query = counter._get_sql_clause()
+        except EmptyResultSet:
+            return 0
+
         cursor = connection.cursor()
         if self._distinct:
             id_col = "%s.%s" % (backend.quote_name(self.model._meta.db_table),
@@ -532,7 +545,12 @@ class ValuesQuerySet(QuerySet):
             field_names = [f.attname for f in self.model._meta.fields]
 
         cursor = connection.cursor()
-        select, sql, params, full_query = self._get_sql_clause()
+
+        try:
+            select, sql, params, full_query = self._get_sql_clause()
+        except EmptyResultSet:
+            raise StopIteration
+
         select = ['%s.%s' % (backend.quote_name(self.model._meta.db_table), backend.quote_name(c)) for c in columns]
         cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
         while 1:
@@ -554,19 +572,25 @@ class DateQuerySet(QuerySet):
         if self._field.null:
             self._where.append('%s.%s IS NOT NULL' % \
                 (backend.quote_name(self.model._meta.db_table), backend.quote_name(self._field.column)))
-        select, sql, params, full_query = self._get_sql_clause()
+        try:
+            select, sql, params, full_query = self._get_sql_clause()
+        except EmptyResultSet:
+            raise StopIteration
+
         table_name = backend.quote_name(self.model._meta.db_table)
         field_name = backend.quote_name(self._field.column)
-        date_trunc_sql = backend.get_date_trunc_sql(self._kind,
-                                                    '%s.%s' % (table_name, field_name))
+
         if backend.allows_group_by_ordinal:
             group_by = '1'
         else:
-            group_by = date_trunc_sql
-        fmt = 'SELECT %s %s GROUP BY %s ORDER BY 1 %s'
-        stmt = fmt % (date_trunc_sql, sql, group_by, self._order)
+            group_by = backend.get_date_trunc_sql(self._kind,
+                                                  '%s.%s' % (table_name, field_name))
+
+        sql = 'SELECT %s %s GROUP BY %s ORDER BY 1 %s' % \
+            (backend.get_date_trunc_sql(self._kind, '%s.%s' % (backend.quote_name(self.model._meta.db_table),
+            backend.quote_name(self._field.column))), sql, group_by, self._order)
         cursor = connection.cursor()
-        cursor.execute(stmt, params)
+        cursor.execute(sql, params)
         if backend.needs_datetime_string_cast:
             return [typecast_timestamp(str(row[0])) for row in cursor.fetchall()]
         else:
@@ -579,6 +603,25 @@ class DateQuerySet(QuerySet):
         c._order = self._order
         return c
 
+class EmptyQuerySet(QuerySet):
+    def __init__(self, model=None):
+        super(EmptyQuerySet, self).__init__(model)
+        self._result_cache = []
+
+    def iterator(self):
+        raise StopIteration
+
+    def count(self):
+        return 0
+
+    def delete(self):
+        pass
+
+    def _clone(self, klass=None, **kwargs):
+        c = super(EmptyQuerySet, self)._clone(klass, **kwargs)
+        c._result_cache = []
+        return c
+
 class QOperator(object):
     "Base class for QAnd and QOr"
     def __init__(self, *args):
@@ -587,10 +630,14 @@ class QOperator(object):
     def get_sql(self, opts):
         joins, where, params = SortedDict(), [], []
         for val in self.args:
-            joins2, where2, params2 = val.get_sql(opts)
-            joins.update(joins2)
-            where.extend(where2)
-            params.extend(params2)
+            try:
+                joins2, where2, params2 = val.get_sql(opts)
+                joins.update(joins2)
+                where.extend(where2)
+                params.extend(params2)
+            except EmptyResultSet:
+                if not isinstance(self, QOr):
+                    raise EmptyResultSet
         if where:
             return joins, ['(%s)' % self.operator.join(where)], params
         return joins, [], params
@@ -644,8 +691,11 @@ class QNot(Q):
         self.q = q
 
     def get_sql(self, opts):
-        joins, where, params = self.q.get_sql(opts)
-        where2 = ['(NOT (%s))' % " AND ".join(where)]
+        try:
+            joins, where, params = self.q.get_sql(opts)
+            where2 = ['(NOT (%s))' % " AND ".join(where)]
+        except EmptyResultSet:
+            return SortedDict(), [], []
         return joins, where2, params
 
 def get_where_clause(lookup_type, table_prefix, field_name, value):
@@ -662,7 +712,11 @@ def get_where_clause(lookup_type, table_prefix, field_name, value):
     except KeyError:
         pass
     if lookup_type == 'in':
-        return '%s%s IN (%s)' % (table_prefix, field_name, ','.join(['%s' for v in value]))
+        in_string = ','.join(['%s' for id in value])
+        if in_string:
+            return '%s%s IN (%s)' % (table_prefix, field_name, in_string)
+        else:
+            raise EmptyResultSet
     elif lookup_type == 'range':
         return '%s%s BETWEEN %%s AND %%s' % (table_prefix, field_name)
     elif lookup_type in ('year', 'month', 'day'):
@@ -948,18 +1002,26 @@ def delete_objects(seen_objs):
 
         pk_list = [pk for pk,instance in seen_objs[cls]]
         for related in cls._meta.get_all_related_many_to_many_objects():
-            for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
-                cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
-                    (qn(related.field.m2m_db_table()),
-                        qn(related.field.m2m_reverse_name()),
-                        ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
-                    pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
+            if not isinstance(related.field, GenericRelation):
+                for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
+                    cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
+                        (qn(related.field.m2m_db_table()),
+                            qn(related.field.m2m_reverse_name()),
+                            ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
+                        pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
         for f in cls._meta.many_to_many:
+            if isinstance(f, GenericRelation):
+                from django.contrib.contenttypes.models import ContentType
+                query_extra = 'AND %s=%%s' % f.rel.to._meta.get_field(f.content_type_field_name).column
+                args_extra = [ContentType.objects.get_for_model(cls).id]
+            else:
+                query_extra = ''
+                args_extra = []
             for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
-                cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
+                cursor.execute(("DELETE FROM %s WHERE %s IN (%s)" % \
                     (qn(f.m2m_db_table()), qn(f.m2m_column_name()),
-                    ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
-                    pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
+                    ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]]))) + query_extra,
+                    pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE] + args_extra)
         for field in cls._meta.fields:
             if field.rel and field.null and field.rel.to in seen_objs:
                 for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
diff --git a/django/newforms/fields.py b/django/newforms/fields.py
index 408c90df45..e9e1fb7746 100644
--- a/django/newforms/fields.py
+++ b/django/newforms/fields.py
@@ -3,8 +3,8 @@ Field classes
 """
 
 from django.utils.translation import gettext
-from util import ValidationError, smart_unicode
-from widgets import TextInput, PasswordInput, CheckboxInput, Select, SelectMultiple
+from util import ErrorList, ValidationError, smart_unicode
+from widgets import TextInput, PasswordInput, HiddenInput, MultipleHiddenInput, CheckboxInput, Select, NullBooleanSelect, SelectMultiple
 import datetime
 import re
 import time
@@ -15,8 +15,9 @@ __all__ = (
     'DEFAULT_TIME_INPUT_FORMATS', 'TimeField',
     'DEFAULT_DATETIME_INPUT_FORMATS', 'DateTimeField',
     'RegexField', 'EmailField', 'URLField', 'BooleanField',
-    'ChoiceField', 'MultipleChoiceField',
-    'ComboField',
+    'ChoiceField', 'NullBooleanField', 'MultipleChoiceField',
+    'ComboField', 'MultiValueField',
+    'SplitDateTimeField',
 )
 
 # These values, if given to to_python(), will trigger the self.required check.
@@ -29,11 +30,12 @@ except NameError:
 
 class Field(object):
     widget = TextInput # Default widget to use when rendering this type of Field.
+    hidden_widget = HiddenInput # Default widget to use when rendering this as "hidden".
 
     # Tracks each time a Field instance is created. Used to retain order.
     creation_counter = 0
 
-    def __init__(self, required=True, widget=None, label=None, initial=None):
+    def __init__(self, required=True, widget=None, label=None, initial=None, help_text=None):
         # required -- Boolean that specifies whether the field is required.
         #             True by default.
         # widget -- A Widget class, or instance of a Widget class, that should be
@@ -45,9 +47,11 @@ class Field(object):
         #         field name, if the Field is part of a Form.
         # initial -- A value to use in this Field's initial display. This value is
         #            *not* used as a fallback if data isn't given.
+        # help_text -- An optional string to use as "help text" for this Field.
         if label is not None:
             label = smart_unicode(label)
         self.required, self.label, self.initial = required, label, initial
+        self.help_text = help_text
         widget = widget or self.widget
         if isinstance(widget, type):
             widget = widget()
@@ -83,17 +87,15 @@ class Field(object):
         return {}
 
 class CharField(Field):
-    def __init__(self, max_length=None, min_length=None, required=True, widget=None, label=None, initial=None):
+    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
         self.max_length, self.min_length = max_length, min_length
-        Field.__init__(self, required, widget, label, initial)
+        super(CharField, self).__init__(*args, **kwargs)
 
     def clean(self, value):
         "Validates max_length and min_length. Returns a Unicode object."
-        Field.clean(self, value)
+        super(CharField, self).clean(value)
         if value in EMPTY_VALUES:
-            value = u''
-            if not self.required:
-                return value
+            return u''
         value = smart_unicode(value)
         if self.max_length is not None and len(value) > self.max_length:
             raise ValidationError(gettext(u'Ensure this value has at most %d characters.') % self.max_length)
@@ -106,18 +108,18 @@ class CharField(Field):
             return {'maxlength': str(self.max_length)}
 
 class IntegerField(Field):
-    def __init__(self, max_value=None, min_value=None, required=True, widget=None, label=None, initial=None):
+    def __init__(self, max_value=None, min_value=None, *args, **kwargs):
         self.max_value, self.min_value = max_value, min_value
-        Field.__init__(self, required, widget, label, initial)
+        super(IntegerField, self).__init__(*args, **kwargs)
 
     def clean(self, value):
         """
         Validates that int() can be called on the input. Returns the result
-        of int().
+        of int(). Returns None for empty values.
         """
         super(IntegerField, self).clean(value)
-        if not self.required and value in EMPTY_VALUES:
-            return u''
+        if value in EMPTY_VALUES:
+            return None
         try:
             value = int(value)
         except (ValueError, TypeError):
@@ -137,8 +139,8 @@ DEFAULT_DATE_INPUT_FORMATS = (
 )
 
 class DateField(Field):
-    def __init__(self, input_formats=None, required=True, widget=None, label=None, initial=None):
-        Field.__init__(self, required, widget, label, initial)
+    def __init__(self, input_formats=None, *args, **kwargs):
+        super(DateField, self).__init__(*args, **kwargs)
         self.input_formats = input_formats or DEFAULT_DATE_INPUT_FORMATS
 
     def clean(self, value):
@@ -146,7 +148,7 @@ class DateField(Field):
         Validates that the input can be converted to a date. Returns a Python
         datetime.date object.
         """
-        Field.clean(self, value)
+        super(DateField, self).clean(value)
         if value in EMPTY_VALUES:
             return None
         if isinstance(value, datetime.datetime):
@@ -166,8 +168,8 @@ DEFAULT_TIME_INPUT_FORMATS = (
 )
 
 class TimeField(Field):
-    def __init__(self, input_formats=None, required=True, widget=None, label=None, initial=None):
-        Field.__init__(self, required, widget, label, initial)
+    def __init__(self, input_formats=None, *args, **kwargs):
+        super(TimeField, self).__init__(*args, **kwargs)
         self.input_formats = input_formats or DEFAULT_TIME_INPUT_FORMATS
 
     def clean(self, value):
@@ -175,7 +177,7 @@ class TimeField(Field):
         Validates that the input can be converted to a time. Returns a Python
         datetime.time object.
         """
-        Field.clean(self, value)
+        super(TimeField, self).clean(value)
         if value in EMPTY_VALUES:
             return None
         if isinstance(value, datetime.time):
@@ -200,8 +202,8 @@ DEFAULT_DATETIME_INPUT_FORMATS = (
 )
 
 class DateTimeField(Field):
-    def __init__(self, input_formats=None, required=True, widget=None, label=None, initial=None):
-        Field.__init__(self, required, widget, label, initial)
+    def __init__(self, input_formats=None, *args, **kwargs):
+        super(DateTimeField, self).__init__(*args, **kwargs)
         self.input_formats = input_formats or DEFAULT_DATETIME_INPUT_FORMATS
 
     def clean(self, value):
@@ -209,7 +211,7 @@ class DateTimeField(Field):
         Validates that the input can be converted to a datetime. Returns a
         Python datetime.datetime object.
         """
-        Field.clean(self, value)
+        super(DateTimeField, self).clean(value)
         if value in EMPTY_VALUES:
             return None
         if isinstance(value, datetime.datetime):
@@ -224,14 +226,13 @@ class DateTimeField(Field):
         raise ValidationError(gettext(u'Enter a valid date/time.'))
 
 class RegexField(Field):
-    def __init__(self, regex, max_length=None, min_length=None, error_message=None,
-            required=True, widget=None, label=None, initial=None):
+    def __init__(self, regex, max_length=None, min_length=None, error_message=None, *args, **kwargs):
         """
         regex can be either a string or a compiled regular expression object.
         error_message is an optional error message to use, if
         'Enter a valid value' is too generic for you.
         """
-        Field.__init__(self, required, widget, label, initial)
+        super(RegexField, self).__init__(*args, **kwargs)
         if isinstance(regex, basestring):
             regex = re.compile(regex)
         self.regex = regex
@@ -243,10 +244,11 @@ class RegexField(Field):
         Validates that the input matches the regular expression. Returns a
         Unicode object.
         """
-        Field.clean(self, value)
-        if value in EMPTY_VALUES: value = u''
+        super(RegexField, self).clean(value)
+        if value in EMPTY_VALUES:
+            value = u''
         value = smart_unicode(value)
-        if not self.required and value == u'':
+        if value == u'':
             return value
         if self.max_length is not None and len(value) > self.max_length:
             raise ValidationError(gettext(u'Ensure this value has at most %d characters.') % self.max_length)
@@ -262,8 +264,9 @@ email_re = re.compile(
     r')@(?:[A-Z0-9-]+\.)+[A-Z]{2,6}$', re.IGNORECASE)  # domain
 
 class EmailField(RegexField):
-    def __init__(self, max_length=None, min_length=None, required=True, widget=None, label=None, initial=None):
-        RegexField.__init__(self, email_re, max_length, min_length, gettext(u'Enter a valid e-mail address.'), required, widget, label, initial)
+    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
+        RegexField.__init__(self, email_re, max_length, min_length,
+            gettext(u'Enter a valid e-mail address.'), *args, **kwargs)
 
 url_re = re.compile(
     r'^https?://' # http:// or https://
@@ -279,14 +282,16 @@ except ImportError:
     URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
 
 class URLField(RegexField):
-    def __init__(self, max_length=None, min_length=None, required=True, verify_exists=False, widget=None, label=None,
-            initial=None, validator_user_agent=URL_VALIDATOR_USER_AGENT):
-        RegexField.__init__(self, url_re, max_length, min_length, gettext(u'Enter a valid URL.'), required, widget, label, initial)
+    def __init__(self, max_length=None, min_length=None, verify_exists=False,
+            validator_user_agent=URL_VALIDATOR_USER_AGENT, *args, **kwargs):
+        super(URLField, self).__init__(url_re, max_length, min_length, gettext(u'Enter a valid URL.'), *args, **kwargs)
         self.verify_exists = verify_exists
         self.user_agent = validator_user_agent
 
     def clean(self, value):
-        value = RegexField.clean(self, value)
+        value = super(URLField, self).clean(value)
+        if value == u'':
+            return value
         if self.verify_exists:
             import urllib2
             from django.conf import settings
@@ -311,24 +316,43 @@ class BooleanField(Field):
 
     def clean(self, value):
         "Returns a Python boolean object."
-        Field.clean(self, value)
+        super(BooleanField, self).clean(value)
         return bool(value)
 
+class NullBooleanField(BooleanField):
+    """
+    A field whose valid values are None, True and False. Invalid values are
+    cleaned to None.
+    """
+    widget = NullBooleanSelect
+
+    def clean(self, value):
+        return {True: True, False: False}.get(value, None)
+
 class ChoiceField(Field):
-    def __init__(self, choices=(), required=True, widget=Select, label=None, initial=None):
-        if isinstance(widget, type):
-            widget = widget(choices=choices)
-        Field.__init__(self, required, widget, label, initial)
+    def __init__(self, choices=(), required=True, widget=Select, label=None, initial=None, help_text=None):
+        super(ChoiceField, self).__init__(required, widget, label, initial, help_text)
         self.choices = choices
 
+    def _get_choices(self):
+        return self._choices
+
+    def _set_choices(self, value):
+        # Setting choices also sets the choices on the widget.
+        self._choices = value
+        self.widget.choices = value
+
+    choices = property(_get_choices, _set_choices)
+
     def clean(self, value):
         """
         Validates that the input is in self.choices.
         """
-        value = Field.clean(self, value)
-        if value in EMPTY_VALUES: value = u''
+        value = super(ChoiceField, self).clean(value)
+        if value in EMPTY_VALUES:
+            value = u''
         value = smart_unicode(value)
-        if not self.required and value == u'':
+        if value == u'':
             return value
         valid_values = set([str(k) for k, v in self.choices])
         if value not in valid_values:
@@ -336,8 +360,10 @@ class ChoiceField(Field):
         return value
 
 class MultipleChoiceField(ChoiceField):
-    def __init__(self, choices=(), required=True, widget=SelectMultiple, label=None, initial=None):
-        ChoiceField.__init__(self, choices, required, widget, label, initial)
+    hidden_widget = MultipleHiddenInput
+
+    def __init__(self, choices=(), required=True, widget=SelectMultiple, label=None, initial=None, help_text=None):
+        super(MultipleChoiceField, self).__init__(choices, required, widget, label, initial, help_text)
 
     def clean(self, value):
         """
@@ -361,8 +387,11 @@ class MultipleChoiceField(ChoiceField):
         return new_value
 
 class ComboField(Field):
-    def __init__(self, fields=(), required=True, widget=None, label=None, initial=None):
-        Field.__init__(self, required, widget, label, initial)
+    """
+    A Field whose clean() method calls multiple Field clean() methods.
+    """
+    def __init__(self, fields=(), *args, **kwargs):
+        super(ComboField, self).__init__(*args, **kwargs)
         # Set 'required' to False on the individual fields, because the
         # required validation will be handled by ComboField, not by those
         # individual fields.
@@ -375,7 +404,88 @@ class ComboField(Field):
         Validates the given value against all of self.fields, which is a
         list of Field instances.
         """
-        Field.clean(self, value)
+        super(ComboField, self).clean(value)
         for field in self.fields:
             value = field.clean(value)
         return value
+
+class MultiValueField(Field):
+    """
+    A Field that is composed of multiple Fields.
+
+    Its clean() method takes a "decompressed" list of values. Each value in
+    this list is cleaned by the corresponding field -- the first value is
+    cleaned by the first field, the second value is cleaned by the second
+    field, etc. Once all fields are cleaned, the list of clean values is
+    "compressed" into a single value.
+
+    Subclasses should implement compress(), which specifies how a list of
+    valid values should be converted to a single value. Subclasses should not
+    have to implement clean().
+
+    You'll probably want to use this with MultiWidget.
+    """
+    def __init__(self, fields=(), *args, **kwargs):
+        super(MultiValueField, self).__init__(*args, **kwargs)
+        # Set 'required' to False on the individual fields, because the
+        # required validation will be handled by MultiValueField, not by those
+        # individual fields.
+        for f in fields:
+            f.required = False
+        self.fields = fields
+
+    def clean(self, value):
+        """
+        Validates every value in the given list. A value is validated against
+        the corresponding Field in self.fields.
+
+        For example, if this MultiValueField was instantiated with
+        fields=(DateField(), TimeField()), clean() would call
+        DateField.clean(value[0]) and TimeField.clean(value[1]).
+        """
+        clean_data = []
+        errors = ErrorList()
+        if self.required and not value:
+            raise ValidationError(gettext(u'This field is required.'))
+        elif not self.required and not value:
+            return self.compress([])
+        if not isinstance(value, (list, tuple)):
+            raise ValidationError(gettext(u'Enter a list of values.'))
+        for i, field in enumerate(self.fields):
+            try:
+                field_value = value[i]
+            except KeyError:
+                field_value = None
+            if self.required and field_value in EMPTY_VALUES:
+                raise ValidationError(gettext(u'This field is required.'))
+            try:
+                clean_data.append(field.clean(field_value))
+            except ValidationError, e:
+                # Collect all validation errors in a single list, which we'll
+                # raise at the end of clean(), rather than raising a single
+                # exception for the first error we encounter.
+                errors.extend(e.messages)
+        if errors:
+            raise ValidationError(errors)
+        return self.compress(clean_data)
+
+    def compress(self, data_list):
+        """
+        Returns a single value for the given list of values. The values can be
+        assumed to be valid.
+
+        For example, if this MultiValueField was instantiated with
+        fields=(DateField(), TimeField()), this might return a datetime
+        object created by combining the date and time in data_list.
+        """
+        raise NotImplementedError('Subclasses must implement this method.')
+
+class SplitDateTimeField(MultiValueField):
+    def __init__(self, *args, **kwargs):
+        fields = (DateField(), TimeField())
+        super(SplitDateTimeField, self).__init__(fields, *args, **kwargs)
+
+    def compress(self, data_list):
+        if data_list:
+            return datetime.datetime.combine(*data_list)
+        return None
diff --git a/django/newforms/forms.py b/django/newforms/forms.py
index 4928065e9d..7e29941bef 100644
--- a/django/newforms/forms.py
+++ b/django/newforms/forms.py
@@ -5,8 +5,8 @@ Form classes
 from django.utils.datastructures import SortedDict, MultiValueDict
 from django.utils.html import escape
 from fields import Field
-from widgets import TextInput, Textarea, HiddenInput
-from util import StrAndUnicode, ErrorDict, ErrorList, ValidationError
+from widgets import TextInput, Textarea, HiddenInput, MultipleHiddenInput
+from util import flatatt, StrAndUnicode, ErrorDict, ErrorList, ValidationError
 
 __all__ = ('BaseForm', 'Form')
 
@@ -26,12 +26,15 @@ class SortedDictFromList(SortedDict):
         self.keyOrder = [d[0] for d in data]
         dict.__init__(self, dict(data))
 
+    def copy(self):
+        return SortedDictFromList(self.items())
+
 class DeclarativeFieldsMetaclass(type):
-    "Metaclass that converts Field attributes to a dictionary called 'fields'."
+    "Metaclass that converts Field attributes to a dictionary called 'base_fields'."
     def __new__(cls, name, bases, attrs):
-        fields = [(name, attrs.pop(name)) for name, obj in attrs.items() if isinstance(obj, Field)]
+        fields = [(field_name, attrs.pop(field_name)) for field_name, obj in attrs.items() if isinstance(obj, Field)]
         fields.sort(lambda x, y: cmp(x[1].creation_counter, y[1].creation_counter))
-        attrs['fields'] = SortedDictFromList(fields)
+        attrs['base_fields'] = SortedDictFromList(fields)
         return type.__new__(cls, name, bases, attrs)
 
 class BaseForm(StrAndUnicode):
@@ -39,13 +42,19 @@ class BaseForm(StrAndUnicode):
     # class is different than Form. See the comments by the Form class for more
     # information. Any improvements to the form API should be made to *this*
     # class, not to the Form class.
-    def __init__(self, data=None, auto_id='id_%s', prefix=None):
-        self.ignore_errors = data is None
+    def __init__(self, data=None, auto_id='id_%s', prefix=None, initial=None):
+        self.is_bound = data is not None
         self.data = data or {}
         self.auto_id = auto_id
         self.prefix = prefix
-        self.clean_data = None # Stores the data after clean() has been called.
+        self.initial = initial or {}
         self.__errors = None # Stores the errors after clean() has been called.
+        # The base_fields class attribute is the *class-wide* definition of
+        # fields. Because a particular *instance* of the class might want to
+        # alter self.fields, we create self.fields here by copying base_fields.
+        # Instances should always modify self.fields; they should not modify
+        # self.base_fields.
+        self.fields = self.base_fields.copy()
 
     def __unicode__(self):
         return self.as_table()
@@ -74,7 +83,7 @@ class BaseForm(StrAndUnicode):
         Returns True if the form has no errors. Otherwise, False. If errors are
         being ignored, returns False.
         """
-        return not self.ignore_errors and not bool(self.errors)
+        return self.is_bound and not bool(self.errors)
 
     def add_prefix(self, field_name):
         """
@@ -85,7 +94,7 @@ class BaseForm(StrAndUnicode):
         """
         return self.prefix and ('%s-%s' % (self.prefix, field_name)) or field_name
 
-    def _html_output(self, normal_row, error_row, row_ender, errors_on_separate_row):
+    def _html_output(self, normal_row, error_row, row_ender, help_text_html, errors_on_separate_row):
         "Helper function for outputting HTML. Used by as_table(), as_ul(), as_p()."
         top_errors = self.non_field_errors() # Errors that should be displayed above all fields.
         output, hidden_fields = [], []
@@ -100,7 +109,11 @@ class BaseForm(StrAndUnicode):
                 if errors_on_separate_row and bf_errors:
                     output.append(error_row % bf_errors)
                 label = bf.label and bf.label_tag(escape(bf.label + ':')) or ''
-                output.append(normal_row % {'errors': bf_errors, 'label': label, 'field': bf})
+                if field.help_text:
+                    help_text = help_text_html % field.help_text
+                else:
+                    help_text = u''
+                output.append(normal_row % {'errors': bf_errors, 'label': label, 'field': unicode(bf), 'help_text': help_text})
         if top_errors:
             output.insert(0, error_row % top_errors)
         if hidden_fields: # Insert any hidden fields in the last row.
@@ -115,15 +128,15 @@ class BaseForm(StrAndUnicode):
 
     def as_table(self):
         "Returns this form rendered as HTML <tr>s -- excluding the <table></table>."
-        return self._html_output(u'<tr><th>%(label)s</th><td>%(errors)s%(field)s</td></tr>', u'<tr><td colspan="2">%s</td></tr>', '</td></tr>', False)
+        return self._html_output(u'<tr><th>%(label)s</th><td>%(errors)s%(field)s%(help_text)s</td></tr>', u'<tr><td colspan="2">%s</td></tr>', '</td></tr>', u'<br />%s', False)
 
     def as_ul(self):
         "Returns this form rendered as HTML <li>s -- excluding the <ul></ul>."
-        return self._html_output(u'<li>%(errors)s%(label)s %(field)s</li>', u'<li>%s</li>', '</li>', False)
+        return self._html_output(u'<li>%(errors)s%(label)s %(field)s%(help_text)s</li>', u'<li>%s</li>', '</li>', u' %s', False)
 
     def as_p(self):
         "Returns this form rendered as HTML <p>s."
-        return self._html_output(u'<p>%(label)s %(field)s</p>', u'<p>%s</p>', '</p>', True)
+        return self._html_output(u'<p>%(label)s %(field)s%(help_text)s</p>', u'<p>%s</p>', '</p>', u' %s', True)
 
     def non_field_errors(self):
         """
@@ -137,11 +150,11 @@ class BaseForm(StrAndUnicode):
         """
         Cleans all of self.data and populates self.__errors and self.clean_data.
         """
-        self.clean_data = {}
         errors = ErrorDict()
-        if self.ignore_errors: # Stop further processing.
+        if not self.is_bound: # Stop further processing.
             self.__errors = errors
             return
+        self.clean_data = {}
         for name, field in self.fields.items():
             # value_from_datadict() gets the data from the dictionary.
             # Each widget type knows how to retrieve its own data, because some
@@ -160,7 +173,7 @@ class BaseForm(StrAndUnicode):
         except ValidationError, e:
             errors[NON_FIELD_ERRORS] = e.messages
         if errors:
-            self.clean_data = None
+            delattr(self, 'clean_data')
         self.__errors = errors
 
     def clean(self):
@@ -218,8 +231,8 @@ class BoundField(StrAndUnicode):
         auto_id = self.auto_id
         if auto_id and not attrs.has_key('id') and not widget.attrs.has_key('id'):
             attrs['id'] = auto_id
-        if self.form.ignore_errors:
-            data = self.field.initial
+        if not self.form.is_bound:
+            data = self.form.initial.get(self.name, self.field.initial)
         else:
             data = self.data
         return widget.render(self.html_name, data, attrs=attrs)
@@ -238,7 +251,7 @@ class BoundField(StrAndUnicode):
         """
         Returns a string of HTML for representing this as an <input type="hidden">.
         """
-        return self.as_widget(HiddenInput(), attrs)
+        return self.as_widget(self.field.hidden_widget(), attrs)
 
     def _data(self):
         """
@@ -247,17 +260,20 @@ class BoundField(StrAndUnicode):
         return self.field.widget.value_from_datadict(self.form.data, self.html_name)
     data = property(_data)
 
-    def label_tag(self, contents=None):
+    def label_tag(self, contents=None, attrs=None):
         """
         Wraps the given contents in a <label>, if the field has an ID attribute.
         Does not HTML-escape the contents. If contents aren't given, uses the
         field's HTML-escaped label.
+
+        If attrs are given, they're used as HTML attributes on the <label> tag.
         """
         contents = contents or escape(self.label)
         widget = self.field.widget
         id_ = widget.attrs.get('id') or self.auto_id
         if id_:
-            contents = '<label for="%s">%s</label>' % (widget.id_for_label(id_), contents)
+            attrs = attrs and flatatt(attrs) or ''
+            contents = '<label for="%s"%s>%s</label>' % (widget.id_for_label(id_), attrs, contents)
         return contents
 
     def _is_hidden(self):
diff --git a/django/newforms/models.py b/django/newforms/models.py
index e000e394e1..a938b6350e 100644
--- a/django/newforms/models.py
+++ b/django/newforms/models.py
@@ -5,9 +5,9 @@ and database field objects.
 
 from forms import BaseForm, DeclarativeFieldsMetaclass, SortedDictFromList
 
-__all__ = ('form_for_model', 'form_for_instance', 'form_for_fields')
+__all__ = ('save_instance', 'form_for_model', 'form_for_instance', 'form_for_fields')
 
-def create(self, save=True):
+def model_save(self, commit=True):
     """
     Creates and returns model instance according to self.clean_data.
 
@@ -15,61 +15,84 @@ def create(self, save=True):
     """
     if self.errors:
         raise ValueError("The %s could not be created because the data didn't validate." % self._model._meta.object_name)
-    obj = self._model(**self.clean_data)
-    if save:
-        obj.save()
-    return obj
+    return save_instance(self, self._model(), commit)
 
-def make_apply_changes(opts, instance):
-    "Returns the apply_changes() method for a form_for_instance Form."
+def save_instance(form, instance, commit=True):
+    """
+    Saves bound Form ``form``'s clean_data into model instance ``instance``.
+
+    Assumes ``form`` has a field for every non-AutoField database field in
+    ``instance``. If commit=True, then the changes to ``instance`` will be
+    saved to the database. Returns ``instance``.
+    """
     from django.db import models
-    def apply_changes(self, save=True):
-        if self.errors:
-            raise ValueError("The %s could not be changed because the data didn't validate." % opts.object_name)
-        clean_data = self.clean_data
-        for f in opts.fields + opts.many_to_many:
-            if isinstance(f, models.AutoField):
-                continue
+    opts = instance.__class__._meta
+    if form.errors:
+        raise ValueError("The %s could not be changed because the data didn't validate." % opts.object_name)
+    clean_data = form.clean_data
+    for f in opts.fields:
+        if isinstance(f, models.AutoField):
+            continue
+        setattr(instance, f.attname, clean_data[f.name])
+    if commit:
+        instance.save()
+        for f in opts.many_to_many:
             setattr(instance, f.attname, clean_data[f.name])
-        if save:
-            instance.save()
-        return instance
-    return apply_changes
+    # GOTCHA: If many-to-many data is given and commit=False, the many-to-many
+    # data will be lost. This happens because a many-to-many options cannot be
+    # set on an object until after it's saved. Maybe we should raise an
+    # exception in that case.
+    return instance
 
-def form_for_model(model, form=BaseForm):
+def make_instance_save(instance):
+    "Returns the save() method for a form_for_instance Form."
+    def save(self, commit=True):
+        return save_instance(self, instance, commit)
+    return save
+
+def form_for_model(model, form=BaseForm, formfield_callback=lambda f: f.formfield()):
     """
     Returns a Form class for the given Django model class.
 
-    Provide 'form' if you want to use a custom BaseForm subclass.
+    Provide ``form`` if you want to use a custom BaseForm subclass.
+
+    Provide ``formfield_callback`` if you want to define different logic for
+    determining the formfield for a given database field. It's a callable that
+    takes a database Field instance and returns a form Field instance.
     """
     opts = model._meta
     field_list = []
     for f in opts.fields + opts.many_to_many:
-        formfield = f.formfield()
+        formfield = formfield_callback(f)
         if formfield:
             field_list.append((f.name, formfield))
     fields = SortedDictFromList(field_list)
-    return type(opts.object_name + 'Form', (form,), {'fields': fields, '_model': model, 'create': create})
+    return type(opts.object_name + 'Form', (form,), {'base_fields': fields, '_model': model, 'save': model_save})
 
-def form_for_instance(instance, form=BaseForm):
+def form_for_instance(instance, form=BaseForm, formfield_callback=lambda f, **kwargs: f.formfield(**kwargs)):
     """
     Returns a Form class for the given Django model instance.
 
-    Provide 'form' if you want to use a custom BaseForm subclass.
+    Provide ``form`` if you want to use a custom BaseForm subclass.
+
+    Provide ``formfield_callback`` if you want to define different logic for
+    determining the formfield for a given database field. It's a callable that
+    takes a database Field instance, plus **kwargs, and returns a form Field
+    instance with the given kwargs (i.e. 'initial').
     """
     model = instance.__class__
     opts = model._meta
     field_list = []
     for f in opts.fields + opts.many_to_many:
         current_value = f.value_from_object(instance)
-        formfield = f.formfield(initial=current_value)
+        formfield = formfield_callback(f, initial=current_value)
         if formfield:
             field_list.append((f.name, formfield))
     fields = SortedDictFromList(field_list)
     return type(opts.object_name + 'InstanceForm', (form,),
-        {'fields': fields, '_model': model, 'apply_changes': make_apply_changes(opts, instance)})
+        {'base_fields': fields, '_model': model, 'save': make_instance_save(instance)})
 
 def form_for_fields(field_list):
     "Returns a Form class for the given list of Django database field instances."
     fields = SortedDictFromList([(f.name, f.formfield()) for f in field_list])
-    return type('FormForFields', (BaseForm,), {'fields': fields})
+    return type('FormForFields', (BaseForm,), {'base_fields': fields})
diff --git a/django/newforms/util.py b/django/newforms/util.py
index a78623a17b..2fec9e4e2e 100644
--- a/django/newforms/util.py
+++ b/django/newforms/util.py
@@ -1,4 +1,9 @@
 from django.conf import settings
+from django.utils.html import escape
+
+# Converts a dictionary to a single string with key="value", XML-style with
+# a leading space. Assumes keys do not need to be XML-escaped.
+flatatt = lambda attrs: u''.join([u' %s="%s"' % (k, escape(v)) for k, v in attrs.items()])
 
 def smart_unicode(s):
     if not isinstance(s, basestring):
diff --git a/django/newforms/widgets.py b/django/newforms/widgets.py
index 996e353775..585e12cc18 100644
--- a/django/newforms/widgets.py
+++ b/django/newforms/widgets.py
@@ -3,14 +3,16 @@ HTML Widget classes
 """
 
 __all__ = (
-    'Widget', 'TextInput', 'PasswordInput', 'HiddenInput', 'FileInput',
-    'Textarea', 'CheckboxInput',
-    'Select', 'SelectMultiple', 'RadioSelect', 'CheckboxSelectMultiple',
+    'Widget', 'TextInput', 'PasswordInput', 'HiddenInput', 'MultipleHiddenInput',
+    'FileInput', 'Textarea', 'CheckboxInput',
+    'Select', 'NullBooleanSelect', 'SelectMultiple', 'RadioSelect', 'CheckboxSelectMultiple',
+    'MultiWidget', 'SplitDateTimeWidget',
 )
 
-from util import StrAndUnicode, smart_unicode
+from util import flatatt, StrAndUnicode, smart_unicode
 from django.utils.datastructures import MultiValueDict
 from django.utils.html import escape
+from django.utils.translation import gettext
 from itertools import chain
 
 try:
@@ -18,10 +20,6 @@ try:
 except NameError:
     from sets import Set as set # Python 2.3 fallback
 
-# Converts a dictionary to a single string with key="value", XML-style with
-# a leading space. Assumes keys do not need to be XML-escaped.
-flatatt = lambda attrs: u''.join([u' %s="%s"' % (k, escape(v)) for k, v in attrs.items()])
-
 class Widget(object):
     is_hidden = False          # Determines whether this corresponds to an <input type="hidden">.
 
@@ -87,6 +85,26 @@ class HiddenInput(Input):
     input_type = 'hidden'
     is_hidden = True
 
+class MultipleHiddenInput(HiddenInput):
+    """
+    A widget that handles <input type="hidden"> for fields that have a list
+    of values.
+    """
+    def __init__(self, attrs=None, choices=()):
+        # choices can be any iterable
+        self.attrs = attrs or {}
+        self.choices = choices
+
+    def render(self, name, value, attrs=None, choices=()):
+        if value is None: value = []
+        final_attrs = self.build_attrs(attrs, type=self.input_type, name=name)
+        return u'\n'.join([(u'<input%s />' % flatatt(dict(value=smart_unicode(v), **final_attrs))) for v in value])
+
+    def value_from_datadict(self, data, name):
+        if isinstance(data, MultiValueDict):
+            return data.getlist(name)
+        return data.get(name, None)
+
 class FileInput(Input):
     input_type = 'file'
 
@@ -118,9 +136,11 @@ class CheckboxInput(Widget):
 
 class Select(Widget):
     def __init__(self, attrs=None, choices=()):
-        # choices can be any iterable
         self.attrs = attrs or {}
-        self.choices = choices
+        # choices can be any iterable, but we may need to render this widget
+        # multiple times. Thus, collapse it into a list so it can be consumed
+        # more than once.
+        self.choices = list(choices)
 
     def render(self, name, value, attrs=None, choices=()):
         if value is None: value = ''
@@ -134,6 +154,25 @@ class Select(Widget):
         output.append(u'</select>')
         return u'\n'.join(output)
 
+class NullBooleanSelect(Select):
+    """
+    A Select Widget intended to be used with NullBooleanField.
+    """
+    def __init__(self, attrs=None):
+        choices = ((u'1', gettext('Unknown')), (u'2', gettext('Yes')), (u'3', gettext('No')))
+        super(NullBooleanSelect, self).__init__(attrs, choices)
+
+    def render(self, name, value, attrs=None, choices=()):
+        try:
+            value = {True: u'2', False: u'3', u'2': u'2', u'3': u'3'}[value]
+        except KeyError:
+            value = u'1'
+        return super(NullBooleanSelect, self).render(name, value, attrs, choices)
+
+    def value_from_datadict(self, data, name):
+        value = data.get(name, None)
+        return {u'2': True, u'3': False, True: True, False: False}.get(value, None)
+
 class SelectMultiple(Widget):
     def __init__(self, attrs=None, choices=()):
         # choices can be any iterable
@@ -162,14 +201,15 @@ class RadioInput(StrAndUnicode):
     def __init__(self, name, value, attrs, choice, index):
         self.name, self.value = name, value
         self.attrs = attrs
-        self.choice_value, self.choice_label = choice
+        self.choice_value = smart_unicode(choice[0])
+        self.choice_label = smart_unicode(choice[1])
         self.index = index
 
     def __unicode__(self):
         return u'<label>%s %s</label>' % (self.tag(), self.choice_label)
 
     def is_checked(self):
-        return self.value == smart_unicode(self.choice_value)
+        return self.value == self.choice_value
 
     def tag(self):
         if self.attrs.has_key('id'):
@@ -218,11 +258,16 @@ class RadioSelect(Select):
 class CheckboxSelectMultiple(SelectMultiple):
     def render(self, name, value, attrs=None, choices=()):
         if value is None: value = []
+        has_id = attrs and attrs.has_key('id')
         final_attrs = self.build_attrs(attrs, name=name)
         output = [u'<ul>']
         str_values = set([smart_unicode(v) for v in value]) # Normalize to strings.
-        cb = CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
-        for option_value, option_label in chain(self.choices, choices):
+        for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
+            # If an ID attribute was given, add a numeric index as a suffix,
+            # so that the checkboxes don't all have the same ID attribute.
+            if has_id:
+                final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
+            cb = CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
             option_value = smart_unicode(option_value)
             rendered_cb = cb.render(name, option_value)
             output.append(u'<li><label>%s %s</label></li>' % (rendered_cb, escape(smart_unicode(option_label))))
@@ -235,3 +280,66 @@ class CheckboxSelectMultiple(SelectMultiple):
             id_ += '_0'
         return id_
     id_for_label = classmethod(id_for_label)
+
+class MultiWidget(Widget):
+    """
+    A widget that is composed of multiple widgets.
+
+    Its render() method takes a "decompressed" list of values, not a single
+    value. Each value in this list is rendered in the corresponding widget --
+    the first value is rendered in the first widget, the second value is
+    rendered in the second widget, etc.
+
+    Subclasses should implement decompress(), which specifies how a single
+    value should be converted to a list of values. Subclasses should not
+    have to implement clean().
+
+    Subclasses may implement format_output(), which takes the list of rendered
+    widgets and returns HTML that formats them any way you'd like.
+
+    You'll probably want to use this with MultiValueField.
+    """
+    def __init__(self, widgets, attrs=None):
+        self.widgets = [isinstance(w, type) and w() or w for w in widgets]
+        super(MultiWidget, self).__init__(attrs)
+
+    def render(self, name, value, attrs=None):
+        # value is a list of values, each corresponding to a widget
+        # in self.widgets.
+        if not isinstance(value, list):
+            value = self.decompress(value)
+        output = []
+        for i, widget in enumerate(self.widgets):
+            try:
+                widget_value = value[i]
+            except KeyError:
+                widget_value = None
+            output.append(widget.render(name + '_%s' % i, widget_value, attrs))
+        return self.format_output(output)
+
+    def value_from_datadict(self, data, name):
+        return [data.get(name + '_%s' % i) for i in range(len(self.widgets))]
+
+    def format_output(self, rendered_widgets):
+        return u''.join(rendered_widgets)
+
+    def decompress(self, value):
+        """
+        Returns a list of decompressed values for the given compressed value.
+        The given value can be assumed to be valid, but not necessarily
+        non-empty.
+        """
+        raise NotImplementedError('Subclasses must implement this method.')
+
+class SplitDateTimeWidget(MultiWidget):
+    """
+    A Widget that splits datetime input into two <input type="text"> boxes.
+    """
+    def __init__(self, attrs=None):
+        widgets = (TextInput(attrs=attrs), TextInput(attrs=attrs))
+        super(SplitDateTimeWidget, self).__init__(widgets, attrs)
+
+    def decompress(self, value):
+        if value:
+            return [value.date(), value.time()]
+        return [None, None]
diff --git a/django/oldforms/__init__.py b/django/oldforms/__init__.py
index 0b9ac05edb..decf0f7064 100644
--- a/django/oldforms/__init__.py
+++ b/django/oldforms/__init__.py
@@ -569,7 +569,7 @@ class NullBooleanField(SelectField):
     "This SelectField provides 'Yes', 'No' and 'Unknown', mapping results to True, False or None"
     def __init__(self, field_name, is_required=False, validator_list=None):
         if validator_list is None: validator_list = []
-        SelectField.__init__(self, field_name, choices=[('1', 'Unknown'), ('2', 'Yes'), ('3', 'No')],
+        SelectField.__init__(self, field_name, choices=[('1', _('Unknown')), ('2', _('Yes')), ('3', _('No'))],
             is_required=is_required, validator_list=validator_list)
 
     def render(self, data):
@@ -958,7 +958,9 @@ class USStateField(TextField):
             raise validators.CriticalValidationError, e.messages
 
     def html2python(data):
-        return data.upper() # Should always be stored in upper case
+        if data:
+            return data.upper() # Should always be stored in upper case
+        return data
     html2python = staticmethod(html2python)
 
 class CommaSeparatedIntegerField(TextField):
diff --git a/django/utils/datastructures.py b/django/utils/datastructures.py
index cecb4da170..9bec7a5df7 100644
--- a/django/utils/datastructures.py
+++ b/django/utils/datastructures.py
@@ -70,7 +70,7 @@ class SortedDict(dict):
         return self.keyOrder[:]
 
     def values(self):
-        return [dict.__getitem__(self,k) for k in self.keyOrder]
+        return [dict.__getitem__(self, k) for k in self.keyOrder]
 
     def update(self, dict):
         for k, v in dict.items():
@@ -81,6 +81,10 @@ class SortedDict(dict):
             self.keyOrder.append(key)
         return dict.setdefault(self, key, default)
 
+    def value_for_index(self, index):
+        "Returns the value of the item at the given zero-based index."
+        return self[self.keyOrder[index]]
+
 class MultiValueDictKeyError(KeyError):
     pass
 
diff --git a/django/utils/simplejson/LICENSE.txt b/django/utils/simplejson/LICENSE.txt
index 90251a9f62..1fa4fd5ba2 100644
--- a/django/utils/simplejson/LICENSE.txt
+++ b/django/utils/simplejson/LICENSE.txt
@@ -1,4 +1,4 @@
-simplejson 1.3
+simplejson 1.5
 Copyright (c) 2006 Bob Ippolito
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
diff --git a/django/utils/simplejson/__init__.py b/django/utils/simplejson/__init__.py
index f88329b950..15b7173976 100644
--- a/django/utils/simplejson/__init__.py
+++ b/django/utils/simplejson/__init__.py
@@ -27,6 +27,21 @@ Encoding basic Python object hierarchies::
     >>> io.getvalue()
     '["streaming API"]'
 
+Compact encoding::
+
+    >>> import simplejson
+    >>> simplejson.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
+    '[1,2,3,{"4":5,"6":7}]'
+
+Pretty printing::
+
+    >>> import simplejson
+    >>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
+    {
+        "4": 5, 
+        "6": 7
+    }
+
 Decoding JSON::
     
     >>> import simplejson
@@ -68,10 +83,10 @@ Extending JSONEncoder::
     ['[', '2.0', ', ', '1.0', ']']
     
 
-Note that the JSON produced by this module is a subset of YAML,
-so it may be used as a serializer for that as well.
+Note that the JSON produced by this module's default settings
+is a subset of YAML, so it may be used as a serializer for that as well.
 """
-__version__ = '1.3'
+__version__ = '1.5'
 __all__ = [
     'dump', 'dumps', 'load', 'loads',
     'JSONDecoder', 'JSONEncoder',
@@ -81,7 +96,7 @@ from django.utils.simplejson.decoder import JSONDecoder
 from django.utils.simplejson.encoder import JSONEncoder
 
 def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
-        allow_nan=True, cls=None, **kw):
+        allow_nan=True, cls=None, indent=None, **kw):
     """
     Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
     ``.write()``-supporting file-like object).
@@ -105,6 +120,10 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     in strict compliance of the JSON specification, instead of using the
     JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
 
+    If ``indent`` is a non-negative integer, then JSON array elements and object
+    members will be pretty-printed with that indent level.  An indent level
+    of 0 will only insert newlines.  ``None`` is the most compact representation.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg.
@@ -112,7 +131,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     if cls is None:
         cls = JSONEncoder
     iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
-        check_circular=check_circular, allow_nan=allow_nan,
+        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
         **kw).iterencode(obj)
     # could accelerate with writelines in some versions of Python, at
     # a debuggability cost
@@ -120,7 +139,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
         fp.write(chunk)
 
 def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
-        allow_nan=True, cls=None, **kw):
+        allow_nan=True, cls=None, indent=None, separators=None, **kw):
     """
     Serialize ``obj`` to a JSON formatted ``str``.
 
@@ -141,14 +160,26 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     strict compliance of the JSON specification, instead of using the
     JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
 
+    If ``indent`` is a non-negative integer, then JSON array elements and
+    object members will be pretty-printed with that indent level.  An indent
+    level of 0 will only insert newlines.  ``None`` is the most compact
+    representation.
+
+    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
+    then it will be used instead of the default ``(', ', ': ')`` separators.
+    ``(',', ':')`` is the most compact JSON representation.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg.
     """
     if cls is None:
         cls = JSONEncoder
-    return cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
-        check_circular=check_circular, allow_nan=allow_nan, **kw).encode(obj)
+    return cls(
+        skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+        separators=separators,
+        **kw).encode(obj)
 
 def load(fp, encoding=None, cls=None, object_hook=None, **kw):
     """
diff --git a/django/utils/simplejson/decoder.py b/django/utils/simplejson/decoder.py
index 684af8c9ad..66f68a200b 100644
--- a/django/utils/simplejson/decoder.py
+++ b/django/utils/simplejson/decoder.py
@@ -127,6 +127,7 @@ def JSONObject(match, context, _w=WHITESPACE.match):
         raise ValueError(errmsg("Expecting property name", s, end))
     end += 1
     encoding = getattr(context, 'encoding', None)
+    iterscan = JSONScanner.iterscan
     while True:
         key, end = scanstring(s, end, encoding)
         end = _w(s, end).end()
@@ -134,7 +135,7 @@ def JSONObject(match, context, _w=WHITESPACE.match):
             raise ValueError(errmsg("Expecting : delimiter", s, end))
         end = _w(s, end + 1).end()
         try:
-            value, end = JSONScanner.iterscan(s, idx=end).next()
+            value, end = iterscan(s, idx=end, context=context).next()
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
         pairs[key] = value
@@ -164,9 +165,10 @@ def JSONArray(match, context, _w=WHITESPACE.match):
     nextchar = s[end:end + 1]
     if nextchar == ']':
         return values, end + 1
+    iterscan = JSONScanner.iterscan
     while True:
         try:
-            value, end = JSONScanner.iterscan(s, idx=end).next()
+            value, end = iterscan(s, idx=end, context=context).next()
         except StopIteration:
             raise ValueError(errmsg("Expecting object", s, end))
         values.append(value)
diff --git a/django/utils/simplejson/encoder.py b/django/utils/simplejson/encoder.py
index bb1aba09f0..c83c6873eb 100644
--- a/django/utils/simplejson/encoder.py
+++ b/django/utils/simplejson/encoder.py
@@ -3,11 +3,11 @@ Implementation of JSONEncoder
 """
 import re
 
-# this should match any kind of infinity
-INFCHARS = re.compile(r'[infINF]')
 ESCAPE = re.compile(r'[\x00-\x19\\"\b\f\n\r\t]')
-ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
+ESCAPE_ASCII = re.compile(r'([\\"/]|[^\ -~])')
 ESCAPE_DCT = {
+    # escape all forward slashes to prevent </script> attack
+    '/': '\\/',
     '\\': '\\\\',
     '"': '\\"',
     '\b': '\\b',
@@ -16,31 +16,31 @@ ESCAPE_DCT = {
     '\r': '\\r',
     '\t': '\\t',
 }
-for i in range(20):
+for i in range(0x20):
     ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
+# assume this produces an infinity on all machines (probably not guaranteed)
+INFINITY = float('1e66666')
+
 def floatstr(o, allow_nan=True):
-    s = str(o)
-    # If the first non-sign is a digit then it's not a special value
-    if (o < 0.0 and s[1].isdigit()) or s[0].isdigit():
-        return s
-    elif not allow_nan:
+    # Check for specials.  Note that this type of test is processor- and/or
+    # platform-specific, so do tests which don't depend on the internals.
+
+    if o != o:
+        text = 'NaN'
+    elif o == INFINITY:
+        text = 'Infinity'
+    elif o == -INFINITY:
+        text = '-Infinity'
+    else:
+        return str(o)
+
+    if not allow_nan:
         raise ValueError("Out of range float values are not JSON compliant: %r"
             % (o,))
-    # These are the string representations on the platforms I've tried
-    if s == 'nan':
-        return 'NaN'
-    if s == 'inf':
-        return 'Infinity'
-    if s == '-inf':
-        return '-Infinity'
-    # NaN should either be inequal to itself, or equal to everything
-    if o != o or o == 0.0:
-        return 'NaN'
-    # Last ditch effort, assume inf
-    if o < 0:
-        return '-Infinity'
-    return 'Infinity'
+
+    return text
+
 
 def encode_basestring(s):
     """
@@ -90,8 +90,11 @@ class JSONEncoder(object):
     implementation (to raise ``TypeError``).
     """
     __all__ = ['__init__', 'default', 'encode', 'iterencode']
+    item_separator = ', '
+    key_separator = ': '
     def __init__(self, skipkeys=False, ensure_ascii=True,
-            check_circular=True, allow_nan=True, sort_keys=False):
+            check_circular=True, allow_nan=True, sort_keys=False,
+            indent=None, separators=None):
         """
         Constructor for JSONEncoder, with sensible defaults.
 
@@ -116,6 +119,15 @@ class JSONEncoder(object):
         If sort_keys is True, then the output of dictionaries will be
         sorted by key; this is useful for regression tests to ensure
         that JSON serializations can be compared on a day-to-day basis.
+
+        If indent is a non-negative integer, then JSON array
+        elements and object members will be pretty-printed with that
+        indent level.  An indent level of 0 will only insert newlines.
+        None is the most compact representation.
+
+        If specified, separators should be a (item_separator, key_separator)
+        tuple. The default is (', ', ': '). To get the most compact JSON
+        representation you should specify (',', ':') to eliminate whitespace.
         """
 
         self.skipkeys = skipkeys
@@ -123,6 +135,13 @@ class JSONEncoder(object):
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
+        self.indent = indent
+        self.current_indent_level = 0
+        if separators is not None:
+            self.item_separator, self.key_separator = separators
+
+    def _newline_indent(self):
+        return '\n' + (' ' * (self.indent * self.current_indent_level))
 
     def _iterencode_list(self, lst, markers=None):
         if not lst:
@@ -134,14 +153,25 @@ class JSONEncoder(object):
                 raise ValueError("Circular reference detected")
             markers[markerid] = lst
         yield '['
+        if self.indent is not None:
+            self.current_indent_level += 1
+            newline_indent = self._newline_indent()
+            separator = self.item_separator + newline_indent
+            yield newline_indent
+        else:
+            newline_indent = None
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
                 first = False
             else:
-                yield ', '
+                yield separator
             for chunk in self._iterencode(value, markers):
                 yield chunk
+        if newline_indent is not None:
+            self.current_indent_level -= 1
+            yield self._newline_indent()
         yield ']'
         if markers is not None:
             del markers[markerid]
@@ -156,6 +186,15 @@ class JSONEncoder(object):
                 raise ValueError("Circular reference detected")
             markers[markerid] = dct
         yield '{'
+        key_separator = self.key_separator
+        if self.indent is not None:
+            self.current_indent_level += 1
+            newline_indent = self._newline_indent()
+            item_separator = self.item_separator + newline_indent
+            yield newline_indent
+        else:
+            newline_indent = None
+            item_separator = self.item_separator
         first = True
         if self.ensure_ascii:
             encoder = encode_basestring_ascii
@@ -165,7 +204,7 @@ class JSONEncoder(object):
         if self.sort_keys:
             keys = dct.keys()
             keys.sort()
-            items = [(k,dct[k]) for k in keys]
+            items = [(k, dct[k]) for k in keys]
         else:
             items = dct.iteritems()
         for key, value in items:
@@ -190,11 +229,14 @@ class JSONEncoder(object):
             if first:
                 first = False
             else:
-                yield ', '
+                yield item_separator
             yield encoder(key)
-            yield ': '
+            yield key_separator
             for chunk in self._iterencode(value, markers):
                 yield chunk
+        if newline_indent is not None:
+            self.current_indent_level -= 1
+            yield self._newline_indent()
         yield '}'
         if markers is not None:
             del markers[markerid]
diff --git a/django/utils/simplejson/jsonfilter.py b/django/utils/simplejson/jsonfilter.py
new file mode 100644
index 0000000000..d02ae2033a
--- /dev/null
+++ b/django/utils/simplejson/jsonfilter.py
@@ -0,0 +1,40 @@
+from django.utils import simplejson
+import cgi
+
+class JSONFilter(object):
+    def __init__(self, app, mime_type='text/x-json'):
+        self.app = app
+        self.mime_type = mime_type
+
+    def __call__(self, environ, start_response):
+        # Read JSON POST input to jsonfilter.json if matching mime type
+        response = {'status': '200 OK', 'headers': []}
+        def json_start_response(status, headers):
+            response['status'] = status
+            response['headers'].extend(headers)
+        environ['jsonfilter.mime_type'] = self.mime_type
+        if environ.get('REQUEST_METHOD', '') == 'POST':
+            if environ.get('CONTENT_TYPE', '') == self.mime_type:
+                args = [_ for _ in [environ.get('CONTENT_LENGTH')] if _]
+                data = environ['wsgi.input'].read(*map(int, args))
+                environ['jsonfilter.json'] = simplejson.loads(data)
+        res = simplejson.dumps(self.app(environ, json_start_response))
+        jsonp = cgi.parse_qs(environ.get('QUERY_STRING', '')).get('jsonp')
+        if jsonp:
+            content_type = 'text/javascript'
+            res = ''.join(jsonp + ['(', res, ')'])
+        elif 'Opera' in environ.get('HTTP_USER_AGENT', ''):
+            # Opera has bunk XMLHttpRequest support for most mime types
+            content_type = 'text/plain'
+        else:
+            content_type = self.mime_type
+        headers = [
+            ('Content-type', content_type),
+            ('Content-length', len(res)),
+        ]
+        headers.extend(response['headers'])
+        start_response(response['status'], headers)
+        return [res]
+
+def factory(app, global_conf, **kw):
+    return JSONFilter(app, **kw)
diff --git a/django/utils/simplejson/scanner.py b/django/utils/simplejson/scanner.py
index b9244cfed1..64f4999fb5 100644
--- a/django/utils/simplejson/scanner.py
+++ b/django/utils/simplejson/scanner.py
@@ -3,11 +3,12 @@ Iterator based sre token scanner
 """
 import sre_parse, sre_compile, sre_constants
 from sre_constants import BRANCH, SUBPATTERN
+from re import VERBOSE, MULTILINE, DOTALL
 import re
 
 __all__ = ['Scanner', 'pattern']
 
-FLAGS = (re.VERBOSE | re.MULTILINE | re.DOTALL)
+FLAGS = (VERBOSE | MULTILINE | DOTALL)
 class Scanner(object):
     def __init__(self, lexicon, flags=FLAGS):
         self.actions = [None]