diff options
| author | django-bot <ops@djangoproject.com> | 2022-02-03 20:24:19 +0100 |
|---|---|---|
| committer | Mariusz Felisiak <felisiak.mariusz@gmail.com> | 2022-02-07 20:37:05 +0100 |
| commit | 9c19aff7c7561e3a82978a272ecdaad40dda5c00 (patch) | |
| tree | f0506b668a013d0063e5fba3dbf4863b466713ba /django/core | |
| parent | f68fa8b45dfac545cfc4111d4e52804c86db68d3 (diff) | |
Refs #33476 -- Reformatted code with Black.
Diffstat (limited to 'django/core')
85 files changed, 3792 insertions, 2445 deletions
diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py index a311b50af6..f09c9ecc4b 100644 --- a/django/core/cache/__init__.py +++ b/django/core/cache/__init__.py @@ -14,27 +14,35 @@ See docs/topics/cache.txt for information on the public API. """ from django.core import signals from django.core.cache.backends.base import ( - BaseCache, CacheKeyWarning, InvalidCacheBackendError, InvalidCacheKey, + BaseCache, + CacheKeyWarning, + InvalidCacheBackendError, + InvalidCacheKey, ) from django.utils.connection import BaseConnectionHandler, ConnectionProxy from django.utils.module_loading import import_string __all__ = [ - 'cache', 'caches', 'DEFAULT_CACHE_ALIAS', 'InvalidCacheBackendError', - 'CacheKeyWarning', 'BaseCache', 'InvalidCacheKey', + "cache", + "caches", + "DEFAULT_CACHE_ALIAS", + "InvalidCacheBackendError", + "CacheKeyWarning", + "BaseCache", + "InvalidCacheKey", ] -DEFAULT_CACHE_ALIAS = 'default' +DEFAULT_CACHE_ALIAS = "default" class CacheHandler(BaseConnectionHandler): - settings_name = 'CACHES' + settings_name = "CACHES" exception_class = InvalidCacheBackendError def create_connection(self, alias): params = self.settings[alias].copy() - backend = params.pop('BACKEND') - location = params.pop('LOCATION', '') + backend = params.pop("BACKEND") + location = params.pop("LOCATION", "") try: backend_cls = import_string(backend) except ImportError as e: @@ -45,7 +53,8 @@ class CacheHandler(BaseConnectionHandler): def all(self, initialized_only=False): return [ - self[alias] for alias in self + self[alias] + for alias in self # If initialized_only is True, return only initialized caches. if not initialized_only or hasattr(self._connections, alias) ] diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index f632d851ea..eb4b3eac6d 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -36,7 +36,7 @@ def default_key_func(key, key_prefix, version): the `key_prefix`. KEY_FUNCTION can be used to specify an alternate function with custom key making behavior. """ - return '%s:%s:%s' % (key_prefix, version, key) + return "%s:%s:%s" % (key_prefix, version, key) def get_key_func(key_func): @@ -57,7 +57,7 @@ class BaseCache: _missing_key = object() def __init__(self, params): - timeout = params.get('timeout', params.get('TIMEOUT', 300)) + timeout = params.get("timeout", params.get("TIMEOUT", 300)) if timeout is not None: try: timeout = int(timeout) @@ -65,22 +65,22 @@ class BaseCache: timeout = 300 self.default_timeout = timeout - options = params.get('OPTIONS', {}) - max_entries = params.get('max_entries', options.get('MAX_ENTRIES', 300)) + options = params.get("OPTIONS", {}) + max_entries = params.get("max_entries", options.get("MAX_ENTRIES", 300)) try: self._max_entries = int(max_entries) except (ValueError, TypeError): self._max_entries = 300 - cull_frequency = params.get('cull_frequency', options.get('CULL_FREQUENCY', 3)) + cull_frequency = params.get("cull_frequency", options.get("CULL_FREQUENCY", 3)) try: self._cull_frequency = int(cull_frequency) except (ValueError, TypeError): self._cull_frequency = 3 - self.key_prefix = params.get('KEY_PREFIX', '') - self.version = params.get('VERSION', 1) - self.key_func = get_key_func(params.get('KEY_FUNCTION')) + self.key_prefix = params.get("KEY_PREFIX", "") + self.version = params.get("VERSION", 1) + self.key_func = get_key_func(params.get("KEY_FUNCTION")) def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): """ @@ -130,47 +130,61 @@ class BaseCache: Return True if the value was stored, False otherwise. """ - raise NotImplementedError('subclasses of BaseCache must provide an add() method') + raise NotImplementedError( + "subclasses of BaseCache must provide an add() method" + ) async def aadd(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - return await sync_to_async(self.add, thread_sensitive=True)(key, value, timeout, version) + return await sync_to_async(self.add, thread_sensitive=True)( + key, value, timeout, version + ) def get(self, key, default=None, version=None): """ Fetch a given key from the cache. If the key does not exist, return default, which itself defaults to None. """ - raise NotImplementedError('subclasses of BaseCache must provide a get() method') + raise NotImplementedError("subclasses of BaseCache must provide a get() method") async def aget(self, key, default=None, version=None): - return await sync_to_async(self.get, thread_sensitive=True)(key, default, version) + return await sync_to_async(self.get, thread_sensitive=True)( + key, default, version + ) def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): """ Set a value in the cache. If timeout is given, use that timeout for the key; otherwise use the default cache timeout. """ - raise NotImplementedError('subclasses of BaseCache must provide a set() method') + raise NotImplementedError("subclasses of BaseCache must provide a set() method") async def aset(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - return await sync_to_async(self.set, thread_sensitive=True)(key, value, timeout, version) + return await sync_to_async(self.set, thread_sensitive=True)( + key, value, timeout, version + ) def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): """ Update the key's expiry time using timeout. Return True if successful or False if the key does not exist. """ - raise NotImplementedError('subclasses of BaseCache must provide a touch() method') + raise NotImplementedError( + "subclasses of BaseCache must provide a touch() method" + ) async def atouch(self, key, timeout=DEFAULT_TIMEOUT, version=None): - return await sync_to_async(self.touch, thread_sensitive=True)(key, timeout, version) + return await sync_to_async(self.touch, thread_sensitive=True)( + key, timeout, version + ) def delete(self, key, version=None): """ Delete a key from the cache and return whether it succeeded, failing silently. """ - raise NotImplementedError('subclasses of BaseCache must provide a delete() method') + raise NotImplementedError( + "subclasses of BaseCache must provide a delete() method" + ) async def adelete(self, key, version=None): return await sync_to_async(self.delete, thread_sensitive=True)(key, version) @@ -234,7 +248,9 @@ class BaseCache: """ Return True if the key is in the cache and has not expired. """ - return self.get(key, self._missing_key, version=version) is not self._missing_key + return ( + self.get(key, self._missing_key, version=version) is not self._missing_key + ) async def ahas_key(self, key, version=None): return ( @@ -318,7 +334,9 @@ class BaseCache: def clear(self): """Remove *all* values from the cache at once.""" - raise NotImplementedError('subclasses of BaseCache must provide a clear() method') + raise NotImplementedError( + "subclasses of BaseCache must provide a clear() method" + ) async def aclear(self): return await sync_to_async(self.clear, thread_sensitive=True)() @@ -373,13 +391,13 @@ class BaseCache: def memcache_key_warnings(key): if len(key) > MEMCACHE_MAX_KEY_LENGTH: yield ( - 'Cache key will cause errors if used with memcached: %r ' - '(longer than %s)' % (key, MEMCACHE_MAX_KEY_LENGTH) + "Cache key will cause errors if used with memcached: %r " + "(longer than %s)" % (key, MEMCACHE_MAX_KEY_LENGTH) ) for char in key: if ord(char) < 33 or ord(char) == 127: yield ( - 'Cache key contains characters that will cause errors if ' - 'used with memcached: %r' % key + "Cache key contains characters that will cause errors if " + "used with memcached: %r" % key ) break diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py index 5bb1c5aec5..e3d055084c 100644 --- a/django/core/cache/backends/db.py +++ b/django/core/cache/backends/db.py @@ -14,13 +14,14 @@ class Options: This allows cache operations to be controlled by the router """ + def __init__(self, table): self.db_table = table - self.app_label = 'django_cache' - self.model_name = 'cacheentry' - self.verbose_name = 'cache entry' - self.verbose_name_plural = 'cache entries' - self.object_name = 'CacheEntry' + self.app_label = "django_cache" + self.model_name = "cacheentry" + self.verbose_name = "cache entry" + self.verbose_name_plural = "cache entries" + self.object_name = "CacheEntry" self.abstract = False self.managed = True self.proxy = False @@ -34,6 +35,7 @@ class BaseDatabaseCache(BaseCache): class CacheEntry: _meta = Options(table) + self.cache_model_class = CacheEntry @@ -54,7 +56,9 @@ class DatabaseCache(BaseDatabaseCache): if not keys: return {} - key_map = {self.make_and_validate_key(key, version=version): key for key in keys} + key_map = { + self.make_and_validate_key(key, version=version): key for key in keys + } db = router.db_for_read(self.cache_model_class) connection = connections[db] @@ -63,13 +67,14 @@ class DatabaseCache(BaseDatabaseCache): with connection.cursor() as cursor: cursor.execute( - 'SELECT %s, %s, %s FROM %s WHERE %s IN (%s)' % ( - quote_name('cache_key'), - quote_name('value'), - quote_name('expires'), + "SELECT %s, %s, %s FROM %s WHERE %s IN (%s)" + % ( + quote_name("cache_key"), + quote_name("value"), + quote_name("expires"), table, - quote_name('cache_key'), - ', '.join(['%s'] * len(key_map)), + quote_name("cache_key"), + ", ".join(["%s"] * len(key_map)), ), list(key_map), ) @@ -78,7 +83,9 @@ class DatabaseCache(BaseDatabaseCache): result = {} expired_keys = [] expression = models.Expression(output_field=models.DateTimeField()) - converters = (connection.ops.get_db_converters(expression) + expression.get_db_converters(connection)) + converters = connection.ops.get_db_converters( + expression + ) + expression.get_db_converters(connection) for key, value, expires in rows: for converter in converters: expires = converter(expires, expression, connection) @@ -93,15 +100,15 @@ class DatabaseCache(BaseDatabaseCache): def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_and_validate_key(key, version=version) - self._base_set('set', key, value, timeout) + self._base_set("set", key, value, timeout) def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_and_validate_key(key, version=version) - return self._base_set('add', key, value, timeout) + return self._base_set("add", key, value, timeout) def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): key = self.make_and_validate_key(key, version=version) - return self._base_set('touch', key, None, timeout) + return self._base_set("touch", key, None, timeout) def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT): timeout = self.get_backend_timeout(timeout) @@ -126,7 +133,7 @@ class DatabaseCache(BaseDatabaseCache): pickled = pickle.dumps(value, self.pickle_protocol) # The DB column is expecting a string, so make sure the value is a # string, not bytes. Refs #19274. - b64encoded = base64.b64encode(pickled).decode('latin1') + b64encoded = base64.b64encode(pickled).decode("latin1") try: # Note: typecasting for datetimes is needed by some 3rd party # database backends. All core backends work without typecasting, @@ -134,52 +141,59 @@ class DatabaseCache(BaseDatabaseCache): # regressions. with transaction.atomic(using=db): cursor.execute( - 'SELECT %s, %s FROM %s WHERE %s = %%s' % ( - quote_name('cache_key'), - quote_name('expires'), + "SELECT %s, %s FROM %s WHERE %s = %%s" + % ( + quote_name("cache_key"), + quote_name("expires"), table, - quote_name('cache_key'), + quote_name("cache_key"), ), - [key] + [key], ) result = cursor.fetchone() if result: current_expires = result[1] - expression = models.Expression(output_field=models.DateTimeField()) - for converter in (connection.ops.get_db_converters(expression) + - expression.get_db_converters(connection)): - current_expires = converter(current_expires, expression, connection) + expression = models.Expression( + output_field=models.DateTimeField() + ) + for converter in connection.ops.get_db_converters( + expression + ) + expression.get_db_converters(connection): + current_expires = converter( + current_expires, expression, connection + ) exp = connection.ops.adapt_datetimefield_value(exp) - if result and mode == 'touch': + if result and mode == "touch": cursor.execute( - 'UPDATE %s SET %s = %%s WHERE %s = %%s' % ( - table, - quote_name('expires'), - quote_name('cache_key') - ), - [exp, key] + "UPDATE %s SET %s = %%s WHERE %s = %%s" + % (table, quote_name("expires"), quote_name("cache_key")), + [exp, key], ) - elif result and (mode == 'set' or (mode == 'add' and current_expires < now)): + elif result and ( + mode == "set" or (mode == "add" and current_expires < now) + ): cursor.execute( - 'UPDATE %s SET %s = %%s, %s = %%s WHERE %s = %%s' % ( + "UPDATE %s SET %s = %%s, %s = %%s WHERE %s = %%s" + % ( table, - quote_name('value'), - quote_name('expires'), - quote_name('cache_key'), + quote_name("value"), + quote_name("expires"), + quote_name("cache_key"), ), - [b64encoded, exp, key] + [b64encoded, exp, key], ) - elif mode != 'touch': + elif mode != "touch": cursor.execute( - 'INSERT INTO %s (%s, %s, %s) VALUES (%%s, %%s, %%s)' % ( + "INSERT INTO %s (%s, %s, %s) VALUES (%%s, %%s, %%s)" + % ( table, - quote_name('cache_key'), - quote_name('value'), - quote_name('expires'), + quote_name("cache_key"), + quote_name("value"), + quote_name("expires"), ), - [key, b64encoded, exp] + [key, b64encoded, exp], ) else: return False # touch failed. @@ -208,10 +222,11 @@ class DatabaseCache(BaseDatabaseCache): with connection.cursor() as cursor: cursor.execute( - 'DELETE FROM %s WHERE %s IN (%s)' % ( + "DELETE FROM %s WHERE %s IN (%s)" + % ( table, - quote_name('cache_key'), - ', '.join(['%s'] * len(keys)), + quote_name("cache_key"), + ", ".join(["%s"] * len(keys)), ), keys, ) @@ -228,13 +243,14 @@ class DatabaseCache(BaseDatabaseCache): with connection.cursor() as cursor: cursor.execute( - 'SELECT %s FROM %s WHERE %s = %%s and %s > %%s' % ( - quote_name('cache_key'), + "SELECT %s FROM %s WHERE %s = %%s and %s > %%s" + % ( + quote_name("cache_key"), quote_name(self._table), - quote_name('cache_key'), - quote_name('expires'), + quote_name("cache_key"), + quote_name("expires"), ), - [key, connection.ops.adapt_datetimefield_value(now)] + [key, connection.ops.adapt_datetimefield_value(now)], ) return cursor.fetchone() is not None @@ -244,23 +260,28 @@ class DatabaseCache(BaseDatabaseCache): else: connection = connections[db] table = connection.ops.quote_name(self._table) - cursor.execute('DELETE FROM %s WHERE %s < %%s' % ( - table, - connection.ops.quote_name('expires'), - ), [connection.ops.adapt_datetimefield_value(now)]) + cursor.execute( + "DELETE FROM %s WHERE %s < %%s" + % ( + table, + connection.ops.quote_name("expires"), + ), + [connection.ops.adapt_datetimefield_value(now)], + ) deleted_count = cursor.rowcount remaining_num = num - deleted_count if remaining_num > self._max_entries: cull_num = remaining_num // self._cull_frequency cursor.execute( - connection.ops.cache_key_culling_sql() % table, - [cull_num]) + connection.ops.cache_key_culling_sql() % table, [cull_num] + ) last_cache_key = cursor.fetchone() if last_cache_key: cursor.execute( - 'DELETE FROM %s WHERE %s < %%s' % ( + "DELETE FROM %s WHERE %s < %%s" + % ( table, - connection.ops.quote_name('cache_key'), + connection.ops.quote_name("cache_key"), ), [last_cache_key[0]], ) @@ -270,4 +291,4 @@ class DatabaseCache(BaseDatabaseCache): connection = connections[db] table = connection.ops.quote_name(self._table) with connection.cursor() as cursor: - cursor.execute('DELETE FROM %s' % table) + cursor.execute("DELETE FROM %s" % table) diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index fc99d11687..631da49444 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -14,7 +14,7 @@ from django.utils.crypto import md5 class FileBasedCache(BaseCache): - cache_suffix = '.djcache' + cache_suffix = ".djcache" pickle_protocol = pickle.HIGHEST_PROTOCOL def __init__(self, dir, params): @@ -31,7 +31,7 @@ class FileBasedCache(BaseCache): def get(self, key, default=None, version=None): fname = self._key_to_file(key, version) try: - with open(fname, 'rb') as f: + with open(fname, "rb") as f: if not self._is_expired(f): return pickle.loads(zlib.decompress(f.read())) except FileNotFoundError: @@ -50,7 +50,7 @@ class FileBasedCache(BaseCache): fd, tmp_path = tempfile.mkstemp(dir=self._dir) renamed = False try: - with open(fd, 'wb') as f: + with open(fd, "wb") as f: self._write_content(f, timeout, value) file_move_safe(tmp_path, fname, allow_overwrite=True) renamed = True @@ -60,7 +60,7 @@ class FileBasedCache(BaseCache): def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): try: - with open(self._key_to_file(key, version), 'r+b') as f: + with open(self._key_to_file(key, version), "r+b") as f: try: locks.lock(f, locks.LOCK_EX) if self._is_expired(f): @@ -91,7 +91,7 @@ class FileBasedCache(BaseCache): def has_key(self, key, version=None): fname = self._key_to_file(key, version) if os.path.exists(fname): - with open(fname, 'rb') as f: + with open(fname, "rb") as f: return not self._is_expired(f) return False @@ -108,8 +108,7 @@ class FileBasedCache(BaseCache): if self._cull_frequency == 0: return self.clear() # Clear the cache when CULL_FREQUENCY = 0 # Delete a random selection of entries - filelist = random.sample(filelist, - int(num_entries / self._cull_frequency)) + filelist = random.sample(filelist, int(num_entries / self._cull_frequency)) for fname in filelist: self._delete(fname) @@ -128,10 +127,15 @@ class FileBasedCache(BaseCache): root cache path joined with the md5sum of the key and a suffix. """ key = self.make_and_validate_key(key, version=version) - return os.path.join(self._dir, ''.join([ - md5(key.encode(), usedforsecurity=False).hexdigest(), - self.cache_suffix, - ])) + return os.path.join( + self._dir, + "".join( + [ + md5(key.encode(), usedforsecurity=False).hexdigest(), + self.cache_suffix, + ] + ), + ) def clear(self): """ @@ -161,5 +165,5 @@ class FileBasedCache(BaseCache): """ return [ os.path.join(self._dir, fname) - for fname in glob.glob1(self._dir, '*%s' % self.cache_suffix) + for fname in glob.glob1(self._dir, "*%s" % self.cache_suffix) ] diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py index 472a28179c..2416168634 100644 --- a/django/core/cache/backends/memcached.py +++ b/django/core/cache/backends/memcached.py @@ -4,7 +4,10 @@ import re import time from django.core.cache.backends.base import ( - DEFAULT_TIMEOUT, BaseCache, InvalidCacheKey, memcache_key_warnings, + DEFAULT_TIMEOUT, + BaseCache, + InvalidCacheKey, + memcache_key_warnings, ) from django.utils.functional import cached_property @@ -13,7 +16,7 @@ class BaseMemcachedCache(BaseCache): def __init__(self, server, params, library, value_not_found_exception): super().__init__(params) if isinstance(server, str): - self._servers = re.split('[;,]', server) + self._servers = re.split("[;,]", server) else: self._servers = server @@ -23,7 +26,7 @@ class BaseMemcachedCache(BaseCache): self._lib = library self._class = library.Client - self._options = params.get('OPTIONS') or {} + self._options = params.get("OPTIONS") or {} @property def client_servers(self): @@ -86,7 +89,9 @@ class BaseMemcachedCache(BaseCache): return bool(self._cache.delete(key)) def get_many(self, keys, version=None): - key_map = {self.make_and_validate_key(key, version=version): key for key in keys} + key_map = { + self.make_and_validate_key(key, version=version): key for key in keys + } ret = self._cache.get_multi(key_map.keys()) return {key_map[k]: v for k, v in ret.items()} @@ -118,7 +123,9 @@ class BaseMemcachedCache(BaseCache): safe_key = self.make_and_validate_key(key, version=version) safe_data[safe_key] = value original_keys[safe_key] = key - failed_keys = self._cache.set_multi(safe_data, self.get_backend_timeout(timeout)) + failed_keys = self._cache.set_multi( + safe_data, self.get_backend_timeout(timeout) + ) return [original_keys[k] for k in failed_keys] def delete_many(self, keys, version=None): @@ -135,15 +142,19 @@ class BaseMemcachedCache(BaseCache): class PyLibMCCache(BaseMemcachedCache): "An implementation of a cache binding using pylibmc" + def __init__(self, server, params): import pylibmc - super().__init__(server, params, library=pylibmc, value_not_found_exception=pylibmc.NotFound) + + super().__init__( + server, params, library=pylibmc, value_not_found_exception=pylibmc.NotFound + ) @property def client_servers(self): output = [] for server in self._servers: - output.append(server[5:] if server.startswith('unix:') else server) + output.append(server[5:] if server.startswith("unix:") else server) return output def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None): @@ -160,13 +171,17 @@ class PyLibMCCache(BaseMemcachedCache): class PyMemcacheCache(BaseMemcachedCache): """An implementation of a cache binding using pymemcache.""" + def __init__(self, server, params): import pymemcache.serde - super().__init__(server, params, library=pymemcache, value_not_found_exception=KeyError) + + super().__init__( + server, params, library=pymemcache, value_not_found_exception=KeyError + ) self._class = self._lib.HashClient self._options = { - 'allow_unicode_keys': True, - 'default_noreply': False, - 'serde': pymemcache.serde.pickle_serde, + "allow_unicode_keys": True, + "default_noreply": False, + "serde": pymemcache.serde.pickle_serde, **self._options, } diff --git a/django/core/cache/backends/redis.py b/django/core/cache/backends/redis.py index f168e93737..e0d30784ff 100644 --- a/django/core/cache/backends/redis.py +++ b/django/core/cache/backends/redis.py @@ -58,7 +58,7 @@ class RedisCacheClient: parser_class = import_string(parser_class) parser_class = parser_class or self._lib.connection.DefaultParser - self._pool_options = {'parser_class': parser_class, 'db': db} + self._pool_options = {"parser_class": parser_class, "db": db} def _get_connection_pool_index(self, write): # Write to the first server. Read from other servers if there are more, @@ -71,7 +71,8 @@ class RedisCacheClient: index = self._get_connection_pool_index(write) if index not in self._pools: self._pools[index] = self._pool_class.from_url( - self._servers[index], **self._pool_options, + self._servers[index], + **self._pool_options, ) return self._pools[index] @@ -159,12 +160,12 @@ class RedisCache(BaseCache): def __init__(self, server, params): super().__init__(params) if isinstance(server, str): - self._servers = re.split('[;,]', server) + self._servers = re.split("[;,]", server) else: self._servers = server self._class = RedisCacheClient - self._options = params.get('OPTIONS', {}) + self._options = params.get("OPTIONS", {}) @cached_property def _cache(self): @@ -198,7 +199,9 @@ class RedisCache(BaseCache): return self._cache.delete(key) def get_many(self, keys, version=None): - key_map = {self.make_and_validate_key(key, version=version): key for key in keys} + key_map = { + self.make_and_validate_key(key, version=version): key for key in keys + } ret = self._cache.get_many(key_map.keys()) return {key_map[k]: v for k, v in ret.items()} diff --git a/django/core/cache/utils.py b/django/core/cache/utils.py index d41960f6e4..ff2a23aa6f 100644 --- a/django/core/cache/utils.py +++ b/django/core/cache/utils.py @@ -1,6 +1,6 @@ from django.utils.crypto import md5 -TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s' +TEMPLATE_FRAGMENT_KEY_TEMPLATE = "template.cache.%s.%s" def make_template_fragment_key(fragment_name, vary_on=None): @@ -8,5 +8,5 @@ def make_template_fragment_key(fragment_name, vary_on=None): if vary_on is not None: for arg in vary_on: hasher.update(str(arg).encode()) - hasher.update(b':') + hasher.update(b":") return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, hasher.hexdigest()) diff --git a/django/core/checks/__init__.py b/django/core/checks/__init__.py index 296e991ddc..998ab9dee2 100644 --- a/django/core/checks/__init__.py +++ b/django/core/checks/__init__.py @@ -1,6 +1,15 @@ from .messages import ( - CRITICAL, DEBUG, ERROR, INFO, WARNING, CheckMessage, Critical, Debug, - Error, Info, Warning, + CRITICAL, + DEBUG, + ERROR, + INFO, + WARNING, + CheckMessage, + Critical, + Debug, + Error, + Info, + Warning, ) from .registry import Tags, register, run_checks, tag_exists @@ -20,8 +29,19 @@ import django.core.checks.urls # NOQA isort:skip __all__ = [ - 'CheckMessage', - 'Debug', 'Info', 'Warning', 'Error', 'Critical', - 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', - 'register', 'run_checks', 'tag_exists', 'Tags', + "CheckMessage", + "Debug", + "Info", + "Warning", + "Error", + "Critical", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL", + "register", + "run_checks", + "tag_exists", + "Tags", ] diff --git a/django/core/checks/async_checks.py b/django/core/checks/async_checks.py index fbb5267358..a0e01867d9 100644 --- a/django/core/checks/async_checks.py +++ b/django/core/checks/async_checks.py @@ -3,14 +3,14 @@ import os from . import Error, Tags, register E001 = Error( - 'You should not set the DJANGO_ALLOW_ASYNC_UNSAFE environment variable in ' - 'deployment. This disables async safety protection.', - id='async.E001', + "You should not set the DJANGO_ALLOW_ASYNC_UNSAFE environment variable in " + "deployment. This disables async safety protection.", + id="async.E001", ) @register(Tags.async_support, deploy=True) def check_async_unsafe(app_configs, **kwargs): - if os.environ.get('DJANGO_ALLOW_ASYNC_UNSAFE'): + if os.environ.get("DJANGO_ALLOW_ASYNC_UNSAFE"): return [E001] return [] diff --git a/django/core/checks/caches.py b/django/core/checks/caches.py index b755e0035a..c288a6ff4a 100644 --- a/django/core/checks/caches.py +++ b/django/core/checks/caches.py @@ -8,7 +8,7 @@ from . import Error, Tags, Warning, register E001 = Error( "You must define a '%s' cache in your CACHES setting." % DEFAULT_CACHE_ALIAS, - id='caches.E001', + id="caches.E001", ) @@ -22,11 +22,11 @@ def check_default_cache_is_configured(app_configs, **kwargs): @register(Tags.caches, deploy=True) def check_cache_location_not_exposed(app_configs, **kwargs): errors = [] - for name in ('MEDIA_ROOT', 'STATIC_ROOT', 'STATICFILES_DIRS'): + for name in ("MEDIA_ROOT", "STATIC_ROOT", "STATICFILES_DIRS"): setting = getattr(settings, name, None) if not setting: continue - if name == 'STATICFILES_DIRS': + if name == "STATICFILES_DIRS": paths = set() for staticfiles_dir in setting: if isinstance(staticfiles_dir, (list, tuple)): @@ -40,19 +40,21 @@ def check_cache_location_not_exposed(app_configs, **kwargs): continue cache_path = pathlib.Path(cache._dir).resolve() if any(path == cache_path for path in paths): - relation = 'matches' + relation = "matches" elif any(path in cache_path.parents for path in paths): - relation = 'is inside' + relation = "is inside" elif any(cache_path in path.parents for path in paths): - relation = 'contains' + relation = "contains" else: continue - errors.append(Warning( - f"Your '{alias}' cache configuration might expose your cache " - f"or lead to corruption of your data because its LOCATION " - f"{relation} {name}.", - id='caches.W002', - )) + errors.append( + Warning( + f"Your '{alias}' cache configuration might expose your cache " + f"or lead to corruption of your data because its LOCATION " + f"{relation} {name}.", + id="caches.W002", + ) + ) return errors @@ -63,10 +65,12 @@ def check_file_based_cache_is_absolute(app_configs, **kwargs): cache = caches[alias] if not isinstance(cache, FileBasedCache): continue - if not pathlib.Path(config['LOCATION']).is_absolute(): - errors.append(Warning( - f"Your '{alias}' cache LOCATION path is relative. Use an " - f"absolute path instead.", - id='caches.W003', - )) + if not pathlib.Path(config["LOCATION"]).is_absolute(): + errors.append( + Warning( + f"Your '{alias}' cache LOCATION path is relative. Use an " + f"absolute path instead.", + id="caches.W003", + ) + ) return errors diff --git a/django/core/checks/compatibility/django_4_0.py b/django/core/checks/compatibility/django_4_0.py index 7788629735..79ee5fa6b3 100644 --- a/django/core/checks/compatibility/django_4_0.py +++ b/django/core/checks/compatibility/django_4_0.py @@ -7,12 +7,14 @@ from .. import Error, Tags, register def check_csrf_trusted_origins(app_configs, **kwargs): errors = [] for origin in settings.CSRF_TRUSTED_ORIGINS: - if '://' not in origin: - errors.append(Error( - 'As of Django 4.0, the values in the CSRF_TRUSTED_ORIGINS ' - 'setting must start with a scheme (usually http:// or ' - 'https://) but found %s. See the release notes for details.' - % origin, - id='4_0.E001', - )) + if "://" not in origin: + errors.append( + Error( + "As of Django 4.0, the values in the CSRF_TRUSTED_ORIGINS " + "setting must start with a scheme (usually http:// or " + "https://) but found %s. See the release notes for details." + % origin, + id="4_0.E001", + ) + ) return errors diff --git a/django/core/checks/files.py b/django/core/checks/files.py index 5f76ae5a17..40dc745840 100644 --- a/django/core/checks/files.py +++ b/django/core/checks/files.py @@ -7,7 +7,7 @@ from . import Error, Tags, register @register(Tags.files) def check_setting_file_upload_temp_dir(app_configs, **kwargs): - setting = getattr(settings, 'FILE_UPLOAD_TEMP_DIR', None) + setting = getattr(settings, "FILE_UPLOAD_TEMP_DIR", None) if setting and not Path(setting).is_dir(): return [ Error( diff --git a/django/core/checks/messages.py b/django/core/checks/messages.py index 0987c2d118..db7aa55119 100644 --- a/django/core/checks/messages.py +++ b/django/core/checks/messages.py @@ -7,10 +7,9 @@ CRITICAL = 50 class CheckMessage: - def __init__(self, level, msg, hint=None, obj=None, id=None): if not isinstance(level, int): - raise TypeError('The first argument should be level.') + raise TypeError("The first argument should be level.") self.level = level self.msg = msg self.hint = hint @@ -18,10 +17,9 @@ class CheckMessage: self.id = id def __eq__(self, other): - return ( - isinstance(other, self.__class__) and - all(getattr(self, attr) == getattr(other, attr) - for attr in ['level', 'msg', 'hint', 'obj', 'id']) + return isinstance(other, self.__class__) and all( + getattr(self, attr) == getattr(other, attr) + for attr in ["level", "msg", "hint", "obj", "id"] ) def __str__(self): @@ -36,18 +34,25 @@ class CheckMessage: else: obj = str(self.obj) id = "(%s) " % self.id if self.id else "" - hint = "\n\tHINT: %s" % self.hint if self.hint else '' + hint = "\n\tHINT: %s" % self.hint if self.hint else "" return "%s: %s%s%s" % (obj, id, self.msg, hint) def __repr__(self): - return "<%s: level=%r, msg=%r, hint=%r, obj=%r, id=%r>" % \ - (self.__class__.__name__, self.level, self.msg, self.hint, self.obj, self.id) + return "<%s: level=%r, msg=%r, hint=%r, obj=%r, id=%r>" % ( + self.__class__.__name__, + self.level, + self.msg, + self.hint, + self.obj, + self.id, + ) def is_serious(self, level=ERROR): return self.level >= level def is_silenced(self): from django.conf import settings + return self.id in settings.SILENCED_SYSTEM_CHECKS diff --git a/django/core/checks/model_checks.py b/django/core/checks/model_checks.py index 15d9b7fd86..7a5bef9b26 100644 --- a/django/core/checks/model_checks.py +++ b/django/core/checks/model_checks.py @@ -17,7 +17,9 @@ def check_all_models(app_configs=None, **kwargs): if app_configs is None: models = apps.get_models() else: - models = chain.from_iterable(app_config.get_models() for app_config in app_configs) + models = chain.from_iterable( + app_config.get_models() for app_config in app_configs + ) for model in models: if model._meta.managed and not model._meta.proxy: db_table_models[model._meta.db_table].append(model._meta.label) @@ -27,7 +29,7 @@ def check_all_models(app_configs=None, **kwargs): "The '%s.check()' class method is currently overridden by %r." % (model.__name__, model.check), obj=model, - id='models.E020' + id="models.E020", ) ) else: @@ -37,17 +39,17 @@ def check_all_models(app_configs=None, **kwargs): for model_constraint in model._meta.constraints: constraints[model_constraint.name].append(model._meta.label) if settings.DATABASE_ROUTERS: - error_class, error_id = Warning, 'models.W035' + error_class, error_id = Warning, "models.W035" error_hint = ( - 'You have configured settings.DATABASE_ROUTERS. Verify that %s ' - 'are correctly routed to separate databases.' + "You have configured settings.DATABASE_ROUTERS. Verify that %s " + "are correctly routed to separate databases." ) else: - error_class, error_id = Error, 'models.E028' + error_class, error_id = Error, "models.E028" error_hint = None for db_table, model_labels in db_table_models.items(): if len(model_labels) != 1: - model_labels_str = ', '.join(model_labels) + model_labels_str = ", ".join(model_labels) errors.append( error_class( "db_table '%s' is used by multiple models: %s." @@ -62,12 +64,13 @@ def check_all_models(app_configs=None, **kwargs): model_labels = set(model_labels) errors.append( Error( - "index name '%s' is not unique %s %s." % ( + "index name '%s' is not unique %s %s." + % ( index_name, - 'for model' if len(model_labels) == 1 else 'among models:', - ', '.join(sorted(model_labels)), + "for model" if len(model_labels) == 1 else "among models:", + ", ".join(sorted(model_labels)), ), - id='models.E029' if len(model_labels) == 1 else 'models.E030', + id="models.E029" if len(model_labels) == 1 else "models.E030", ), ) for constraint_name, model_labels in constraints.items(): @@ -75,12 +78,13 @@ def check_all_models(app_configs=None, **kwargs): model_labels = set(model_labels) errors.append( Error( - "constraint name '%s' is not unique %s %s." % ( + "constraint name '%s' is not unique %s %s." + % ( constraint_name, - 'for model' if len(model_labels) == 1 else 'among models:', - ', '.join(sorted(model_labels)), + "for model" if len(model_labels) == 1 else "among models:", + ", ".join(sorted(model_labels)), ), - id='models.E031' if len(model_labels) == 1 else 'models.E032', + id="models.E031" if len(model_labels) == 1 else "models.E032", ), ) return errors @@ -104,8 +108,10 @@ def _check_lazy_references(apps, ignore=None): return [] from django.db.models import signals + model_signals = { - signal: name for name, signal in vars(signals).items() + signal: name + for name, signal in vars(signals).items() if isinstance(signal, signals.ModelSignal) } @@ -120,9 +126,9 @@ def _check_lazy_references(apps, ignore=None): annotated there with a `func` attribute so as to imitate a partial. """ operation, args, keywords = obj, [], {} - while hasattr(operation, 'func'): - args.extend(getattr(operation, 'args', [])) - keywords.update(getattr(operation, 'keywords', {})) + while hasattr(operation, "func"): + args.extend(getattr(operation, "args", [])) + keywords.update(getattr(operation, "keywords", {})) operation = operation.func return operation, args, keywords @@ -146,11 +152,11 @@ def _check_lazy_references(apps, ignore=None): "to '%(model)s', but %(model_error)s." ) params = { - 'model': '.'.join(model_key), - 'field': keywords['field'], - 'model_error': app_model_error(model_key), + "model": ".".join(model_key), + "field": keywords["field"], + "model_error": app_model_error(model_key), } - return Error(error_msg % params, obj=keywords['field'], id='fields.E307') + return Error(error_msg % params, obj=keywords["field"], id="fields.E307") def signal_connect_error(model_key, func, args, keywords): error_msg = ( @@ -163,34 +169,39 @@ def _check_lazy_references(apps, ignore=None): if isinstance(receiver, types.FunctionType): description = "The function '%s'" % receiver.__name__ elif isinstance(receiver, types.MethodType): - description = "Bound method '%s.%s'" % (receiver.__self__.__class__.__name__, receiver.__name__) + description = "Bound method '%s.%s'" % ( + receiver.__self__.__class__.__name__, + receiver.__name__, + ) else: description = "An instance of class '%s'" % receiver.__class__.__name__ - signal_name = model_signals.get(func.__self__, 'unknown') + signal_name = model_signals.get(func.__self__, "unknown") params = { - 'model': '.'.join(model_key), - 'receiver': description, - 'signal': signal_name, - 'model_error': app_model_error(model_key), + "model": ".".join(model_key), + "receiver": description, + "signal": signal_name, + "model_error": app_model_error(model_key), } - return Error(error_msg % params, obj=receiver.__module__, id='signals.E001') + return Error(error_msg % params, obj=receiver.__module__, id="signals.E001") def default_error(model_key, func, args, keywords): - error_msg = "%(op)s contains a lazy reference to %(model)s, but %(model_error)s." + error_msg = ( + "%(op)s contains a lazy reference to %(model)s, but %(model_error)s." + ) params = { - 'op': func, - 'model': '.'.join(model_key), - 'model_error': app_model_error(model_key), + "op": func, + "model": ".".join(model_key), + "model_error": app_model_error(model_key), } - return Error(error_msg % params, obj=func, id='models.E022') + return Error(error_msg % params, obj=func, id="models.E022") # Maps common uses of lazy operations to corresponding error functions # defined above. If a key maps to None, no error will be produced. # default_error() will be used for usages that don't appear in this dict. known_lazy = { - ('django.db.models.fields.related', 'resolve_related_class'): field_error, - ('django.db.models.fields.related', 'set_managed'): None, - ('django.dispatch.dispatcher', 'connect'): signal_connect_error, + ("django.db.models.fields.related", "resolve_related_class"): field_error, + ("django.db.models.fields.related", "set_managed"): None, + ("django.dispatch.dispatcher", "connect"): signal_connect_error, } def build_error(model_key, func, args, keywords): @@ -198,11 +209,17 @@ def _check_lazy_references(apps, ignore=None): error_fn = known_lazy.get(key, default_error) return error_fn(model_key, func, args, keywords) if error_fn else None - return sorted(filter(None, ( - build_error(model_key, *extract_operation(func)) - for model_key in pending_models - for func in apps._pending_operations[model_key] - )), key=lambda error: error.msg) + return sorted( + filter( + None, + ( + build_error(model_key, *extract_operation(func)) + for model_key in pending_models + for func in apps._pending_operations[model_key] + ), + ), + key=lambda error: error.msg, + ) @register(Tags.models) diff --git a/django/core/checks/registry.py b/django/core/checks/registry.py index d7bfa49548..f4bdea8691 100644 --- a/django/core/checks/registry.py +++ b/django/core/checks/registry.py @@ -8,24 +8,24 @@ class Tags: """ Built-in tags for internal checks. """ - admin = 'admin' - async_support = 'async_support' - caches = 'caches' - compatibility = 'compatibility' - database = 'database' - files = 'files' - models = 'models' - security = 'security' - signals = 'signals' - sites = 'sites' - staticfiles = 'staticfiles' - templates = 'templates' - translation = 'translation' - urls = 'urls' + admin = "admin" + async_support = "async_support" + caches = "caches" + compatibility = "compatibility" + database = "database" + files = "files" + models = "models" + security = "security" + signals = "signals" + sites = "sites" + staticfiles = "staticfiles" + templates = "templates" + translation = "translation" + urls = "urls" -class CheckRegistry: +class CheckRegistry: def __init__(self): self.registered_checks = set() self.deployment_checks = set() @@ -46,13 +46,18 @@ class CheckRegistry: # or registry.register(my_check, 'mytag', 'anothertag') """ + def inner(check): if not func_accepts_kwargs(check): raise TypeError( - 'Check functions must accept keyword arguments (**kwargs).' + "Check functions must accept keyword arguments (**kwargs)." ) check.tags = tags - checks = self.deployment_checks if kwargs.get('deploy') else self.registered_checks + checks = ( + self.deployment_checks + if kwargs.get("deploy") + else self.registered_checks + ) checks.add(check) return check @@ -63,7 +68,13 @@ class CheckRegistry: tags += (check,) return inner - def run_checks(self, app_configs=None, tags=None, include_deployment_checks=False, databases=None): + def run_checks( + self, + app_configs=None, + tags=None, + include_deployment_checks=False, + databases=None, + ): """ Run all registered checks and return list of Errors and Warnings. """ @@ -77,9 +88,8 @@ class CheckRegistry: new_errors = check(app_configs=app_configs, databases=databases) if not is_iterable(new_errors): raise TypeError( - 'The function %r did not return a list. All functions ' - 'registered with the checks registry must return a list.' - % check, + "The function %r did not return a list. All functions " + "registered with the checks registry must return a list." % check, ) errors.extend(new_errors) return errors @@ -88,9 +98,11 @@ class CheckRegistry: return tag in self.tags_available(include_deployment_checks) def tags_available(self, deployment_checks=False): - return set(chain.from_iterable( - check.tags for check in self.get_checks(deployment_checks) - )) + return set( + chain.from_iterable( + check.tags for check in self.get_checks(deployment_checks) + ) + ) def get_checks(self, include_deployment_checks=False): checks = list(self.registered_checks) diff --git a/django/core/checks/security/base.py b/django/core/checks/security/base.py index d37b968a7d..f85adabd1a 100644 --- a/django/core/checks/security/base.py +++ b/django/core/checks/security/base.py @@ -4,15 +4,22 @@ from django.core.exceptions import ImproperlyConfigured from .. import Error, Tags, Warning, register CROSS_ORIGIN_OPENER_POLICY_VALUES = { - 'same-origin', 'same-origin-allow-popups', 'unsafe-none', + "same-origin", + "same-origin-allow-popups", + "unsafe-none", } REFERRER_POLICY_VALUES = { - 'no-referrer', 'no-referrer-when-downgrade', 'origin', - 'origin-when-cross-origin', 'same-origin', 'strict-origin', - 'strict-origin-when-cross-origin', 'unsafe-url', + "no-referrer", + "no-referrer-when-downgrade", + "origin", + "origin-when-cross-origin", + "same-origin", + "strict-origin", + "strict-origin-when-cross-origin", + "unsafe-url", } -SECRET_KEY_INSECURE_PREFIX = 'django-insecure-' +SECRET_KEY_INSECURE_PREFIX = "django-insecure-" SECRET_KEY_MIN_LENGTH = 50 SECRET_KEY_MIN_UNIQUE_CHARACTERS = 5 @@ -31,7 +38,7 @@ W001 = Warning( "SECURE_CONTENT_TYPE_NOSNIFF, SECURE_REFERRER_POLICY, " "SECURE_CROSS_ORIGIN_OPENER_POLICY, and SECURE_SSL_REDIRECT settings will " "have no effect.", - id='security.W001', + id="security.W001", ) W002 = Warning( @@ -41,7 +48,7 @@ W002 = Warning( "'x-frame-options' header. Unless there is a good reason for your " "site to be served in a frame, you should consider enabling this " "header to help prevent clickjacking attacks.", - id='security.W002', + id="security.W002", ) W004 = Warning( @@ -50,7 +57,7 @@ W004 = Warning( "setting a value and enabling HTTP Strict Transport Security. " "Be sure to read the documentation first; enabling HSTS carelessly " "can cause serious, irreversible problems.", - id='security.W004', + id="security.W004", ) W005 = Warning( @@ -59,7 +66,7 @@ W005 = Warning( "via an insecure connection to a subdomain. Only set this to True if " "you are certain that all subdomains of your domain should be served " "exclusively via SSL.", - id='security.W005', + id="security.W005", ) W006 = Warning( @@ -68,7 +75,7 @@ W006 = Warning( "'X-Content-Type-Options: nosniff' header. " "You should consider enabling this header to prevent the " "browser from identifying content types incorrectly.", - id='security.W006', + id="security.W006", ) W008 = Warning( @@ -77,17 +84,17 @@ W008 = Warning( "connections, you may want to either set this setting True " "or configure a load balancer or reverse-proxy server " "to redirect all connections to HTTPS.", - id='security.W008', + id="security.W008", ) W009 = Warning( - SECRET_KEY_WARNING_MSG % 'SECRET_KEY', - id='security.W009', + SECRET_KEY_WARNING_MSG % "SECRET_KEY", + id="security.W009", ) W018 = Warning( "You should not have DEBUG set to True in deployment.", - id='security.W018', + id="security.W018", ) W019 = Warning( @@ -96,51 +103,53 @@ W019 = Warning( "MIDDLEWARE, but X_FRAME_OPTIONS is not set to 'DENY'. " "Unless there is a good reason for your site to serve other parts of " "itself in a frame, you should change it to 'DENY'.", - id='security.W019', + id="security.W019", ) W020 = Warning( "ALLOWED_HOSTS must not be empty in deployment.", - id='security.W020', + id="security.W020", ) W021 = Warning( "You have not set the SECURE_HSTS_PRELOAD setting to True. Without this, " "your site cannot be submitted to the browser preload list.", - id='security.W021', + id="security.W021", ) W022 = Warning( - 'You have not set the SECURE_REFERRER_POLICY setting. Without this, your ' - 'site will not send a Referrer-Policy header. You should consider ' - 'enabling this header to protect user privacy.', - id='security.W022', + "You have not set the SECURE_REFERRER_POLICY setting. Without this, your " + "site will not send a Referrer-Policy header. You should consider " + "enabling this header to protect user privacy.", + id="security.W022", ) E023 = Error( - 'You have set the SECURE_REFERRER_POLICY setting to an invalid value.', - hint='Valid values are: {}.'.format(', '.join(sorted(REFERRER_POLICY_VALUES))), - id='security.E023', + "You have set the SECURE_REFERRER_POLICY setting to an invalid value.", + hint="Valid values are: {}.".format(", ".join(sorted(REFERRER_POLICY_VALUES))), + id="security.E023", ) E024 = Error( - 'You have set the SECURE_CROSS_ORIGIN_OPENER_POLICY setting to an invalid ' - 'value.', - hint='Valid values are: {}.'.format( - ', '.join(sorted(CROSS_ORIGIN_OPENER_POLICY_VALUES)), + "You have set the SECURE_CROSS_ORIGIN_OPENER_POLICY setting to an invalid " + "value.", + hint="Valid values are: {}.".format( + ", ".join(sorted(CROSS_ORIGIN_OPENER_POLICY_VALUES)), ), - id='security.E024', + id="security.E024", ) -W025 = Warning(SECRET_KEY_WARNING_MSG, id='security.W025') +W025 = Warning(SECRET_KEY_WARNING_MSG, id="security.W025") def _security_middleware(): - return 'django.middleware.security.SecurityMiddleware' in settings.MIDDLEWARE + return "django.middleware.security.SecurityMiddleware" in settings.MIDDLEWARE def _xframe_middleware(): - return 'django.middleware.clickjacking.XFrameOptionsMiddleware' in settings.MIDDLEWARE + return ( + "django.middleware.clickjacking.XFrameOptionsMiddleware" in settings.MIDDLEWARE + ) @register(Tags.security, deploy=True) @@ -164,9 +173,9 @@ def check_sts(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_sts_include_subdomains(app_configs, **kwargs): passed_check = ( - not _security_middleware() or - not settings.SECURE_HSTS_SECONDS or - settings.SECURE_HSTS_INCLUDE_SUBDOMAINS is True + not _security_middleware() + or not settings.SECURE_HSTS_SECONDS + or settings.SECURE_HSTS_INCLUDE_SUBDOMAINS is True ) return [] if passed_check else [W005] @@ -174,9 +183,9 @@ def check_sts_include_subdomains(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_sts_preload(app_configs, **kwargs): passed_check = ( - not _security_middleware() or - not settings.SECURE_HSTS_SECONDS or - settings.SECURE_HSTS_PRELOAD is True + not _security_middleware() + or not settings.SECURE_HSTS_SECONDS + or settings.SECURE_HSTS_PRELOAD is True ) return [] if passed_check else [W021] @@ -184,26 +193,22 @@ def check_sts_preload(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_content_type_nosniff(app_configs, **kwargs): passed_check = ( - not _security_middleware() or - settings.SECURE_CONTENT_TYPE_NOSNIFF is True + not _security_middleware() or settings.SECURE_CONTENT_TYPE_NOSNIFF is True ) return [] if passed_check else [W006] @register(Tags.security, deploy=True) def check_ssl_redirect(app_configs, **kwargs): - passed_check = ( - not _security_middleware() or - settings.SECURE_SSL_REDIRECT is True - ) + passed_check = not _security_middleware() or settings.SECURE_SSL_REDIRECT is True return [] if passed_check else [W008] def _check_secret_key(secret_key): return ( - len(set(secret_key)) >= SECRET_KEY_MIN_UNIQUE_CHARACTERS and - len(secret_key) >= SECRET_KEY_MIN_LENGTH and - not secret_key.startswith(SECRET_KEY_INSECURE_PREFIX) + len(set(secret_key)) >= SECRET_KEY_MIN_UNIQUE_CHARACTERS + and len(secret_key) >= SECRET_KEY_MIN_LENGTH + and not secret_key.startswith(SECRET_KEY_INSECURE_PREFIX) ) @@ -224,14 +229,12 @@ def check_secret_key_fallbacks(app_configs, **kwargs): try: fallbacks = settings.SECRET_KEY_FALLBACKS except (ImproperlyConfigured, AttributeError): - warnings.append( - Warning(W025.msg % 'SECRET_KEY_FALLBACKS', id=W025.id) - ) + warnings.append(Warning(W025.msg % "SECRET_KEY_FALLBACKS", id=W025.id)) else: for index, key in enumerate(fallbacks): if not _check_secret_key(key): warnings.append( - Warning(W025.msg % f'SECRET_KEY_FALLBACKS[{index}]', id=W025.id) + Warning(W025.msg % f"SECRET_KEY_FALLBACKS[{index}]", id=W025.id) ) return warnings @@ -244,10 +247,7 @@ def check_debug(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_xframe_deny(app_configs, **kwargs): - passed_check = ( - not _xframe_middleware() or - settings.X_FRAME_OPTIONS == 'DENY' - ) + passed_check = not _xframe_middleware() or settings.X_FRAME_OPTIONS == "DENY" return [] if passed_check else [W019] @@ -263,7 +263,7 @@ def check_referrer_policy(app_configs, **kwargs): return [W022] # Support a comma-separated string or iterable of values to allow fallback. if isinstance(settings.SECURE_REFERRER_POLICY, str): - values = {v.strip() for v in settings.SECURE_REFERRER_POLICY.split(',')} + values = {v.strip() for v in settings.SECURE_REFERRER_POLICY.split(",")} else: values = set(settings.SECURE_REFERRER_POLICY) if not values <= REFERRER_POLICY_VALUES: @@ -274,9 +274,10 @@ def check_referrer_policy(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_cross_origin_opener_policy(app_configs, **kwargs): if ( - _security_middleware() and - settings.SECURE_CROSS_ORIGIN_OPENER_POLICY is not None and - settings.SECURE_CROSS_ORIGIN_OPENER_POLICY not in CROSS_ORIGIN_OPENER_POLICY_VALUES + _security_middleware() + and settings.SECURE_CROSS_ORIGIN_OPENER_POLICY is not None + and settings.SECURE_CROSS_ORIGIN_OPENER_POLICY + not in CROSS_ORIGIN_OPENER_POLICY_VALUES ): return [E024] return [] diff --git a/django/core/checks/security/csrf.py b/django/core/checks/security/csrf.py index 2b70d363e2..ea65e48c94 100644 --- a/django/core/checks/security/csrf.py +++ b/django/core/checks/security/csrf.py @@ -10,7 +10,7 @@ W003 = Warning( "('django.middleware.csrf.CsrfViewMiddleware' is not in your " "MIDDLEWARE). Enabling the middleware is the safest approach " "to ensure you don't leave any holes.", - id='security.W003', + id="security.W003", ) W016 = Warning( @@ -18,12 +18,12 @@ W016 = Warning( "MIDDLEWARE, but you have not set CSRF_COOKIE_SECURE to True. " "Using a secure-only CSRF cookie makes it more difficult for network " "traffic sniffers to steal the CSRF token.", - id='security.W016', + id="security.W016", ) def _csrf_middleware(): - return 'django.middleware.csrf.CsrfViewMiddleware' in settings.MIDDLEWARE + return "django.middleware.csrf.CsrfViewMiddleware" in settings.MIDDLEWARE @register(Tags.security, deploy=True) @@ -35,9 +35,9 @@ def check_csrf_middleware(app_configs, **kwargs): @register(Tags.security, deploy=True) def check_csrf_cookie_secure(app_configs, **kwargs): passed_check = ( - settings.CSRF_USE_SESSIONS or - not _csrf_middleware() or - settings.CSRF_COOKIE_SECURE + settings.CSRF_USE_SESSIONS + or not _csrf_middleware() + or settings.CSRF_COOKIE_SECURE ) return [] if passed_check else [W016] @@ -51,17 +51,17 @@ def check_csrf_failure_view(app_configs, **kwargs): view = _get_failure_view() except ImportError: msg = ( - "The CSRF failure view '%s' could not be imported." % - settings.CSRF_FAILURE_VIEW + "The CSRF failure view '%s' could not be imported." + % settings.CSRF_FAILURE_VIEW ) - errors.append(Error(msg, id='security.E102')) + errors.append(Error(msg, id="security.E102")) else: try: inspect.signature(view).bind(None, reason=None) except TypeError: msg = ( - "The CSRF failure view '%s' does not take the correct number of arguments." % - settings.CSRF_FAILURE_VIEW + "The CSRF failure view '%s' does not take the correct number of arguments." + % settings.CSRF_FAILURE_VIEW ) - errors.append(Error(msg, id='security.E101')) + errors.append(Error(msg, id="security.E101")) return errors diff --git a/django/core/checks/security/sessions.py b/django/core/checks/security/sessions.py index 1f31a167fa..7c251c0601 100644 --- a/django/core/checks/security/sessions.py +++ b/django/core/checks/security/sessions.py @@ -15,7 +15,7 @@ W010 = Warning( "You have 'django.contrib.sessions' in your INSTALLED_APPS, " "but you have not set SESSION_COOKIE_SECURE to True." ), - id='security.W010', + id="security.W010", ) W011 = Warning( @@ -24,12 +24,12 @@ W011 = Warning( "in your MIDDLEWARE, but you have not set " "SESSION_COOKIE_SECURE to True." ), - id='security.W011', + id="security.W011", ) W012 = Warning( add_session_cookie_message("SESSION_COOKIE_SECURE is not set to True."), - id='security.W012', + id="security.W012", ) @@ -45,7 +45,7 @@ W013 = Warning( "You have 'django.contrib.sessions' in your INSTALLED_APPS, " "but you have not set SESSION_COOKIE_HTTPONLY to True.", ), - id='security.W013', + id="security.W013", ) W014 = Warning( @@ -54,12 +54,12 @@ W014 = Warning( "in your MIDDLEWARE, but you have not set " "SESSION_COOKIE_HTTPONLY to True." ), - id='security.W014', + id="security.W014", ) W015 = Warning( add_httponly_message("SESSION_COOKIE_HTTPONLY is not set to True."), - id='security.W015', + id="security.W015", ) @@ -90,7 +90,7 @@ def check_session_cookie_httponly(app_configs, **kwargs): def _session_middleware(): - return 'django.contrib.sessions.middleware.SessionMiddleware' in settings.MIDDLEWARE + return "django.contrib.sessions.middleware.SessionMiddleware" in settings.MIDDLEWARE def _session_app(): diff --git a/django/core/checks/templates.py b/django/core/checks/templates.py index 14325bd3e0..5214276987 100644 --- a/django/core/checks/templates.py +++ b/django/core/checks/templates.py @@ -9,34 +9,40 @@ from . import Error, Tags, register E001 = Error( "You have 'APP_DIRS': True in your TEMPLATES but also specify 'loaders' " "in OPTIONS. Either remove APP_DIRS or remove the 'loaders' option.", - id='templates.E001', + id="templates.E001", ) E002 = Error( "'string_if_invalid' in TEMPLATES OPTIONS must be a string but got: {} ({}).", id="templates.E002", ) E003 = Error( - '{} is used for multiple template tag modules: {}', - id='templates.E003', + "{} is used for multiple template tag modules: {}", + id="templates.E003", ) @register(Tags.templates) def check_setting_app_dirs_loaders(app_configs, **kwargs): - return [E001] if any( - conf.get('APP_DIRS') and 'loaders' in conf.get('OPTIONS', {}) - for conf in settings.TEMPLATES - ) else [] + return ( + [E001] + if any( + conf.get("APP_DIRS") and "loaders" in conf.get("OPTIONS", {}) + for conf in settings.TEMPLATES + ) + else [] + ) @register(Tags.templates) def check_string_if_invalid_is_string(app_configs, **kwargs): errors = [] for conf in settings.TEMPLATES: - string_if_invalid = conf.get('OPTIONS', {}).get('string_if_invalid', '') + string_if_invalid = conf.get("OPTIONS", {}).get("string_if_invalid", "") if not isinstance(string_if_invalid, str): error = copy.copy(E002) - error.msg = error.msg.format(string_if_invalid, type(string_if_invalid).__name__) + error.msg = error.msg.format( + string_if_invalid, type(string_if_invalid).__name__ + ) errors.append(error) return errors @@ -47,7 +53,7 @@ def check_for_template_tags_with_the_same_name(app_configs, **kwargs): libraries = defaultdict(list) for conf in settings.TEMPLATES: - custom_libraries = conf.get('OPTIONS', {}).get('libraries', {}) + custom_libraries = conf.get("OPTIONS", {}).get("libraries", {}) for module_name, module_path in custom_libraries.items(): libraries[module_name].append(module_path) @@ -56,12 +62,14 @@ def check_for_template_tags_with_the_same_name(app_configs, **kwargs): for library_name, items in libraries.items(): if len(items) > 1: - errors.append(Error( - E003.msg.format( - repr(library_name), - ', '.join(repr(item) for item in items), - ), - id=E003.id, - )) + errors.append( + Error( + E003.msg.format( + repr(library_name), + ", ".join(repr(item) for item in items), + ), + id=E003.id, + ) + ) return errors diff --git a/django/core/checks/translation.py b/django/core/checks/translation.py index 8457a6b89d..214e970373 100644 --- a/django/core/checks/translation.py +++ b/django/core/checks/translation.py @@ -5,24 +5,24 @@ from django.utils.translation.trans_real import language_code_re from . import Error, Tags, register E001 = Error( - 'You have provided an invalid value for the LANGUAGE_CODE setting: {!r}.', - id='translation.E001', + "You have provided an invalid value for the LANGUAGE_CODE setting: {!r}.", + id="translation.E001", ) E002 = Error( - 'You have provided an invalid language code in the LANGUAGES setting: {!r}.', - id='translation.E002', + "You have provided an invalid language code in the LANGUAGES setting: {!r}.", + id="translation.E002", ) E003 = Error( - 'You have provided an invalid language code in the LANGUAGES_BIDI setting: {!r}.', - id='translation.E003', + "You have provided an invalid language code in the LANGUAGES_BIDI setting: {!r}.", + id="translation.E003", ) E004 = Error( - 'You have provided a value for the LANGUAGE_CODE setting that is not in ' - 'the LANGUAGES setting.', - id='translation.E004', + "You have provided a value for the LANGUAGE_CODE setting that is not in " + "the LANGUAGES setting.", + id="translation.E004", ) @@ -40,7 +40,8 @@ def check_setting_languages(app_configs, **kwargs): """Error if LANGUAGES setting is invalid.""" return [ Error(E002.msg.format(tag), id=E002.id) - for tag, _ in settings.LANGUAGES if not isinstance(tag, str) or not language_code_re.match(tag) + for tag, _ in settings.LANGUAGES + if not isinstance(tag, str) or not language_code_re.match(tag) ] @@ -49,7 +50,8 @@ def check_setting_languages_bidi(app_configs, **kwargs): """Error if LANGUAGES_BIDI setting is invalid.""" return [ Error(E003.msg.format(tag), id=E003.id) - for tag in settings.LANGUAGES_BIDI if not isinstance(tag, str) or not language_code_re.match(tag) + for tag in settings.LANGUAGES_BIDI + if not isinstance(tag, str) or not language_code_re.match(tag) ] diff --git a/django/core/checks/urls.py b/django/core/checks/urls.py index e51ca3fc1f..34eff9671d 100644 --- a/django/core/checks/urls.py +++ b/django/core/checks/urls.py @@ -7,8 +7,9 @@ from . import Error, Tags, Warning, register @register(Tags.urls) def check_url_config(app_configs, **kwargs): - if getattr(settings, 'ROOT_URLCONF', None): + if getattr(settings, "ROOT_URLCONF", None): from django.urls import get_resolver + resolver = get_resolver() return check_resolver(resolver) return [] @@ -18,10 +19,10 @@ def check_resolver(resolver): """ Recursively check the resolver. """ - check_method = getattr(resolver, 'check', None) + check_method = getattr(resolver, "check", None) if check_method is not None: return check_method() - elif not hasattr(resolver, 'resolve'): + elif not hasattr(resolver, "resolve"): return get_warning_for_invalid_pattern(resolver) else: return [] @@ -32,21 +33,24 @@ def check_url_namespaces_unique(app_configs, **kwargs): """ Warn if URL namespaces used in applications aren't unique. """ - if not getattr(settings, 'ROOT_URLCONF', None): + if not getattr(settings, "ROOT_URLCONF", None): return [] from django.urls import get_resolver + resolver = get_resolver() all_namespaces = _load_all_namespaces(resolver) counter = Counter(all_namespaces) non_unique_namespaces = [n for n, count in counter.items() if count > 1] errors = [] for namespace in non_unique_namespaces: - errors.append(Warning( - "URL namespace '{}' isn't unique. You may not be able to reverse " - "all URLs in this namespace".format(namespace), - id="urls.W005", - )) + errors.append( + Warning( + "URL namespace '{}' isn't unique. You may not be able to reverse " + "all URLs in this namespace".format(namespace), + id="urls.W005", + ) + ) return errors @@ -54,13 +58,14 @@ def _load_all_namespaces(resolver, parents=()): """ Recursively load all namespaces from URL patterns. """ - url_patterns = getattr(resolver, 'url_patterns', []) + url_patterns = getattr(resolver, "url_patterns", []) namespaces = [ - ':'.join(parents + (url.namespace,)) for url in url_patterns - if getattr(url, 'namespace', None) is not None + ":".join(parents + (url.namespace,)) + for url in url_patterns + if getattr(url, "namespace", None) is not None ] for pattern in url_patterns: - namespace = getattr(pattern, 'namespace', None) + namespace = getattr(pattern, "namespace", None) current = parents if namespace is not None: current += (namespace,) @@ -85,26 +90,28 @@ def get_warning_for_invalid_pattern(pattern): else: hint = None - return [Error( - "Your URL pattern {!r} is invalid. Ensure that urlpatterns is a list " - "of path() and/or re_path() instances.".format(pattern), - hint=hint, - id="urls.E004", - )] + return [ + Error( + "Your URL pattern {!r} is invalid. Ensure that urlpatterns is a list " + "of path() and/or re_path() instances.".format(pattern), + hint=hint, + id="urls.E004", + ) + ] @register(Tags.urls) def check_url_settings(app_configs, **kwargs): errors = [] - for name in ('STATIC_URL', 'MEDIA_URL'): + for name in ("STATIC_URL", "MEDIA_URL"): value = getattr(settings, name) - if value and not value.endswith('/'): + if value and not value.endswith("/"): errors.append(E006(name)) return errors def E006(name): return Error( - 'The {} setting must end with a slash.'.format(name), - id='urls.E006', + "The {} setting must end with a slash.".format(name), + id="urls.E006", ) diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 673d004d57..7be4e16bc5 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -8,21 +8,25 @@ from django.utils.hashable import make_hashable class FieldDoesNotExist(Exception): """The requested model field does not exist""" + pass class AppRegistryNotReady(Exception): """The django.apps registry is not populated yet""" + pass class ObjectDoesNotExist(Exception): """The requested object does not exist""" + silent_variable_failure = True class MultipleObjectsReturned(Exception): """The query returned multiple objects when only one was expected.""" + pass @@ -32,21 +36,25 @@ class SuspiciousOperation(Exception): class SuspiciousMultipartForm(SuspiciousOperation): """Suspect MIME request in multipart form data""" + pass class SuspiciousFileOperation(SuspiciousOperation): """A Suspicious filesystem operation was attempted""" + pass class DisallowedHost(SuspiciousOperation): """HTTP_HOST header contains invalid value""" + pass class DisallowedRedirect(SuspiciousOperation): """Redirect to scheme not in allowed list""" + pass @@ -55,6 +63,7 @@ class TooManyFieldsSent(SuspiciousOperation): The number of fields in a GET or POST request exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS. """ + pass @@ -63,49 +72,58 @@ class RequestDataTooBig(SuspiciousOperation): The size of the request (excluding any file uploads) exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE. """ + pass class RequestAborted(Exception): """The request was closed before it was completed, or timed out.""" + pass class BadRequest(Exception): """The request is malformed and cannot be processed.""" + pass class PermissionDenied(Exception): """The user did not have permission to do that""" + pass class ViewDoesNotExist(Exception): """The requested view does not exist""" + pass class MiddlewareNotUsed(Exception): """This middleware is not used in this server configuration""" + pass class ImproperlyConfigured(Exception): """Django is somehow improperly configured""" + pass class FieldError(Exception): """Some kind of problem with a model field.""" + pass -NON_FIELD_ERRORS = '__all__' +NON_FIELD_ERRORS = "__all__" class ValidationError(Exception): """An error while validating data.""" + def __init__(self, message, code=None, params=None): """ The `message` argument can be a single error, a list of errors, or a @@ -118,9 +136,9 @@ class ValidationError(Exception): super().__init__(message, code, params) if isinstance(message, ValidationError): - if hasattr(message, 'error_dict'): + if hasattr(message, "error_dict"): message = message.error_dict - elif not hasattr(message, 'message'): + elif not hasattr(message, "message"): message = message.error_list else: message, code, params = message.message, message.code, message.params @@ -138,7 +156,7 @@ class ValidationError(Exception): # Normalize plain strings to instances of ValidationError. if not isinstance(message, ValidationError): message = ValidationError(message) - if hasattr(message, 'error_dict'): + if hasattr(message, "error_dict"): self.error_list.extend(sum(message.error_dict.values(), [])) else: self.error_list.extend(message.error_list) @@ -153,18 +171,18 @@ class ValidationError(Exception): def message_dict(self): # Trigger an AttributeError if this ValidationError # doesn't have an error_dict. - getattr(self, 'error_dict') + getattr(self, "error_dict") return dict(self) @property def messages(self): - if hasattr(self, 'error_dict'): + if hasattr(self, "error_dict"): return sum(dict(self).values(), []) return list(self) def update_error_dict(self, error_dict): - if hasattr(self, 'error_dict'): + if hasattr(self, "error_dict"): for field, error_list in self.error_dict.items(): error_dict.setdefault(field, []).extend(error_list) else: @@ -172,7 +190,7 @@ class ValidationError(Exception): return error_dict def __iter__(self): - if hasattr(self, 'error_dict'): + if hasattr(self, "error_dict"): for field, errors in self.error_dict.items(): yield field, list(ValidationError(errors)) else: @@ -183,12 +201,12 @@ class ValidationError(Exception): yield str(message) def __str__(self): - if hasattr(self, 'error_dict'): + if hasattr(self, "error_dict"): return repr(dict(self)) return repr(list(self)) def __repr__(self): - return 'ValidationError(%s)' % self + return "ValidationError(%s)" % self def __eq__(self, other): if not isinstance(other, ValidationError): @@ -196,22 +214,26 @@ class ValidationError(Exception): return hash(self) == hash(other) def __hash__(self): - if hasattr(self, 'message'): - return hash(( - self.message, - self.code, - make_hashable(self.params), - )) - if hasattr(self, 'error_dict'): + if hasattr(self, "message"): + return hash( + ( + self.message, + self.code, + make_hashable(self.params), + ) + ) + if hasattr(self, "error_dict"): return hash(make_hashable(self.error_dict)) - return hash(tuple(sorted(self.error_list, key=operator.attrgetter('message')))) + return hash(tuple(sorted(self.error_list, key=operator.attrgetter("message")))) class EmptyResultSet(Exception): """A database query predicate is impossible.""" + pass class SynchronousOnlyOperation(Exception): """The user tried to call a sync-only function from an async context.""" + pass diff --git a/django/core/files/__init__.py b/django/core/files/__init__.py index 58a6fd8f85..d046aca084 100644 --- a/django/core/files/__init__.py +++ b/django/core/files/__init__.py @@ -1,3 +1,3 @@ from django.core.files.base import File -__all__ = ['File'] +__all__ = ["File"] diff --git a/django/core/files/base.py b/django/core/files/base.py index 2ac662ed7c..3ca43ec254 100644 --- a/django/core/files/base.py +++ b/django/core/files/base.py @@ -6,18 +6,18 @@ from django.utils.functional import cached_property class File(FileProxyMixin): - DEFAULT_CHUNK_SIZE = 64 * 2 ** 10 + DEFAULT_CHUNK_SIZE = 64 * 2**10 def __init__(self, file, name=None): self.file = file if name is None: - name = getattr(file, 'name', None) + name = getattr(file, "name", None) self.name = name - if hasattr(file, 'mode'): + if hasattr(file, "mode"): self.mode = file.mode def __str__(self): - return self.name or '' + return self.name or "" def __repr__(self): return "<%s: %s>" % (self.__class__.__name__, self or "None") @@ -30,14 +30,14 @@ class File(FileProxyMixin): @cached_property def size(self): - if hasattr(self.file, 'size'): + if hasattr(self.file, "size"): return self.file.size - if hasattr(self.file, 'name'): + if hasattr(self.file, "name"): try: return os.path.getsize(self.file.name) except (OSError, TypeError): pass - if hasattr(self.file, 'tell') and hasattr(self.file, 'seek'): + if hasattr(self.file, "tell") and hasattr(self.file, "seek"): pos = self.file.tell() self.file.seek(0, os.SEEK_END) size = self.file.tell() @@ -122,13 +122,14 @@ class ContentFile(File): """ A File-like object that takes just raw content, rather than an actual file. """ + def __init__(self, content, name=None): stream_class = StringIO if isinstance(content, str) else BytesIO super().__init__(stream_class(content), name=name) self.size = len(content) def __str__(self): - return 'Raw content' + return "Raw content" def __bool__(self): return True @@ -141,20 +142,20 @@ class ContentFile(File): pass def write(self, data): - self.__dict__.pop('size', None) # Clear the computed size. + self.__dict__.pop("size", None) # Clear the computed size. return self.file.write(data) def endswith_cr(line): """Return True if line (a text or bytestring) ends with '\r'.""" - return line.endswith('\r' if isinstance(line, str) else b'\r') + return line.endswith("\r" if isinstance(line, str) else b"\r") def endswith_lf(line): """Return True if line (a text or bytestring) ends with '\n'.""" - return line.endswith('\n' if isinstance(line, str) else b'\n') + return line.endswith("\n" if isinstance(line, str) else b"\n") def equals_lf(line): """Return True if line (a text or bytestring) equals '\n'.""" - return line == ('\n' if isinstance(line, str) else b'\n') + return line == ("\n" if isinstance(line, str) else b"\n") diff --git a/django/core/files/images.py b/django/core/files/images.py index a1252f2c8b..6a603f24fc 100644 --- a/django/core/files/images.py +++ b/django/core/files/images.py @@ -14,6 +14,7 @@ class ImageFile(File): A mixin for use alongside django.core.files.base.File, which provides additional features for dealing with images. """ + @property def width(self): return self._get_image_dimensions()[0] @@ -23,7 +24,7 @@ class ImageFile(File): return self._get_image_dimensions()[1] def _get_image_dimensions(self): - if not hasattr(self, '_dimensions_cache'): + if not hasattr(self, "_dimensions_cache"): close = self.closed self.open() self._dimensions_cache = get_image_dimensions(self, close=close) @@ -39,13 +40,13 @@ def get_image_dimensions(file_or_path, close=False): from PIL import ImageFile as PillowImageFile p = PillowImageFile.Parser() - if hasattr(file_or_path, 'read'): + if hasattr(file_or_path, "read"): file = file_or_path file_pos = file.tell() file.seek(0) else: try: - file = open(file_or_path, 'rb') + file = open(file_or_path, "rb") except OSError: return (None, None) close = True diff --git a/django/core/files/locks.py b/django/core/files/locks.py index fdb9665332..da1a25fcf0 100644 --- a/django/core/files/locks.py +++ b/django/core/files/locks.py @@ -18,18 +18,25 @@ Example Usage:: """ import os -__all__ = ('LOCK_EX', 'LOCK_SH', 'LOCK_NB', 'lock', 'unlock') +__all__ = ("LOCK_EX", "LOCK_SH", "LOCK_NB", "lock", "unlock") def _fd(f): """Get a filedescriptor from something which could be a file or an fd.""" - return f.fileno() if hasattr(f, 'fileno') else f + return f.fileno() if hasattr(f, "fileno") else f -if os.name == 'nt': +if os.name == "nt": import msvcrt from ctypes import ( - POINTER, Structure, Union, byref, c_int64, c_ulong, c_void_p, sizeof, + POINTER, + Structure, + Union, + byref, + c_int64, + c_ulong, + c_void_p, + sizeof, windll, ) from ctypes.wintypes import BOOL, DWORD, HANDLE @@ -48,23 +55,20 @@ if os.name == 'nt': # --- Union inside Structure by stackoverflow:3480240 --- class _OFFSET(Structure): - _fields_ = [ - ('Offset', DWORD), - ('OffsetHigh', DWORD)] + _fields_ = [("Offset", DWORD), ("OffsetHigh", DWORD)] class _OFFSET_UNION(Union): - _anonymous_ = ['_offset'] - _fields_ = [ - ('_offset', _OFFSET), - ('Pointer', PVOID)] + _anonymous_ = ["_offset"] + _fields_ = [("_offset", _OFFSET), ("Pointer", PVOID)] class OVERLAPPED(Structure): - _anonymous_ = ['_offset_union'] + _anonymous_ = ["_offset_union"] _fields_ = [ - ('Internal', ULONG_PTR), - ('InternalHigh', ULONG_PTR), - ('_offset_union', _OFFSET_UNION), - ('hEvent', HANDLE)] + ("Internal", ULONG_PTR), + ("InternalHigh", ULONG_PTR), + ("_offset_union", _OFFSET_UNION), + ("hEvent", HANDLE), + ] LPOVERLAPPED = POINTER(OVERLAPPED) @@ -87,9 +91,11 @@ if os.name == 'nt': overlapped = OVERLAPPED() ret = UnlockFileEx(hfile, 0, 0, 0xFFFF0000, byref(overlapped)) return bool(ret) + else: try: import fcntl + LOCK_SH = fcntl.LOCK_SH # shared lock LOCK_NB = fcntl.LOCK_NB # non-blocking LOCK_EX = fcntl.LOCK_EX @@ -105,7 +111,9 @@ else: def unlock(f): # File is unlocked return True + else: + def lock(f, flags): try: fcntl.flock(_fd(f), flags) diff --git a/django/core/files/move.py b/django/core/files/move.py index 2cce7848ca..2d71e11885 100644 --- a/django/core/files/move.py +++ b/django/core/files/move.py @@ -11,23 +11,26 @@ from shutil import copystat from django.core.files import locks -__all__ = ['file_move_safe'] +__all__ = ["file_move_safe"] def _samefile(src, dst): # Macintosh, Unix. - if hasattr(os.path, 'samefile'): + if hasattr(os.path, "samefile"): try: return os.path.samefile(src, dst) except OSError: return False # All other platforms: check for same pathname. - return (os.path.normcase(os.path.abspath(src)) == - os.path.normcase(os.path.abspath(dst))) + return os.path.normcase(os.path.abspath(src)) == os.path.normcase( + os.path.abspath(dst) + ) -def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False): +def file_move_safe( + old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False +): """ Move a file from one location to another in the safest way possible. @@ -43,7 +46,10 @@ def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_ove try: if not allow_overwrite and os.access(new_file_name, os.F_OK): - raise FileExistsError('Destination file %s exists and allow_overwrite is False.' % new_file_name) + raise FileExistsError( + "Destination file %s exists and allow_overwrite is False." + % new_file_name + ) os.rename(old_file_name, new_file_name) return @@ -53,14 +59,21 @@ def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_ove pass # first open the old file, so that it won't go away - with open(old_file_name, 'rb') as old_file: + with open(old_file_name, "rb") as old_file: # now open the new file, not forgetting allow_overwrite - fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) | - (os.O_EXCL if not allow_overwrite else 0))) + fd = os.open( + new_file_name, + ( + os.O_WRONLY + | os.O_CREAT + | getattr(os, "O_BINARY", 0) + | (os.O_EXCL if not allow_overwrite else 0) + ), + ) try: locks.lock(fd, locks.LOCK_EX) current_chunk = None - while current_chunk != b'': + while current_chunk != b"": current_chunk = old_file.read(chunk_size) os.write(fd, current_chunk) finally: @@ -83,5 +96,5 @@ def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_ove # fail when deleting opened files, ignore it. (For the # systems where this happens, temporary files will be auto-deleted # on close anyway.) - if getattr(e, 'winerror', 0) != 32: + if getattr(e, "winerror", 0) != 32: raise diff --git a/django/core/files/storage.py b/django/core/files/storage.py index bbb3e8b21d..690456aa71 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -19,8 +19,11 @@ from django.utils.module_loading import import_string from django.utils.text import get_valid_filename __all__ = ( - 'Storage', 'FileSystemStorage', 'DefaultStorage', 'default_storage', - 'get_storage_class', + "Storage", + "FileSystemStorage", + "DefaultStorage", + "default_storage", + "get_storage_class", ) @@ -33,7 +36,7 @@ class Storage: # The following methods represent a public interface to private methods. # These shouldn't be overridden by subclasses unless absolutely necessary. - def open(self, name, mode='rb'): + def open(self, name, mode="rb"): """Retrieve the specified file from storage.""" return self._open(name, mode) @@ -47,7 +50,7 @@ class Storage: if name is None: name = content.name - if not hasattr(content, 'chunks'): + if not hasattr(content, "chunks"): content = File(content, name) name = self.get_available_name(name, max_length=max_length) @@ -71,17 +74,19 @@ class Storage: character alphanumeric string (before the file extension, if one exists) to the filename. """ - return '%s_%s%s' % (file_root, get_random_string(7), file_ext) + return "%s_%s%s" % (file_root, get_random_string(7), file_ext) def get_available_name(self, name, max_length=None): """ Return a filename that's free on the target storage system and available for new content to be written to. """ - name = str(name).replace('\\', '/') + name = str(name).replace("\\", "/") dir_name, file_name = os.path.split(name) - if '..' in pathlib.PurePath(dir_name).parts: - raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dir_name) + if ".." in pathlib.PurePath(dir_name).parts: + raise SuspiciousFileOperation( + "Detected path traversal attempt in '%s'" % dir_name + ) validate_file_name(file_name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, generate an alternative filename @@ -90,7 +95,9 @@ class Storage: # exceed the max_length. while self.exists(name) or (max_length and len(name) > max_length): # file_ext includes the dot. - name = os.path.join(dir_name, self.get_alternative_name(file_root, file_ext)) + name = os.path.join( + dir_name, self.get_alternative_name(file_root, file_ext) + ) if max_length is None: continue # Truncate file_root if max_length exceeded. @@ -101,10 +108,12 @@ class Storage: if not file_root: raise SuspiciousFileOperation( 'Storage can not find an available filename for "%s". ' - 'Please make sure that the corresponding file field ' + "Please make sure that the corresponding file field " 'allows sufficient "max_length".' % name ) - name = os.path.join(dir_name, self.get_alternative_name(file_root, file_ext)) + name = os.path.join( + dir_name, self.get_alternative_name(file_root, file_ext) + ) return name def generate_filename(self, filename): @@ -112,11 +121,13 @@ class Storage: Validate the filename by calling get_valid_name() and return a filename to be passed to the save() method. """ - filename = str(filename).replace('\\', '/') + filename = str(filename).replace("\\", "/") # `filename` may include a path as returned by FileField.upload_to. dirname, filename = os.path.split(filename) - if '..' in pathlib.PurePath(dirname).parts: - raise SuspiciousFileOperation("Detected path traversal attempt in '%s'" % dirname) + if ".." in pathlib.PurePath(dirname).parts: + raise SuspiciousFileOperation( + "Detected path traversal attempt in '%s'" % dirname + ) return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename))) def path(self, name): @@ -134,55 +145,67 @@ class Storage: """ Delete the specified file from the storage system. """ - raise NotImplementedError('subclasses of Storage must provide a delete() method') + raise NotImplementedError( + "subclasses of Storage must provide a delete() method" + ) def exists(self, name): """ Return True if a file referenced by the given name already exists in the storage system, or False if the name is available for a new file. """ - raise NotImplementedError('subclasses of Storage must provide an exists() method') + raise NotImplementedError( + "subclasses of Storage must provide an exists() method" + ) def listdir(self, path): """ List the contents of the specified path. Return a 2-tuple of lists: the first item being directories, the second item being files. """ - raise NotImplementedError('subclasses of Storage must provide a listdir() method') + raise NotImplementedError( + "subclasses of Storage must provide a listdir() method" + ) def size(self, name): """ Return the total size, in bytes, of the file specified by name. """ - raise NotImplementedError('subclasses of Storage must provide a size() method') + raise NotImplementedError("subclasses of Storage must provide a size() method") def url(self, name): """ Return an absolute URL where the file's contents can be accessed directly by a web browser. """ - raise NotImplementedError('subclasses of Storage must provide a url() method') + raise NotImplementedError("subclasses of Storage must provide a url() method") def get_accessed_time(self, name): """ Return the last accessed time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('subclasses of Storage must provide a get_accessed_time() method') + raise NotImplementedError( + "subclasses of Storage must provide a get_accessed_time() method" + ) def get_created_time(self, name): """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('subclasses of Storage must provide a get_created_time() method') + raise NotImplementedError( + "subclasses of Storage must provide a get_created_time() method" + ) def get_modified_time(self, name): """ Return the last modified time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ - raise NotImplementedError('subclasses of Storage must provide a get_modified_time() method') + raise NotImplementedError( + "subclasses of Storage must provide a get_modified_time() method" + ) @deconstructible @@ -190,12 +213,18 @@ class FileSystemStorage(Storage): """ Standard filesystem storage """ + # The combination of O_CREAT and O_EXCL makes os.open() raise OSError if # the file already exists before it's opened. - OS_OPEN_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0) + OS_OPEN_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) - def __init__(self, location=None, base_url=None, file_permissions_mode=None, - directory_permissions_mode=None): + def __init__( + self, + location=None, + base_url=None, + file_permissions_mode=None, + directory_permissions_mode=None, + ): self._location = location self._base_url = base_url self._file_permissions_mode = file_permissions_mode @@ -204,15 +233,15 @@ class FileSystemStorage(Storage): def _clear_cached_properties(self, setting, **kwargs): """Reset setting based property values.""" - if setting == 'MEDIA_ROOT': - self.__dict__.pop('base_location', None) - self.__dict__.pop('location', None) - elif setting == 'MEDIA_URL': - self.__dict__.pop('base_url', None) - elif setting == 'FILE_UPLOAD_PERMISSIONS': - self.__dict__.pop('file_permissions_mode', None) - elif setting == 'FILE_UPLOAD_DIRECTORY_PERMISSIONS': - self.__dict__.pop('directory_permissions_mode', None) + if setting == "MEDIA_ROOT": + self.__dict__.pop("base_location", None) + self.__dict__.pop("location", None) + elif setting == "MEDIA_URL": + self.__dict__.pop("base_url", None) + elif setting == "FILE_UPLOAD_PERMISSIONS": + self.__dict__.pop("file_permissions_mode", None) + elif setting == "FILE_UPLOAD_DIRECTORY_PERMISSIONS": + self.__dict__.pop("directory_permissions_mode", None) def _value_or_setting(self, value, setting): return setting if value is None else value @@ -227,19 +256,23 @@ class FileSystemStorage(Storage): @cached_property def base_url(self): - if self._base_url is not None and not self._base_url.endswith('/'): - self._base_url += '/' + if self._base_url is not None and not self._base_url.endswith("/"): + self._base_url += "/" return self._value_or_setting(self._base_url, settings.MEDIA_URL) @cached_property def file_permissions_mode(self): - return self._value_or_setting(self._file_permissions_mode, settings.FILE_UPLOAD_PERMISSIONS) + return self._value_or_setting( + self._file_permissions_mode, settings.FILE_UPLOAD_PERMISSIONS + ) @cached_property def directory_permissions_mode(self): - return self._value_or_setting(self._directory_permissions_mode, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + return self._value_or_setting( + self._directory_permissions_mode, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS + ) - def _open(self, name, mode='rb'): + def _open(self, name, mode="rb"): return File(open(self.path(name), mode)) def _save(self, name, content): @@ -253,13 +286,15 @@ class FileSystemStorage(Storage): # argument to intermediate-level directories. old_umask = os.umask(0o777 & ~self.directory_permissions_mode) try: - os.makedirs(directory, self.directory_permissions_mode, exist_ok=True) + os.makedirs( + directory, self.directory_permissions_mode, exist_ok=True + ) finally: os.umask(old_umask) else: os.makedirs(directory, exist_ok=True) except FileExistsError: - raise FileExistsError('%s exists and is not a directory.' % directory) + raise FileExistsError("%s exists and is not a directory." % directory) # There's a potential race condition between get_available_name and # saving the file; it's possible that two threads might return the @@ -270,7 +305,7 @@ class FileSystemStorage(Storage): while True: try: # This file has a file path that we can move. - if hasattr(content, 'temporary_file_path'): + if hasattr(content, "temporary_file_path"): file_move_safe(content.temporary_file_path(), full_path) # This is a normal uploadedfile that we can stream. @@ -282,7 +317,7 @@ class FileSystemStorage(Storage): locks.lock(fd, locks.LOCK_EX) for chunk in content.chunks(): if _file is None: - mode = 'wb' if isinstance(chunk, bytes) else 'wt' + mode = "wb" if isinstance(chunk, bytes) else "wt" _file = os.fdopen(fd, mode) _file.write(chunk) finally: @@ -305,11 +340,11 @@ class FileSystemStorage(Storage): # Ensure the saved path is always relative to the storage root. name = os.path.relpath(full_path, self.location) # Store filenames with forward slashes, even on Windows. - return str(name).replace('\\', '/') + return str(name).replace("\\", "/") def delete(self, name): if not name: - raise ValueError('The name must be given to delete().') + raise ValueError("The name must be given to delete().") name = self.path(name) # If the file or directory exists, delete it from the filesystem. try: @@ -347,7 +382,7 @@ class FileSystemStorage(Storage): raise ValueError("This file is not accessible via a URL.") url = filepath_to_uri(name) if url is not None: - url = url.lstrip('/') + url = url.lstrip("/") return urljoin(self.base_url, url) def _datetime_from_timestamp(self, ts): diff --git a/django/core/files/temp.py b/django/core/files/temp.py index 57a8107b37..5bd31dd5f2 100644 --- a/django/core/files/temp.py +++ b/django/core/files/temp.py @@ -21,10 +21,14 @@ import tempfile from django.core.files.utils import FileProxyMixin -__all__ = ('NamedTemporaryFile', 'gettempdir',) +__all__ = ( + "NamedTemporaryFile", + "gettempdir", +) -if os.name == 'nt': +if os.name == "nt": + class TemporaryFile(FileProxyMixin): """ Temporary file object constructor that supports reopening of the @@ -34,7 +38,8 @@ if os.name == 'nt': __init__() doesn't support the 'delete', 'buffering', 'encoding', or 'newline' keyword arguments. """ - def __init__(self, mode='w+b', bufsize=-1, suffix='', prefix='', dir=None): + + def __init__(self, mode="w+b", bufsize=-1, suffix="", prefix="", dir=None): fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir) self.name = name self.file = os.fdopen(fd, mode, bufsize) diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py index f452bcd9a4..efbfcac4c8 100644 --- a/django/core/files/uploadedfile.py +++ b/django/core/files/uploadedfile.py @@ -10,8 +10,12 @@ from django.core.files import temp as tempfile from django.core.files.base import File from django.core.files.utils import validate_file_name -__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', - 'SimpleUploadedFile') +__all__ = ( + "UploadedFile", + "TemporaryUploadedFile", + "InMemoryUploadedFile", + "SimpleUploadedFile", +) class UploadedFile(File): @@ -23,7 +27,15 @@ class UploadedFile(File): represents some file data that the user submitted with a form. """ - def __init__(self, file=None, name=None, content_type=None, size=None, charset=None, content_type_extra=None): + def __init__( + self, + file=None, + name=None, + content_type=None, + size=None, + charset=None, + content_type_extra=None, + ): super().__init__(file, name) self.size = size self.content_type = content_type @@ -46,7 +58,7 @@ class UploadedFile(File): if len(name) > 255: name, ext = os.path.splitext(name) ext = ext[:255] - name = name[:255 - len(ext)] + ext + name = name[: 255 - len(ext)] + ext name = validate_file_name(name) @@ -59,9 +71,12 @@ class TemporaryUploadedFile(UploadedFile): """ A file uploaded to a temporary location (i.e. stream-to-disk). """ + def __init__(self, name, content_type, size, charset, content_type_extra=None): _, ext = os.path.splitext(name) - file = tempfile.NamedTemporaryFile(suffix='.upload' + ext, dir=settings.FILE_UPLOAD_TEMP_DIR) + file = tempfile.NamedTemporaryFile( + suffix=".upload" + ext, dir=settings.FILE_UPLOAD_TEMP_DIR + ) super().__init__(file, name, content_type, size, charset, content_type_extra) def temporary_file_path(self): @@ -82,7 +97,17 @@ class InMemoryUploadedFile(UploadedFile): """ A file uploaded into memory (i.e. stream-to-memory). """ - def __init__(self, file, field_name, name, content_type, size, charset, content_type_extra=None): + + def __init__( + self, + file, + field_name, + name, + content_type, + size, + charset, + content_type_extra=None, + ): super().__init__(file, name, content_type, size, charset, content_type_extra) self.field_name = field_name @@ -103,9 +128,12 @@ class SimpleUploadedFile(InMemoryUploadedFile): """ A simple representation of a file, which just has content, size, and a name. """ - def __init__(self, name, content, content_type='text/plain'): - content = content or b'' - super().__init__(BytesIO(content), None, name, content_type, len(content), None, None) + + def __init__(self, name, content, content_type="text/plain"): + content = content or b"" + super().__init__( + BytesIO(content), None, name, content_type, len(content), None, None + ) @classmethod def from_dict(cls, file_dict): @@ -115,6 +143,8 @@ class SimpleUploadedFile(InMemoryUploadedFile): - content-type - content """ - return cls(file_dict['filename'], - file_dict['content'], - file_dict.get('content-type', 'text/plain')) + return cls( + file_dict["filename"], + file_dict["content"], + file_dict.get("content-type", "text/plain"), + ) diff --git a/django/core/files/uploadhandler.py b/django/core/files/uploadhandler.py index ee6bb31fce..64781f811b 100644 --- a/django/core/files/uploadhandler.py +++ b/django/core/files/uploadhandler.py @@ -5,15 +5,18 @@ import os from io import BytesIO from django.conf import settings -from django.core.files.uploadedfile import ( - InMemoryUploadedFile, TemporaryUploadedFile, -) +from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile from django.utils.module_loading import import_string __all__ = [ - 'UploadFileException', 'StopUpload', 'SkipFile', 'FileUploadHandler', - 'TemporaryFileUploadHandler', 'MemoryFileUploadHandler', 'load_handler', - 'StopFutureHandlers' + "UploadFileException", + "StopUpload", + "SkipFile", + "FileUploadHandler", + "TemporaryFileUploadHandler", + "MemoryFileUploadHandler", + "load_handler", + "StopFutureHandlers", ] @@ -21,6 +24,7 @@ class UploadFileException(Exception): """ Any error having to do with uploading files. """ + pass @@ -28,6 +32,7 @@ class StopUpload(UploadFileException): """ This exception is raised when an upload must abort. """ + def __init__(self, connection_reset=False): """ If ``connection_reset`` is ``True``, Django knows will halt the upload @@ -38,15 +43,16 @@ class StopUpload(UploadFileException): def __str__(self): if self.connection_reset: - return 'StopUpload: Halt current upload.' + return "StopUpload: Halt current upload." else: - return 'StopUpload: Consume request data, then halt.' + return "StopUpload: Consume request data, then halt." class SkipFile(UploadFileException): """ This exception is raised by an upload handler that wants to skip a given file. """ + pass @@ -55,6 +61,7 @@ class StopFutureHandlers(UploadFileException): Upload handlers that have handled a file and do not want future handlers to run should raise this exception instead of returning None. """ + pass @@ -62,7 +69,8 @@ class FileUploadHandler: """ Base class for streaming upload handlers. """ - chunk_size = 64 * 2 ** 10 # : The default chunk size is 64 KB. + + chunk_size = 64 * 2**10 # : The default chunk size is 64 KB. def __init__(self, request=None): self.file_name = None @@ -72,7 +80,9 @@ class FileUploadHandler: self.content_type_extra = None self.request = request - def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + def handle_raw_input( + self, input_data, META, content_length, boundary, encoding=None + ): """ Handle the raw input from the client. @@ -90,7 +100,15 @@ class FileUploadHandler: """ pass - def new_file(self, field_name, file_name, content_type, content_length, charset=None, content_type_extra=None): + def new_file( + self, + field_name, + file_name, + content_type, + content_length, + charset=None, + content_type_extra=None, + ): """ Signal that a new file has been started. @@ -109,7 +127,9 @@ class FileUploadHandler: Receive data from the streamed upload parser. ``start`` is the position in the file of the chunk. """ - raise NotImplementedError('subclasses of FileUploadHandler must provide a receive_data_chunk() method') + raise NotImplementedError( + "subclasses of FileUploadHandler must provide a receive_data_chunk() method" + ) def file_complete(self, file_size): """ @@ -118,7 +138,9 @@ class FileUploadHandler: Subclasses should return a valid ``UploadedFile`` object. """ - raise NotImplementedError('subclasses of FileUploadHandler must provide a file_complete() method') + raise NotImplementedError( + "subclasses of FileUploadHandler must provide a file_complete() method" + ) def upload_complete(self): """ @@ -139,12 +161,15 @@ class TemporaryFileUploadHandler(FileUploadHandler): """ Upload handler that streams data into a temporary file. """ + def new_file(self, *args, **kwargs): """ Create the file object to append to as data is coming in. """ super().new_file(*args, **kwargs) - self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset, self.content_type_extra) + self.file = TemporaryUploadedFile( + self.file_name, self.content_type, 0, self.charset, self.content_type_extra + ) def receive_data_chunk(self, raw_data, start): self.file.write(raw_data) @@ -155,7 +180,7 @@ class TemporaryFileUploadHandler(FileUploadHandler): return self.file def upload_interrupted(self): - if hasattr(self, 'file'): + if hasattr(self, "file"): temp_location = self.file.temporary_file_path() try: self.file.close() @@ -169,7 +194,9 @@ class MemoryFileUploadHandler(FileUploadHandler): File upload handler to stream uploads into memory (used for small files). """ - def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + def handle_raw_input( + self, input_data, META, content_length, boundary, encoding=None + ): """ Use the content_length to signal whether or not this handler should be used. @@ -204,7 +231,7 @@ class MemoryFileUploadHandler(FileUploadHandler): content_type=self.content_type, size=file_size, charset=self.charset, - content_type_extra=self.content_type_extra + content_type_extra=self.content_type_extra, ) diff --git a/django/core/files/utils.py b/django/core/files/utils.py index f28cea1077..85342b2f3f 100644 --- a/django/core/files/utils.py +++ b/django/core/files/utils.py @@ -6,7 +6,7 @@ from django.core.exceptions import SuspiciousFileOperation def validate_file_name(name, allow_relative_path=False): # Remove potentially dangerous names - if os.path.basename(name) in {'', '.', '..'}: + if os.path.basename(name) in {"", ".", ".."}: raise SuspiciousFileOperation("Could not derive file name from '%s'" % name) if allow_relative_path: @@ -14,7 +14,7 @@ def validate_file_name(name, allow_relative_path=False): # FileField.generate_filename() where all file paths are expected to be # Unix style (with forward slashes). path = pathlib.PurePosixPath(name) - if path.is_absolute() or '..' in path.parts: + if path.is_absolute() or ".." in path.parts: raise SuspiciousFileOperation( "Detected path traversal attempt in '%s'" % name ) @@ -56,21 +56,21 @@ class FileProxyMixin: def readable(self): if self.closed: return False - if hasattr(self.file, 'readable'): + if hasattr(self.file, "readable"): return self.file.readable() return True def writable(self): if self.closed: return False - if hasattr(self.file, 'writable'): + if hasattr(self.file, "writable"): return self.file.writable() - return 'w' in getattr(self.file, 'mode', '') + return "w" in getattr(self.file, "mode", "") def seekable(self): if self.closed: return False - if hasattr(self.file, 'seekable'): + if hasattr(self.file, "seekable"): return self.file.seekable() return True diff --git a/django/core/handlers/asgi.py b/django/core/handlers/asgi.py index 9d84efc964..7b17c58153 100644 --- a/django/core/handlers/asgi.py +++ b/django/core/handlers/asgi.py @@ -10,13 +10,18 @@ from django.core import signals from django.core.exceptions import RequestAborted, RequestDataTooBig from django.core.handlers import base from django.http import ( - FileResponse, HttpRequest, HttpResponse, HttpResponseBadRequest, - HttpResponseServerError, QueryDict, parse_cookie, + FileResponse, + HttpRequest, + HttpResponse, + HttpResponseBadRequest, + HttpResponseServerError, + QueryDict, + parse_cookie, ) from django.urls import set_script_prefix from django.utils.functional import cached_property -logger = logging.getLogger('django.request') +logger = logging.getLogger("django.request") class ASGIRequest(HttpRequest): @@ -24,6 +29,7 @@ class ASGIRequest(HttpRequest): Custom request subclass that decodes from an ASGI-standard request dict and wraps request body handling. """ + # Number of seconds until a Request gives up on trying to read a request # body and aborts. body_receive_timeout = 60 @@ -33,60 +39,60 @@ class ASGIRequest(HttpRequest): self._post_parse_error = False self._read_started = False self.resolver_match = None - self.script_name = self.scope.get('root_path', '') - if self.script_name and scope['path'].startswith(self.script_name): + self.script_name = self.scope.get("root_path", "") + if self.script_name and scope["path"].startswith(self.script_name): # TODO: Better is-prefix checking, slash handling? - self.path_info = scope['path'][len(self.script_name):] + self.path_info = scope["path"][len(self.script_name) :] else: - self.path_info = scope['path'] + self.path_info = scope["path"] # The Django path is different from ASGI scope path args, it should # combine with script name. if self.script_name: - self.path = '%s/%s' % ( - self.script_name.rstrip('/'), - self.path_info.replace('/', '', 1), + self.path = "%s/%s" % ( + self.script_name.rstrip("/"), + self.path_info.replace("/", "", 1), ) else: - self.path = scope['path'] + self.path = scope["path"] # HTTP basics. - self.method = self.scope['method'].upper() + self.method = self.scope["method"].upper() # Ensure query string is encoded correctly. - query_string = self.scope.get('query_string', '') + query_string = self.scope.get("query_string", "") if isinstance(query_string, bytes): query_string = query_string.decode() self.META = { - 'REQUEST_METHOD': self.method, - 'QUERY_STRING': query_string, - 'SCRIPT_NAME': self.script_name, - 'PATH_INFO': self.path_info, + "REQUEST_METHOD": self.method, + "QUERY_STRING": query_string, + "SCRIPT_NAME": self.script_name, + "PATH_INFO": self.path_info, # WSGI-expecting code will need these for a while - 'wsgi.multithread': True, - 'wsgi.multiprocess': True, + "wsgi.multithread": True, + "wsgi.multiprocess": True, } - if self.scope.get('client'): - self.META['REMOTE_ADDR'] = self.scope['client'][0] - self.META['REMOTE_HOST'] = self.META['REMOTE_ADDR'] - self.META['REMOTE_PORT'] = self.scope['client'][1] - if self.scope.get('server'): - self.META['SERVER_NAME'] = self.scope['server'][0] - self.META['SERVER_PORT'] = str(self.scope['server'][1]) + if self.scope.get("client"): + self.META["REMOTE_ADDR"] = self.scope["client"][0] + self.META["REMOTE_HOST"] = self.META["REMOTE_ADDR"] + self.META["REMOTE_PORT"] = self.scope["client"][1] + if self.scope.get("server"): + self.META["SERVER_NAME"] = self.scope["server"][0] + self.META["SERVER_PORT"] = str(self.scope["server"][1]) else: - self.META['SERVER_NAME'] = 'unknown' - self.META['SERVER_PORT'] = '0' + self.META["SERVER_NAME"] = "unknown" + self.META["SERVER_PORT"] = "0" # Headers go into META. - for name, value in self.scope.get('headers', []): - name = name.decode('latin1') - if name == 'content-length': - corrected_name = 'CONTENT_LENGTH' - elif name == 'content-type': - corrected_name = 'CONTENT_TYPE' + for name, value in self.scope.get("headers", []): + name = name.decode("latin1") + if name == "content-length": + corrected_name = "CONTENT_LENGTH" + elif name == "content-type": + corrected_name = "CONTENT_TYPE" else: - corrected_name = 'HTTP_%s' % name.upper().replace('-', '_') + corrected_name = "HTTP_%s" % name.upper().replace("-", "_") # HTTP/2 say only ASCII chars are allowed in headers, but decode # latin1 just in case. - value = value.decode('latin1') + value = value.decode("latin1") if corrected_name in self.META: - value = self.META[corrected_name] + ',' + value + value = self.META[corrected_name] + "," + value self.META[corrected_name] = value # Pull out request encoding, if provided. self._set_content_type_params(self.META) @@ -97,13 +103,13 @@ class ASGIRequest(HttpRequest): @cached_property def GET(self): - return QueryDict(self.META['QUERY_STRING']) + return QueryDict(self.META["QUERY_STRING"]) def _get_scheme(self): - return self.scope.get('scheme') or super()._get_scheme() + return self.scope.get("scheme") or super()._get_scheme() def _get_post(self): - if not hasattr(self, '_post'): + if not hasattr(self, "_post"): self._load_post_and_files() return self._post @@ -111,7 +117,7 @@ class ASGIRequest(HttpRequest): self._post = post def _get_files(self): - if not hasattr(self, '_files'): + if not hasattr(self, "_files"): self._load_post_and_files() return self._files @@ -120,14 +126,15 @@ class ASGIRequest(HttpRequest): @cached_property def COOKIES(self): - return parse_cookie(self.META.get('HTTP_COOKIE', '')) + return parse_cookie(self.META.get("HTTP_COOKIE", "")) class ASGIHandler(base.BaseHandler): """Handler for ASGI requests.""" + request_class = ASGIRequest # Size to chunk response bodies into for multiple response messages. - chunk_size = 2 ** 16 + chunk_size = 2**16 def __init__(self): super().__init__() @@ -139,10 +146,9 @@ class ASGIHandler(base.BaseHandler): """ # Serve only HTTP connections. # FIXME: Allow to override this. - if scope['type'] != 'http': + if scope["type"] != "http": raise ValueError( - 'Django can only handle ASGI/HTTP connections, not %s.' - % scope['type'] + "Django can only handle ASGI/HTTP connections, not %s." % scope["type"] ) async with ThreadSensitiveContext(): @@ -159,7 +165,9 @@ class ASGIHandler(base.BaseHandler): return # Request is complete and can be served. set_script_prefix(self.get_script_prefix(scope)) - await sync_to_async(signals.request_started.send, thread_sensitive=True)(sender=self.__class__, scope=scope) + await sync_to_async(signals.request_started.send, thread_sensitive=True)( + sender=self.__class__, scope=scope + ) # Get the request and check for basic issues. request, error_response = self.create_request(scope, body_file) if request is None: @@ -178,17 +186,19 @@ class ASGIHandler(base.BaseHandler): async def read_body(self, receive): """Reads an HTTP body from an ASGI connection.""" # Use the tempfile that auto rolls-over to a disk file as it fills up. - body_file = tempfile.SpooledTemporaryFile(max_size=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, mode='w+b') + body_file = tempfile.SpooledTemporaryFile( + max_size=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, mode="w+b" + ) while True: message = await receive() - if message['type'] == 'http.disconnect': + if message["type"] == "http.disconnect": # Early client disconnect. raise RequestAborted() # Add a body chunk from the message, if provided. - if 'body' in message: - body_file.write(message['body']) + if "body" in message: + body_file.write(message["body"]) # Quit out if that's the end. - if not message.get('more_body', False): + if not message.get("more_body", False): break body_file.seek(0) return body_file @@ -202,13 +212,13 @@ class ASGIHandler(base.BaseHandler): return self.request_class(scope, body_file), None except UnicodeDecodeError: logger.warning( - 'Bad Request (UnicodeDecodeError)', + "Bad Request (UnicodeDecodeError)", exc_info=sys.exc_info(), - extra={'status_code': 400}, + extra={"status_code": 400}, ) return None, HttpResponseBadRequest() except RequestDataTooBig: - return None, HttpResponse('413 Payload too large', status=413) + return None, HttpResponse("413 Payload too large", status=413) def handle_uncaught_exception(self, request, resolver, exc_info): """Last-chance handler for exceptions.""" @@ -218,8 +228,8 @@ class ASGIHandler(base.BaseHandler): return super().handle_uncaught_exception(request, resolver, exc_info) except Exception: return HttpResponseServerError( - traceback.format_exc() if settings.DEBUG else 'Internal Server Error', - content_type='text/plain', + traceback.format_exc() if settings.DEBUG else "Internal Server Error", + content_type="text/plain", ) async def send_response(self, response, send): @@ -229,44 +239,50 @@ class ASGIHandler(base.BaseHandler): response_headers = [] for header, value in response.items(): if isinstance(header, str): - header = header.encode('ascii') + header = header.encode("ascii") if isinstance(value, str): - value = value.encode('latin1') + value = value.encode("latin1") response_headers.append((bytes(header), bytes(value))) for c in response.cookies.values(): response_headers.append( - (b'Set-Cookie', c.output(header='').encode('ascii').strip()) + (b"Set-Cookie", c.output(header="").encode("ascii").strip()) ) # Initial response message. - await send({ - 'type': 'http.response.start', - 'status': response.status_code, - 'headers': response_headers, - }) + await send( + { + "type": "http.response.start", + "status": response.status_code, + "headers": response_headers, + } + ) # Streaming responses need to be pinned to their iterator. if response.streaming: # Access `__iter__` and not `streaming_content` directly in case # it has been overridden in a subclass. for part in response: for chunk, _ in self.chunk_bytes(part): - await send({ - 'type': 'http.response.body', - 'body': chunk, - # Ignore "more" as there may be more parts; instead, - # use an empty final closing message with False. - 'more_body': True, - }) + await send( + { + "type": "http.response.body", + "body": chunk, + # Ignore "more" as there may be more parts; instead, + # use an empty final closing message with False. + "more_body": True, + } + ) # Final closing message. - await send({'type': 'http.response.body'}) + await send({"type": "http.response.body"}) # Other responses just need chunking. else: # Yield chunks of response. for chunk, last in self.chunk_bytes(response.content): - await send({ - 'type': 'http.response.body', - 'body': chunk, - 'more_body': not last, - }) + await send( + { + "type": "http.response.body", + "body": chunk, + "more_body": not last, + } + ) await sync_to_async(response.close, thread_sensitive=True)() @classmethod @@ -281,7 +297,7 @@ class ASGIHandler(base.BaseHandler): return while position < len(data): yield ( - data[position:position + cls.chunk_size], + data[position : position + cls.chunk_size], (position + cls.chunk_size) >= len(data), ) position += cls.chunk_size @@ -292,4 +308,4 @@ class ASGIHandler(base.BaseHandler): """ if settings.FORCE_SCRIPT_NAME: return settings.FORCE_SCRIPT_NAME - return scope.get('root_path', '') or '' + return scope.get("root_path", "") or "" diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py index 728e449703..7c863bb5c1 100644 --- a/django/core/handlers/base.py +++ b/django/core/handlers/base.py @@ -14,7 +14,7 @@ from django.utils.module_loading import import_string from .exception import convert_exception_to_response -logger = logging.getLogger('django.request') +logger = logging.getLogger("django.request") class BaseHandler: @@ -38,12 +38,12 @@ class BaseHandler: handler_is_async = is_async for middleware_path in reversed(settings.MIDDLEWARE): middleware = import_string(middleware_path) - middleware_can_sync = getattr(middleware, 'sync_capable', True) - middleware_can_async = getattr(middleware, 'async_capable', False) + middleware_can_sync = getattr(middleware, "sync_capable", True) + middleware_can_async = getattr(middleware, "async_capable", False) if not middleware_can_sync and not middleware_can_async: raise RuntimeError( - 'Middleware %s must have at least one of ' - 'sync_capable/async_capable set to True.' % middleware_path + "Middleware %s must have at least one of " + "sync_capable/async_capable set to True." % middleware_path ) elif not handler_is_async and middleware_can_sync: middleware_is_async = False @@ -52,35 +52,40 @@ class BaseHandler: try: # Adapt handler, if needed. adapted_handler = self.adapt_method_mode( - middleware_is_async, handler, handler_is_async, - debug=settings.DEBUG, name='middleware %s' % middleware_path, + middleware_is_async, + handler, + handler_is_async, + debug=settings.DEBUG, + name="middleware %s" % middleware_path, ) mw_instance = middleware(adapted_handler) except MiddlewareNotUsed as exc: if settings.DEBUG: if str(exc): - logger.debug('MiddlewareNotUsed(%r): %s', middleware_path, exc) + logger.debug("MiddlewareNotUsed(%r): %s", middleware_path, exc) else: - logger.debug('MiddlewareNotUsed: %r', middleware_path) + logger.debug("MiddlewareNotUsed: %r", middleware_path) continue else: handler = adapted_handler if mw_instance is None: raise ImproperlyConfigured( - 'Middleware factory %s returned None.' % middleware_path + "Middleware factory %s returned None." % middleware_path ) - if hasattr(mw_instance, 'process_view'): + if hasattr(mw_instance, "process_view"): self._view_middleware.insert( 0, self.adapt_method_mode(is_async, mw_instance.process_view), ) - if hasattr(mw_instance, 'process_template_response'): + if hasattr(mw_instance, "process_template_response"): self._template_response_middleware.append( - self.adapt_method_mode(is_async, mw_instance.process_template_response), + self.adapt_method_mode( + is_async, mw_instance.process_template_response + ), ) - if hasattr(mw_instance, 'process_exception'): + if hasattr(mw_instance, "process_exception"): # The exception-handling stack is still always synchronous for # now, so adapt that way. self._exception_middleware.append( @@ -97,7 +102,12 @@ class BaseHandler: self._middleware_chain = handler def adapt_method_mode( - self, is_async, method, method_is_async=None, debug=False, name=None, + self, + is_async, + method, + method_is_async=None, + debug=False, + name=None, ): """ Adapt a method to be in the correct "mode": @@ -111,15 +121,15 @@ class BaseHandler: if method_is_async is None: method_is_async = asyncio.iscoroutinefunction(method) if debug and not name: - name = name or 'method %s()' % method.__qualname__ + name = name or "method %s()" % method.__qualname__ if is_async: if not method_is_async: if debug: - logger.debug('Synchronous %s adapted.', name) + logger.debug("Synchronous %s adapted.", name) return sync_to_async(method, thread_sensitive=True) elif method_is_async: if debug: - logger.debug('Asynchronous %s adapted.', name) + logger.debug("Asynchronous %s adapted.", name) return async_to_sync(method) return method @@ -131,7 +141,9 @@ class BaseHandler: response._resource_closers.append(request.close) if response.status_code >= 400: log_response( - '%s: %s', response.reason_phrase, request.path, + "%s: %s", + response.reason_phrase, + request.path, response=response, request=request, ) @@ -151,7 +163,9 @@ class BaseHandler: response._resource_closers.append(request.close) if response.status_code >= 400: await sync_to_async(log_response, thread_sensitive=False)( - '%s: %s', response.reason_phrase, request.path, + "%s: %s", + response.reason_phrase, + request.path, response=response, request=request, ) @@ -168,7 +182,9 @@ class BaseHandler: # Apply view middleware for middleware_method in self._view_middleware: - response = middleware_method(request, callback, callback_args, callback_kwargs) + response = middleware_method( + request, callback, callback_args, callback_kwargs + ) if response: break @@ -189,16 +205,15 @@ class BaseHandler: # If the response supports deferred rendering, apply template # response middleware and then render the response - if hasattr(response, 'render') and callable(response.render): + if hasattr(response, "render") and callable(response.render): for middleware_method in self._template_response_middleware: response = middleware_method(request, response) # Complain if the template response middleware returned None (a common error). self.check_response( response, middleware_method, - name='%s.process_template_response' % ( - middleware_method.__self__.__class__.__name__, - ) + name="%s.process_template_response" + % (middleware_method.__self__.__class__.__name__,), ) try: response = response.render() @@ -220,7 +235,9 @@ class BaseHandler: # Apply view middleware. for middleware_method in self._view_middleware: - response = await middleware_method(request, callback, callback_args, callback_kwargs) + response = await middleware_method( + request, callback, callback_args, callback_kwargs + ) if response: break @@ -228,9 +245,13 @@ class BaseHandler: wrapped_callback = self.make_view_atomic(callback) # If it is a synchronous view, run it in a subthread if not asyncio.iscoroutinefunction(wrapped_callback): - wrapped_callback = sync_to_async(wrapped_callback, thread_sensitive=True) + wrapped_callback = sync_to_async( + wrapped_callback, thread_sensitive=True + ) try: - response = await wrapped_callback(request, *callback_args, **callback_kwargs) + response = await wrapped_callback( + request, *callback_args, **callback_kwargs + ) except Exception as e: response = await sync_to_async( self.process_exception_by_middleware, @@ -244,7 +265,7 @@ class BaseHandler: # If the response supports deferred rendering, apply template # response middleware and then render the response - if hasattr(response, 'render') and callable(response.render): + if hasattr(response, "render") and callable(response.render): for middleware_method in self._template_response_middleware: response = await middleware_method(request, response) # Complain if the template response middleware returned None or @@ -252,15 +273,16 @@ class BaseHandler: self.check_response( response, middleware_method, - name='%s.process_template_response' % ( - middleware_method.__self__.__class__.__name__, - ) + name="%s.process_template_response" + % (middleware_method.__self__.__class__.__name__,), ) try: if asyncio.iscoroutinefunction(response.render): response = await response.render() else: - response = await sync_to_async(response.render, thread_sensitive=True)() + response = await sync_to_async( + response.render, thread_sensitive=True + )() except Exception as e: response = await sync_to_async( self.process_exception_by_middleware, @@ -271,7 +293,7 @@ class BaseHandler: # Make sure the response is not a coroutine if asyncio.iscoroutine(response): - raise RuntimeError('Response is still a coroutine.') + raise RuntimeError("Response is still a coroutine.") return response def resolve_request(self, request): @@ -280,7 +302,7 @@ class BaseHandler: with its args and kwargs. """ # Work out the resolver. - if hasattr(request, 'urlconf'): + if hasattr(request, "urlconf"): urlconf = request.urlconf set_urlconf(urlconf) resolver = get_resolver(urlconf) @@ -295,13 +317,13 @@ class BaseHandler: """ Raise an error if the view returned None or an uncalled coroutine. """ - if not(response is None or asyncio.iscoroutine(response)): + if not (response is None or asyncio.iscoroutine(response)): return if not name: if isinstance(callback, types.FunctionType): # FBV - name = 'The view %s.%s' % (callback.__module__, callback.__name__) + name = "The view %s.%s" % (callback.__module__, callback.__name__) else: # CBV - name = 'The view %s.%s.__call__' % ( + name = "The view %s.%s.__call__" % ( callback.__module__, callback.__class__.__name__, ) @@ -320,12 +342,15 @@ class BaseHandler: # Other utility methods. def make_view_atomic(self, view): - non_atomic_requests = getattr(view, '_non_atomic_requests', set()) + non_atomic_requests = getattr(view, "_non_atomic_requests", set()) for db in connections.all(): - if db.settings_dict['ATOMIC_REQUESTS'] and db.alias not in non_atomic_requests: + if ( + db.settings_dict["ATOMIC_REQUESTS"] + and db.alias not in non_atomic_requests + ): if asyncio.iscoroutinefunction(view): raise RuntimeError( - 'You cannot use ATOMIC_REQUESTS with async views.' + "You cannot use ATOMIC_REQUESTS with async views." ) view = transaction.atomic(using=db.alias)(view) return view diff --git a/django/core/handlers/exception.py b/django/core/handlers/exception.py index 5470b3dd53..79577c2d0a 100644 --- a/django/core/handlers/exception.py +++ b/django/core/handlers/exception.py @@ -8,7 +8,10 @@ from asgiref.sync import sync_to_async from django.conf import settings from django.core import signals from django.core.exceptions import ( - BadRequest, PermissionDenied, RequestDataTooBig, SuspiciousOperation, + BadRequest, + PermissionDenied, + RequestDataTooBig, + SuspiciousOperation, TooManyFieldsSent, ) from django.http import Http404 @@ -32,15 +35,20 @@ def convert_exception_to_response(get_response): can rely on getting a response instead of an exception. """ if asyncio.iscoroutinefunction(get_response): + @wraps(get_response) async def inner(request): try: response = await get_response(request) except Exception as exc: - response = await sync_to_async(response_for_exception, thread_sensitive=False)(request, exc) + response = await sync_to_async( + response_for_exception, thread_sensitive=False + )(request, exc) return response + return inner else: + @wraps(get_response) def inner(request): try: @@ -48,6 +56,7 @@ def convert_exception_to_response(get_response): except Exception as exc: response = response_for_exception(request, exc) return response + return inner @@ -56,21 +65,29 @@ def response_for_exception(request, exc): if settings.DEBUG: response = debug.technical_404_response(request, exc) else: - response = get_exception_response(request, get_resolver(get_urlconf()), 404, exc) + response = get_exception_response( + request, get_resolver(get_urlconf()), 404, exc + ) elif isinstance(exc, PermissionDenied): - response = get_exception_response(request, get_resolver(get_urlconf()), 403, exc) + response = get_exception_response( + request, get_resolver(get_urlconf()), 403, exc + ) log_response( - 'Forbidden (Permission denied): %s', request.path, + "Forbidden (Permission denied): %s", + request.path, response=response, request=request, exception=exc, ) elif isinstance(exc, MultiPartParserError): - response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc) + response = get_exception_response( + request, get_resolver(get_urlconf()), 400, exc + ) log_response( - 'Bad request (Unable to parse request body): %s', request.path, + "Bad request (Unable to parse request body): %s", + request.path, response=response, request=request, exception=exc, @@ -78,11 +95,17 @@ def response_for_exception(request, exc): elif isinstance(exc, BadRequest): if settings.DEBUG: - response = debug.technical_500_response(request, *sys.exc_info(), status_code=400) + response = debug.technical_500_response( + request, *sys.exc_info(), status_code=400 + ) else: - response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc) + response = get_exception_response( + request, get_resolver(get_urlconf()), 400, exc + ) log_response( - '%s: %s', str(exc), request.path, + "%s: %s", + str(exc), + request.path, response=response, request=request, exception=exc, @@ -95,29 +118,41 @@ def response_for_exception(request, exc): # The request logger receives events for any problematic request # The security logger receives events for all SuspiciousOperations - security_logger = logging.getLogger('django.security.%s' % exc.__class__.__name__) + security_logger = logging.getLogger( + "django.security.%s" % exc.__class__.__name__ + ) security_logger.error( str(exc), exc_info=exc, - extra={'status_code': 400, 'request': request}, + extra={"status_code": 400, "request": request}, ) if settings.DEBUG: - response = debug.technical_500_response(request, *sys.exc_info(), status_code=400) + response = debug.technical_500_response( + request, *sys.exc_info(), status_code=400 + ) else: - response = get_exception_response(request, get_resolver(get_urlconf()), 400, exc) + response = get_exception_response( + request, get_resolver(get_urlconf()), 400, exc + ) else: signals.got_request_exception.send(sender=None, request=request) - response = handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info()) + response = handle_uncaught_exception( + request, get_resolver(get_urlconf()), sys.exc_info() + ) log_response( - '%s: %s', response.reason_phrase, request.path, + "%s: %s", + response.reason_phrase, + request.path, response=response, request=request, exception=exc, ) # Force a TemplateResponse to be rendered. - if not getattr(response, 'is_rendered', True) and callable(getattr(response, 'render', None)): + if not getattr(response, "is_rendered", True) and callable( + getattr(response, "render", None) + ): response = response.render() return response diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py index 30920da6d7..126e795fab 100644 --- a/django/core/handlers/wsgi.py +++ b/django/core/handlers/wsgi.py @@ -9,21 +9,22 @@ from django.utils.encoding import repercent_broken_unicode from django.utils.functional import cached_property from django.utils.regex_helper import _lazy_re_compile -_slashes_re = _lazy_re_compile(br'/+') +_slashes_re = _lazy_re_compile(rb"/+") class LimitedStream: """Wrap another stream to disallow reading it past a number of bytes.""" + def __init__(self, stream, limit): self.stream = stream self.remaining = limit - self.buffer = b'' + self.buffer = b"" def _read_limited(self, size=None): if size is None or size > self.remaining: size = self.remaining if size == 0: - return b'' + return b"" result = self.stream.read(size) self.remaining -= len(result) return result @@ -31,18 +32,17 @@ class LimitedStream: def read(self, size=None): if size is None: result = self.buffer + self._read_limited() - self.buffer = b'' + self.buffer = b"" elif size < len(self.buffer): result = self.buffer[:size] self.buffer = self.buffer[size:] else: # size >= len(self.buffer) result = self.buffer + self._read_limited(size - len(self.buffer)) - self.buffer = b'' + self.buffer = b"" return result def readline(self, size=None): - while b'\n' not in self.buffer and \ - (size is None or len(self.buffer) < size): + while b"\n" not in self.buffer and (size is None or len(self.buffer) < size): if size: # since size is not None here, len(self.buffer) < size chunk = self._read_limited(size - len(self.buffer)) @@ -65,39 +65,38 @@ class WSGIRequest(HttpRequest): script_name = get_script_name(environ) # If PATH_INFO is empty (e.g. accessing the SCRIPT_NAME URL without a # trailing slash), operate as if '/' was requested. - path_info = get_path_info(environ) or '/' + path_info = get_path_info(environ) or "/" self.environ = environ self.path_info = path_info # be careful to only replace the first slash in the path because of # http://test/something and http://test//something being different as # stated in https://www.ietf.org/rfc/rfc2396.txt - self.path = '%s/%s' % (script_name.rstrip('/'), - path_info.replace('/', '', 1)) + self.path = "%s/%s" % (script_name.rstrip("/"), path_info.replace("/", "", 1)) self.META = environ - self.META['PATH_INFO'] = path_info - self.META['SCRIPT_NAME'] = script_name - self.method = environ['REQUEST_METHOD'].upper() + self.META["PATH_INFO"] = path_info + self.META["SCRIPT_NAME"] = script_name + self.method = environ["REQUEST_METHOD"].upper() # Set content_type, content_params, and encoding. self._set_content_type_params(environ) try: - content_length = int(environ.get('CONTENT_LENGTH')) + content_length = int(environ.get("CONTENT_LENGTH")) except (ValueError, TypeError): content_length = 0 - self._stream = LimitedStream(self.environ['wsgi.input'], content_length) + self._stream = LimitedStream(self.environ["wsgi.input"], content_length) self._read_started = False self.resolver_match = None def _get_scheme(self): - return self.environ.get('wsgi.url_scheme') + return self.environ.get("wsgi.url_scheme") @cached_property def GET(self): # The WSGI spec says 'QUERY_STRING' may be absent. - raw_query_string = get_bytes_from_wsgi(self.environ, 'QUERY_STRING', '') + raw_query_string = get_bytes_from_wsgi(self.environ, "QUERY_STRING", "") return QueryDict(raw_query_string, encoding=self._encoding) def _get_post(self): - if not hasattr(self, '_post'): + if not hasattr(self, "_post"): self._load_post_and_files() return self._post @@ -106,12 +105,12 @@ class WSGIRequest(HttpRequest): @cached_property def COOKIES(self): - raw_cookie = get_str_from_wsgi(self.environ, 'HTTP_COOKIE', '') + raw_cookie = get_str_from_wsgi(self.environ, "HTTP_COOKIE", "") return parse_cookie(raw_cookie) @property def FILES(self): - if not hasattr(self, '_files'): + if not hasattr(self, "_files"): self._load_post_and_files() return self._files @@ -133,24 +132,28 @@ class WSGIHandler(base.BaseHandler): response._handler_class = self.__class__ - status = '%d %s' % (response.status_code, response.reason_phrase) + status = "%d %s" % (response.status_code, response.reason_phrase) response_headers = [ *response.items(), - *(('Set-Cookie', c.output(header='')) for c in response.cookies.values()), + *(("Set-Cookie", c.output(header="")) for c in response.cookies.values()), ] start_response(status, response_headers) - if getattr(response, 'file_to_stream', None) is not None and environ.get('wsgi.file_wrapper'): + if getattr(response, "file_to_stream", None) is not None and environ.get( + "wsgi.file_wrapper" + ): # If `wsgi.file_wrapper` is used the WSGI server does not call # .close on the response, but on the file wrapper. Patch it to use # response.close instead which takes care of closing all files. response.file_to_stream.close = response.close - response = environ['wsgi.file_wrapper'](response.file_to_stream, response.block_size) + response = environ["wsgi.file_wrapper"]( + response.file_to_stream, response.block_size + ) return response def get_path_info(environ): """Return the HTTP request's PATH_INFO as a string.""" - path_info = get_bytes_from_wsgi(environ, 'PATH_INFO', '/') + path_info = get_bytes_from_wsgi(environ, "PATH_INFO", "/") return repercent_broken_unicode(path_info).decode() @@ -171,17 +174,19 @@ def get_script_name(environ): # rewrites. Unfortunately not every web server (lighttpd!) passes this # information through all the time, so FORCE_SCRIPT_NAME, above, is still # needed. - script_url = get_bytes_from_wsgi(environ, 'SCRIPT_URL', '') or get_bytes_from_wsgi(environ, 'REDIRECT_URL', '') + script_url = get_bytes_from_wsgi(environ, "SCRIPT_URL", "") or get_bytes_from_wsgi( + environ, "REDIRECT_URL", "" + ) if script_url: - if b'//' in script_url: + if b"//" in script_url: # mod_wsgi squashes multiple successive slashes in PATH_INFO, # do the same with script_url before manipulating paths (#17133). - script_url = _slashes_re.sub(b'/', script_url) - path_info = get_bytes_from_wsgi(environ, 'PATH_INFO', '') - script_name = script_url[:-len(path_info)] if path_info else script_url + script_url = _slashes_re.sub(b"/", script_url) + path_info = get_bytes_from_wsgi(environ, "PATH_INFO", "") + script_name = script_url[: -len(path_info)] if path_info else script_url else: - script_name = get_bytes_from_wsgi(environ, 'SCRIPT_NAME', '') + script_name = get_bytes_from_wsgi(environ, "SCRIPT_NAME", "") return script_name.decode() @@ -196,7 +201,7 @@ def get_bytes_from_wsgi(environ, key, default): # Non-ASCII values in the WSGI environ are arbitrarily decoded with # ISO-8859-1. This is wrong for Django websites where UTF-8 is the default. # Re-encode to recover the original bytestring. - return value.encode('iso-8859-1') + return value.encode("iso-8859-1") def get_str_from_wsgi(environ, key, default): @@ -206,4 +211,4 @@ def get_str_from_wsgi(environ, key, default): key and default should be str objects. """ value = get_bytes_from_wsgi(environ, key, default) - return value.decode(errors='replace') + return value.decode(errors="replace") diff --git a/django/core/mail/__init__.py b/django/core/mail/__init__.py index f49cd07dce..dc63e8702c 100644 --- a/django/core/mail/__init__.py +++ b/django/core/mail/__init__.py @@ -2,24 +2,40 @@ Tools for sending email. """ from django.conf import settings + # Imported for backwards compatibility and for the sake # of a cleaner namespace. These symbols used to be in # django/core/mail.py before the introduction of email # backends and the subsequent reorganization (See #10355) from django.core.mail.message import ( - DEFAULT_ATTACHMENT_MIME_TYPE, BadHeaderError, EmailMessage, - EmailMultiAlternatives, SafeMIMEMultipart, SafeMIMEText, - forbid_multi_line_headers, make_msgid, + DEFAULT_ATTACHMENT_MIME_TYPE, + BadHeaderError, + EmailMessage, + EmailMultiAlternatives, + SafeMIMEMultipart, + SafeMIMEText, + forbid_multi_line_headers, + make_msgid, ) from django.core.mail.utils import DNS_NAME, CachedDnsName from django.utils.module_loading import import_string __all__ = [ - 'CachedDnsName', 'DNS_NAME', 'EmailMessage', 'EmailMultiAlternatives', - 'SafeMIMEText', 'SafeMIMEMultipart', 'DEFAULT_ATTACHMENT_MIME_TYPE', - 'make_msgid', 'BadHeaderError', 'forbid_multi_line_headers', - 'get_connection', 'send_mail', 'send_mass_mail', 'mail_admins', - 'mail_managers', + "CachedDnsName", + "DNS_NAME", + "EmailMessage", + "EmailMultiAlternatives", + "SafeMIMEText", + "SafeMIMEMultipart", + "DEFAULT_ATTACHMENT_MIME_TYPE", + "make_msgid", + "BadHeaderError", + "forbid_multi_line_headers", + "get_connection", + "send_mail", + "send_mass_mail", + "mail_admins", + "mail_managers", ] @@ -35,9 +51,17 @@ def get_connection(backend=None, fail_silently=False, **kwds): return klass(fail_silently=fail_silently, **kwds) -def send_mail(subject, message, from_email, recipient_list, - fail_silently=False, auth_user=None, auth_password=None, - connection=None, html_message=None): +def send_mail( + subject, + message, + from_email, + recipient_list, + fail_silently=False, + auth_user=None, + auth_password=None, + connection=None, + html_message=None, +): """ Easy wrapper for sending a single message to a recipient list. All members of the recipient list will see the other recipients in the 'To' field. @@ -54,15 +78,18 @@ def send_mail(subject, message, from_email, recipient_list, password=auth_password, fail_silently=fail_silently, ) - mail = EmailMultiAlternatives(subject, message, from_email, recipient_list, connection=connection) + mail = EmailMultiAlternatives( + subject, message, from_email, recipient_list, connection=connection + ) if html_message: - mail.attach_alternative(html_message, 'text/html') + mail.attach_alternative(html_message, "text/html") return mail.send() -def send_mass_mail(datatuple, fail_silently=False, auth_user=None, - auth_password=None, connection=None): +def send_mass_mail( + datatuple, fail_silently=False, auth_user=None, auth_password=None, connection=None +): """ Given a datatuple of (subject, message, from_email, recipient_list), send each message to each recipient list. Return the number of emails sent. @@ -87,35 +114,41 @@ def send_mass_mail(datatuple, fail_silently=False, auth_user=None, return connection.send_messages(messages) -def mail_admins(subject, message, fail_silently=False, connection=None, - html_message=None): +def mail_admins( + subject, message, fail_silently=False, connection=None, html_message=None +): """Send a message to the admins, as defined by the ADMINS setting.""" if not settings.ADMINS: return if not all(isinstance(a, (list, tuple)) and len(a) == 2 for a in settings.ADMINS): - raise ValueError('The ADMINS setting must be a list of 2-tuples.') + raise ValueError("The ADMINS setting must be a list of 2-tuples.") mail = EmailMultiAlternatives( - '%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject), message, - settings.SERVER_EMAIL, [a[1] for a in settings.ADMINS], + "%s%s" % (settings.EMAIL_SUBJECT_PREFIX, subject), + message, + settings.SERVER_EMAIL, + [a[1] for a in settings.ADMINS], connection=connection, ) if html_message: - mail.attach_alternative(html_message, 'text/html') + mail.attach_alternative(html_message, "text/html") mail.send(fail_silently=fail_silently) -def mail_managers(subject, message, fail_silently=False, connection=None, - html_message=None): +def mail_managers( + subject, message, fail_silently=False, connection=None, html_message=None +): """Send a message to the managers, as defined by the MANAGERS setting.""" if not settings.MANAGERS: return if not all(isinstance(a, (list, tuple)) and len(a) == 2 for a in settings.MANAGERS): - raise ValueError('The MANAGERS setting must be a list of 2-tuples.') + raise ValueError("The MANAGERS setting must be a list of 2-tuples.") mail = EmailMultiAlternatives( - '%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject), message, - settings.SERVER_EMAIL, [a[1] for a in settings.MANAGERS], + "%s%s" % (settings.EMAIL_SUBJECT_PREFIX, subject), + message, + settings.SERVER_EMAIL, + [a[1] for a in settings.MANAGERS], connection=connection, ) if html_message: - mail.attach_alternative(html_message, 'text/html') + mail.attach_alternative(html_message, "text/html") mail.send(fail_silently=fail_silently) diff --git a/django/core/mail/backends/base.py b/django/core/mail/backends/base.py index d687703332..b35b964cb1 100644 --- a/django/core/mail/backends/base.py +++ b/django/core/mail/backends/base.py @@ -14,6 +14,7 @@ class BaseEmailBackend: # do something with connection pass """ + def __init__(self, fail_silently=False, **kwargs): self.fail_silently = fail_silently @@ -56,4 +57,6 @@ class BaseEmailBackend: Send one or more EmailMessage objects and return the number of email messages sent. """ - raise NotImplementedError('subclasses of BaseEmailBackend must override send_messages() method') + raise NotImplementedError( + "subclasses of BaseEmailBackend must override send_messages() method" + ) diff --git a/django/core/mail/backends/console.py b/django/core/mail/backends/console.py index a8bdcbd2c0..ee5dd28504 100644 --- a/django/core/mail/backends/console.py +++ b/django/core/mail/backends/console.py @@ -9,18 +9,20 @@ from django.core.mail.backends.base import BaseEmailBackend class EmailBackend(BaseEmailBackend): def __init__(self, *args, **kwargs): - self.stream = kwargs.pop('stream', sys.stdout) + self.stream = kwargs.pop("stream", sys.stdout) self._lock = threading.RLock() super().__init__(*args, **kwargs) def write_message(self, message): msg = message.message() msg_data = msg.as_bytes() - charset = msg.get_charset().get_output_charset() if msg.get_charset() else 'utf-8' + charset = ( + msg.get_charset().get_output_charset() if msg.get_charset() else "utf-8" + ) msg_data = msg_data.decode(charset) - self.stream.write('%s\n' % msg_data) - self.stream.write('-' * 79) - self.stream.write('\n') + self.stream.write("%s\n" % msg_data) + self.stream.write("-" * 79) + self.stream.write("\n") def send_messages(self, email_messages): """Write all messages to the stream in a thread-safe way.""" diff --git a/django/core/mail/backends/filebased.py b/django/core/mail/backends/filebased.py index 498d86fba8..3b2b037150 100644 --- a/django/core/mail/backends/filebased.py +++ b/django/core/mail/backends/filebased.py @@ -5,9 +5,7 @@ import os from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.core.mail.backends.console import ( - EmailBackend as ConsoleEmailBackend, -) +from django.core.mail.backends.console import EmailBackend as ConsoleEmailBackend class EmailBackend(ConsoleEmailBackend): @@ -16,31 +14,35 @@ class EmailBackend(ConsoleEmailBackend): if file_path is not None: self.file_path = file_path else: - self.file_path = getattr(settings, 'EMAIL_FILE_PATH', None) + self.file_path = getattr(settings, "EMAIL_FILE_PATH", None) self.file_path = os.path.abspath(self.file_path) try: os.makedirs(self.file_path, exist_ok=True) except FileExistsError: raise ImproperlyConfigured( - 'Path for saving email messages exists, but is not a directory: %s' % self.file_path + "Path for saving email messages exists, but is not a directory: %s" + % self.file_path ) except OSError as err: raise ImproperlyConfigured( - 'Could not create directory for saving email messages: %s (%s)' % (self.file_path, err) + "Could not create directory for saving email messages: %s (%s)" + % (self.file_path, err) ) # Make sure that self.file_path is writable. if not os.access(self.file_path, os.W_OK): - raise ImproperlyConfigured('Could not write to directory: %s' % self.file_path) + raise ImproperlyConfigured( + "Could not write to directory: %s" % self.file_path + ) # Finally, call super(). # Since we're using the console-based backend as a base, # force the stream to be None, so we don't default to stdout - kwargs['stream'] = None + kwargs["stream"] = None super().__init__(*args, **kwargs) def write_message(self, message): - self.stream.write(message.message().as_bytes() + b'\n') - self.stream.write(b'-' * 79) - self.stream.write(b'\n') + self.stream.write(message.message().as_bytes() + b"\n") + self.stream.write(b"-" * 79) + self.stream.write(b"\n") def _get_filename(self): """Return a unique file name.""" @@ -52,7 +54,7 @@ class EmailBackend(ConsoleEmailBackend): def open(self): if self.stream is None: - self.stream = open(self._get_filename(), 'ab') + self.stream = open(self._get_filename(), "ab") return True return False diff --git a/django/core/mail/backends/locmem.py b/django/core/mail/backends/locmem.py index 84732e997b..76676973a4 100644 --- a/django/core/mail/backends/locmem.py +++ b/django/core/mail/backends/locmem.py @@ -15,9 +15,10 @@ class EmailBackend(BaseEmailBackend): The dummy outbox is accessible through the outbox instance attribute. """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if not hasattr(mail, 'outbox'): + if not hasattr(mail, "outbox"): mail.outbox = [] def send_messages(self, messages): diff --git a/django/core/mail/backends/smtp.py b/django/core/mail/backends/smtp.py index 13ed4a2798..5df7c20ae0 100644 --- a/django/core/mail/backends/smtp.py +++ b/django/core/mail/backends/smtp.py @@ -13,10 +13,21 @@ class EmailBackend(BaseEmailBackend): """ A wrapper that manages the SMTP network connection. """ - def __init__(self, host=None, port=None, username=None, password=None, - use_tls=None, fail_silently=False, use_ssl=None, timeout=None, - ssl_keyfile=None, ssl_certfile=None, - **kwargs): + + def __init__( + self, + host=None, + port=None, + username=None, + password=None, + use_tls=None, + fail_silently=False, + use_ssl=None, + timeout=None, + ssl_keyfile=None, + ssl_certfile=None, + **kwargs, + ): super().__init__(fail_silently=fail_silently) self.host = host or settings.EMAIL_HOST self.port = port or settings.EMAIL_PORT @@ -25,12 +36,17 @@ class EmailBackend(BaseEmailBackend): self.use_tls = settings.EMAIL_USE_TLS if use_tls is None else use_tls self.use_ssl = settings.EMAIL_USE_SSL if use_ssl is None else use_ssl self.timeout = settings.EMAIL_TIMEOUT if timeout is None else timeout - self.ssl_keyfile = settings.EMAIL_SSL_KEYFILE if ssl_keyfile is None else ssl_keyfile - self.ssl_certfile = settings.EMAIL_SSL_CERTFILE if ssl_certfile is None else ssl_certfile + self.ssl_keyfile = ( + settings.EMAIL_SSL_KEYFILE if ssl_keyfile is None else ssl_keyfile + ) + self.ssl_certfile = ( + settings.EMAIL_SSL_CERTFILE if ssl_certfile is None else ssl_certfile + ) if self.use_ssl and self.use_tls: raise ValueError( "EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set " - "one of those settings to True.") + "one of those settings to True." + ) self.connection = None self._lock = threading.RLock() @@ -50,21 +66,27 @@ class EmailBackend(BaseEmailBackend): # If local_hostname is not specified, socket.getfqdn() gets used. # For performance, we use the cached FQDN for local_hostname. - connection_params = {'local_hostname': DNS_NAME.get_fqdn()} + connection_params = {"local_hostname": DNS_NAME.get_fqdn()} if self.timeout is not None: - connection_params['timeout'] = self.timeout + connection_params["timeout"] = self.timeout if self.use_ssl: - connection_params.update({ - 'keyfile': self.ssl_keyfile, - 'certfile': self.ssl_certfile, - }) + connection_params.update( + { + "keyfile": self.ssl_keyfile, + "certfile": self.ssl_certfile, + } + ) try: - self.connection = self.connection_class(self.host, self.port, **connection_params) + self.connection = self.connection_class( + self.host, self.port, **connection_params + ) # TLS/SSL are mutually exclusive, so only attempt TLS over # non-secure connections. if not self.use_ssl and self.use_tls: - self.connection.starttls(keyfile=self.ssl_keyfile, certfile=self.ssl_certfile) + self.connection.starttls( + keyfile=self.ssl_keyfile, certfile=self.ssl_certfile + ) if self.username and self.password: self.connection.login(self.username, self.password) return True @@ -119,10 +141,14 @@ class EmailBackend(BaseEmailBackend): return False encoding = email_message.encoding or settings.DEFAULT_CHARSET from_email = sanitize_address(email_message.from_email, encoding) - recipients = [sanitize_address(addr, encoding) for addr in email_message.recipients()] + recipients = [ + sanitize_address(addr, encoding) for addr in email_message.recipients() + ] message = email_message.message() try: - self.connection.sendmail(from_email, recipients, message.as_bytes(linesep='\r\n')) + self.connection.sendmail( + from_email, recipients, message.as_bytes(linesep="\r\n") + ) except smtplib.SMTPException: if not self.fail_silently: raise diff --git a/django/core/mail/message.py b/django/core/mail/message.py index ccc8a769ea..cd5b71ad51 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -1,7 +1,7 @@ import mimetypes -from email import ( - charset as Charset, encoders as Encoders, generator, message_from_string, -) +from email import charset as Charset +from email import encoders as Encoders +from email import generator, message_from_string from email.errors import HeaderParseError from email.header import Header from email.headerregistry import Address, parser @@ -20,14 +20,14 @@ from django.utils.encoding import force_str, punycode # Don't BASE64-encode UTF-8 messages so that we avoid unwanted attention from # some spam filters. -utf8_charset = Charset.Charset('utf-8') +utf8_charset = Charset.Charset("utf-8") utf8_charset.body_encoding = None # Python defaults to BASE64 -utf8_charset_qp = Charset.Charset('utf-8') +utf8_charset_qp = Charset.Charset("utf-8") utf8_charset_qp.body_encoding = Charset.QP # Default MIME type to use on attachments (if it is not explicitly given # and cannot be guessed). -DEFAULT_ATTACHMENT_MIME_TYPE = 'application/octet-stream' +DEFAULT_ATTACHMENT_MIME_TYPE = "application/octet-stream" RFC5322_EMAIL_LINE_LENGTH_LIMIT = 998 @@ -38,17 +38,17 @@ class BadHeaderError(ValueError): # Header names that contain structured address data (RFC #5322) ADDRESS_HEADERS = { - 'from', - 'sender', - 'reply-to', - 'to', - 'cc', - 'bcc', - 'resent-from', - 'resent-sender', - 'resent-to', - 'resent-cc', - 'resent-bcc', + "from", + "sender", + "reply-to", + "to", + "cc", + "bcc", + "resent-from", + "resent-sender", + "resent-to", + "resent-cc", + "resent-bcc", } @@ -56,17 +56,21 @@ def forbid_multi_line_headers(name, val, encoding): """Forbid multi-line headers to prevent header injection.""" encoding = encoding or settings.DEFAULT_CHARSET val = str(val) # val may be lazy - if '\n' in val or '\r' in val: - raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name)) + if "\n" in val or "\r" in val: + raise BadHeaderError( + "Header values can't contain newlines (got %r for header %r)" % (val, name) + ) try: - val.encode('ascii') + val.encode("ascii") except UnicodeEncodeError: if name.lower() in ADDRESS_HEADERS: - val = ', '.join(sanitize_address(addr, encoding) for addr in getaddresses((val,))) + val = ", ".join( + sanitize_address(addr, encoding) for addr in getaddresses((val,)) + ) else: val = Header(val, encoding).encode() else: - if name.lower() == 'subject': + if name.lower() == "subject": val = Header(val).encode() return name, val @@ -86,28 +90,27 @@ def sanitize_address(addr, encoding): if rest: # The entire email address must be parsed. raise ValueError( - 'Invalid address; only %s could be parsed from "%s"' - % (token, addr) + 'Invalid address; only %s could be parsed from "%s"' % (token, addr) ) - nm = token.display_name or '' + nm = token.display_name or "" localpart = token.local_part - domain = token.domain or '' + domain = token.domain or "" else: nm, address = addr - localpart, domain = address.rsplit('@', 1) + localpart, domain = address.rsplit("@", 1) address_parts = nm + localpart + domain - if '\n' in address_parts or '\r' in address_parts: - raise ValueError('Invalid address; address parts cannot contain newlines.') + if "\n" in address_parts or "\r" in address_parts: + raise ValueError("Invalid address; address parts cannot contain newlines.") # Avoid UTF-8 encode, if it's possible. try: - nm.encode('ascii') + nm.encode("ascii") nm = Header(nm).encode() except UnicodeEncodeError: nm = Header(nm, encoding).encode() try: - localpart.encode('ascii') + localpart.encode("ascii") except UnicodeEncodeError: localpart = Header(localpart, encoding).encode() domain = punycode(domain) @@ -117,7 +120,7 @@ def sanitize_address(addr, encoding): class MIMEMixin: - def as_string(self, unixfrom=False, linesep='\n'): + def as_string(self, unixfrom=False, linesep="\n"): """Return the entire formatted message as a string. Optional `unixfrom' when True, means include the Unix From_ envelope header. @@ -130,7 +133,7 @@ class MIMEMixin: g.flatten(self, unixfrom=unixfrom, linesep=linesep) return fp.getvalue() - def as_bytes(self, unixfrom=False, linesep='\n'): + def as_bytes(self, unixfrom=False, linesep="\n"): """Return the entire formatted message as bytes. Optional `unixfrom' when True, means include the Unix From_ envelope header. @@ -145,16 +148,14 @@ class MIMEMixin: class SafeMIMEMessage(MIMEMixin, MIMEMessage): - def __setitem__(self, name, val): # message/rfc822 attachments must be ASCII - name, val = forbid_multi_line_headers(name, val, 'ascii') + name, val = forbid_multi_line_headers(name, val, "ascii") MIMEMessage.__setitem__(self, name, val) class SafeMIMEText(MIMEMixin, MIMEText): - - def __init__(self, _text, _subtype='plain', _charset=None): + def __init__(self, _text, _subtype="plain", _charset=None): self.encoding = _charset MIMEText.__init__(self, _text, _subtype=_subtype, _charset=_charset) @@ -163,7 +164,7 @@ class SafeMIMEText(MIMEMixin, MIMEText): MIMEText.__setitem__(self, name, val) def set_payload(self, payload, charset=None): - if charset == 'utf-8' and not isinstance(charset, Charset.Charset): + if charset == "utf-8" and not isinstance(charset, Charset.Charset): has_long_lines = any( len(line.encode()) > RFC5322_EMAIL_LINE_LENGTH_LIMIT for line in payload.splitlines() @@ -175,8 +176,9 @@ class SafeMIMEText(MIMEMixin, MIMEText): class SafeMIMEMultipart(MIMEMixin, MIMEMultipart): - - def __init__(self, _subtype='mixed', boundary=None, _subparts=None, encoding=None, **_params): + def __init__( + self, _subtype="mixed", boundary=None, _subparts=None, encoding=None, **_params + ): self.encoding = encoding MIMEMultipart.__init__(self, _subtype, boundary, _subparts, **_params) @@ -187,13 +189,24 @@ class SafeMIMEMultipart(MIMEMixin, MIMEMultipart): class EmailMessage: """A container for email information.""" - content_subtype = 'plain' - mixed_subtype = 'mixed' - encoding = None # None => use settings default - def __init__(self, subject='', body='', from_email=None, to=None, bcc=None, - connection=None, attachments=None, headers=None, cc=None, - reply_to=None): + content_subtype = "plain" + mixed_subtype = "mixed" + encoding = None # None => use settings default + + def __init__( + self, + subject="", + body="", + from_email=None, + to=None, + bcc=None, + connection=None, + attachments=None, + headers=None, + cc=None, + reply_to=None, + ): """ Initialize a single email message (which can be sent to multiple recipients). @@ -224,7 +237,7 @@ class EmailMessage: self.reply_to = [] self.from_email = from_email or settings.DEFAULT_FROM_EMAIL self.subject = subject - self.body = body or '' + self.body = body or "" self.attachments = [] if attachments: for attachment in attachments: @@ -237,6 +250,7 @@ class EmailMessage: def get_connection(self, fail_silently=False): from django.core.mail import get_connection + if not self.connection: self.connection = get_connection(fail_silently=fail_silently) return self.connection @@ -245,26 +259,26 @@ class EmailMessage: encoding = self.encoding or settings.DEFAULT_CHARSET msg = SafeMIMEText(self.body, self.content_subtype, encoding) msg = self._create_message(msg) - msg['Subject'] = self.subject - msg['From'] = self.extra_headers.get('From', self.from_email) - self._set_list_header_if_not_empty(msg, 'To', self.to) - self._set_list_header_if_not_empty(msg, 'Cc', self.cc) - self._set_list_header_if_not_empty(msg, 'Reply-To', self.reply_to) + msg["Subject"] = self.subject + msg["From"] = self.extra_headers.get("From", self.from_email) + self._set_list_header_if_not_empty(msg, "To", self.to) + self._set_list_header_if_not_empty(msg, "Cc", self.cc) + self._set_list_header_if_not_empty(msg, "Reply-To", self.reply_to) # Email header names are case-insensitive (RFC 2045), so we have to # accommodate that when doing comparisons. header_names = [key.lower() for key in self.extra_headers] - if 'date' not in header_names: + if "date" not in header_names: # formatdate() uses stdlib methods to format the date, which use # the stdlib/OS concept of a timezone, however, Django sets the # TZ environment variable based on the TIME_ZONE setting which # will get picked up by formatdate(). - msg['Date'] = formatdate(localtime=settings.EMAIL_USE_LOCALTIME) - if 'message-id' not in header_names: + msg["Date"] = formatdate(localtime=settings.EMAIL_USE_LOCALTIME) + if "message-id" not in header_names: # Use cached DNS_NAME for performance - msg['Message-ID'] = make_msgid(domain=DNS_NAME) + msg["Message-ID"] = make_msgid(domain=DNS_NAME) for name, value in self.extra_headers.items(): - if name.lower() != 'from': # From is already handled + if name.lower() != "from": # From is already handled msg[name] = value return msg @@ -298,17 +312,21 @@ class EmailMessage: if isinstance(filename, MIMEBase): if content is not None or mimetype is not None: raise ValueError( - 'content and mimetype must not be given when a MIMEBase ' - 'instance is provided.' + "content and mimetype must not be given when a MIMEBase " + "instance is provided." ) self.attachments.append(filename) elif content is None: - raise ValueError('content must be provided.') + raise ValueError("content must be provided.") else: - mimetype = mimetype or mimetypes.guess_type(filename)[0] or DEFAULT_ATTACHMENT_MIME_TYPE - basetype, subtype = mimetype.split('/', 1) + mimetype = ( + mimetype + or mimetypes.guess_type(filename)[0] + or DEFAULT_ATTACHMENT_MIME_TYPE + ) + basetype, subtype = mimetype.split("/", 1) - if basetype == 'text': + if basetype == "text": if isinstance(content, bytes): try: content = content.decode() @@ -331,7 +349,7 @@ class EmailMessage: DEFAULT_ATTACHMENT_MIME_TYPE and don't decode the content. """ path = Path(path) - with path.open('rb') as file: + with path.open("rb") as file: content = file.read() self.attach(path.name, content, mimetype) @@ -359,11 +377,11 @@ class EmailMessage: If the mimetype is message/rfc822, content may be an email.Message or EmailMessage object, as well as a str. """ - basetype, subtype = mimetype.split('/', 1) - if basetype == 'text': + basetype, subtype = mimetype.split("/", 1) + if basetype == "text": encoding = self.encoding or settings.DEFAULT_CHARSET attachment = SafeMIMEText(content, subtype, encoding) - elif basetype == 'message' and subtype == 'rfc822': + elif basetype == "message" and subtype == "rfc822": # Bug #18967: per RFC2046 s5.2.1, message/rfc822 attachments # must not be base64 encoded. if isinstance(content, EmailMessage): @@ -390,10 +408,12 @@ class EmailMessage: attachment = self._create_mime_attachment(content, mimetype) if filename: try: - filename.encode('ascii') + filename.encode("ascii") except UnicodeEncodeError: - filename = ('utf-8', '', filename) - attachment.add_header('Content-Disposition', 'attachment', filename=filename) + filename = ("utf-8", "", filename) + attachment.add_header( + "Content-Disposition", "attachment", filename=filename + ) return attachment def _set_list_header_if_not_empty(self, msg, header, values): @@ -405,7 +425,7 @@ class EmailMessage: try: value = self.extra_headers[header] except KeyError: - value = ', '.join(str(v) for v in values) + value = ", ".join(str(v) for v in values) msg[header] = value @@ -415,25 +435,45 @@ class EmailMultiAlternatives(EmailMessage): messages. For example, including text and HTML versions of the text is made easier. """ - alternative_subtype = 'alternative' - def __init__(self, subject='', body='', from_email=None, to=None, bcc=None, - connection=None, attachments=None, headers=None, alternatives=None, - cc=None, reply_to=None): + alternative_subtype = "alternative" + + def __init__( + self, + subject="", + body="", + from_email=None, + to=None, + bcc=None, + connection=None, + attachments=None, + headers=None, + alternatives=None, + cc=None, + reply_to=None, + ): """ Initialize a single email message (which can be sent to multiple recipients). """ super().__init__( - subject, body, from_email, to, bcc, connection, attachments, - headers, cc, reply_to, + subject, + body, + from_email, + to, + bcc, + connection, + attachments, + headers, + cc, + reply_to, ) self.alternatives = alternatives or [] def attach_alternative(self, content, mimetype): """Attach an alternative content representation.""" if content is None or mimetype is None: - raise ValueError('Both content and mimetype must be provided.') + raise ValueError("Both content and mimetype must be provided.") self.alternatives.append((content, mimetype)) def _create_message(self, msg): @@ -443,7 +483,9 @@ class EmailMultiAlternatives(EmailMessage): encoding = self.encoding or settings.DEFAULT_CHARSET if self.alternatives: body_msg = msg - msg = SafeMIMEMultipart(_subtype=self.alternative_subtype, encoding=encoding) + msg = SafeMIMEMultipart( + _subtype=self.alternative_subtype, encoding=encoding + ) if self.body: msg.attach(body_msg) for alternative in self.alternatives: diff --git a/django/core/mail/utils.py b/django/core/mail/utils.py index 1e48faa366..8143c236d5 100644 --- a/django/core/mail/utils.py +++ b/django/core/mail/utils.py @@ -14,7 +14,7 @@ class CachedDnsName: return self.get_fqdn() def get_fqdn(self): - if not hasattr(self, '_fqdn'): + if not hasattr(self, "_fqdn"): self._fqdn = punycode(socket.getfqdn()) return self._fqdn diff --git a/django/core/management/__init__.py b/django/core/management/__init__.py index 6133e71c50..7049e06474 100644 --- a/django/core/management/__init__.py +++ b/django/core/management/__init__.py @@ -3,7 +3,10 @@ import os import pkgutil import sys from argparse import ( - _AppendConstAction, _CountAction, _StoreConstAction, _SubParsersAction, + _AppendConstAction, + _CountAction, + _StoreConstAction, + _SubParsersAction, ) from collections import defaultdict from difflib import get_close_matches @@ -14,7 +17,10 @@ from django.apps import apps from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.management.base import ( - BaseCommand, CommandError, CommandParser, handle_default_options, + BaseCommand, + CommandError, + CommandParser, + handle_default_options, ) from django.core.management.color import color_style from django.utils import autoreload @@ -25,9 +31,12 @@ def find_commands(management_dir): Given a path to a management directory, return a list of all the command names that are available. """ - command_dir = os.path.join(management_dir, 'commands') - return [name for _, name, is_pkg in pkgutil.iter_modules([command_dir]) - if not is_pkg and not name.startswith('_')] + command_dir = os.path.join(management_dir, "commands") + return [ + name + for _, name, is_pkg in pkgutil.iter_modules([command_dir]) + if not is_pkg and not name.startswith("_") + ] def load_command_class(app_name, name): @@ -36,7 +45,7 @@ def load_command_class(app_name, name): class instance. Allow all errors raised by the import process (ImportError, AttributeError) to propagate. """ - module = import_module('%s.management.commands.%s' % (app_name, name)) + module = import_module("%s.management.commands.%s" % (app_name, name)) return module.Command() @@ -63,13 +72,13 @@ def get_commands(): The dictionary is cached on the first call and reused on subsequent calls. """ - commands = {name: 'django.core' for name in find_commands(__path__[0])} + commands = {name: "django.core" for name in find_commands(__path__[0])} if not settings.configured: return commands for app_config in reversed(apps.get_app_configs()): - path = os.path.join(app_config.path, 'management') + path = os.path.join(app_config.path, "management") commands.update({name: app_config.name for name in find_commands(path)}) return commands @@ -98,7 +107,7 @@ def call_command(command_name, *args, **options): if isinstance(command_name, BaseCommand): # Command object passed in. command = command_name - command_name = command.__class__.__module__.split('.')[-1] + command_name = command.__class__.__module__.split(".")[-1] else: # Load the command object by name. try: @@ -113,11 +122,12 @@ def call_command(command_name, *args, **options): command = load_command_class(app_name, command_name) # Simulate argument parsing to get the option defaults (see #10080 for details). - parser = command.create_parser('', command_name) + parser = command.create_parser("", command_name) # Use the `dest` option name from the parser option opt_mapping = { - min(s_opt.option_strings).lstrip('-').replace('-', '_'): s_opt.dest - for s_opt in parser._actions if s_opt.option_strings + min(s_opt.option_strings).lstrip("-").replace("-", "_"): s_opt.dest + for s_opt in parser._actions + if s_opt.option_strings } arg_options = {opt_mapping.get(key, key): value for key, value in options.items()} parse_args = [] @@ -140,20 +150,20 @@ def call_command(command_name, *args, **options): mutually_exclusive_required_options = { opt for group in parser._mutually_exclusive_groups - for opt in group._group_actions if group.required + for opt in group._group_actions + if group.required } # Any required arguments which are passed in via **options must be passed # to parse_args(). for opt in parser_actions: - if ( - opt.dest in options and - (opt.required or opt in mutually_exclusive_required_options) + if opt.dest in options and ( + opt.required or opt in mutually_exclusive_required_options ): opt_dest_count = sum(v == opt.dest for v in opt_mapping.values()) if opt_dest_count > 1: raise TypeError( - f'Cannot pass the dest {opt.dest!r} that matches multiple ' - f'arguments via **options.' + f"Cannot pass the dest {opt.dest!r} that matches multiple " + f"arguments via **options." ) parse_args.append(min(opt.option_strings)) if isinstance(opt, (_AppendConstAction, _CountAction, _StoreConstAction)): @@ -173,16 +183,17 @@ def call_command(command_name, *args, **options): if unknown_options: raise TypeError( "Unknown option(s) for %s command: %s. " - "Valid options are: %s." % ( + "Valid options are: %s." + % ( command_name, - ', '.join(sorted(unknown_options)), - ', '.join(sorted(valid_options)), + ", ".join(sorted(unknown_options)), + ", ".join(sorted(valid_options)), ) ) # Move positional args out of options to mimic legacy optparse - args = defaults.pop('args', ()) - if 'skip_checks' not in options: - defaults['skip_checks'] = True + args = defaults.pop("args", ()) + if "skip_checks" not in options: + defaults["skip_checks"] = True return command.execute(*args, **defaults) @@ -191,11 +202,12 @@ class ManagementUtility: """ Encapsulate the logic of the django-admin and manage.py utilities. """ + def __init__(self, argv=None): self.argv = argv or sys.argv[:] self.prog_name = os.path.basename(self.argv[0]) - if self.prog_name == '__main__.py': - self.prog_name = 'python -m django' + if self.prog_name == "__main__.py": + self.prog_name = "python -m django" self.settings_exception = None def main_help_text(self, commands_only=False): @@ -205,16 +217,17 @@ class ManagementUtility: else: usage = [ "", - "Type '%s help <subcommand>' for help on a specific subcommand." % self.prog_name, + "Type '%s help <subcommand>' for help on a specific subcommand." + % self.prog_name, "", "Available subcommands:", ] commands_dict = defaultdict(lambda: []) for name, app in get_commands().items(): - if app == 'django.core': - app = 'django' + if app == "django.core": + app = "django" else: - app = app.rpartition('.')[-1] + app = app.rpartition(".")[-1] commands_dict[app].append(name) style = color_style() for app in sorted(commands_dict): @@ -224,12 +237,15 @@ class ManagementUtility: usage.append(" %s" % name) # Output an extra note if settings are not properly configured if self.settings_exception is not None: - usage.append(style.NOTICE( - "Note that only Django core commands are listed " - "as settings are not properly configured (error: %s)." - % self.settings_exception)) + usage.append( + style.NOTICE( + "Note that only Django core commands are listed " + "as settings are not properly configured (error: %s)." + % self.settings_exception + ) + ) - return '\n'.join(usage) + return "\n".join(usage) def fetch_command(self, subcommand): """ @@ -242,7 +258,7 @@ class ManagementUtility: try: app_name = commands[subcommand] except KeyError: - if os.environ.get('DJANGO_SETTINGS_MODULE'): + if os.environ.get("DJANGO_SETTINGS_MODULE"): # If `subcommand` is missing due to misconfigured settings, the # following line will retrigger an ImproperlyConfigured exception # (get_commands() swallows the original one) so the user is @@ -251,9 +267,9 @@ class ManagementUtility: elif not settings.configured: sys.stderr.write("No Django settings specified.\n") possible_matches = get_close_matches(subcommand, commands) - sys.stderr.write('Unknown command: %r' % subcommand) + sys.stderr.write("Unknown command: %r" % subcommand) if possible_matches: - sys.stderr.write('. Did you mean %s?' % possible_matches[0]) + sys.stderr.write(". Did you mean %s?" % possible_matches[0]) sys.stderr.write("\nType '%s help' for usage.\n" % self.prog_name) sys.exit(1) if isinstance(app_name, BaseCommand): @@ -285,29 +301,29 @@ class ManagementUtility: and formatted as potential completion suggestions. """ # Don't complete if user hasn't sourced bash_completion file. - if 'DJANGO_AUTO_COMPLETE' not in os.environ: + if "DJANGO_AUTO_COMPLETE" not in os.environ: return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) + cwords = os.environ["COMP_WORDS"].split()[1:] + cword = int(os.environ["COMP_CWORD"]) try: curr = cwords[cword - 1] except IndexError: - curr = '' + curr = "" - subcommands = [*get_commands(), 'help'] - options = [('--help', False)] + subcommands = [*get_commands(), "help"] + options = [("--help", False)] # subcommand if cword == 1: - print(' '.join(sorted(filter(lambda x: x.startswith(curr), subcommands)))) + print(" ".join(sorted(filter(lambda x: x.startswith(curr), subcommands)))) # subcommand options # special case: the 'help' subcommand has no options - elif cwords[0] in subcommands and cwords[0] != 'help': + elif cwords[0] in subcommands and cwords[0] != "help": subcommand_cls = self.fetch_command(cwords[0]) # special case: add the names of installed apps to options - if cwords[0] in ('dumpdata', 'sqlmigrate', 'sqlsequencereset', 'test'): + if cwords[0] in ("dumpdata", "sqlmigrate", "sqlsequencereset", "test"): try: app_configs = apps.get_app_configs() # Get the last part of the dotted path as the app name. @@ -316,13 +332,14 @@ class ManagementUtility: # Fail silently if DJANGO_SETTINGS_MODULE isn't set. The # user will find out once they execute the command. pass - parser = subcommand_cls.create_parser('', cwords[0]) + parser = subcommand_cls.create_parser("", cwords[0]) options.extend( (min(s_opt.option_strings), s_opt.nargs != 0) - for s_opt in parser._actions if s_opt.option_strings + for s_opt in parser._actions + if s_opt.option_strings ) # filter out previously specified options from available options - prev_opts = {x.split('=')[0] for x in cwords[1:cword - 1]} + prev_opts = {x.split("=")[0] for x in cwords[1 : cword - 1]} options = (opt for opt in options if opt[0] not in prev_opts) # filter options by current input @@ -330,7 +347,7 @@ class ManagementUtility: for opt_label, require_arg in options: # append '=' to options which require args if require_arg: - opt_label += '=' + opt_label += "=" print(opt_label) # Exit code of the bash completion function is never passed back to # the user, so it's safe to always exit with 0. @@ -345,20 +362,20 @@ class ManagementUtility: try: subcommand = self.argv[1] except IndexError: - subcommand = 'help' # Display help if no arguments were given. + subcommand = "help" # Display help if no arguments were given. # Preprocess options to extract --settings and --pythonpath. # These options could affect the commands that are available, so they # must be processed early. parser = CommandParser( prog=self.prog_name, - usage='%(prog)s subcommand [options] [args]', + usage="%(prog)s subcommand [options] [args]", add_help=False, allow_abbrev=False, ) - parser.add_argument('--settings') - parser.add_argument('--pythonpath') - parser.add_argument('args', nargs='*') # catch-all + parser.add_argument("--settings") + parser.add_argument("--pythonpath") + parser.add_argument("args", nargs="*") # catch-all try: options, args = parser.parse_known_args(self.argv[2:]) handle_default_options(options) @@ -376,7 +393,7 @@ class ManagementUtility: # Start the auto-reloading dev server even if the code is broken. # The hardcoded condition is a code smell but we can't rely on a # flag on the command class because we haven't located it yet. - if subcommand == 'runserver' and '--noreload' not in self.argv: + if subcommand == "runserver" and "--noreload" not in self.argv: try: autoreload.check_errors(django.setup)() except Exception: @@ -391,7 +408,9 @@ class ManagementUtility: # (e.g. options for the contrib.staticfiles' runserver). # Changes here require manually testing as described in # #27522. - _parser = self.fetch_command('runserver').create_parser('django', 'runserver') + _parser = self.fetch_command("runserver").create_parser( + "django", "runserver" + ) _options, _args = _parser.parse_known_args(self.argv[2:]) for _arg in _args: self.argv.remove(_arg) @@ -402,19 +421,21 @@ class ManagementUtility: self.autocomplete() - if subcommand == 'help': - if '--commands' in args: - sys.stdout.write(self.main_help_text(commands_only=True) + '\n') + if subcommand == "help": + if "--commands" in args: + sys.stdout.write(self.main_help_text(commands_only=True) + "\n") elif not options.args: - sys.stdout.write(self.main_help_text() + '\n') + sys.stdout.write(self.main_help_text() + "\n") else: - self.fetch_command(options.args[0]).print_help(self.prog_name, options.args[0]) + self.fetch_command(options.args[0]).print_help( + self.prog_name, options.args[0] + ) # Special-cases: We want 'django-admin --version' and # 'django-admin --help' to work, for backwards compatibility. - elif subcommand == 'version' or self.argv[1:] == ['--version']: - sys.stdout.write(django.get_version() + '\n') - elif self.argv[1:] in (['--help'], ['-h']): - sys.stdout.write(self.main_help_text() + '\n') + elif subcommand == "version" or self.argv[1:] == ["--version"]: + sys.stdout.write(django.get_version() + "\n") + elif self.argv[1:] in (["--help"], ["-h"]): + sys.stdout.write(self.main_help_text() + "\n") else: self.fetch_command(subcommand).run_from_argv(self.argv) diff --git a/django/core/management/base.py b/django/core/management/base.py index 197230fc14..ab1f7e9f70 100644 --- a/django/core/management/base.py +++ b/django/core/management/base.py @@ -14,7 +14,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.management.color import color_style, no_style from django.db import DEFAULT_DB_ALIAS, connections -ALL_CHECKS = '__all__' +ALL_CHECKS = "__all__" class CommandError(Exception): @@ -29,6 +29,7 @@ class CommandError(Exception): error) is the preferred way to indicate that something has gone wrong in the execution of a command. """ + def __init__(self, *args, returncode=1, **kwargs): self.returncode = returncode super().__init__(*args, **kwargs) @@ -38,6 +39,7 @@ class SystemCheckError(CommandError): """ The system check framework detected unrecoverable errors. """ + pass @@ -47,15 +49,19 @@ class CommandParser(ArgumentParser): SystemExit in several occasions, as SystemExit is unacceptable when a command is called programmatically. """ - def __init__(self, *, missing_args_message=None, called_from_command_line=None, **kwargs): + + def __init__( + self, *, missing_args_message=None, called_from_command_line=None, **kwargs + ): self.missing_args_message = missing_args_message self.called_from_command_line = called_from_command_line super().__init__(**kwargs) def parse_args(self, args=None, namespace=None): # Catch missing argument for a better error message - if (self.missing_args_message and - not (args or any(not arg.startswith('-') for arg in args))): + if self.missing_args_message and not ( + args or any(not arg.startswith("-") for arg in args) + ): self.error(self.missing_args_message) return super().parse_args(args, namespace) @@ -73,15 +79,17 @@ def handle_default_options(options): user commands. """ if options.settings: - os.environ['DJANGO_SETTINGS_MODULE'] = options.settings + os.environ["DJANGO_SETTINGS_MODULE"] = options.settings if options.pythonpath: sys.path.insert(0, options.pythonpath) def no_translations(handle_func): """Decorator that forces a command to run with translations deactivated.""" + def wrapped(*args, **kwargs): from django.utils import translation + saved_locale = translation.get_language() translation.deactivate_all() try: @@ -90,6 +98,7 @@ def no_translations(handle_func): if saved_locale is not None: translation.activate(saved_locale) return res + return wrapped @@ -98,15 +107,21 @@ class DjangoHelpFormatter(HelpFormatter): Customized formatter so that command-specific arguments appear in the --help output before arguments common to all commands. """ + show_last = { - '--version', '--verbosity', '--traceback', '--settings', '--pythonpath', - '--no-color', '--force-color', '--skip-checks', + "--version", + "--verbosity", + "--traceback", + "--settings", + "--pythonpath", + "--no-color", + "--force-color", + "--skip-checks", } def _reordered_actions(self, actions): return sorted( - actions, - key=lambda a: set(a.option_strings) & self.show_last != set() + actions, key=lambda a: set(a.option_strings) & self.show_last != set() ) def add_usage(self, usage, actions, *args, **kwargs): @@ -120,6 +135,7 @@ class OutputWrapper(TextIOBase): """ Wrapper around stdout/stderr """ + @property def style_func(self): return self._style_func @@ -131,7 +147,7 @@ class OutputWrapper(TextIOBase): else: self._style_func = lambda x: x - def __init__(self, out, ending='\n'): + def __init__(self, out, ending="\n"): self._out = out self.style_func = None self.ending = ending @@ -140,13 +156,13 @@ class OutputWrapper(TextIOBase): return getattr(self._out, name) def flush(self): - if hasattr(self._out, 'flush'): + if hasattr(self._out, "flush"): self._out.flush() def isatty(self): - return hasattr(self._out, 'isatty') and self._out.isatty() + return hasattr(self._out, "isatty") and self._out.isatty() - def write(self, msg='', style_func=None, ending=None): + def write(self, msg="", style_func=None, ending=None): ending = self.ending if ending is None else ending if ending and not msg.endswith(ending): msg += ending @@ -225,17 +241,18 @@ class BaseCommand: A tuple of any options the command uses which aren't defined by the argument parser. """ + # Metadata about this command. - help = '' + help = "" # Configuration shortcuts that alter various logic. _called_from_command_line = False output_transaction = False # Whether to wrap the output in a "BEGIN; COMMIT;" requires_migrations_checks = False - requires_system_checks = '__all__' + requires_system_checks = "__all__" # Arguments, common to all commands, which aren't defined by the argument # parser. - base_stealth_options = ('stderr', 'stdout') + base_stealth_options = ("stderr", "stdout") # Command-specific options not defined by the argument parser. stealth_options = () suppressed_base_arguments = set() @@ -251,10 +268,10 @@ class BaseCommand: self.style = color_style(force_color) self.stderr.style_func = self.style.ERROR if ( - not isinstance(self.requires_system_checks, (list, tuple)) and - self.requires_system_checks != ALL_CHECKS + not isinstance(self.requires_system_checks, (list, tuple)) + and self.requires_system_checks != ALL_CHECKS ): - raise TypeError('requires_system_checks must be a list or tuple.') + raise TypeError("requires_system_checks must be a list or tuple.") def get_version(self): """ @@ -270,50 +287,66 @@ class BaseCommand: parse the arguments to this command. """ parser = CommandParser( - prog='%s %s' % (os.path.basename(prog_name), subcommand), + prog="%s %s" % (os.path.basename(prog_name), subcommand), description=self.help or None, formatter_class=DjangoHelpFormatter, - missing_args_message=getattr(self, 'missing_args_message', None), - called_from_command_line=getattr(self, '_called_from_command_line', None), - **kwargs + missing_args_message=getattr(self, "missing_args_message", None), + called_from_command_line=getattr(self, "_called_from_command_line", None), + **kwargs, ) self.add_base_argument( - parser, '--version', action='version', version=self.get_version(), + parser, + "--version", + action="version", + version=self.get_version(), help="Show program's version number and exit.", ) self.add_base_argument( - parser, '-v', '--verbosity', default=1, - type=int, choices=[0, 1, 2, 3], - help='Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output', + parser, + "-v", + "--verbosity", + default=1, + type=int, + choices=[0, 1, 2, 3], + help="Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output", ) self.add_base_argument( - parser, '--settings', + parser, + "--settings", help=( - 'The Python path to a settings module, e.g. ' + "The Python path to a settings module, e.g. " '"myproject.settings.main". If this isn\'t provided, the ' - 'DJANGO_SETTINGS_MODULE environment variable will be used.' + "DJANGO_SETTINGS_MODULE environment variable will be used." ), ) self.add_base_argument( - parser, '--pythonpath', + parser, + "--pythonpath", help='A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".', ) self.add_base_argument( - parser, '--traceback', action='store_true', - help='Raise on CommandError exceptions.', + parser, + "--traceback", + action="store_true", + help="Raise on CommandError exceptions.", ) self.add_base_argument( - parser, '--no-color', action='store_true', + parser, + "--no-color", + action="store_true", help="Don't colorize the command output.", ) self.add_base_argument( - parser, '--force-color', action='store_true', - help='Force colorization of the command output.', + parser, + "--force-color", + action="store_true", + help="Force colorization of the command output.", ) if self.requires_system_checks: parser.add_argument( - '--skip-checks', action='store_true', - help='Skip system checks.', + "--skip-checks", + action="store_true", + help="Skip system checks.", ) self.add_arguments(parser) return parser @@ -331,7 +364,7 @@ class BaseCommand: """ for arg in args: if arg in self.suppressed_base_arguments: - kwargs['help'] = argparse.SUPPRESS + kwargs["help"] = argparse.SUPPRESS break parser.add_argument(*args, **kwargs) @@ -357,7 +390,7 @@ class BaseCommand: options = parser.parse_args(argv[2:]) cmd_options = vars(options) # Move positional args out of options to mimic legacy optparse - args = cmd_options.pop('args', ()) + args = cmd_options.pop("args", ()) handle_default_options(options) try: self.execute(*args, **cmd_options) @@ -369,7 +402,7 @@ class BaseCommand: if isinstance(e, SystemCheckError): self.stderr.write(str(e), lambda x: x) else: - self.stderr.write('%s: %s' % (e.__class__.__name__, e)) + self.stderr.write("%s: %s" % (e.__class__.__name__, e)) sys.exit(e.returncode) finally: try: @@ -385,19 +418,21 @@ class BaseCommand: controlled by the ``requires_system_checks`` attribute, except if force-skipped). """ - if options['force_color'] and options['no_color']: - raise CommandError("The --no-color and --force-color options can't be used together.") - if options['force_color']: + if options["force_color"] and options["no_color"]: + raise CommandError( + "The --no-color and --force-color options can't be used together." + ) + if options["force_color"]: self.style = color_style(force_color=True) - elif options['no_color']: + elif options["no_color"]: self.style = no_style() self.stderr.style_func = None - if options.get('stdout'): - self.stdout = OutputWrapper(options['stdout']) - if options.get('stderr'): - self.stderr = OutputWrapper(options['stderr']) + if options.get("stdout"): + self.stdout = OutputWrapper(options["stdout"]) + if options.get("stderr"): + self.stderr = OutputWrapper(options["stderr"]) - if self.requires_system_checks and not options['skip_checks']: + if self.requires_system_checks and not options["skip_checks"]: if self.requires_system_checks == ALL_CHECKS: self.check() else: @@ -407,8 +442,8 @@ class BaseCommand: output = self.handle(*args, **options) if output: if self.output_transaction: - connection = connections[options.get('database', DEFAULT_DB_ALIAS)] - output = '%s\n%s\n%s' % ( + connection = connections[options.get("database", DEFAULT_DB_ALIAS)] + output = "%s\n%s\n%s" % ( self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()), output, self.style.SQL_KEYWORD(connection.ops.end_transaction_sql()), @@ -416,9 +451,15 @@ class BaseCommand: self.stdout.write(output) return output - def check(self, app_configs=None, tags=None, display_num_errors=False, - include_deployment_checks=False, fail_level=checks.ERROR, - databases=None): + def check( + self, + app_configs=None, + tags=None, + display_num_errors=False, + include_deployment_checks=False, + fail_level=checks.ERROR, + databases=None, + ): """ Use the system check framework to validate entire Django project. Raise CommandError for any serious message (error or critical errors). @@ -436,17 +477,35 @@ class BaseCommand: visible_issue_count = 0 # excludes silenced warnings if all_issues: - debugs = [e for e in all_issues if e.level < checks.INFO and not e.is_silenced()] - infos = [e for e in all_issues if checks.INFO <= e.level < checks.WARNING and not e.is_silenced()] - warnings = [e for e in all_issues if checks.WARNING <= e.level < checks.ERROR and not e.is_silenced()] - errors = [e for e in all_issues if checks.ERROR <= e.level < checks.CRITICAL and not e.is_silenced()] - criticals = [e for e in all_issues if checks.CRITICAL <= e.level and not e.is_silenced()] + debugs = [ + e for e in all_issues if e.level < checks.INFO and not e.is_silenced() + ] + infos = [ + e + for e in all_issues + if checks.INFO <= e.level < checks.WARNING and not e.is_silenced() + ] + warnings = [ + e + for e in all_issues + if checks.WARNING <= e.level < checks.ERROR and not e.is_silenced() + ] + errors = [ + e + for e in all_issues + if checks.ERROR <= e.level < checks.CRITICAL and not e.is_silenced() + ] + criticals = [ + e + for e in all_issues + if checks.CRITICAL <= e.level and not e.is_silenced() + ] sorted_issues = [ - (criticals, 'CRITICALS'), - (errors, 'ERRORS'), - (warnings, 'WARNINGS'), - (infos, 'INFOS'), - (debugs, 'DEBUGS'), + (criticals, "CRITICALS"), + (errors, "ERRORS"), + (warnings, "WARNINGS"), + (infos, "INFOS"), + (debugs, "DEBUGS"), ] for issues, group_name in sorted_issues: @@ -456,20 +515,23 @@ class BaseCommand: self.style.ERROR(str(e)) if e.is_serious() else self.style.WARNING(str(e)) - for e in issues) + for e in issues + ) formatted = "\n".join(sorted(formatted)) - body += '\n%s:\n%s\n' % (group_name, formatted) + body += "\n%s:\n%s\n" % (group_name, formatted) if visible_issue_count: header = "System check identified some issues:\n" if display_num_errors: if visible_issue_count: - footer += '\n' + footer += "\n" footer += "System check identified %s (%s silenced)." % ( - "no issues" if visible_issue_count == 0 else - "1 issue" if visible_issue_count == 1 else - "%s issues" % visible_issue_count, + "no issues" + if visible_issue_count == 0 + else "1 issue" + if visible_issue_count == 1 + else "%s issues" % visible_issue_count, len(all_issues) - visible_issue_count, ) @@ -491,6 +553,7 @@ class BaseCommand: migrations in the database. """ from django.db.migrations.executor import MigrationExecutor + try: executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) except ImproperlyConfigured: @@ -499,25 +562,32 @@ class BaseCommand: plan = executor.migration_plan(executor.loader.graph.leaf_nodes()) if plan: - apps_waiting_migration = sorted({migration.app_label for migration, backwards in plan}) + apps_waiting_migration = sorted( + {migration.app_label for migration, backwards in plan} + ) self.stdout.write( self.style.NOTICE( "\nYou have %(unapplied_migration_count)s unapplied migration(s). " "Your project may not work properly until you apply the " - "migrations for app(s): %(apps_waiting_migration)s." % { + "migrations for app(s): %(apps_waiting_migration)s." + % { "unapplied_migration_count": len(plan), "apps_waiting_migration": ", ".join(apps_waiting_migration), } ) ) - self.stdout.write(self.style.NOTICE("Run 'python manage.py migrate' to apply them.")) + self.stdout.write( + self.style.NOTICE("Run 'python manage.py migrate' to apply them.") + ) def handle(self, *args, **options): """ The actual logic of the command. Subclasses must implement this method. """ - raise NotImplementedError('subclasses of BaseCommand must provide a handle() method') + raise NotImplementedError( + "subclasses of BaseCommand must provide a handle() method" + ) class AppCommand(BaseCommand): @@ -528,23 +598,32 @@ class AppCommand(BaseCommand): Rather than implementing ``handle()``, subclasses must implement ``handle_app_config()``, which will be called once for each application. """ + missing_args_message = "Enter at least one application label." def add_arguments(self, parser): - parser.add_argument('args', metavar='app_label', nargs='+', help='One or more application label.') + parser.add_argument( + "args", + metavar="app_label", + nargs="+", + help="One or more application label.", + ) def handle(self, *app_labels, **options): from django.apps import apps + try: app_configs = [apps.get_app_config(app_label) for app_label in app_labels] except (LookupError, ImportError) as e: - raise CommandError("%s. Are you sure your INSTALLED_APPS setting is correct?" % e) + raise CommandError( + "%s. Are you sure your INSTALLED_APPS setting is correct?" % e + ) output = [] for app_config in app_configs: app_output = self.handle_app_config(app_config, **options) if app_output: output.append(app_output) - return '\n'.join(output) + return "\n".join(output) def handle_app_config(self, app_config, **options): """ @@ -568,11 +647,12 @@ class LabelCommand(BaseCommand): If the arguments should be names of installed applications, use ``AppCommand`` instead. """ - label = 'label' + + label = "label" missing_args_message = "Enter at least one %s." % label def add_arguments(self, parser): - parser.add_argument('args', metavar=self.label, nargs='+') + parser.add_argument("args", metavar=self.label, nargs="+") def handle(self, *labels, **options): output = [] @@ -580,11 +660,13 @@ class LabelCommand(BaseCommand): label_output = self.handle_label(label, **options) if label_output: output.append(label_output) - return '\n'.join(output) + return "\n".join(output) def handle_label(self, label, **options): """ Perform the command's actions for ``label``, which will be the string as given on the command line. """ - raise NotImplementedError('subclasses of LabelCommand must provide a handle_label() method') + raise NotImplementedError( + "subclasses of LabelCommand must provide a handle_label() method" + ) diff --git a/django/core/management/color.py b/django/core/management/color.py index be8c31bb95..d2255d2282 100644 --- a/django/core/management/color.py +++ b/django/core/management/color.py @@ -10,6 +10,7 @@ from django.utils import termcolors try: import colorama + colorama.init() except (ImportError, OSError): HAS_COLORAMA = False @@ -22,6 +23,7 @@ def supports_color(): Return True if the running system's terminal supports color, and False otherwise. """ + def vt_codes_enabled_in_windows_registry(): """ Check the Windows Registry to see if VT code handling has been enabled @@ -33,26 +35,28 @@ def supports_color(): except ImportError: return False else: - reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, 'Console') + reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Console") try: - reg_key_value, _ = winreg.QueryValueEx(reg_key, 'VirtualTerminalLevel') + reg_key_value, _ = winreg.QueryValueEx(reg_key, "VirtualTerminalLevel") except FileNotFoundError: return False else: return reg_key_value == 1 # isatty is not always implemented, #6223. - is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + is_a_tty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty() return is_a_tty and ( - sys.platform != 'win32' or - HAS_COLORAMA or - 'ANSICON' in os.environ or + sys.platform != "win32" + or HAS_COLORAMA + or "ANSICON" in os.environ + or # Windows Terminal supports VT codes. - 'WT_SESSION' in os.environ or + "WT_SESSION" in os.environ + or # Microsoft Visual Studio Code's built-in terminal supports colors. - os.environ.get('TERM_PROGRAM') == 'vscode' or - vt_codes_enabled_in_windows_registry() + os.environ.get("TERM_PROGRAM") == "vscode" + or vt_codes_enabled_in_windows_registry() ) @@ -60,7 +64,7 @@ class Style: pass -def make_style(config_string=''): +def make_style(config_string=""): """ Create a Style object from the given config_string. @@ -79,8 +83,10 @@ def make_style(config_string=''): format = color_settings.get(role, {}) style_func = termcolors.make_style(**format) else: + def style_func(x): return x + setattr(style, role, style_func) # For backwards compatibility, @@ -95,7 +101,7 @@ def no_style(): """ Return a Style object with no color scheme. """ - return make_style('nocolor') + return make_style("nocolor") def color_style(force_color=False): @@ -104,4 +110,4 @@ def color_style(force_color=False): """ if not force_color and not supports_color(): return no_style() - return make_style(os.environ.get('DJANGO_COLORS', '')) + return make_style(os.environ.get("DJANGO_COLORS", "")) diff --git a/django/core/management/commands/check.py b/django/core/management/commands/check.py index a92563641f..7624b85390 100644 --- a/django/core/management/commands/check.py +++ b/django/core/management/commands/check.py @@ -10,37 +10,46 @@ class Command(BaseCommand): requires_system_checks = [] def add_arguments(self, parser): - parser.add_argument('args', metavar='app_label', nargs='*') + parser.add_argument("args", metavar="app_label", nargs="*") parser.add_argument( - '--tag', '-t', action='append', dest='tags', - help='Run only checks labeled with given tag.', + "--tag", + "-t", + action="append", + dest="tags", + help="Run only checks labeled with given tag.", ) parser.add_argument( - '--list-tags', action='store_true', - help='List available tags.', + "--list-tags", + action="store_true", + help="List available tags.", ) parser.add_argument( - '--deploy', action='store_true', - help='Check deployment settings.', + "--deploy", + action="store_true", + help="Check deployment settings.", ) parser.add_argument( - '--fail-level', - default='ERROR', - choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], + "--fail-level", + default="ERROR", + choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"], help=( - 'Message level that will cause the command to exit with a ' - 'non-zero status. Default is ERROR.' + "Message level that will cause the command to exit with a " + "non-zero status. Default is ERROR." ), ) parser.add_argument( - '--database', action='append', dest='databases', - help='Run database related checks against these aliases.', + "--database", + action="append", + dest="databases", + help="Run database related checks against these aliases.", ) def handle(self, *app_labels, **options): - include_deployment_checks = options['deploy'] - if options['list_tags']: - self.stdout.write('\n'.join(sorted(registry.tags_available(include_deployment_checks)))) + include_deployment_checks = options["deploy"] + if options["list_tags"]: + self.stdout.write( + "\n".join(sorted(registry.tags_available(include_deployment_checks))) + ) return if app_labels: @@ -48,23 +57,27 @@ class Command(BaseCommand): else: app_configs = None - tags = options['tags'] + tags = options["tags"] if tags: try: invalid_tag = next( - tag for tag in tags if not checks.tag_exists(tag, include_deployment_checks) + tag + for tag in tags + if not checks.tag_exists(tag, include_deployment_checks) ) except StopIteration: # no invalid tags pass else: - raise CommandError('There is no system check with the "%s" tag.' % invalid_tag) + raise CommandError( + 'There is no system check with the "%s" tag.' % invalid_tag + ) self.check( app_configs=app_configs, tags=tags, display_num_errors=True, include_deployment_checks=include_deployment_checks, - fail_level=getattr(checks, options['fail_level']), - databases=options['databases'], + fail_level=getattr(checks, options["fail_level"]), + databases=options["databases"], ) diff --git a/django/core/management/commands/compilemessages.py b/django/core/management/commands/compilemessages.py index 308fa8831b..bd055d087f 100644 --- a/django/core/management/commands/compilemessages.py +++ b/django/core/management/commands/compilemessages.py @@ -5,22 +5,22 @@ import os from pathlib import Path from django.core.management.base import BaseCommand, CommandError -from django.core.management.utils import ( - find_command, is_ignored_path, popen_wrapper, -) +from django.core.management.utils import find_command, is_ignored_path, popen_wrapper def has_bom(fn): - with fn.open('rb') as f: + with fn.open("rb") as f: sample = f.read(4) - return sample.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)) + return sample.startswith( + (codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE) + ) def is_writable(path): # Known side effect: updating file access/modified time to current time if # it is writable. try: - with open(path, 'a'): + with open(path, "a"): os.utime(path, None) except OSError: return False @@ -28,71 +28,91 @@ def is_writable(path): class Command(BaseCommand): - help = 'Compiles .po files to .mo files for use with builtin gettext support.' + help = "Compiles .po files to .mo files for use with builtin gettext support." requires_system_checks = [] - program = 'msgfmt' - program_options = ['--check-format'] + program = "msgfmt" + program_options = ["--check-format"] def add_arguments(self, parser): parser.add_argument( - '--locale', '-l', action='append', default=[], - help='Locale(s) to process (e.g. de_AT). Default is to process all. ' - 'Can be used multiple times.', + "--locale", + "-l", + action="append", + default=[], + help="Locale(s) to process (e.g. de_AT). Default is to process all. " + "Can be used multiple times.", ) parser.add_argument( - '--exclude', '-x', action='append', default=[], - help='Locales to exclude. Default is none. Can be used multiple times.', + "--exclude", + "-x", + action="append", + default=[], + help="Locales to exclude. Default is none. Can be used multiple times.", ) parser.add_argument( - '--use-fuzzy', '-f', dest='fuzzy', action='store_true', - help='Use fuzzy translations.', + "--use-fuzzy", + "-f", + dest="fuzzy", + action="store_true", + help="Use fuzzy translations.", ) parser.add_argument( - '--ignore', '-i', action='append', dest='ignore_patterns', - default=[], metavar='PATTERN', - help='Ignore directories matching this glob-style pattern. ' - 'Use multiple times to ignore more.', + "--ignore", + "-i", + action="append", + dest="ignore_patterns", + default=[], + metavar="PATTERN", + help="Ignore directories matching this glob-style pattern. " + "Use multiple times to ignore more.", ) def handle(self, **options): - locale = options['locale'] - exclude = options['exclude'] - ignore_patterns = set(options['ignore_patterns']) - self.verbosity = options['verbosity'] - if options['fuzzy']: - self.program_options = self.program_options + ['-f'] + locale = options["locale"] + exclude = options["exclude"] + ignore_patterns = set(options["ignore_patterns"]) + self.verbosity = options["verbosity"] + if options["fuzzy"]: + self.program_options = self.program_options + ["-f"] if find_command(self.program) is None: - raise CommandError("Can't find %s. Make sure you have GNU gettext " - "tools 0.15 or newer installed." % self.program) + raise CommandError( + "Can't find %s. Make sure you have GNU gettext " + "tools 0.15 or newer installed." % self.program + ) - basedirs = [os.path.join('conf', 'locale'), 'locale'] - if os.environ.get('DJANGO_SETTINGS_MODULE'): + basedirs = [os.path.join("conf", "locale"), "locale"] + if os.environ.get("DJANGO_SETTINGS_MODULE"): from django.conf import settings + basedirs.extend(settings.LOCALE_PATHS) # Walk entire tree, looking for locale directories - for dirpath, dirnames, filenames in os.walk('.', topdown=True): + for dirpath, dirnames, filenames in os.walk(".", topdown=True): for dirname in dirnames: - if is_ignored_path(os.path.normpath(os.path.join(dirpath, dirname)), ignore_patterns): + if is_ignored_path( + os.path.normpath(os.path.join(dirpath, dirname)), ignore_patterns + ): dirnames.remove(dirname) - elif dirname == 'locale': + elif dirname == "locale": basedirs.append(os.path.join(dirpath, dirname)) # Gather existing directories. basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs))) if not basedirs: - raise CommandError("This script should be run from the Django Git " - "checkout or your project or app tree, or with " - "the settings module specified.") + raise CommandError( + "This script should be run from the Django Git " + "checkout or your project or app tree, or with " + "the settings module specified." + ) # Build locale list all_locales = [] for basedir in basedirs: - locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % basedir)) + locale_dirs = filter(os.path.isdir, glob.glob("%s/*" % basedir)) all_locales.extend(map(os.path.basename, locale_dirs)) # Account for excluded locales @@ -102,18 +122,22 @@ class Command(BaseCommand): self.has_errors = False for basedir in basedirs: if locales: - dirs = [os.path.join(basedir, locale, 'LC_MESSAGES') for locale in locales] + dirs = [ + os.path.join(basedir, locale, "LC_MESSAGES") for locale in locales + ] else: dirs = [basedir] locations = [] for ldir in dirs: for dirpath, dirnames, filenames in os.walk(ldir): - locations.extend((dirpath, f) for f in filenames if f.endswith('.po')) + locations.extend( + (dirpath, f) for f in filenames if f.endswith(".po") + ) if locations: self.compile_messages(locations) if self.has_errors: - raise CommandError('compilemessages generated one or more errors.') + raise CommandError("compilemessages generated one or more errors.") def compile_messages(self, locations): """ @@ -123,24 +147,25 @@ class Command(BaseCommand): futures = [] for i, (dirpath, f) in enumerate(locations): po_path = Path(dirpath) / f - mo_path = po_path.with_suffix('.mo') + mo_path = po_path.with_suffix(".mo") try: if mo_path.stat().st_mtime >= po_path.stat().st_mtime: if self.verbosity > 0: self.stdout.write( - 'File “%s” is already compiled and up to date.' + "File “%s” is already compiled and up to date." % po_path ) continue except FileNotFoundError: pass if self.verbosity > 0: - self.stdout.write('processing file %s in %s' % (f, dirpath)) + self.stdout.write("processing file %s in %s" % (f, dirpath)) if has_bom(po_path): self.stderr.write( - 'The %s file has a BOM (Byte Order Mark). Django only ' - 'supports .po files encoded in UTF-8 and without any BOM.' % po_path + "The %s file has a BOM (Byte Order Mark). Django only " + "supports .po files encoded in UTF-8 and without any BOM." + % po_path ) self.has_errors = True continue @@ -148,13 +173,13 @@ class Command(BaseCommand): # Check writability on first location if i == 0 and not is_writable(mo_path): self.stderr.write( - 'The po files under %s are in a seemingly not writable location. ' - 'mo files will not be updated/created.' % dirpath + "The po files under %s are in a seemingly not writable location. " + "mo files will not be updated/created." % dirpath ) self.has_errors = True return - args = [self.program, *self.program_options, '-o', mo_path, po_path] + args = [self.program, *self.program_options, "-o", mo_path, po_path] futures.append(executor.submit(popen_wrapper, args)) for future in concurrent.futures.as_completed(futures): @@ -162,7 +187,9 @@ class Command(BaseCommand): if status: if self.verbosity > 0: if errors: - self.stderr.write("Execution of %s failed: %s" % (self.program, errors)) + self.stderr.write( + "Execution of %s failed: %s" % (self.program, errors) + ) else: self.stderr.write("Execution of %s failed" % self.program) self.has_errors = True diff --git a/django/core/management/commands/createcachetable.py b/django/core/management/commands/createcachetable.py index 84f61049cd..99dc3da040 100644 --- a/django/core/management/commands/createcachetable.py +++ b/django/core/management/commands/createcachetable.py @@ -3,7 +3,12 @@ from django.core.cache import caches from django.core.cache.backends.db import BaseDatabaseCache from django.core.management.base import BaseCommand, CommandError from django.db import ( - DEFAULT_DB_ALIAS, DatabaseError, connections, models, router, transaction, + DEFAULT_DB_ALIAS, + DatabaseError, + connections, + models, + router, + transaction, ) @@ -14,24 +19,27 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'args', metavar='table_name', nargs='*', - help='Optional table names. Otherwise, settings.CACHES is used to find cache tables.', + "args", + metavar="table_name", + nargs="*", + help="Optional table names. Otherwise, settings.CACHES is used to find cache tables.", ) parser.add_argument( - '--database', + "--database", default=DEFAULT_DB_ALIAS, - help='Nominates a database onto which the cache tables will be ' - 'installed. Defaults to the "default" database.', + help="Nominates a database onto which the cache tables will be " + 'installed. Defaults to the "default" database.', ) parser.add_argument( - '--dry-run', action='store_true', - help='Does not create the table, just prints the SQL that would be run.', + "--dry-run", + action="store_true", + help="Does not create the table, just prints the SQL that would be run.", ) def handle(self, *tablenames, **options): - db = options['database'] - self.verbosity = options['verbosity'] - dry_run = options['dry_run'] + db = options["database"] + self.verbosity = options["verbosity"] + dry_run = options["dry_run"] if tablenames: # Legacy behavior, tablename specified as argument for tablename in tablenames: @@ -55,9 +63,11 @@ class Command(BaseCommand): fields = ( # "key" is a reserved word in MySQL, so use "cache_key" instead. - models.CharField(name='cache_key', max_length=255, unique=True, primary_key=True), - models.TextField(name='value'), - models.DateTimeField(name='expires', db_index=True), + models.CharField( + name="cache_key", max_length=255, unique=True, primary_key=True + ), + models.TextField(name="value"), + models.DateTimeField(name="expires", db_index=True), ) table_output = [] index_output = [] @@ -66,7 +76,7 @@ class Command(BaseCommand): field_output = [ qn(f.name), f.db_type(connection=connection), - '%sNULL' % ('NOT ' if not f.null else ''), + "%sNULL" % ("NOT " if not f.null else ""), ] if f.primary_key: field_output.append("PRIMARY KEY") @@ -75,14 +85,21 @@ class Command(BaseCommand): if f.db_index: unique = "UNIQUE " if f.unique else "" index_output.append( - "CREATE %sINDEX %s ON %s (%s);" % - (unique, qn('%s_%s' % (tablename, f.name)), qn(tablename), qn(f.name)) + "CREATE %sINDEX %s ON %s (%s);" + % ( + unique, + qn("%s_%s" % (tablename, f.name)), + qn(tablename), + qn(f.name), + ) ) table_output.append(" ".join(field_output)) full_statement = ["CREATE TABLE %s (" % qn(tablename)] for i, line in enumerate(table_output): - full_statement.append(' %s%s' % (line, ',' if i < len(table_output) - 1 else '')) - full_statement.append(');') + full_statement.append( + " %s%s" % (line, "," if i < len(table_output) - 1 else "") + ) + full_statement.append(");") full_statement = "\n".join(full_statement) @@ -92,14 +109,17 @@ class Command(BaseCommand): self.stdout.write(statement) return - with transaction.atomic(using=database, savepoint=connection.features.can_rollback_ddl): + with transaction.atomic( + using=database, savepoint=connection.features.can_rollback_ddl + ): with connection.cursor() as curs: try: curs.execute(full_statement) except DatabaseError as e: raise CommandError( - "Cache table '%s' could not be created.\nThe error was: %s." % - (tablename, e)) + "Cache table '%s' could not be created.\nThe error was: %s." + % (tablename, e) + ) for statement in index_output: curs.execute(statement) diff --git a/django/core/management/commands/dbshell.py b/django/core/management/commands/dbshell.py index cd94787f3d..9cdd64f190 100644 --- a/django/core/management/commands/dbshell.py +++ b/django/core/management/commands/dbshell.py @@ -14,29 +14,31 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database onto which to open a shell. Defaults to the "default" database.', ) - parameters = parser.add_argument_group('parameters', prefix_chars='--') - parameters.add_argument('parameters', nargs='*') + parameters = parser.add_argument_group("parameters", prefix_chars="--") + parameters.add_argument("parameters", nargs="*") def handle(self, **options): - connection = connections[options['database']] + connection = connections[options["database"]] try: - connection.client.runshell(options['parameters']) + connection.client.runshell(options["parameters"]) except FileNotFoundError: # Note that we're assuming the FileNotFoundError relates to the # command missing. It could be raised for some other reason, in # which case this error message would be inaccurate. Still, this # message catches the common case. raise CommandError( - 'You appear not to have the %r program installed or on your path.' % - connection.client.executable_name + "You appear not to have the %r program installed or on your path." + % connection.client.executable_name ) except subprocess.CalledProcessError as e: raise CommandError( - '"%s" returned non-zero exit status %s.' % ( - ' '.join(e.cmd), + '"%s" returned non-zero exit status %s.' + % ( + " ".join(e.cmd), e.returncode, ), returncode=e.returncode, diff --git a/django/core/management/commands/diffsettings.py b/django/core/management/commands/diffsettings.py index 5adf35eb66..27cd575294 100644 --- a/django/core/management/commands/diffsettings.py +++ b/django/core/management/commands/diffsettings.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand -def module_to_dict(module, omittable=lambda k: k.startswith('_') or not k.isupper()): +def module_to_dict(module, omittable=lambda k: k.startswith("_") or not k.isupper()): """Convert a module namespace to a Python dictionary.""" return {k: repr(getattr(module, k)) for k in dir(module) if not omittable(k)} @@ -14,21 +14,25 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--all', action='store_true', + "--all", + action="store_true", help=( 'Display all settings, regardless of their value. In "hash" ' 'mode, default values are prefixed by "###".' ), ) parser.add_argument( - '--default', metavar='MODULE', + "--default", + metavar="MODULE", help=( "The settings module to compare the current settings against. Leave empty to " "compare against Django's default settings." ), ) parser.add_argument( - '--output', default='hash', choices=('hash', 'unified'), + "--output", + default="hash", + choices=("hash", "unified"), help=( "Selects the output format. 'hash' mode displays each changed " "setting, with the settings that don't appear in the defaults " @@ -46,13 +50,15 @@ class Command(BaseCommand): settings._setup() user_settings = module_to_dict(settings._wrapped) - default = options['default'] - default_settings = module_to_dict(Settings(default) if default else global_settings) + default = options["default"] + default_settings = module_to_dict( + Settings(default) if default else global_settings + ) output_func = { - 'hash': self.output_hash, - 'unified': self.output_unified, - }[options['output']] - return '\n'.join(output_func(user_settings, default_settings, **options)) + "hash": self.output_hash, + "unified": self.output_unified, + }[options["output"]] + return "\n".join(output_func(user_settings, default_settings, **options)) def output_hash(self, user_settings, default_settings, **options): # Inspired by Postfix's "postconf -n". @@ -62,7 +68,7 @@ class Command(BaseCommand): output.append("%s = %s ###" % (key, user_settings[key])) elif user_settings[key] != default_settings[key]: output.append("%s = %s" % (key, user_settings[key])) - elif options['all']: + elif options["all"]: output.append("### %s = %s" % (key, user_settings[key])) return output @@ -70,10 +76,16 @@ class Command(BaseCommand): output = [] for key in sorted(user_settings): if key not in default_settings: - output.append(self.style.SUCCESS("+ %s = %s" % (key, user_settings[key]))) + output.append( + self.style.SUCCESS("+ %s = %s" % (key, user_settings[key])) + ) elif user_settings[key] != default_settings[key]: - output.append(self.style.ERROR("- %s = %s" % (key, default_settings[key]))) - output.append(self.style.SUCCESS("+ %s = %s" % (key, user_settings[key]))) - elif options['all']: + output.append( + self.style.ERROR("- %s = %s" % (key, default_settings[key])) + ) + output.append( + self.style.SUCCESS("+ %s = %s" % (key, user_settings[key])) + ) + elif options["all"]: output.append(" %s = %s" % (key, user_settings[key])) return output diff --git a/django/core/management/commands/dumpdata.py b/django/core/management/commands/dumpdata.py index 925a23a56d..74a5b2d22a 100644 --- a/django/core/management/commands/dumpdata.py +++ b/django/core/management/commands/dumpdata.py @@ -10,12 +10,14 @@ from django.db import DEFAULT_DB_ALIAS, router try: import bz2 + has_bz2 = True except ImportError: has_bz2 = False try: import lzma + has_lzma = True except ImportError: has_lzma = False @@ -33,65 +35,79 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'args', metavar='app_label[.ModelName]', nargs='*', - help='Restricts dumped data to the specified app_label or app_label.ModelName.', + "args", + metavar="app_label[.ModelName]", + nargs="*", + help="Restricts dumped data to the specified app_label or app_label.ModelName.", ) parser.add_argument( - '--format', default='json', - help='Specifies the output serialization format for fixtures.', + "--format", + default="json", + help="Specifies the output serialization format for fixtures.", ) parser.add_argument( - '--indent', type=int, - help='Specifies the indent level to use when pretty-printing output.', + "--indent", + type=int, + help="Specifies the indent level to use when pretty-printing output.", ) parser.add_argument( - '--database', + "--database", default=DEFAULT_DB_ALIAS, - help='Nominates a specific database to dump fixtures from. ' - 'Defaults to the "default" database.', + help="Nominates a specific database to dump fixtures from. " + 'Defaults to the "default" database.', ) parser.add_argument( - '-e', '--exclude', action='append', default=[], - help='An app_label or app_label.ModelName to exclude ' - '(use multiple --exclude to exclude multiple apps/models).', + "-e", + "--exclude", + action="append", + default=[], + help="An app_label or app_label.ModelName to exclude " + "(use multiple --exclude to exclude multiple apps/models).", ) parser.add_argument( - '--natural-foreign', action='store_true', dest='use_natural_foreign_keys', - help='Use natural foreign keys if they are available.', + "--natural-foreign", + action="store_true", + dest="use_natural_foreign_keys", + help="Use natural foreign keys if they are available.", ) parser.add_argument( - '--natural-primary', action='store_true', dest='use_natural_primary_keys', - help='Use natural primary keys if they are available.', + "--natural-primary", + action="store_true", + dest="use_natural_primary_keys", + help="Use natural primary keys if they are available.", ) parser.add_argument( - '-a', '--all', action='store_true', dest='use_base_manager', + "-a", + "--all", + action="store_true", + dest="use_base_manager", help="Use Django's base manager to dump all models stored in the database, " - "including those that would otherwise be filtered or modified by a custom manager.", + "including those that would otherwise be filtered or modified by a custom manager.", ) parser.add_argument( - '--pks', dest='primary_keys', + "--pks", + dest="primary_keys", help="Only dump objects with given primary keys. Accepts a comma-separated " - "list of keys. This option only works when you specify one model.", + "list of keys. This option only works when you specify one model.", ) parser.add_argument( - '-o', '--output', - help='Specifies file to which the output is written.' + "-o", "--output", help="Specifies file to which the output is written." ) def handle(self, *app_labels, **options): - format = options['format'] - indent = options['indent'] - using = options['database'] - excludes = options['exclude'] - output = options['output'] - show_traceback = options['traceback'] - use_natural_foreign_keys = options['use_natural_foreign_keys'] - use_natural_primary_keys = options['use_natural_primary_keys'] - use_base_manager = options['use_base_manager'] - pks = options['primary_keys'] + format = options["format"] + indent = options["indent"] + using = options["database"] + excludes = options["exclude"] + output = options["output"] + show_traceback = options["traceback"] + use_natural_foreign_keys = options["use_natural_foreign_keys"] + use_natural_primary_keys = options["use_natural_primary_keys"] + use_base_manager = options["use_base_manager"] + pks = options["primary_keys"] if pks: - primary_keys = [pk.strip() for pk in pks.split(',')] + primary_keys = [pk.strip() for pk in pks.split(",")] else: primary_keys = [] @@ -101,8 +117,10 @@ class Command(BaseCommand): if primary_keys: raise CommandError("You can only use --pks option with one model") app_list = dict.fromkeys( - app_config for app_config in apps.get_app_configs() - if app_config.models_module is not None and app_config not in excluded_apps + app_config + for app_config in apps.get_app_configs() + if app_config.models_module is not None + and app_config not in excluded_apps ) else: if len(app_labels) > 1 and primary_keys: @@ -110,7 +128,7 @@ class Command(BaseCommand): app_list = {} for label in app_labels: try: - app_label, model_label = label.split('.') + app_label, model_label = label.split(".") try: app_config = apps.get_app_config(app_label) except LookupError as e: @@ -120,7 +138,9 @@ class Command(BaseCommand): try: model = app_config.get_model(model_label) except LookupError: - raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) + raise CommandError( + "Unknown model: %s.%s" % (app_label, model_label) + ) app_list_value = app_list.setdefault(app_config, []) @@ -131,7 +151,9 @@ class Command(BaseCommand): app_list_value.append(model) except ValueError: if primary_keys: - raise CommandError("You can only use --pks option with one model") + raise CommandError( + "You can only use --pks option with one model" + ) # This is just an app - no model qualifier app_label = label try: @@ -158,7 +180,9 @@ class Command(BaseCommand): count the number of objects to be serialized. """ if use_natural_foreign_keys: - models = serializers.sort_dependencies(app_list.items(), allow_cycles=True) + models = serializers.sort_dependencies( + app_list.items(), allow_cycles=True + ) else: # There is no need to sort dependencies when natural foreign # keys are not used. @@ -173,7 +197,8 @@ class Command(BaseCommand): continue if model._meta.proxy and model._meta.proxy_for_model not in models: warnings.warn( - "%s is a proxy model and won't be serialized." % model._meta.label, + "%s is a proxy model and won't be serialized." + % model._meta.label, category=ProxyModelWarning, ) if not model._meta.proxy and router.allow_migrate_model(using, model): @@ -195,25 +220,27 @@ class Command(BaseCommand): progress_output = None object_count = 0 # If dumpdata is outputting to stdout, there is no way to display progress - if output and self.stdout.isatty() and options['verbosity'] > 0: + if output and self.stdout.isatty() and options["verbosity"] > 0: progress_output = self.stdout object_count = sum(get_objects(count_only=True)) if output: file_root, file_ext = os.path.splitext(output) compression_formats = { - '.bz2': (open, {}, file_root), - '.gz': (gzip.open, {}, output), - '.lzma': (open, {}, file_root), - '.xz': (open, {}, file_root), - '.zip': (open, {}, file_root), + ".bz2": (open, {}, file_root), + ".gz": (gzip.open, {}, output), + ".lzma": (open, {}, file_root), + ".xz": (open, {}, file_root), + ".zip": (open, {}, file_root), } if has_bz2: - compression_formats['.bz2'] = (bz2.open, {}, output) + compression_formats[".bz2"] = (bz2.open, {}, output) if has_lzma: - compression_formats['.lzma'] = ( - lzma.open, {'format': lzma.FORMAT_ALONE}, output + compression_formats[".lzma"] = ( + lzma.open, + {"format": lzma.FORMAT_ALONE}, + output, ) - compression_formats['.xz'] = (lzma.open, {}, output) + compression_formats[".xz"] = (lzma.open, {}, output) try: open_method, kwargs, file_path = compression_formats[file_ext] except KeyError: @@ -225,15 +252,18 @@ class Command(BaseCommand): f"Fixtures saved in '{file_name}'.", RuntimeWarning, ) - stream = open_method(file_path, 'wt', **kwargs) + stream = open_method(file_path, "wt", **kwargs) else: stream = None try: serializers.serialize( - format, get_objects(), indent=indent, + format, + get_objects(), + indent=indent, use_natural_foreign_keys=use_natural_foreign_keys, use_natural_primary_keys=use_natural_primary_keys, - stream=stream or self.stdout, progress_output=progress_output, + stream=stream or self.stdout, + progress_output=progress_output, object_count=object_count, ) finally: diff --git a/django/core/management/commands/flush.py b/django/core/management/commands/flush.py index 6737b9be40..8bad63d41d 100644 --- a/django/core/management/commands/flush.py +++ b/django/core/management/commands/flush.py @@ -9,30 +9,34 @@ from django.db import DEFAULT_DB_ALIAS, connections class Command(BaseCommand): help = ( - 'Removes ALL DATA from the database, including data added during ' + "Removes ALL DATA from the database, including data added during " 'migrations. Does not achieve a "fresh install" state.' ) - stealth_options = ('reset_sequences', 'allow_cascade', 'inhibit_post_migrate') + stealth_options = ("reset_sequences", "allow_cascade", "inhibit_post_migrate") def add_arguments(self, parser): parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database to flush. Defaults to the "default" database.', ) def handle(self, **options): - database = options['database'] + database = options["database"] connection = connections[database] - verbosity = options['verbosity'] - interactive = options['interactive'] + verbosity = options["verbosity"] + interactive = options["interactive"] # The following are stealth options used by Django's internals. - reset_sequences = options.get('reset_sequences', True) - allow_cascade = options.get('allow_cascade', False) - inhibit_post_migrate = options.get('inhibit_post_migrate', False) + reset_sequences = options.get("reset_sequences", True) + allow_cascade = options.get("allow_cascade", False) + inhibit_post_migrate = options.get("inhibit_post_migrate", False) self.style = no_style() @@ -40,25 +44,31 @@ class Command(BaseCommand): # dispatcher events. for app_config in apps.get_app_configs(): try: - import_module('.management', app_config.name) + import_module(".management", app_config.name) except ImportError: pass - sql_list = sql_flush(self.style, connection, - reset_sequences=reset_sequences, - allow_cascade=allow_cascade) + sql_list = sql_flush( + self.style, + connection, + reset_sequences=reset_sequences, + allow_cascade=allow_cascade, + ) if interactive: - confirm = input("""You have requested a flush of the database. + confirm = input( + """You have requested a flush of the database. This will IRREVERSIBLY DESTROY all data currently in the "%s" database, and return each table to an empty state. Are you sure you want to do this? - Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME']) + Type 'yes' to continue, or 'no' to cancel: """ + % connection.settings_dict["NAME"] + ) else: - confirm = 'yes' + confirm = "yes" - if confirm == 'yes': + if confirm == "yes": try: connection.ops.execute_sql_flush(sql_list) except Exception as exc: @@ -68,9 +78,8 @@ Are you sure you want to do this? " * At least one of the expected database tables doesn't exist.\n" " * The SQL was invalid.\n" "Hint: Look at the output of 'django-admin sqlflush'. " - "That's the SQL this command wasn't able to run." % ( - connection.settings_dict['NAME'], - ) + "That's the SQL this command wasn't able to run." + % (connection.settings_dict["NAME"],) ) from exc # Empty sql_list may signify an empty database and post_migrate would then crash @@ -79,4 +88,4 @@ Are you sure you want to do this? # respond as if the database had been migrated from scratch. emit_post_migrate_signal(verbosity, interactive, database) else: - self.stdout.write('Flush cancelled.') + self.stdout.write("Flush cancelled.") diff --git a/django/core/management/commands/inspectdb.py b/django/core/management/commands/inspectdb.py index d64725ca73..753290d574 100644 --- a/django/core/management/commands/inspectdb.py +++ b/django/core/management/commands/inspectdb.py @@ -9,23 +9,30 @@ from django.db.models.constants import LOOKUP_SEP class Command(BaseCommand): help = "Introspects the database tables in the given database and outputs a Django model module." requires_system_checks = [] - stealth_options = ('table_name_filter',) - db_module = 'django.db' + stealth_options = ("table_name_filter",) + db_module = "django.db" def add_arguments(self, parser): parser.add_argument( - 'table', nargs='*', type=str, - help='Selects what tables or views should be introspected.', + "table", + nargs="*", + type=str, + help="Selects what tables or views should be introspected.", ) parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database to introspect. Defaults to using the "default" database.', ) parser.add_argument( - '--include-partitions', action='store_true', help='Also output models for partition tables.', + "--include-partitions", + action="store_true", + help="Also output models for partition tables.", ) parser.add_argument( - '--include-views', action='store_true', help='Also output models for database views.', + "--include-views", + action="store_true", + help="Also output models for database views.", ) def handle(self, **options): @@ -33,15 +40,17 @@ class Command(BaseCommand): for line in self.handle_inspection(options): self.stdout.write(line) except NotImplementedError: - raise CommandError("Database inspection isn't supported for the currently selected database backend.") + raise CommandError( + "Database inspection isn't supported for the currently selected database backend." + ) def handle_inspection(self, options): - connection = connections[options['database']] + connection = connections[options["database"]] # 'table_name_filter' is a stealth option - table_name_filter = options.get('table_name_filter') + table_name_filter = options.get("table_name_filter") def table2model(table_name): - return re.sub(r'[^a-zA-Z0-9]', '', table_name.title()) + return re.sub(r"[^a-zA-Z0-9]", "", table_name.title()) with connection.cursor() as cursor: yield "# This is an auto-generated Django model module." @@ -54,55 +63,71 @@ class Command(BaseCommand): "Django to create, modify, and delete the table" ) yield "# Feel free to rename the models, but don't rename db_table values or field names." - yield 'from %s import models' % self.db_module + yield "from %s import models" % self.db_module known_models = [] table_info = connection.introspection.get_table_list(cursor) # Determine types of tables and/or views to be introspected. - types = {'t'} - if options['include_partitions']: - types.add('p') - if options['include_views']: - types.add('v') + types = {"t"} + if options["include_partitions"]: + types.add("p") + if options["include_views"]: + types.add("v") - for table_name in (options['table'] or sorted(info.name for info in table_info if info.type in types)): + for table_name in options["table"] or sorted( + info.name for info in table_info if info.type in types + ): if table_name_filter is not None and callable(table_name_filter): if not table_name_filter(table_name): continue try: try: - relations = connection.introspection.get_relations(cursor, table_name) + relations = connection.introspection.get_relations( + cursor, table_name + ) except NotImplementedError: relations = {} try: - constraints = connection.introspection.get_constraints(cursor, table_name) + constraints = connection.introspection.get_constraints( + cursor, table_name + ) except NotImplementedError: constraints = {} - primary_key_column = connection.introspection.get_primary_key_column(cursor, table_name) + primary_key_column = ( + connection.introspection.get_primary_key_column( + cursor, table_name + ) + ) unique_columns = [ - c['columns'][0] for c in constraints.values() - if c['unique'] and len(c['columns']) == 1 + c["columns"][0] + for c in constraints.values() + if c["unique"] and len(c["columns"]) == 1 ] - table_description = connection.introspection.get_table_description(cursor, table_name) + table_description = connection.introspection.get_table_description( + cursor, table_name + ) except Exception as e: yield "# Unable to inspect table '%s'" % table_name yield "# The error was: %s" % e continue - yield '' - yield '' - yield 'class %s(models.Model):' % table2model(table_name) + yield "" + yield "" + yield "class %s(models.Model):" % table2model(table_name) known_models.append(table2model(table_name)) used_column_names = [] # Holds column names used in the table so far column_to_field_name = {} # Maps column names to names of model fields for row in table_description: - comment_notes = [] # Holds Field notes, to be displayed in a Python comment. + comment_notes = ( + [] + ) # Holds Field notes, to be displayed in a Python comment. extra_params = {} # Holds Field parameters such as 'db_column'. column_name = row.name is_relation = column_name in relations att_name, params, notes = self.normalize_col_name( - column_name, used_column_names, is_relation) + column_name, used_column_names, is_relation + ) extra_params.update(params) comment_notes.extend(notes) @@ -111,70 +136,91 @@ class Command(BaseCommand): # Add primary_key and unique, if necessary. if column_name == primary_key_column: - extra_params['primary_key'] = True + extra_params["primary_key"] = True elif column_name in unique_columns: - extra_params['unique'] = True + extra_params["unique"] = True if is_relation: ref_db_column, ref_db_table = relations[column_name] - if extra_params.pop('unique', False) or extra_params.get('primary_key'): - rel_type = 'OneToOneField' + if extra_params.pop("unique", False) or extra_params.get( + "primary_key" + ): + rel_type = "OneToOneField" else: - rel_type = 'ForeignKey' - ref_pk_column = connection.introspection.get_primary_key_column(cursor, ref_db_table) + rel_type = "ForeignKey" + ref_pk_column = ( + connection.introspection.get_primary_key_column( + cursor, ref_db_table + ) + ) if ref_pk_column and ref_pk_column != ref_db_column: - extra_params['to_field'] = ref_db_column + extra_params["to_field"] = ref_db_column rel_to = ( - 'self' if ref_db_table == table_name + "self" + if ref_db_table == table_name else table2model(ref_db_table) ) if rel_to in known_models: - field_type = '%s(%s' % (rel_type, rel_to) + field_type = "%s(%s" % (rel_type, rel_to) else: field_type = "%s('%s'" % (rel_type, rel_to) else: # Calling `get_field_type` to get the field type string and any # additional parameters and notes. - field_type, field_params, field_notes = self.get_field_type(connection, table_name, row) + field_type, field_params, field_notes = self.get_field_type( + connection, table_name, row + ) extra_params.update(field_params) comment_notes.extend(field_notes) - field_type += '(' + field_type += "(" # Don't output 'id = meta.AutoField(primary_key=True)', because # that's assumed if it doesn't exist. - if att_name == 'id' and extra_params == {'primary_key': True}: - if field_type == 'AutoField(': + if att_name == "id" and extra_params == {"primary_key": True}: + if field_type == "AutoField(": continue - elif field_type == connection.features.introspected_field_types['AutoField'] + '(': - comment_notes.append('AutoField?') + elif ( + field_type + == connection.features.introspected_field_types["AutoField"] + + "(" + ): + comment_notes.append("AutoField?") # Add 'null' and 'blank', if the 'null_ok' flag was present in the # table description. if row.null_ok: # If it's NULL... - extra_params['blank'] = True - extra_params['null'] = True + extra_params["blank"] = True + extra_params["null"] = True - field_desc = '%s = %s%s' % ( + field_desc = "%s = %s%s" % ( att_name, # Custom fields will have a dotted path - '' if '.' in field_type else 'models.', + "" if "." in field_type else "models.", field_type, ) - if field_type.startswith(('ForeignKey(', 'OneToOneField(')): - field_desc += ', models.DO_NOTHING' + if field_type.startswith(("ForeignKey(", "OneToOneField(")): + field_desc += ", models.DO_NOTHING" if extra_params: - if not field_desc.endswith('('): - field_desc += ', ' - field_desc += ', '.join('%s=%r' % (k, v) for k, v in extra_params.items()) - field_desc += ')' + if not field_desc.endswith("("): + field_desc += ", " + field_desc += ", ".join( + "%s=%r" % (k, v) for k, v in extra_params.items() + ) + field_desc += ")" if comment_notes: - field_desc += ' # ' + ' '.join(comment_notes) - yield ' %s' % field_desc - is_view = any(info.name == table_name and info.type == 'v' for info in table_info) - is_partition = any(info.name == table_name and info.type == 'p' for info in table_info) - yield from self.get_meta(table_name, constraints, column_to_field_name, is_view, is_partition) + field_desc += " # " + " ".join(comment_notes) + yield " %s" % field_desc + is_view = any( + info.name == table_name and info.type == "v" for info in table_info + ) + is_partition = any( + info.name == table_name and info.type == "p" for info in table_info + ) + yield from self.get_meta( + table_name, constraints, column_to_field_name, is_view, is_partition + ) def normalize_col_name(self, col_name, used_column_names, is_relation): """ @@ -185,50 +231,54 @@ class Command(BaseCommand): new_name = col_name.lower() if new_name != col_name: - field_notes.append('Field name made lowercase.') + field_notes.append("Field name made lowercase.") if is_relation: - if new_name.endswith('_id'): + if new_name.endswith("_id"): new_name = new_name[:-3] else: - field_params['db_column'] = col_name + field_params["db_column"] = col_name - new_name, num_repl = re.subn(r'\W', '_', new_name) + new_name, num_repl = re.subn(r"\W", "_", new_name) if num_repl > 0: - field_notes.append('Field renamed to remove unsuitable characters.') + field_notes.append("Field renamed to remove unsuitable characters.") if new_name.find(LOOKUP_SEP) >= 0: while new_name.find(LOOKUP_SEP) >= 0: - new_name = new_name.replace(LOOKUP_SEP, '_') + new_name = new_name.replace(LOOKUP_SEP, "_") if col_name.lower().find(LOOKUP_SEP) >= 0: # Only add the comment if the double underscore was in the original name - field_notes.append("Field renamed because it contained more than one '_' in a row.") + field_notes.append( + "Field renamed because it contained more than one '_' in a row." + ) - if new_name.startswith('_'): - new_name = 'field%s' % new_name + if new_name.startswith("_"): + new_name = "field%s" % new_name field_notes.append("Field renamed because it started with '_'.") - if new_name.endswith('_'): - new_name = '%sfield' % new_name + if new_name.endswith("_"): + new_name = "%sfield" % new_name field_notes.append("Field renamed because it ended with '_'.") if keyword.iskeyword(new_name): - new_name += '_field' - field_notes.append('Field renamed because it was a Python reserved word.') + new_name += "_field" + field_notes.append("Field renamed because it was a Python reserved word.") if new_name[0].isdigit(): - new_name = 'number_%s' % new_name - field_notes.append("Field renamed because it wasn't a valid Python identifier.") + new_name = "number_%s" % new_name + field_notes.append( + "Field renamed because it wasn't a valid Python identifier." + ) if new_name in used_column_names: num = 0 - while '%s_%d' % (new_name, num) in used_column_names: + while "%s_%d" % (new_name, num) in used_column_names: num += 1 - new_name = '%s_%d' % (new_name, num) - field_notes.append('Field renamed because of name conflict.') + new_name = "%s_%d" % (new_name, num) + field_notes.append("Field renamed because of name conflict.") if col_name != new_name and field_notes: - field_params['db_column'] = col_name + field_params["db_column"] = col_name return new_name, field_params, field_notes @@ -244,30 +294,37 @@ class Command(BaseCommand): try: field_type = connection.introspection.get_field_type(row.type_code, row) except KeyError: - field_type = 'TextField' - field_notes.append('This field type is a guess.') + field_type = "TextField" + field_notes.append("This field type is a guess.") # Add max_length for all CharFields. - if field_type == 'CharField' and row.internal_size: - field_params['max_length'] = int(row.internal_size) + if field_type == "CharField" and row.internal_size: + field_params["max_length"] = int(row.internal_size) - if field_type in {'CharField', 'TextField'} and row.collation: - field_params['db_collation'] = row.collation + if field_type in {"CharField", "TextField"} and row.collation: + field_params["db_collation"] = row.collation - if field_type == 'DecimalField': + if field_type == "DecimalField": if row.precision is None or row.scale is None: field_notes.append( - 'max_digits and decimal_places have been guessed, as this ' - 'database handles decimal fields as float') - field_params['max_digits'] = row.precision if row.precision is not None else 10 - field_params['decimal_places'] = row.scale if row.scale is not None else 5 + "max_digits and decimal_places have been guessed, as this " + "database handles decimal fields as float" + ) + field_params["max_digits"] = ( + row.precision if row.precision is not None else 10 + ) + field_params["decimal_places"] = ( + row.scale if row.scale is not None else 5 + ) else: - field_params['max_digits'] = row.precision - field_params['decimal_places'] = row.scale + field_params["max_digits"] = row.precision + field_params["decimal_places"] = row.scale return field_type, field_params, field_notes - def get_meta(self, table_name, constraints, column_to_field_name, is_view, is_partition): + def get_meta( + self, table_name, constraints, column_to_field_name, is_view, is_partition + ): """ Return a sequence comprising the lines of code necessary to construct the inner Meta class for the model corresponding @@ -276,28 +333,30 @@ class Command(BaseCommand): unique_together = [] has_unsupported_constraint = False for params in constraints.values(): - if params['unique']: - columns = params['columns'] + if params["unique"]: + columns = params["columns"] if None in columns: has_unsupported_constraint = True columns = [x for x in columns if x is not None] if len(columns) > 1: - unique_together.append(str(tuple(column_to_field_name[c] for c in columns))) + unique_together.append( + str(tuple(column_to_field_name[c] for c in columns)) + ) if is_view: managed_comment = " # Created from a view. Don't remove." elif is_partition: managed_comment = " # Created from a partition. Don't remove." else: - managed_comment = '' - meta = [''] + managed_comment = "" + meta = [""] if has_unsupported_constraint: - meta.append(' # A unique constraint could not be introspected.') + meta.append(" # A unique constraint could not be introspected.") meta += [ - ' class Meta:', - ' managed = False%s' % managed_comment, - ' db_table = %r' % table_name + " class Meta:", + " managed = False%s" % managed_comment, + " db_table = %r" % table_name, ] if unique_together: - tup = '(' + ', '.join(unique_together) + ',)' + tup = "(" + ", ".join(unique_together) + ",)" meta += [" unique_together = %s" % tup] return meta diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 20428f9f10..38a2818d5c 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -15,64 +15,82 @@ from django.core.management.base import BaseCommand, CommandError from django.core.management.color import no_style from django.core.management.utils import parse_apps_and_model_labels from django.db import ( - DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections, router, + DEFAULT_DB_ALIAS, + DatabaseError, + IntegrityError, + connections, + router, transaction, ) from django.utils.functional import cached_property try: import bz2 + has_bz2 = True except ImportError: has_bz2 = False try: import lzma + has_lzma = True except ImportError: has_lzma = False -READ_STDIN = '-' +READ_STDIN = "-" class Command(BaseCommand): - help = 'Installs the named fixture(s) in the database.' + help = "Installs the named fixture(s) in the database." missing_args_message = ( "No database fixture specified. Please provide the path of at least " "one fixture in the command line." ) def add_arguments(self, parser): - parser.add_argument('args', metavar='fixture', nargs='+', help='Fixture labels.') parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "args", metavar="fixture", nargs="+", help="Fixture labels." + ) + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a specific database to load fixtures into. Defaults to the "default" database.', ) parser.add_argument( - '--app', dest='app_label', - help='Only look for fixtures in the specified app.', + "--app", + dest="app_label", + help="Only look for fixtures in the specified app.", ) parser.add_argument( - '--ignorenonexistent', '-i', action='store_true', dest='ignore', - help='Ignores entries in the serialized data for fields that do not ' - 'currently exist on the model.', + "--ignorenonexistent", + "-i", + action="store_true", + dest="ignore", + help="Ignores entries in the serialized data for fields that do not " + "currently exist on the model.", ) parser.add_argument( - '-e', '--exclude', action='append', default=[], - help='An app_label or app_label.ModelName to exclude. Can be used multiple times.', + "-e", + "--exclude", + action="append", + default=[], + help="An app_label or app_label.ModelName to exclude. Can be used multiple times.", ) parser.add_argument( - '--format', - help='Format of serialized data when reading from stdin.', + "--format", + help="Format of serialized data when reading from stdin.", ) def handle(self, *fixture_labels, **options): - self.ignore = options['ignore'] - self.using = options['database'] - self.app_label = options['app_label'] - self.verbosity = options['verbosity'] - self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(options['exclude']) - self.format = options['format'] + self.ignore = options["ignore"] + self.using = options["database"] + self.app_label = options["app_label"] + self.verbosity = options["verbosity"] + self.excluded_models, self.excluded_apps = parse_apps_and_model_labels( + options["exclude"] + ) + self.format = options["format"] with transaction.atomic(using=self.using): self.loaddata(fixture_labels) @@ -89,16 +107,16 @@ class Command(BaseCommand): """A dict mapping format names to (open function, mode arg) tuples.""" # Forcing binary mode may be revisited after dropping Python 2 support (see #22399) compression_formats = { - None: (open, 'rb'), - 'gz': (gzip.GzipFile, 'rb'), - 'zip': (SingleZipReader, 'r'), - 'stdin': (lambda *args: sys.stdin, None), + None: (open, "rb"), + "gz": (gzip.GzipFile, "rb"), + "zip": (SingleZipReader, "r"), + "stdin": (lambda *args: sys.stdin, None), } if has_bz2: - compression_formats['bz2'] = (bz2.BZ2File, 'r') + compression_formats["bz2"] = (bz2.BZ2File, "r") if has_lzma: - compression_formats['lzma'] = (lzma.LZMAFile, 'r') - compression_formats['xz'] = (lzma.LZMAFile, 'r') + compression_formats["lzma"] = (lzma.LZMAFile, "r") + compression_formats["xz"] = (lzma.LZMAFile, "r") return compression_formats def reset_sequences(self, connection, models): @@ -106,7 +124,7 @@ class Command(BaseCommand): sequence_sql = connection.ops.sequence_reset_sql(no_style(), models) if sequence_sql: if self.verbosity >= 2: - self.stdout.write('Resetting sequences') + self.stdout.write("Resetting sequences") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) @@ -162,14 +180,18 @@ class Command(BaseCommand): else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" - % (self.loaded_object_count, self.fixture_object_count, self.fixture_count) + % ( + self.loaded_object_count, + self.fixture_object_count, + self.fixture_count, + ) ) def save_obj(self, obj): """Save an object if permitted.""" if ( - obj.object._meta.app_config in self.excluded_apps or - type(obj.object) in self.excluded_models + obj.object._meta.app_config in self.excluded_apps + or type(obj.object) in self.excluded_models ): return False saved = False @@ -180,11 +202,14 @@ class Command(BaseCommand): obj.save(using=self.using) # psycopg2 raises ValueError if data contains NUL chars. except (DatabaseError, IntegrityError, ValueError) as e: - e.args = ('Could not load %(object_label)s(pk=%(pk)s): %(error_msg)s' % { - 'object_label': obj.object._meta.label, - 'pk': obj.object.pk, - 'error_msg': e, - },) + e.args = ( + "Could not load %(object_label)s(pk=%(pk)s): %(error_msg)s" + % { + "object_label": obj.object._meta.label, + "pk": obj.object.pk, + "error_msg": e, + }, + ) raise if obj.deferred_fields: self.objs_with_deferred_fields.append(obj) @@ -193,7 +218,9 @@ class Command(BaseCommand): def load_label(self, fixture_label): """Load fixtures files for a given label.""" show_progress = self.verbosity >= 3 - for fixture_file, fixture_dir, fixture_name in self.find_fixtures(fixture_label): + for fixture_file, fixture_dir, fixture_name in self.find_fixtures( + fixture_label + ): _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file)) open_method, mode = self.compression_formats[cmp_fmt] fixture = open_method(fixture_file, mode) @@ -207,7 +234,10 @@ class Command(BaseCommand): ) try: objects = serializers.deserialize( - ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore, + ser_fmt, + fixture, + using=self.using, + ignorenonexistent=self.ignore, handle_forward_references=True, ) @@ -217,12 +247,14 @@ class Command(BaseCommand): loaded_objects_in_fixture += 1 if show_progress: self.stdout.write( - '\rProcessed %i object(s).' % loaded_objects_in_fixture, - ending='' + "\rProcessed %i object(s)." % loaded_objects_in_fixture, + ending="", ) except Exception as e: if not isinstance(e, CommandError): - e.args = ("Problem installing fixture '%s': %s" % (fixture_file, e),) + e.args = ( + "Problem installing fixture '%s': %s" % (fixture_file, e), + ) raise finally: fixture.close() @@ -236,7 +268,7 @@ class Command(BaseCommand): warnings.warn( "No fixture data found for '%s'. (File format may be " "invalid.)" % fixture_name, - RuntimeWarning + RuntimeWarning, ) def get_fixture_name_and_dirs(self, fixture_name): @@ -254,16 +286,18 @@ class Command(BaseCommand): cmp_fmts = self.compression_formats if cmp_fmt is None else [cmp_fmt] ser_fmts = self.serialization_formats if ser_fmt is None else [ser_fmt] return { - '%s.%s' % ( + "%s.%s" + % ( fixture_name, - '.'.join([ext for ext in combo if ext]), - ) for combo in product(databases, ser_fmts, cmp_fmts) + ".".join([ext for ext in combo if ext]), + ) + for combo in product(databases, ser_fmts, cmp_fmts) } def find_fixture_files_in_dir(self, fixture_dir, fixture_name, targets): fixture_files_in_dir = [] path = os.path.join(fixture_dir, fixture_name) - for candidate in glob.iglob(glob.escape(path) + '*'): + for candidate in glob.iglob(glob.escape(path) + "*"): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error # messages. @@ -287,18 +321,22 @@ class Command(BaseCommand): if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = self.find_fixture_files_in_dir( - fixture_dir, fixture_name, targets, + fixture_dir, + fixture_name, + targets, ) if self.verbosity >= 2 and not fixture_files_in_dir: - self.stdout.write("No fixture '%s' in %s." % - (fixture_name, humanize(fixture_dir))) + self.stdout.write( + "No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir)) + ) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( - "Multiple fixtures named '%s' in %s. Aborting." % - (fixture_name, humanize(fixture_dir))) + "Multiple fixtures named '%s' in %s. Aborting." + % (fixture_name, humanize(fixture_dir)) + ) fixture_files.extend(fixture_files_in_dir) if not fixture_files: @@ -321,11 +359,12 @@ class Command(BaseCommand): raise ImproperlyConfigured("settings.FIXTURE_DIRS contains duplicates.") for app_config in apps.get_app_configs(): app_label = app_config.label - app_dir = os.path.join(app_config.path, 'fixtures') + app_dir = os.path.join(app_config.path, "fixtures") if app_dir in fixture_dirs: raise ImproperlyConfigured( "'%s' is a default fixture directory for the '%s' app " - "and cannot be listed in settings.FIXTURE_DIRS." % (app_dir, app_label) + "and cannot be listed in settings.FIXTURE_DIRS." + % (app_dir, app_label) ) if self.app_label and app_label != self.app_label: @@ -333,7 +372,7 @@ class Command(BaseCommand): if os.path.isdir(app_dir): dirs.append(app_dir) dirs.extend(fixture_dirs) - dirs.append('') + dirs.append("") return [os.path.realpath(d) for d in dirs] def parse_name(self, fixture_name): @@ -342,10 +381,12 @@ class Command(BaseCommand): """ if fixture_name == READ_STDIN: if not self.format: - raise CommandError('--format must be specified when reading from stdin.') - return READ_STDIN, self.format, 'stdin' + raise CommandError( + "--format must be specified when reading from stdin." + ) + return READ_STDIN, self.format, "stdin" - parts = fixture_name.rsplit('.', 2) + parts = fixture_name.rsplit(".", 2) if len(parts) > 1 and parts[-1] in self.compression_formats: cmp_fmt = parts[-1] @@ -360,17 +401,17 @@ class Command(BaseCommand): else: raise CommandError( "Problem installing fixture '%s': %s is not a known " - "serialization format." % ('.'.join(parts[:-1]), parts[-1])) + "serialization format." % (".".join(parts[:-1]), parts[-1]) + ) else: ser_fmt = None - name = '.'.join(parts) + name = ".".join(parts) return name, ser_fmt, cmp_fmt class SingleZipReader(zipfile.ZipFile): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if len(self.namelist()) != 1: @@ -381,4 +422,4 @@ class SingleZipReader(zipfile.ZipFile): def humanize(dirname): - return "'%s'" % dirname if dirname else 'absolute path' + return "'%s'" % dirname if dirname else "absolute path" diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 0070342181..af1b43b7c7 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -12,7 +12,10 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files.temp import NamedTemporaryFile from django.core.management.base import BaseCommand, CommandError from django.core.management.utils import ( - find_command, handle_extensions, is_ignored_path, popen_wrapper, + find_command, + handle_extensions, + is_ignored_path, + popen_wrapper, ) from django.utils.encoding import DEFAULT_LOCALE_ENCODING from django.utils.functional import cached_property @@ -21,7 +24,9 @@ from django.utils.regex_helper import _lazy_re_compile from django.utils.text import get_text_list from django.utils.translation import templatize -plural_forms_re = _lazy_re_compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL) +plural_forms_re = _lazy_re_compile( + r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL +) STATUS_OK = 0 NO_LOCALE_DIR = object() @@ -63,6 +68,7 @@ class BuildFile: """ Represent the state of a translatable file during the build process. """ + def __init__(self, command, domain, translatable): self.command = command self.domain = domain @@ -70,11 +76,11 @@ class BuildFile: @cached_property def is_templatized(self): - if self.domain == 'djangojs': + if self.domain == "djangojs": return self.command.gettext_version < (0, 18, 3) - elif self.domain == 'django': + elif self.domain == "django": file_ext = os.path.splitext(self.translatable.file)[1] - return file_ext != '.py' + return file_ext != ".py" return False @cached_property @@ -90,10 +96,10 @@ class BuildFile: if not self.is_templatized: return self.path extension = { - 'djangojs': 'c', - 'django': 'py', + "djangojs": "c", + "django": "py", }.get(self.domain) - filename = '%s.%s' % (self.translatable.file, extension) + filename = "%s.%s" % (self.translatable.file, extension) return os.path.join(self.translatable.dirpath, filename) def preprocess(self): @@ -104,15 +110,15 @@ class BuildFile: if not self.is_templatized: return - with open(self.path, encoding='utf-8') as fp: + with open(self.path, encoding="utf-8") as fp: src_data = fp.read() - if self.domain == 'djangojs': + if self.domain == "djangojs": content = prepare_js_for_gettext(src_data) - elif self.domain == 'django': + elif self.domain == "django": content = templatize(src_data, origin=self.path[2:]) - with open(self.work_path, 'w', encoding='utf-8') as fp: + with open(self.work_path, "w", encoding="utf-8") as fp: fp.write(content) def postprocess_messages(self, msgs): @@ -126,7 +132,7 @@ class BuildFile: return msgs # Remove '.py' suffix - if os.name == 'nt': + if os.name == "nt": # Preserve '.\' prefix on Windows to respect gettext behavior old_path = self.work_path new_path = self.path @@ -135,10 +141,10 @@ class BuildFile: new_path = self.path[2:] return re.sub( - r'^(#: .*)(' + re.escape(old_path) + r')', + r"^(#: .*)(" + re.escape(old_path) + r")", lambda match: match[0].replace(old_path, new_path), msgs, - flags=re.MULTILINE + flags=re.MULTILINE, ) def cleanup(self): @@ -164,8 +170,8 @@ def normalize_eols(raw_contents): lines_list = raw_contents.splitlines() # Ensure last line has its EOL if lines_list and lines_list[-1]: - lines_list.append('') - return '\n'.join(lines_list) + lines_list.append("") + return "\n".join(lines_list) def write_pot_file(potfile, msgs): @@ -182,16 +188,16 @@ def write_pot_file(potfile, msgs): found, header_read = False, False for line in pot_lines: if not found and not header_read: - if 'charset=CHARSET' in line: + if "charset=CHARSET" in line: found = True - line = line.replace('charset=CHARSET', 'charset=UTF-8') + line = line.replace("charset=CHARSET", "charset=UTF-8") if not line and not found: header_read = True lines.append(line) - msgs = '\n'.join(lines) + msgs = "\n".join(lines) # Force newlines of POT files to '\n' to work around # https://savannah.gnu.org/bugs/index.php?52395 - with open(potfile, 'a', encoding='utf-8', newline='\n') as fp: + with open(potfile, "a", encoding="utf-8", newline="\n") as fp: fp.write(msgs) @@ -209,61 +215,86 @@ class Command(BaseCommand): requires_system_checks = [] - msgmerge_options = ['-q', '--backup=none', '--previous', '--update'] - msguniq_options = ['--to-code=utf-8'] - msgattrib_options = ['--no-obsolete'] - xgettext_options = ['--from-code=UTF-8', '--add-comments=Translators'] + msgmerge_options = ["-q", "--backup=none", "--previous", "--update"] + msguniq_options = ["--to-code=utf-8"] + msgattrib_options = ["--no-obsolete"] + xgettext_options = ["--from-code=UTF-8", "--add-comments=Translators"] def add_arguments(self, parser): parser.add_argument( - '--locale', '-l', default=[], action='append', - help='Creates or updates the message files for the given locale(s) (e.g. pt_BR). ' - 'Can be used multiple times.', + "--locale", + "-l", + default=[], + action="append", + help="Creates or updates the message files for the given locale(s) (e.g. pt_BR). " + "Can be used multiple times.", ) parser.add_argument( - '--exclude', '-x', default=[], action='append', - help='Locales to exclude. Default is none. Can be used multiple times.', + "--exclude", + "-x", + default=[], + action="append", + help="Locales to exclude. Default is none. Can be used multiple times.", ) parser.add_argument( - '--domain', '-d', default='django', + "--domain", + "-d", + default="django", help='The domain of the message files (default: "django").', ) parser.add_argument( - '--all', '-a', action='store_true', - help='Updates the message files for all existing locales.', + "--all", + "-a", + action="store_true", + help="Updates the message files for all existing locales.", ) parser.add_argument( - '--extension', '-e', dest='extensions', action='append', + "--extension", + "-e", + dest="extensions", + action="append", help='The file extension(s) to examine (default: "html,txt,py", or "js" ' - 'if the domain is "djangojs"). Separate multiple extensions with ' - 'commas, or use -e multiple times.', + 'if the domain is "djangojs"). Separate multiple extensions with ' + "commas, or use -e multiple times.", ) parser.add_argument( - '--symlinks', '-s', action='store_true', - help='Follows symlinks to directories when examining source code ' - 'and templates for translation strings.', + "--symlinks", + "-s", + action="store_true", + help="Follows symlinks to directories when examining source code " + "and templates for translation strings.", ) parser.add_argument( - '--ignore', '-i', action='append', dest='ignore_patterns', - default=[], metavar='PATTERN', - help='Ignore files or directories matching this glob-style pattern. ' - 'Use multiple times to ignore more.', + "--ignore", + "-i", + action="append", + dest="ignore_patterns", + default=[], + metavar="PATTERN", + help="Ignore files or directories matching this glob-style pattern. " + "Use multiple times to ignore more.", ) parser.add_argument( - '--no-default-ignore', action='store_false', dest='use_default_ignore_patterns', + "--no-default-ignore", + action="store_false", + dest="use_default_ignore_patterns", help="Don't ignore the common glob-style patterns 'CVS', '.*', '*~' and '*.pyc'.", ) parser.add_argument( - '--no-wrap', action='store_true', + "--no-wrap", + action="store_true", help="Don't break long message lines into several lines.", ) parser.add_argument( - '--no-location', action='store_true', + "--no-location", + action="store_true", help="Don't write '#: filename:line' lines.", ) parser.add_argument( - '--add-location', - choices=('full', 'file', 'never'), const='full', nargs='?', + "--add-location", + choices=("full", "file", "never"), + const="full", + nargs="?", help=( "Controls '#: filename:line' lines. If the option is 'full' " "(the default if not given), the lines include both file name " @@ -273,61 +304,65 @@ class Command(BaseCommand): ), ) parser.add_argument( - '--no-obsolete', action='store_true', + "--no-obsolete", + action="store_true", help="Remove obsolete message strings.", ) parser.add_argument( - '--keep-pot', action='store_true', + "--keep-pot", + action="store_true", help="Keep .pot file after making messages. Useful when debugging.", ) def handle(self, *args, **options): - locale = options['locale'] - exclude = options['exclude'] - self.domain = options['domain'] - self.verbosity = options['verbosity'] - process_all = options['all'] - extensions = options['extensions'] - self.symlinks = options['symlinks'] + locale = options["locale"] + exclude = options["exclude"] + self.domain = options["domain"] + self.verbosity = options["verbosity"] + process_all = options["all"] + extensions = options["extensions"] + self.symlinks = options["symlinks"] - ignore_patterns = options['ignore_patterns'] - if options['use_default_ignore_patterns']: - ignore_patterns += ['CVS', '.*', '*~', '*.pyc'] + ignore_patterns = options["ignore_patterns"] + if options["use_default_ignore_patterns"]: + ignore_patterns += ["CVS", ".*", "*~", "*.pyc"] self.ignore_patterns = list(set(ignore_patterns)) # Avoid messing with mutable class variables - if options['no_wrap']: - self.msgmerge_options = self.msgmerge_options[:] + ['--no-wrap'] - self.msguniq_options = self.msguniq_options[:] + ['--no-wrap'] - self.msgattrib_options = self.msgattrib_options[:] + ['--no-wrap'] - self.xgettext_options = self.xgettext_options[:] + ['--no-wrap'] - if options['no_location']: - self.msgmerge_options = self.msgmerge_options[:] + ['--no-location'] - self.msguniq_options = self.msguniq_options[:] + ['--no-location'] - self.msgattrib_options = self.msgattrib_options[:] + ['--no-location'] - self.xgettext_options = self.xgettext_options[:] + ['--no-location'] - if options['add_location']: + if options["no_wrap"]: + self.msgmerge_options = self.msgmerge_options[:] + ["--no-wrap"] + self.msguniq_options = self.msguniq_options[:] + ["--no-wrap"] + self.msgattrib_options = self.msgattrib_options[:] + ["--no-wrap"] + self.xgettext_options = self.xgettext_options[:] + ["--no-wrap"] + if options["no_location"]: + self.msgmerge_options = self.msgmerge_options[:] + ["--no-location"] + self.msguniq_options = self.msguniq_options[:] + ["--no-location"] + self.msgattrib_options = self.msgattrib_options[:] + ["--no-location"] + self.xgettext_options = self.xgettext_options[:] + ["--no-location"] + if options["add_location"]: if self.gettext_version < (0, 19): raise CommandError( "The --add-location option requires gettext 0.19 or later. " - "You have %s." % '.'.join(str(x) for x in self.gettext_version) + "You have %s." % ".".join(str(x) for x in self.gettext_version) ) - arg_add_location = "--add-location=%s" % options['add_location'] + arg_add_location = "--add-location=%s" % options["add_location"] self.msgmerge_options = self.msgmerge_options[:] + [arg_add_location] self.msguniq_options = self.msguniq_options[:] + [arg_add_location] self.msgattrib_options = self.msgattrib_options[:] + [arg_add_location] self.xgettext_options = self.xgettext_options[:] + [arg_add_location] - self.no_obsolete = options['no_obsolete'] - self.keep_pot = options['keep_pot'] + self.no_obsolete = options["no_obsolete"] + self.keep_pot = options["keep_pot"] - if self.domain not in ('django', 'djangojs'): - raise CommandError("currently makemessages only supports domains " - "'django' and 'djangojs'") - if self.domain == 'djangojs': - exts = extensions or ['js'] + if self.domain not in ("django", "djangojs"): + raise CommandError( + "currently makemessages only supports domains " + "'django' and 'djangojs'" + ) + if self.domain == "djangojs": + exts = extensions or ["js"] else: - exts = extensions or ['html', 'txt', 'py'] + exts = extensions or ["html", "txt", "py"] self.extensions = handle_extensions(exts) if (not locale and not exclude and not process_all) or self.domain is None: @@ -338,32 +373,35 @@ class Command(BaseCommand): if self.verbosity > 1: self.stdout.write( - 'examining files with the extensions: %s' - % get_text_list(list(self.extensions), 'and') + "examining files with the extensions: %s" + % get_text_list(list(self.extensions), "and") ) self.invoked_for_django = False self.locale_paths = [] self.default_locale_path = None - if os.path.isdir(os.path.join('conf', 'locale')): - self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))] + if os.path.isdir(os.path.join("conf", "locale")): + self.locale_paths = [os.path.abspath(os.path.join("conf", "locale"))] self.default_locale_path = self.locale_paths[0] self.invoked_for_django = True else: if self.settings_available: self.locale_paths.extend(settings.LOCALE_PATHS) # Allow to run makemessages inside an app dir - if os.path.isdir('locale'): - self.locale_paths.append(os.path.abspath('locale')) + if os.path.isdir("locale"): + self.locale_paths.append(os.path.abspath("locale")) if self.locale_paths: self.default_locale_path = self.locale_paths[0] os.makedirs(self.default_locale_path, exist_ok=True) # Build locale list - looks_like_locale = re.compile(r'[a-z]{2}') - locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path)) + looks_like_locale = re.compile(r"[a-z]{2}") + locale_dirs = filter( + os.path.isdir, glob.glob("%s/*" % self.default_locale_path) + ) all_locales = [ - lang_code for lang_code in map(os.path.basename, locale_dirs) + lang_code + for lang_code in map(os.path.basename, locale_dirs) if looks_like_locale.match(lang_code) ] @@ -375,25 +413,26 @@ class Command(BaseCommand): locales = set(locales).difference(exclude) if locales: - check_programs('msguniq', 'msgmerge', 'msgattrib') + check_programs("msguniq", "msgmerge", "msgattrib") - check_programs('xgettext') + check_programs("xgettext") try: potfiles = self.build_potfiles() # Build po files for each selected locale for locale in locales: - if '-' in locale: + if "-" in locale: self.stdout.write( - 'invalid locale %s, did you mean %s?' % ( + "invalid locale %s, did you mean %s?" + % ( locale, - locale.replace('-', '_'), + locale.replace("-", "_"), ), ) continue if self.verbosity > 0: - self.stdout.write('processing locale %s' % locale) + self.stdout.write("processing locale %s" % locale) for potfile in potfiles: self.write_po_file(potfile, locale) finally: @@ -405,10 +444,10 @@ class Command(BaseCommand): # Gettext tools will output system-encoded bytestrings instead of UTF-8, # when looking up the version. It's especially a problem on Windows. out, err, status = popen_wrapper( - ['xgettext', '--version'], + ["xgettext", "--version"], stdout_encoding=DEFAULT_LOCALE_ENCODING, ) - m = re.search(r'(\d+)\.(\d+)\.?(\d+)?', out) + m = re.search(r"(\d+)\.(\d+)\.?(\d+)?", out) if m: return tuple(int(d) for d in m.groups() if d is not None) else: @@ -433,26 +472,27 @@ class Command(BaseCommand): self.process_files(file_list) potfiles = [] for path in self.locale_paths: - potfile = os.path.join(path, '%s.pot' % self.domain) + potfile = os.path.join(path, "%s.pot" % self.domain) if not os.path.exists(potfile): continue - args = ['msguniq'] + self.msguniq_options + [potfile] + args = ["msguniq"] + self.msguniq_options + [potfile] msgs, errors, status = popen_wrapper(args) if errors: if status != STATUS_OK: raise CommandError( - "errors happened while running msguniq\n%s" % errors) + "errors happened while running msguniq\n%s" % errors + ) elif self.verbosity > 0: self.stdout.write(errors) msgs = normalize_eols(msgs) - with open(potfile, 'w', encoding='utf-8') as fp: + with open(potfile, "w", encoding="utf-8") as fp: fp.write(msgs) potfiles.append(potfile) return potfiles def remove_potfiles(self): for path in self.locale_paths: - pot_path = os.path.join(path, '%s.pot' % self.domain) + pot_path = os.path.join(path, "%s.pot" % self.domain) if os.path.exists(pot_path): os.unlink(pot_path) @@ -464,23 +504,40 @@ class Command(BaseCommand): all_files = [] ignored_roots = [] if self.settings_available: - ignored_roots = [os.path.normpath(p) for p in (settings.MEDIA_ROOT, settings.STATIC_ROOT) if p] - for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=self.symlinks): + ignored_roots = [ + os.path.normpath(p) + for p in (settings.MEDIA_ROOT, settings.STATIC_ROOT) + if p + ] + for dirpath, dirnames, filenames in os.walk( + root, topdown=True, followlinks=self.symlinks + ): for dirname in dirnames[:]: - if (is_ignored_path(os.path.normpath(os.path.join(dirpath, dirname)), self.ignore_patterns) or - os.path.join(os.path.abspath(dirpath), dirname) in ignored_roots): + if ( + is_ignored_path( + os.path.normpath(os.path.join(dirpath, dirname)), + self.ignore_patterns, + ) + or os.path.join(os.path.abspath(dirpath), dirname) in ignored_roots + ): dirnames.remove(dirname) if self.verbosity > 1: - self.stdout.write('ignoring directory %s' % dirname) - elif dirname == 'locale': + self.stdout.write("ignoring directory %s" % dirname) + elif dirname == "locale": dirnames.remove(dirname) - self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname)) + self.locale_paths.insert( + 0, os.path.join(os.path.abspath(dirpath), dirname) + ) for filename in filenames: file_path = os.path.normpath(os.path.join(dirpath, filename)) file_ext = os.path.splitext(filename)[1] - if file_ext not in self.extensions or is_ignored_path(file_path, self.ignore_patterns): + if file_ext not in self.extensions or is_ignored_path( + file_path, self.ignore_patterns + ): if self.verbosity > 1: - self.stdout.write('ignoring file %s in %s' % (filename, dirpath)) + self.stdout.write( + "ignoring file %s in %s" % (filename, dirpath) + ) else: locale_dir = None for path in self.locale_paths: @@ -488,7 +545,9 @@ class Command(BaseCommand): locale_dir = path break locale_dir = locale_dir or self.default_locale_path or NO_LOCALE_DIR - all_files.append(self.translatable_file_class(dirpath, filename, locale_dir)) + all_files.append( + self.translatable_file_class(dirpath, filename, locale_dir) + ) return sorted(all_files) def process_files(self, file_list): @@ -513,18 +572,22 @@ class Command(BaseCommand): build_files = [] for translatable in files: if self.verbosity > 1: - self.stdout.write('processing file %s in %s' % ( - translatable.file, translatable.dirpath - )) - if self.domain not in ('djangojs', 'django'): + self.stdout.write( + "processing file %s in %s" + % (translatable.file, translatable.dirpath) + ) + if self.domain not in ("djangojs", "django"): continue build_file = self.build_file_class(self, self.domain, translatable) try: build_file.preprocess() except UnicodeDecodeError as e: self.stdout.write( - 'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % ( - translatable.file, translatable.dirpath, e, + "UnicodeDecodeError: skipped file %s in %s (reason: %s)" + % ( + translatable.file, + translatable.dirpath, + e, ) ) continue @@ -535,41 +598,43 @@ class Command(BaseCommand): raise build_files.append(build_file) - if self.domain == 'djangojs': + if self.domain == "djangojs": is_templatized = build_file.is_templatized args = [ - 'xgettext', - '-d', self.domain, - '--language=%s' % ('C' if is_templatized else 'JavaScript',), - '--keyword=gettext_noop', - '--keyword=gettext_lazy', - '--keyword=ngettext_lazy:1,2', - '--keyword=pgettext:1c,2', - '--keyword=npgettext:1c,2,3', - '--output=-', + "xgettext", + "-d", + self.domain, + "--language=%s" % ("C" if is_templatized else "JavaScript",), + "--keyword=gettext_noop", + "--keyword=gettext_lazy", + "--keyword=ngettext_lazy:1,2", + "--keyword=pgettext:1c,2", + "--keyword=npgettext:1c,2,3", + "--output=-", ] - elif self.domain == 'django': + elif self.domain == "django": args = [ - 'xgettext', - '-d', self.domain, - '--language=Python', - '--keyword=gettext_noop', - '--keyword=gettext_lazy', - '--keyword=ngettext_lazy:1,2', - '--keyword=pgettext:1c,2', - '--keyword=npgettext:1c,2,3', - '--keyword=pgettext_lazy:1c,2', - '--keyword=npgettext_lazy:1c,2,3', - '--output=-', + "xgettext", + "-d", + self.domain, + "--language=Python", + "--keyword=gettext_noop", + "--keyword=gettext_lazy", + "--keyword=ngettext_lazy:1,2", + "--keyword=pgettext:1c,2", + "--keyword=npgettext:1c,2,3", + "--keyword=pgettext_lazy:1c,2", + "--keyword=npgettext_lazy:1c,2,3", + "--output=-", ] else: return input_files = [bf.work_path for bf in build_files] - with NamedTemporaryFile(mode='w+') as input_files_list: - input_files_list.write('\n'.join(input_files)) + with NamedTemporaryFile(mode="w+") as input_files_list: + input_files_list.write("\n".join(input_files)) input_files_list.flush() - args.extend(['--files-from', input_files_list.name]) + args.extend(["--files-from", input_files_list.name]) args.extend(self.xgettext_options) msgs, errors, status = popen_wrapper(args) @@ -578,8 +643,8 @@ class Command(BaseCommand): for build_file in build_files: build_file.cleanup() raise CommandError( - 'errors happened while running xgettext on %s\n%s' % - ('\n'.join(input_files), errors) + "errors happened while running xgettext on %s\n%s" + % ("\n".join(input_files), errors) ) elif self.verbosity > 0: # Print warnings @@ -597,7 +662,7 @@ class Command(BaseCommand): ) for build_file in build_files: msgs = build_file.postprocess_messages(msgs) - potfile = os.path.join(locale_dir, '%s.pot' % self.domain) + potfile = os.path.join(locale_dir, "%s.pot" % self.domain) write_pot_file(potfile, msgs) for build_file in build_files: @@ -610,38 +675,41 @@ class Command(BaseCommand): Use msgmerge and msgattrib GNU gettext utilities. """ - basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES') + basedir = os.path.join(os.path.dirname(potfile), locale, "LC_MESSAGES") os.makedirs(basedir, exist_ok=True) - pofile = os.path.join(basedir, '%s.po' % self.domain) + pofile = os.path.join(basedir, "%s.po" % self.domain) if os.path.exists(pofile): - args = ['msgmerge'] + self.msgmerge_options + [pofile, potfile] + args = ["msgmerge"] + self.msgmerge_options + [pofile, potfile] _, errors, status = popen_wrapper(args) if errors: if status != STATUS_OK: raise CommandError( - "errors happened while running msgmerge\n%s" % errors) + "errors happened while running msgmerge\n%s" % errors + ) elif self.verbosity > 0: self.stdout.write(errors) - msgs = Path(pofile).read_text(encoding='utf-8') + msgs = Path(pofile).read_text(encoding="utf-8") else: - with open(potfile, encoding='utf-8') as fp: + with open(potfile, encoding="utf-8") as fp: msgs = fp.read() if not self.invoked_for_django: msgs = self.copy_plural_forms(msgs, locale) msgs = normalize_eols(msgs) msgs = msgs.replace( - "#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "") - with open(pofile, 'w', encoding='utf-8') as fp: + "#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "" + ) + with open(pofile, "w", encoding="utf-8") as fp: fp.write(msgs) if self.no_obsolete: - args = ['msgattrib'] + self.msgattrib_options + ['-o', pofile, pofile] + args = ["msgattrib"] + self.msgattrib_options + ["-o", pofile, pofile] msgs, errors, status = popen_wrapper(args) if errors: if status != STATUS_OK: raise CommandError( - "errors happened while running msgattrib\n%s" % errors) + "errors happened while running msgattrib\n%s" % errors + ) elif self.verbosity > 0: self.stdout.write(errors) @@ -652,19 +720,21 @@ class Command(BaseCommand): contents of a newly created .po file. """ django_dir = os.path.normpath(os.path.join(os.path.dirname(django.__file__))) - if self.domain == 'djangojs': - domains = ('djangojs', 'django') + if self.domain == "djangojs": + domains = ("djangojs", "django") else: - domains = ('django',) + domains = ("django",) for domain in domains: - django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', '%s.po' % domain) + django_po = os.path.join( + django_dir, "conf", "locale", locale, "LC_MESSAGES", "%s.po" % domain + ) if os.path.exists(django_po): - with open(django_po, encoding='utf-8') as fp: + with open(django_po, encoding="utf-8") as fp: m = plural_forms_re.search(fp.read()) if m: - plural_form_line = m['value'] + plural_form_line = m["value"] if self.verbosity > 1: - self.stdout.write('copying plural forms: %s' % plural_form_line) + self.stdout.write("copying plural forms: %s" % plural_form_line) lines = [] found = False for line in msgs.splitlines(): @@ -672,6 +742,6 @@ class Command(BaseCommand): line = plural_form_line found = True lines.append(line) - msgs = '\n'.join(lines) + msgs = "\n".join(lines) break return msgs diff --git a/django/core/management/commands/makemigrations.py b/django/core/management/commands/makemigrations.py index 4349f33a61..325848d8b2 100644 --- a/django/core/management/commands/makemigrations.py +++ b/django/core/management/commands/makemigrations.py @@ -5,15 +5,14 @@ from itertools import takewhile from django.apps import apps from django.conf import settings -from django.core.management.base import ( - BaseCommand, CommandError, no_translations, -) +from django.core.management.base import BaseCommand, CommandError, no_translations from django.db import DEFAULT_DB_ALIAS, OperationalError, connections, router from django.db.migrations import Migration from django.db.migrations.autodetector import MigrationAutodetector from django.db.migrations.loader import MigrationLoader from django.db.migrations.questioner import ( - InteractiveMigrationQuestioner, MigrationQuestioner, + InteractiveMigrationQuestioner, + MigrationQuestioner, NonInteractiveMigrationQuestioner, ) from django.db.migrations.state import ProjectState @@ -26,42 +25,57 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'args', metavar='app_label', nargs='*', - help='Specify the app label(s) to create migrations for.', + "args", + metavar="app_label", + nargs="*", + help="Specify the app label(s) to create migrations for.", ) parser.add_argument( - '--dry-run', action='store_true', + "--dry-run", + action="store_true", help="Just show what migrations would be made; don't actually write them.", ) parser.add_argument( - '--merge', action='store_true', + "--merge", + action="store_true", help="Enable fixing of migration conflicts.", ) parser.add_argument( - '--empty', action='store_true', + "--empty", + action="store_true", help="Create an empty migration.", ) parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '-n', '--name', + "-n", + "--name", help="Use this name for migration file(s).", ) parser.add_argument( - '--no-header', action='store_false', dest='include_header', - help='Do not add header comments to new migration file(s).', + "--no-header", + action="store_false", + dest="include_header", + help="Do not add header comments to new migration file(s).", ) parser.add_argument( - '--check', action='store_true', dest='check_changes', - help='Exit with a non-zero status if model changes are missing migrations.', + "--check", + action="store_true", + dest="check_changes", + help="Exit with a non-zero status if model changes are missing migrations.", ) parser.add_argument( - '--scriptable', action='store_true', dest='scriptable', + "--scriptable", + action="store_true", + dest="scriptable", help=( - 'Divert log output and input prompts to stderr, writing only ' - 'paths of generated migration files to stdout.' + "Divert log output and input prompts to stderr, writing only " + "paths of generated migration files to stdout." ), ) @@ -74,17 +88,17 @@ class Command(BaseCommand): @no_translations def handle(self, *app_labels, **options): - self.verbosity = options['verbosity'] - self.interactive = options['interactive'] - self.dry_run = options['dry_run'] - self.merge = options['merge'] - self.empty = options['empty'] - self.migration_name = options['name'] + self.verbosity = options["verbosity"] + self.interactive = options["interactive"] + self.dry_run = options["dry_run"] + self.merge = options["merge"] + self.empty = options["empty"] + self.migration_name = options["name"] if self.migration_name and not self.migration_name.isidentifier(): - raise CommandError('The migration name must be a valid Python identifier.') - self.include_header = options['include_header'] - check_changes = options['check_changes'] - self.scriptable = options['scriptable'] + raise CommandError("The migration name must be a valid Python identifier.") + self.include_header = options["include_header"] + check_changes = options["check_changes"] + self.scriptable = options["scriptable"] # If logs and prompts are diverted to stderr, remove the ERROR style. if self.scriptable: self.stderr.style_func = None @@ -108,22 +122,25 @@ class Command(BaseCommand): # Raise an error if any migrations are applied before their dependencies. consistency_check_labels = {config.label for config in apps.get_app_configs()} # Non-default databases are only checked if database routers used. - aliases_to_check = connections if settings.DATABASE_ROUTERS else [DEFAULT_DB_ALIAS] + aliases_to_check = ( + connections if settings.DATABASE_ROUTERS else [DEFAULT_DB_ALIAS] + ) for alias in sorted(aliases_to_check): connection = connections[alias] - if (connection.settings_dict['ENGINE'] != 'django.db.backends.dummy' and any( - # At least one model must be migrated to the database. - router.allow_migrate(connection.alias, app_label, model_name=model._meta.object_name) - for app_label in consistency_check_labels - for model in apps.get_app_config(app_label).get_models() - )): + if connection.settings_dict["ENGINE"] != "django.db.backends.dummy" and any( + # At least one model must be migrated to the database. + router.allow_migrate( + connection.alias, app_label, model_name=model._meta.object_name + ) + for app_label in consistency_check_labels + for model in apps.get_app_config(app_label).get_models() + ): try: loader.check_consistent_history(connection) except OperationalError as error: warnings.warn( "Got an error checking a consistent migration history " - "performed for database connection '%s': %s" - % (alias, error), + "performed for database connection '%s': %s" % (alias, error), RuntimeWarning, ) # Before anything else, see if there's conflicting apps and drop out @@ -133,14 +150,14 @@ class Command(BaseCommand): # If app_labels is specified, filter out conflicting migrations for unspecified apps if app_labels: conflicts = { - app_label: conflict for app_label, conflict in conflicts.items() + app_label: conflict + for app_label, conflict in conflicts.items() if app_label in app_labels } if conflicts and not self.merge: name_str = "; ".join( - "%s in %s" % (", ".join(names), app) - for app, names in conflicts.items() + "%s in %s" % (", ".join(names), app) for app, names in conflicts.items() ) raise CommandError( "Conflicting migrations detected; multiple leaf nodes in the " @@ -150,7 +167,7 @@ class Command(BaseCommand): # If they want to merge and there's nothing to merge, then politely exit if self.merge and not conflicts: - self.log('No conflicts detected to merge.') + self.log("No conflicts detected to merge.") return # If they want to merge and there is something to merge, then @@ -181,12 +198,11 @@ class Command(BaseCommand): # If they want to make an empty migration, make one for each app if self.empty: if not app_labels: - raise CommandError("You must supply at least one app label when using --empty.") + raise CommandError( + "You must supply at least one app label when using --empty." + ) # Make a fake changes() result we can pass to arrange_for_graph - changes = { - app: [Migration("custom", app)] - for app in app_labels - } + changes = {app: [Migration("custom", app)] for app in app_labels} changes = autodetector.arrange_for_graph( changes=changes, graph=loader.graph, @@ -210,9 +226,12 @@ class Command(BaseCommand): if len(app_labels) == 1: self.log("No changes detected in app '%s'" % app_labels.pop()) else: - self.log("No changes detected in apps '%s'" % ("', '".join(app_labels))) + self.log( + "No changes detected in apps '%s'" + % ("', '".join(app_labels)) + ) else: - self.log('No changes detected') + self.log("No changes detected") else: self.write_migration_files(changes) if check_changes: @@ -236,11 +255,11 @@ class Command(BaseCommand): migration_string = os.path.relpath(writer.path) except ValueError: migration_string = writer.path - if migration_string.startswith('..'): + if migration_string.startswith(".."): migration_string = writer.path - self.log(' %s\n' % self.style.MIGRATE_LABEL(migration_string)) + self.log(" %s\n" % self.style.MIGRATE_LABEL(migration_string)) for operation in migration.operations: - self.log(' - %s' % operation.describe()) + self.log(" - %s" % operation.describe()) if self.scriptable: self.stdout.write(migration_string) if not self.dry_run: @@ -254,15 +273,17 @@ class Command(BaseCommand): # We just do this once per app directory_created[app_label] = True migration_string = writer.as_string() - with open(writer.path, "w", encoding='utf-8') as fh: + with open(writer.path, "w", encoding="utf-8") as fh: fh.write(migration_string) elif self.verbosity == 3: # Alternatively, makemigrations --dry-run --verbosity 3 # will log the migrations rather than saving the file to # the disk. - self.log(self.style.MIGRATE_HEADING( - "Full migrations file '%s':" % writer.filename - )) + self.log( + self.style.MIGRATE_HEADING( + "Full migrations file '%s':" % writer.filename + ) + ) self.log(writer.as_string()) def handle_merge(self, loader, conflicts): @@ -273,7 +294,7 @@ class Command(BaseCommand): if self.interactive: questioner = InteractiveMigrationQuestioner(prompt_output=self.log_output) else: - questioner = MigrationQuestioner(defaults={'ask_merge': True}) + questioner = MigrationQuestioner(defaults={"ask_merge": True}) for app_label, migration_names in conflicts.items(): # Grab out the migrations in question, and work out their @@ -282,7 +303,8 @@ class Command(BaseCommand): for migration_name in migration_names: migration = loader.get_migration(app_label, migration_name) migration.ancestry = [ - mig for mig in loader.graph.forwards_plan((app_label, migration_name)) + mig + for mig in loader.graph.forwards_plan((app_label, migration_name)) if mig[0] == migration.app_label ] merge_migrations.append(migration) @@ -291,25 +313,33 @@ class Command(BaseCommand): return all(item == seq[0] for item in seq[1:]) merge_migrations_generations = zip(*(m.ancestry for m in merge_migrations)) - common_ancestor_count = sum(1 for common_ancestor_generation - in takewhile(all_items_equal, merge_migrations_generations)) + common_ancestor_count = sum( + 1 + for common_ancestor_generation in takewhile( + all_items_equal, merge_migrations_generations + ) + ) if not common_ancestor_count: - raise ValueError("Could not find common ancestor of %s" % migration_names) + raise ValueError( + "Could not find common ancestor of %s" % migration_names + ) # Now work out the operations along each divergent branch for migration in merge_migrations: migration.branch = migration.ancestry[common_ancestor_count:] - migrations_ops = (loader.get_migration(node_app, node_name).operations - for node_app, node_name in migration.branch) + migrations_ops = ( + loader.get_migration(node_app, node_name).operations + for node_app, node_name in migration.branch + ) migration.merged_operations = sum(migrations_ops, []) # In future, this could use some of the Optimizer code # (can_optimize_through) to automatically see if they're # mergeable. For now, we always just prompt the user. if self.verbosity > 0: - self.log(self.style.MIGRATE_HEADING('Merging %s' % app_label)) + self.log(self.style.MIGRATE_HEADING("Merging %s" % app_label)) for migration in merge_migrations: - self.log(self.style.MIGRATE_LABEL(' Branch %s' % migration.name)) + self.log(self.style.MIGRATE_LABEL(" Branch %s" % migration.name)) for operation in migration.merged_operations: - self.log(' - %s' % operation.describe()) + self.log(" - %s" % operation.describe()) if questioner.ask_merge(app_label): # If they still want to merge it, then write out an empty # file depending on the migrations needing merging. @@ -321,36 +351,47 @@ class Command(BaseCommand): biggest_number = max(x for x in numbers if x is not None) except ValueError: biggest_number = 1 - subclass = type("Migration", (Migration,), { - "dependencies": [(app_label, migration.name) for migration in merge_migrations], - }) - parts = ['%04i' % (biggest_number + 1)] + subclass = type( + "Migration", + (Migration,), + { + "dependencies": [ + (app_label, migration.name) + for migration in merge_migrations + ], + }, + ) + parts = ["%04i" % (biggest_number + 1)] if self.migration_name: parts.append(self.migration_name) else: - parts.append('merge') - leaf_names = '_'.join(sorted(migration.name for migration in merge_migrations)) + parts.append("merge") + leaf_names = "_".join( + sorted(migration.name for migration in merge_migrations) + ) if len(leaf_names) > 47: parts.append(get_migration_name_timestamp()) else: parts.append(leaf_names) - migration_name = '_'.join(parts) + migration_name = "_".join(parts) new_migration = subclass(migration_name, app_label) writer = MigrationWriter(new_migration, self.include_header) if not self.dry_run: # Write the merge migrations file to the disk - with open(writer.path, "w", encoding='utf-8') as fh: + with open(writer.path, "w", encoding="utf-8") as fh: fh.write(writer.as_string()) if self.verbosity > 0: - self.log('\nCreated new merge migration %s' % writer.path) + self.log("\nCreated new merge migration %s" % writer.path) if self.scriptable: self.stdout.write(writer.path) elif self.verbosity == 3: # Alternatively, makemigrations --merge --dry-run --verbosity 3 # will log the merge migrations rather than saving the file # to the disk. - self.log(self.style.MIGRATE_HEADING( - "Full merge migrations file '%s':" % writer.filename - )) + self.log( + self.style.MIGRATE_HEADING( + "Full merge migrations file '%s':" % writer.filename + ) + ) self.log(writer.as_string()) diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py index a4ad1f3e20..59fd1f0d55 100644 --- a/django/core/management/commands/migrate.py +++ b/django/core/management/commands/migrate.py @@ -3,12 +3,8 @@ import time from importlib import import_module from django.apps import apps -from django.core.management.base import ( - BaseCommand, CommandError, no_translations, -) -from django.core.management.sql import ( - emit_post_migrate_signal, emit_pre_migrate_signal, -) +from django.core.management.base import BaseCommand, CommandError, no_translations +from django.core.management.sql import emit_post_migrate_signal, emit_pre_migrate_signal from django.db import DEFAULT_DB_ALIAS, connections, router from django.db.migrations.autodetector import MigrationAutodetector from django.db.migrations.executor import MigrationExecutor @@ -19,73 +15,89 @@ from django.utils.text import Truncator class Command(BaseCommand): - help = "Updates database schema. Manages both apps with migrations and those without." + help = ( + "Updates database schema. Manages both apps with migrations and those without." + ) requires_system_checks = [] def add_arguments(self, parser): parser.add_argument( - '--skip-checks', action='store_true', - help='Skip system checks.', + "--skip-checks", + action="store_true", + help="Skip system checks.", ) parser.add_argument( - 'app_label', nargs='?', - help='App label of an application to synchronize the state.', + "app_label", + nargs="?", + help="App label of an application to synchronize the state.", ) parser.add_argument( - 'migration_name', nargs='?', - help='Database state will be brought to the state after that ' - 'migration. Use the name "zero" to unapply all migrations.', + "migration_name", + nargs="?", + help="Database state will be brought to the state after that " + 'migration. Use the name "zero" to unapply all migrations.', ) parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '--database', + "--database", default=DEFAULT_DB_ALIAS, help='Nominates a database to synchronize. Defaults to the "default" database.', ) parser.add_argument( - '--fake', action='store_true', - help='Mark migrations as run without actually running them.', + "--fake", + action="store_true", + help="Mark migrations as run without actually running them.", ) parser.add_argument( - '--fake-initial', action='store_true', - help='Detect if tables already exist and fake-apply initial migrations if so. Make sure ' - 'that the current database schema matches your initial migration before using this ' - 'flag. Django will only check for an existing table name.', + "--fake-initial", + action="store_true", + help="Detect if tables already exist and fake-apply initial migrations if so. Make sure " + "that the current database schema matches your initial migration before using this " + "flag. Django will only check for an existing table name.", ) parser.add_argument( - '--plan', action='store_true', - help='Shows a list of the migration actions that will be performed.', + "--plan", + action="store_true", + help="Shows a list of the migration actions that will be performed.", ) parser.add_argument( - '--run-syncdb', action='store_true', - help='Creates tables for apps without migrations.', + "--run-syncdb", + action="store_true", + help="Creates tables for apps without migrations.", ) parser.add_argument( - '--check', action='store_true', dest='check_unapplied', - help='Exits with a non-zero status if unapplied migrations exist.', + "--check", + action="store_true", + dest="check_unapplied", + help="Exits with a non-zero status if unapplied migrations exist.", ) parser.add_argument( - '--prune', action='store_true', dest='prune', - help='Delete nonexistent migrations from the django_migrations table.', + "--prune", + action="store_true", + dest="prune", + help="Delete nonexistent migrations from the django_migrations table.", ) @no_translations def handle(self, *args, **options): - database = options['database'] - if not options['skip_checks']: + database = options["database"] + if not options["skip_checks"]: self.check(databases=[database]) - self.verbosity = options['verbosity'] - self.interactive = options['interactive'] + self.verbosity = options["verbosity"] + self.interactive = options["interactive"] # Import the 'management' module within each installed app, to register # dispatcher events. for app_config in apps.get_app_configs(): if module_has_submodule(app_config.module, "management"): - import_module('.management', app_config.name) + import_module(".management", app_config.name) # Get the database we're operating from connection = connections[database] @@ -103,8 +115,7 @@ class Command(BaseCommand): conflicts = executor.loader.detect_conflicts() if conflicts: name_str = "; ".join( - "%s in %s" % (", ".join(names), app) - for app, names in conflicts.items() + "%s in %s" % (", ".join(names), app) for app, names in conflicts.items() ) raise CommandError( "Conflicting migrations detected; multiple leaf nodes in the " @@ -113,163 +124,185 @@ class Command(BaseCommand): ) # If they supplied command line arguments, work out what they mean. - run_syncdb = options['run_syncdb'] + run_syncdb = options["run_syncdb"] target_app_labels_only = True - if options['app_label']: + if options["app_label"]: # Validate app_label. - app_label = options['app_label'] + app_label = options["app_label"] try: apps.get_app_config(app_label) except LookupError as err: raise CommandError(str(err)) if run_syncdb: if app_label in executor.loader.migrated_apps: - raise CommandError("Can't use run_syncdb with app '%s' as it has migrations." % app_label) + raise CommandError( + "Can't use run_syncdb with app '%s' as it has migrations." + % app_label + ) elif app_label not in executor.loader.migrated_apps: raise CommandError("App '%s' does not have migrations." % app_label) - if options['app_label'] and options['migration_name']: - migration_name = options['migration_name'] + if options["app_label"] and options["migration_name"]: + migration_name = options["migration_name"] if migration_name == "zero": targets = [(app_label, None)] else: try: - migration = executor.loader.get_migration_by_prefix(app_label, migration_name) + migration = executor.loader.get_migration_by_prefix( + app_label, migration_name + ) except AmbiguityError: raise CommandError( "More than one migration matches '%s' in app '%s'. " - "Please be more specific." % - (migration_name, app_label) + "Please be more specific." % (migration_name, app_label) ) except KeyError: - raise CommandError("Cannot find a migration matching '%s' from app '%s'." % ( - migration_name, app_label)) + raise CommandError( + "Cannot find a migration matching '%s' from app '%s'." + % (migration_name, app_label) + ) target = (app_label, migration.name) # Partially applied squashed migrations are not included in the # graph, use the last replacement instead. if ( - target not in executor.loader.graph.nodes and - target in executor.loader.replacements + target not in executor.loader.graph.nodes + and target in executor.loader.replacements ): incomplete_migration = executor.loader.replacements[target] target = incomplete_migration.replaces[-1] targets = [target] target_app_labels_only = False - elif options['app_label']: - targets = [key for key in executor.loader.graph.leaf_nodes() if key[0] == app_label] + elif options["app_label"]: + targets = [ + key for key in executor.loader.graph.leaf_nodes() if key[0] == app_label + ] else: targets = executor.loader.graph.leaf_nodes() - if options['prune']: - if not options['app_label']: + if options["prune"]: + if not options["app_label"]: raise CommandError( - 'Migrations can be pruned only when an app is specified.' + "Migrations can be pruned only when an app is specified." ) if self.verbosity > 0: - self.stdout.write('Pruning migrations:', self.style.MIGRATE_HEADING) - to_prune = set(executor.loader.applied_migrations) - set(executor.loader.disk_migrations) + self.stdout.write("Pruning migrations:", self.style.MIGRATE_HEADING) + to_prune = set(executor.loader.applied_migrations) - set( + executor.loader.disk_migrations + ) squashed_migrations_with_deleted_replaced_migrations = [ migration_key for migration_key, migration_obj in executor.loader.replacements.items() if any(replaced in to_prune for replaced in migration_obj.replaces) ] if squashed_migrations_with_deleted_replaced_migrations: - self.stdout.write(self.style.NOTICE( - " Cannot use --prune because the following squashed " - "migrations have their 'replaces' attributes and may not " - "be recorded as applied:" - )) + self.stdout.write( + self.style.NOTICE( + " Cannot use --prune because the following squashed " + "migrations have their 'replaces' attributes and may not " + "be recorded as applied:" + ) + ) for migration in squashed_migrations_with_deleted_replaced_migrations: app, name = migration - self.stdout.write(f' {app}.{name}') - self.stdout.write(self.style.NOTICE( - " Re-run 'manage.py migrate' if they are not marked as " - "applied, and remove 'replaces' attributes in their " - "Migration classes." - )) + self.stdout.write(f" {app}.{name}") + self.stdout.write( + self.style.NOTICE( + " Re-run 'manage.py migrate' if they are not marked as " + "applied, and remove 'replaces' attributes in their " + "Migration classes." + ) + ) else: to_prune = sorted( - migration - for migration in to_prune - if migration[0] == app_label + migration for migration in to_prune if migration[0] == app_label ) if to_prune: for migration in to_prune: app, name = migration if self.verbosity > 0: - self.stdout.write(self.style.MIGRATE_LABEL( - f' Pruning {app}.{name}' - ), ending='') + self.stdout.write( + self.style.MIGRATE_LABEL(f" Pruning {app}.{name}"), + ending="", + ) executor.recorder.record_unapplied(app, name) if self.verbosity > 0: - self.stdout.write(self.style.SUCCESS(' OK')) + self.stdout.write(self.style.SUCCESS(" OK")) elif self.verbosity > 0: - self.stdout.write(' No migrations to prune.') + self.stdout.write(" No migrations to prune.") plan = executor.migration_plan(targets) - exit_dry = plan and options['check_unapplied'] + exit_dry = plan and options["check_unapplied"] - if options['plan']: - self.stdout.write('Planned operations:', self.style.MIGRATE_LABEL) + if options["plan"]: + self.stdout.write("Planned operations:", self.style.MIGRATE_LABEL) if not plan: - self.stdout.write(' No planned migration operations.') + self.stdout.write(" No planned migration operations.") for migration, backwards in plan: self.stdout.write(str(migration), self.style.MIGRATE_HEADING) for operation in migration.operations: message, is_error = self.describe_operation(operation, backwards) style = self.style.WARNING if is_error else None - self.stdout.write(' ' + message, style) + self.stdout.write(" " + message, style) if exit_dry: sys.exit(1) return if exit_dry: sys.exit(1) - if options['prune']: + if options["prune"]: return # At this point, ignore run_syncdb if there aren't any apps to sync. - run_syncdb = options['run_syncdb'] and executor.loader.unmigrated_apps + run_syncdb = options["run_syncdb"] and executor.loader.unmigrated_apps # Print some useful info if self.verbosity >= 1: self.stdout.write(self.style.MIGRATE_HEADING("Operations to perform:")) if run_syncdb: - if options['app_label']: + if options["app_label"]: self.stdout.write( - self.style.MIGRATE_LABEL(" Synchronize unmigrated app: %s" % app_label) + self.style.MIGRATE_LABEL( + " Synchronize unmigrated app: %s" % app_label + ) ) else: self.stdout.write( - self.style.MIGRATE_LABEL(" Synchronize unmigrated apps: ") + - (", ".join(sorted(executor.loader.unmigrated_apps))) + self.style.MIGRATE_LABEL(" Synchronize unmigrated apps: ") + + (", ".join(sorted(executor.loader.unmigrated_apps))) ) if target_app_labels_only: self.stdout.write( - self.style.MIGRATE_LABEL(" Apply all migrations: ") + - (", ".join(sorted({a for a, n in targets})) or "(none)") + self.style.MIGRATE_LABEL(" Apply all migrations: ") + + (", ".join(sorted({a for a, n in targets})) or "(none)") ) else: if targets[0][1] is None: self.stdout.write( - self.style.MIGRATE_LABEL(' Unapply all migrations: ') + - str(targets[0][0]) + self.style.MIGRATE_LABEL(" Unapply all migrations: ") + + str(targets[0][0]) ) else: - self.stdout.write(self.style.MIGRATE_LABEL( - " Target specific migration: ") + "%s, from %s" - % (targets[0][1], targets[0][0]) + self.stdout.write( + self.style.MIGRATE_LABEL(" Target specific migration: ") + + "%s, from %s" % (targets[0][1], targets[0][0]) ) pre_migrate_state = executor._create_project_state(with_applied_migrations=True) pre_migrate_apps = pre_migrate_state.apps emit_pre_migrate_signal( - self.verbosity, self.interactive, connection.alias, stdout=self.stdout, apps=pre_migrate_apps, plan=plan, + self.verbosity, + self.interactive, + connection.alias, + stdout=self.stdout, + apps=pre_migrate_apps, + plan=plan, ) # Run the syncdb phase. if run_syncdb: if self.verbosity >= 1: - self.stdout.write(self.style.MIGRATE_HEADING("Synchronizing apps without migrations:")) - if options['app_label']: + self.stdout.write( + self.style.MIGRATE_HEADING("Synchronizing apps without migrations:") + ) + if options["app_label"]: self.sync_apps(connection, [app_label]) else: self.sync_apps(connection, executor.loader.unmigrated_apps) @@ -287,23 +320,30 @@ class Command(BaseCommand): ) changes = autodetector.changes(graph=executor.loader.graph) if changes: - self.stdout.write(self.style.NOTICE( - " Your models in app(s): %s have changes that are not " - "yet reflected in a migration, and so won't be " - "applied." % ", ".join(repr(app) for app in sorted(changes)) - )) - self.stdout.write(self.style.NOTICE( - " Run 'manage.py makemigrations' to make new " - "migrations, and then re-run 'manage.py migrate' to " - "apply them." - )) + self.stdout.write( + self.style.NOTICE( + " Your models in app(s): %s have changes that are not " + "yet reflected in a migration, and so won't be " + "applied." % ", ".join(repr(app) for app in sorted(changes)) + ) + ) + self.stdout.write( + self.style.NOTICE( + " Run 'manage.py makemigrations' to make new " + "migrations, and then re-run 'manage.py migrate' to " + "apply them." + ) + ) fake = False fake_initial = False else: - fake = options['fake'] - fake_initial = options['fake_initial'] + fake = options["fake"] + fake_initial = options["fake_initial"] post_migrate_state = executor.migrate( - targets, plan=plan, state=pre_migrate_state.clone(), fake=fake, + targets, + plan=plan, + state=pre_migrate_state.clone(), + fake=fake, fake_initial=fake_initial, ) # post_migrate signals have access to all models. Ensure that all models @@ -320,14 +360,19 @@ class Command(BaseCommand): model_key = model_state.app_label, model_state.name_lower model_keys.append(model_key) post_migrate_apps.unregister_model(*model_key) - post_migrate_apps.render_multiple([ - ModelState.from_model(apps.get_model(*model)) for model in model_keys - ]) + post_migrate_apps.render_multiple( + [ModelState.from_model(apps.get_model(*model)) for model in model_keys] + ) # Send the post_migrate signal, so individual apps can do whatever they need # to do at this point. emit_post_migrate_signal( - self.verbosity, self.interactive, connection.alias, stdout=self.stdout, apps=post_migrate_apps, plan=plan, + self.verbosity, + self.interactive, + connection.alias, + stdout=self.stdout, + apps=post_migrate_apps, + plan=plan, ) def migration_progress_callback(self, action, migration=None, fake=False): @@ -339,7 +384,9 @@ class Command(BaseCommand): self.stdout.write(" Applying %s..." % migration, ending="") self.stdout.flush() elif action == "apply_success": - elapsed = " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + elapsed = ( + " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + ) if fake: self.stdout.write(self.style.SUCCESS(" FAKED" + elapsed)) else: @@ -350,7 +397,9 @@ class Command(BaseCommand): self.stdout.write(" Unapplying %s..." % migration, ending="") self.stdout.flush() elif action == "unapply_success": - elapsed = " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + elapsed = ( + " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + ) if fake: self.stdout.write(self.style.SUCCESS(" FAKED" + elapsed)) else: @@ -361,7 +410,9 @@ class Command(BaseCommand): self.stdout.write(" Rendering model states...", ending="") self.stdout.flush() elif action == "render_success": - elapsed = " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + elapsed = ( + " (%.3fs)" % (time.monotonic() - self.start) if compute_time else "" + ) self.stdout.write(self.style.SUCCESS(" DONE" + elapsed)) def sync_apps(self, connection, app_labels): @@ -373,7 +424,9 @@ class Command(BaseCommand): all_models = [ ( app_config.label, - router.get_migratable_models(app_config, connection.alias, include_auto_created=False), + router.get_migratable_models( + app_config, connection.alias, include_auto_created=False + ), ) for app_config in apps.get_app_configs() if app_config.models_module is not None and app_config.label in app_labels @@ -383,8 +436,11 @@ class Command(BaseCommand): opts = model._meta converter = connection.introspection.identifier_converter return not ( - (converter(opts.db_table) in tables) or - (opts.auto_created and converter(opts.auto_created._meta.db_table) in tables) + (converter(opts.db_table) in tables) + or ( + opts.auto_created + and converter(opts.auto_created._meta.db_table) in tables + ) ) manifest = { @@ -394,7 +450,7 @@ class Command(BaseCommand): # Create the tables for each model if self.verbosity >= 1: - self.stdout.write(' Creating tables...') + self.stdout.write(" Creating tables...") with connection.schema_editor() as editor: for app_name, model_list in manifest.items(): for model in model_list: @@ -403,36 +459,39 @@ class Command(BaseCommand): continue if self.verbosity >= 3: self.stdout.write( - ' Processing %s.%s model' % (app_name, model._meta.object_name) + " Processing %s.%s model" + % (app_name, model._meta.object_name) ) if self.verbosity >= 1: - self.stdout.write(' Creating table %s' % model._meta.db_table) + self.stdout.write( + " Creating table %s" % model._meta.db_table + ) editor.create_model(model) # Deferred SQL is executed when exiting the editor's context. if self.verbosity >= 1: - self.stdout.write(' Running deferred SQL...') + self.stdout.write(" Running deferred SQL...") @staticmethod def describe_operation(operation, backwards): """Return a string that describes a migration operation for --plan.""" - prefix = '' + prefix = "" is_error = False - if hasattr(operation, 'code'): + if hasattr(operation, "code"): code = operation.reverse_code if backwards else operation.code - action = (code.__doc__ or '') if code else None - elif hasattr(operation, 'sql'): + action = (code.__doc__ or "") if code else None + elif hasattr(operation, "sql"): action = operation.reverse_sql if backwards else operation.sql else: - action = '' + action = "" if backwards: - prefix = 'Undo ' + prefix = "Undo " if action is not None: - action = str(action).replace('\n', '') + action = str(action).replace("\n", "") elif backwards: - action = 'IRREVERSIBLE' + action = "IRREVERSIBLE" is_error = True if action: - action = ' -> ' + action + action = " -> " + action truncated = Truncator(action) return prefix + operation.describe() + truncated.chars(40), is_error diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py index 473fde0de0..3c39f57e4d 100644 --- a/django/core/management/commands/runserver.py +++ b/django/core/management/commands/runserver.py @@ -7,18 +7,19 @@ from datetime import datetime from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from django.core.servers.basehttp import ( - WSGIServer, get_internal_wsgi_application, run, -) +from django.core.servers.basehttp import WSGIServer, get_internal_wsgi_application, run from django.utils import autoreload from django.utils.regex_helper import _lazy_re_compile -naiveip_re = _lazy_re_compile(r"""^(?: +naiveip_re = _lazy_re_compile( + r"""^(?: (?P<addr> (?P<ipv4>\d{1,3}(?:\.\d{1,3}){3}) | # IPv4 address (?P<ipv6>\[[a-fA-F0-9:]+\]) | # IPv6 address (?P<fqdn>[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*) # FQDN -):)?(?P<port>\d+)$""", re.X) +):)?(?P<port>\d+)$""", + re.X, +) class Command(BaseCommand): @@ -26,39 +27,46 @@ class Command(BaseCommand): # Validation is called explicitly each time the server is reloaded. requires_system_checks = [] - stealth_options = ('shutdown_message',) - suppressed_base_arguments = {'--verbosity', '--traceback'} + stealth_options = ("shutdown_message",) + suppressed_base_arguments = {"--verbosity", "--traceback"} - default_addr = '127.0.0.1' - default_addr_ipv6 = '::1' - default_port = '8000' - protocol = 'http' + default_addr = "127.0.0.1" + default_addr_ipv6 = "::1" + default_port = "8000" + protocol = "http" server_cls = WSGIServer def add_arguments(self, parser): parser.add_argument( - 'addrport', nargs='?', - help='Optional port number, or ipaddr:port' + "addrport", nargs="?", help="Optional port number, or ipaddr:port" ) parser.add_argument( - '--ipv6', '-6', action='store_true', dest='use_ipv6', - help='Tells Django to use an IPv6 address.', + "--ipv6", + "-6", + action="store_true", + dest="use_ipv6", + help="Tells Django to use an IPv6 address.", ) parser.add_argument( - '--nothreading', action='store_false', dest='use_threading', - help='Tells Django to NOT use threading.', + "--nothreading", + action="store_false", + dest="use_threading", + help="Tells Django to NOT use threading.", ) parser.add_argument( - '--noreload', action='store_false', dest='use_reloader', - help='Tells Django to NOT use the auto-reloader.', + "--noreload", + action="store_false", + dest="use_reloader", + help="Tells Django to NOT use the auto-reloader.", ) parser.add_argument( - '--skip-checks', action='store_true', - help='Skip system checks.', + "--skip-checks", + action="store_true", + help="Skip system checks.", ) def execute(self, *args, **options): - if options['no_color']: + if options["no_color"]: # We rely on the environment because it's currently the only # way to reach WSGIRequestHandler. This seems an acceptable # compromise considering `runserver` runs indefinitely. @@ -71,20 +79,22 @@ class Command(BaseCommand): def handle(self, *args, **options): if not settings.DEBUG and not settings.ALLOWED_HOSTS: - raise CommandError('You must set settings.ALLOWED_HOSTS if DEBUG is False.') + raise CommandError("You must set settings.ALLOWED_HOSTS if DEBUG is False.") - self.use_ipv6 = options['use_ipv6'] + self.use_ipv6 = options["use_ipv6"] if self.use_ipv6 and not socket.has_ipv6: - raise CommandError('Your Python does not support IPv6.') + raise CommandError("Your Python does not support IPv6.") self._raw_ipv6 = False - if not options['addrport']: - self.addr = '' + if not options["addrport"]: + self.addr = "" self.port = self.default_port else: - m = re.match(naiveip_re, options['addrport']) + m = re.match(naiveip_re, options["addrport"]) if m is None: - raise CommandError('"%s" is not a valid port number ' - 'or address:port pair.' % options['addrport']) + raise CommandError( + '"%s" is not a valid port number ' + "or address:port pair." % options["addrport"] + ) self.addr, _ipv4, _ipv6, _fqdn, self.port = m.groups() if not self.port.isdigit(): raise CommandError("%r is not a valid port number." % self.port) @@ -102,7 +112,7 @@ class Command(BaseCommand): def run(self, **options): """Run the server, using the autoreloader if needed.""" - use_reloader = options['use_reloader'] + use_reloader = options["use_reloader"] if use_reloader: autoreload.run_with_reloader(self.inner_run, **options) @@ -114,36 +124,45 @@ class Command(BaseCommand): # to be raised in the child process, raise it now. autoreload.raise_last_exception() - threading = options['use_threading'] + threading = options["use_threading"] # 'shutdown_message' is a stealth option. - shutdown_message = options.get('shutdown_message', '') - quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C' + shutdown_message = options.get("shutdown_message", "") + quit_command = "CTRL-BREAK" if sys.platform == "win32" else "CONTROL-C" - if not options['skip_checks']: - self.stdout.write('Performing system checks...\n\n') + if not options["skip_checks"]: + self.stdout.write("Performing system checks...\n\n") self.check(display_num_errors=True) # Need to check migrations here, so can't use the # requires_migrations_check attribute. self.check_migrations() - now = datetime.now().strftime('%B %d, %Y - %X') + now = datetime.now().strftime("%B %d, %Y - %X") self.stdout.write(now) - self.stdout.write(( - "Django version %(version)s, using settings %(settings)r\n" - "Starting development server at %(protocol)s://%(addr)s:%(port)s/\n" - "Quit the server with %(quit_command)s." - ) % { - "version": self.get_version(), - "settings": settings.SETTINGS_MODULE, - "protocol": self.protocol, - "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, - "port": self.port, - "quit_command": quit_command, - }) + self.stdout.write( + ( + "Django version %(version)s, using settings %(settings)r\n" + "Starting development server at %(protocol)s://%(addr)s:%(port)s/\n" + "Quit the server with %(quit_command)s." + ) + % { + "version": self.get_version(), + "settings": settings.SETTINGS_MODULE, + "protocol": self.protocol, + "addr": "[%s]" % self.addr if self._raw_ipv6 else self.addr, + "port": self.port, + "quit_command": quit_command, + } + ) try: handler = self.get_handler(*args, **options) - run(self.addr, int(self.port), handler, - ipv6=self.use_ipv6, threading=threading, server_cls=self.server_cls) + run( + self.addr, + int(self.port), + handler, + ipv6=self.use_ipv6, + threading=threading, + server_cls=self.server_cls, + ) except OSError as e: # Use helpful error messages instead of ugly tracebacks. ERRORS = { diff --git a/django/core/management/commands/sendtestemail.py b/django/core/management/commands/sendtestemail.py index 9ed1e9600f..6a69849300 100644 --- a/django/core/management/commands/sendtestemail.py +++ b/django/core/management/commands/sendtestemail.py @@ -11,30 +11,33 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'email', nargs='*', - help='One or more email addresses to send a test email to.', + "email", + nargs="*", + help="One or more email addresses to send a test email to.", ) parser.add_argument( - '--managers', action='store_true', - help='Send a test email to the addresses specified in settings.MANAGERS.', + "--managers", + action="store_true", + help="Send a test email to the addresses specified in settings.MANAGERS.", ) parser.add_argument( - '--admins', action='store_true', - help='Send a test email to the addresses specified in settings.ADMINS.', + "--admins", + action="store_true", + help="Send a test email to the addresses specified in settings.ADMINS.", ) def handle(self, *args, **kwargs): - subject = 'Test email from %s on %s' % (socket.gethostname(), timezone.now()) + subject = "Test email from %s on %s" % (socket.gethostname(), timezone.now()) send_mail( subject=subject, - message="If you\'re reading this, it was successful.", + message="If you're reading this, it was successful.", from_email=None, - recipient_list=kwargs['email'], + recipient_list=kwargs["email"], ) - if kwargs['managers']: + if kwargs["managers"]: mail_managers(subject, "This email was sent to the site managers.") - if kwargs['admins']: + if kwargs["admins"]: mail_admins(subject, "This email was sent to the site admins.") diff --git a/django/core/management/commands/shell.py b/django/core/management/commands/shell.py index cbd4c86620..52ab27cb21 100644 --- a/django/core/management/commands/shell.py +++ b/django/core/management/commands/shell.py @@ -15,28 +15,34 @@ class Command(BaseCommand): ) requires_system_checks = [] - shells = ['ipython', 'bpython', 'python'] + shells = ["ipython", "bpython", "python"] def add_arguments(self, parser): parser.add_argument( - '--no-startup', action='store_true', - help='When using plain Python, ignore the PYTHONSTARTUP environment variable and ~/.pythonrc.py script.', + "--no-startup", + action="store_true", + help="When using plain Python, ignore the PYTHONSTARTUP environment variable and ~/.pythonrc.py script.", ) parser.add_argument( - '-i', '--interface', choices=self.shells, + "-i", + "--interface", + choices=self.shells, help='Specify an interactive interpreter interface. Available options: "ipython", "bpython", and "python"', ) parser.add_argument( - '-c', '--command', - help='Instead of opening an interactive shell, run a command as Django and exit.', + "-c", + "--command", + help="Instead of opening an interactive shell, run a command as Django and exit.", ) def ipython(self, options): from IPython import start_ipython + start_ipython(argv=[]) def bpython(self, options): import bpython + bpython.embed() def python(self, options): @@ -47,8 +53,10 @@ class Command(BaseCommand): # We want to honor both $PYTHONSTARTUP and .pythonrc.py, so follow system # conventions and get $PYTHONSTARTUP first then .pythonrc.py. - if not options['no_startup']: - for pythonrc in OrderedSet([os.environ.get("PYTHONSTARTUP"), os.path.expanduser('~/.pythonrc.py')]): + if not options["no_startup"]: + for pythonrc in OrderedSet( + [os.environ.get("PYTHONSTARTUP"), os.path.expanduser("~/.pythonrc.py")] + ): if not pythonrc: continue if not os.path.isfile(pythonrc): @@ -58,7 +66,7 @@ class Command(BaseCommand): # Match the behavior of the cpython shell where an error in # PYTHONSTARTUP prints an exception and continues. try: - exec(compile(pythonrc_code, pythonrc, 'exec'), imported_objects) + exec(compile(pythonrc_code, pythonrc, "exec"), imported_objects) except Exception: traceback.print_exc() @@ -78,7 +86,7 @@ class Command(BaseCommand): # Match the behavior of the cpython shell where an error in # sys.__interactivehook__ prints a warning and the exception # and continues. - print('Failed calling sys.__interactivehook__') + print("Failed calling sys.__interactivehook__") traceback.print_exc() # Set up tab completion for objects imported by $PYTHONSTARTUP or @@ -86,6 +94,7 @@ class Command(BaseCommand): try: import readline import rlcompleter + readline.set_completer(rlcompleter.Completer(imported_objects).complete) except ImportError: pass @@ -95,17 +104,23 @@ class Command(BaseCommand): def handle(self, **options): # Execute the command and exit. - if options['command']: - exec(options['command'], globals()) + if options["command"]: + exec(options["command"], globals()) return # Execute stdin if it has anything to read and exit. # Not supported on Windows due to select.select() limitations. - if sys.platform != 'win32' and not sys.stdin.isatty() and select.select([sys.stdin], [], [], 0)[0]: + if ( + sys.platform != "win32" + and not sys.stdin.isatty() + and select.select([sys.stdin], [], [], 0)[0] + ): exec(sys.stdin.read(), globals()) return - available_shells = [options['interface']] if options['interface'] else self.shells + available_shells = ( + [options["interface"]] if options["interface"] else self.shells + ) for shell in available_shells: try: diff --git a/django/core/management/commands/showmigrations.py b/django/core/management/commands/showmigrations.py index e3227457ce..1f3a64f9b1 100644 --- a/django/core/management/commands/showmigrations.py +++ b/django/core/management/commands/showmigrations.py @@ -12,48 +12,58 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'app_label', nargs='*', - help='App labels of applications to limit the output to.', + "app_label", + nargs="*", + help="App labels of applications to limit the output to.", ) parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help=( - 'Nominates a database to show migrations for. Defaults to the ' + "Nominates a database to show migrations for. Defaults to the " '"default" database.' ), ) formats = parser.add_mutually_exclusive_group() formats.add_argument( - '--list', '-l', action='store_const', dest='format', const='list', + "--list", + "-l", + action="store_const", + dest="format", + const="list", help=( - 'Shows a list of all migrations and which are applied. ' - 'With a verbosity level of 2 or above, the applied datetimes ' - 'will be included.' + "Shows a list of all migrations and which are applied. " + "With a verbosity level of 2 or above, the applied datetimes " + "will be included." ), ) formats.add_argument( - '--plan', '-p', action='store_const', dest='format', const='plan', + "--plan", + "-p", + action="store_const", + dest="format", + const="plan", help=( - 'Shows all migrations in the order they will be applied. ' - 'With a verbosity level of 2 or above all direct migration dependencies ' - 'and reverse dependencies (run_before) will be included.' - ) + "Shows all migrations in the order they will be applied. " + "With a verbosity level of 2 or above all direct migration dependencies " + "and reverse dependencies (run_before) will be included." + ), ) - parser.set_defaults(format='list') + parser.set_defaults(format="list") def handle(self, *args, **options): - self.verbosity = options['verbosity'] + self.verbosity = options["verbosity"] # Get the database we're operating from - db = options['database'] + db = options["database"] connection = connections[db] - if options['format'] == "plan": - return self.show_plan(connection, options['app_label']) + if options["format"] == "plan": + return self.show_plan(connection, options["app_label"]) else: - return self.show_list(connection, options['app_label']) + return self.show_list(connection, options["app_label"]) def _validate_app_names(self, loader, app_names): has_bad_names = False @@ -93,17 +103,26 @@ class Command(BaseCommand): # Give it a nice title if it's a squashed one title = plan_node[1] if graph.nodes[plan_node].replaces: - title += " (%s squashed migrations)" % len(graph.nodes[plan_node].replaces) + title += " (%s squashed migrations)" % len( + graph.nodes[plan_node].replaces + ) applied_migration = loader.applied_migrations.get(plan_node) # Mark it as applied/unapplied if applied_migration: if plan_node in recorded_migrations: - output = ' [X] %s' % title + output = " [X] %s" % title else: title += " Run 'manage.py migrate' to finish recording." - output = ' [-] %s' % title - if self.verbosity >= 2 and hasattr(applied_migration, 'applied'): - output += ' (applied at %s)' % applied_migration.applied.strftime('%Y-%m-%d %H:%M:%S') + output = " [-] %s" % title + if self.verbosity >= 2 and hasattr( + applied_migration, "applied" + ): + output += ( + " (applied at %s)" + % applied_migration.applied.strftime( + "%Y-%m-%d %H:%M:%S" + ) + ) self.stdout.write(output) else: self.stdout.write(" [ ] %s" % title) @@ -154,4 +173,4 @@ class Command(BaseCommand): else: self.stdout.write("[ ] %s.%s%s" % (node.key[0], node.key[1], deps)) if not plan: - self.stdout.write('(no migrations)', self.style.ERROR) + self.stdout.write("(no migrations)", self.style.ERROR) diff --git a/django/core/management/commands/sqlflush.py b/django/core/management/commands/sqlflush.py index 29782607bb..e6701349d6 100644 --- a/django/core/management/commands/sqlflush.py +++ b/django/core/management/commands/sqlflush.py @@ -14,12 +14,13 @@ class Command(BaseCommand): def add_arguments(self, parser): super().add_arguments(parser) parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database to print the SQL for. Defaults to the "default" database.', ) def handle(self, **options): - sql_statements = sql_flush(self.style, connections[options['database']]) - if not sql_statements and options['verbosity'] >= 1: - self.stderr.write('No tables found.') - return '\n'.join(sql_statements) + sql_statements = sql_flush(self.style, connections[options["database"]]) + if not sql_statements and options["verbosity"] >= 1: + self.stderr.write("No tables found.") + return "\n".join(sql_statements) diff --git a/django/core/management/commands/sqlmigrate.py b/django/core/management/commands/sqlmigrate.py index f687360fb4..880eb11d9b 100644 --- a/django/core/management/commands/sqlmigrate.py +++ b/django/core/management/commands/sqlmigrate.py @@ -10,34 +10,40 @@ class Command(BaseCommand): output_transaction = True def add_arguments(self, parser): - parser.add_argument('app_label', help='App label of the application containing the migration.') - parser.add_argument('migration_name', help='Migration name to print the SQL for.') parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "app_label", help="App label of the application containing the migration." + ) + parser.add_argument( + "migration_name", help="Migration name to print the SQL for." + ) + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database to create SQL for. Defaults to the "default" database.', ) parser.add_argument( - '--backwards', action='store_true', - help='Creates SQL to unapply the migration, rather than to apply it', + "--backwards", + action="store_true", + help="Creates SQL to unapply the migration, rather than to apply it", ) def execute(self, *args, **options): # sqlmigrate doesn't support coloring its output but we need to force # no_color=True so that the BEGIN/COMMIT statements added by # output_transaction don't get colored either. - options['no_color'] = True + options["no_color"] = True return super().execute(*args, **options) def handle(self, *args, **options): # Get the database we're operating from - connection = connections[options['database']] + connection = connections[options["database"]] # Load up a loader to get all the migration data, but don't replace # migrations. loader = MigrationLoader(connection, replace_migrations=False) # Resolve command-line arguments into a migration - app_label, migration_name = options['app_label'], options['migration_name'] + app_label, migration_name = options["app_label"], options["migration_name"] # Validate app_label try: apps.get_app_config(app_label) @@ -48,21 +54,27 @@ class Command(BaseCommand): try: migration = loader.get_migration_by_prefix(app_label, migration_name) except AmbiguityError: - raise CommandError("More than one migration matches '%s' in app '%s'. Please be more specific." % ( - migration_name, app_label)) + raise CommandError( + "More than one migration matches '%s' in app '%s'. Please be more specific." + % (migration_name, app_label) + ) except KeyError: - raise CommandError("Cannot find a migration matching '%s' from app '%s'. Is it in INSTALLED_APPS?" % ( - migration_name, app_label)) + raise CommandError( + "Cannot find a migration matching '%s' from app '%s'. Is it in INSTALLED_APPS?" + % (migration_name, app_label) + ) target = (app_label, migration.name) # Show begin/end around output for atomic migrations, if the database # supports transactional DDL. - self.output_transaction = migration.atomic and connection.features.can_rollback_ddl + self.output_transaction = ( + migration.atomic and connection.features.can_rollback_ddl + ) # Make a plan that represents just the requested migrations and show SQL # for it - plan = [(loader.graph.nodes[target], options['backwards'])] + plan = [(loader.graph.nodes[target], options["backwards"])] sql_statements = loader.collect_sql(plan) - if not sql_statements and options['verbosity'] >= 1: - self.stderr.write('No operations found.') - return '\n'.join(sql_statements) + if not sql_statements and options["verbosity"] >= 1: + self.stderr.write("No operations found.") + return "\n".join(sql_statements) diff --git a/django/core/management/commands/sqlsequencereset.py b/django/core/management/commands/sqlsequencereset.py index 1d74ed9f55..454a2ab10c 100644 --- a/django/core/management/commands/sqlsequencereset.py +++ b/django/core/management/commands/sqlsequencereset.py @@ -3,23 +3,26 @@ from django.db import DEFAULT_DB_ALIAS, connections class Command(AppCommand): - help = 'Prints the SQL statements for resetting sequences for the given app name(s).' + help = ( + "Prints the SQL statements for resetting sequences for the given app name(s)." + ) output_transaction = True def add_arguments(self, parser): super().add_arguments(parser) parser.add_argument( - '--database', default=DEFAULT_DB_ALIAS, + "--database", + default=DEFAULT_DB_ALIAS, help='Nominates a database to print the SQL for. Defaults to the "default" database.', ) def handle_app_config(self, app_config, **options): if app_config.models_module is None: return - connection = connections[options['database']] + connection = connections[options["database"]] models = app_config.get_models(include_auto_created=True) statements = connection.ops.sequence_reset_sql(self.style, models) - if not statements and options['verbosity'] >= 1: - self.stderr.write('No sequences found.') - return '\n'.join(statements) + if not statements and options["verbosity"] >= 1: + self.stderr.write("No sequences found.") + return "\n".join(statements) diff --git a/django/core/management/commands/squashmigrations.py b/django/core/management/commands/squashmigrations.py index 80fdc0cbc1..1592e792f8 100644 --- a/django/core/management/commands/squashmigrations.py +++ b/django/core/management/commands/squashmigrations.py @@ -16,44 +16,51 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'app_label', - help='App label of the application to squash migrations for.', + "app_label", + help="App label of the application to squash migrations for.", ) parser.add_argument( - 'start_migration_name', nargs='?', - help='Migrations will be squashed starting from and including this migration.', + "start_migration_name", + nargs="?", + help="Migrations will be squashed starting from and including this migration.", ) parser.add_argument( - 'migration_name', - help='Migrations will be squashed until and including this migration.', + "migration_name", + help="Migrations will be squashed until and including this migration.", ) parser.add_argument( - '--no-optimize', action='store_true', - help='Do not try to optimize the squashed operations.', + "--no-optimize", + action="store_true", + help="Do not try to optimize the squashed operations.", ) parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '--squashed-name', - help='Sets the name of the new squashed migration.', + "--squashed-name", + help="Sets the name of the new squashed migration.", ) parser.add_argument( - '--no-header', action='store_false', dest='include_header', - help='Do not add a header comment to the new squashed migration.', + "--no-header", + action="store_false", + dest="include_header", + help="Do not add a header comment to the new squashed migration.", ) def handle(self, **options): - self.verbosity = options['verbosity'] - self.interactive = options['interactive'] - app_label = options['app_label'] - start_migration_name = options['start_migration_name'] - migration_name = options['migration_name'] - no_optimize = options['no_optimize'] - squashed_name = options['squashed_name'] - include_header = options['include_header'] + self.verbosity = options["verbosity"] + self.interactive = options["interactive"] + app_label = options["app_label"] + start_migration_name = options["start_migration_name"] + migration_name = options["migration_name"] + no_optimize = options["no_optimize"] + squashed_name = options["squashed_name"] + include_header = options["include_header"] # Validate app_label. try: apps.get_app_config(app_label) @@ -72,13 +79,19 @@ class Command(BaseCommand): # Work out the list of predecessor migrations migrations_to_squash = [ loader.get_migration(al, mn) - for al, mn in loader.graph.forwards_plan((migration.app_label, migration.name)) + for al, mn in loader.graph.forwards_plan( + (migration.app_label, migration.name) + ) if al == migration.app_label ] if start_migration_name: - start_migration = self.find_migration(loader, app_label, start_migration_name) - start = loader.get_migration(start_migration.app_label, start_migration.name) + start_migration = self.find_migration( + loader, app_label, start_migration_name + ) + start = loader.get_migration( + start_migration.app_label, start_migration.name + ) try: start_index = migrations_to_squash.index(start) migrations_to_squash = migrations_to_squash[start_index:] @@ -93,7 +106,9 @@ class Command(BaseCommand): # Tell them what we're doing and optionally ask if we should proceed if self.verbosity > 0 or self.interactive: - self.stdout.write(self.style.MIGRATE_HEADING("Will squash the following migrations:")) + self.stdout.write( + self.style.MIGRATE_HEADING("Will squash the following migrations:") + ) for migration in migrations_to_squash: self.stdout.write(" - %s" % migration.name) @@ -122,7 +137,8 @@ class Command(BaseCommand): raise CommandError( "You cannot squash squashed migrations! Please transition " "it to a normal migration first: " - "https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations" % get_docs_version() + "https://docs.djangoproject.com/en/%s/topics/migrations/#squashing-migrations" + % get_docs_version() ) operations.extend(smigration.operations) for dependency in smigration.dependencies: @@ -137,7 +153,9 @@ class Command(BaseCommand): if no_optimize: if self.verbosity > 0: - self.stdout.write(self.style.MIGRATE_HEADING("(Skipping optimization.)")) + self.stdout.write( + self.style.MIGRATE_HEADING("(Skipping optimization.)") + ) new_operations = operations else: if self.verbosity > 0: @@ -151,8 +169,8 @@ class Command(BaseCommand): self.stdout.write(" No optimizations possible.") else: self.stdout.write( - " Optimized from %s operations to %s operations." % - (len(operations), len(new_operations)) + " Optimized from %s operations to %s operations." + % (len(operations), len(new_operations)) ) # Work out the value of replaces (any squashed ones we're re-squashing) @@ -165,22 +183,26 @@ class Command(BaseCommand): replaces.append((migration.app_label, migration.name)) # Make a new migration with those operations - subclass = type("Migration", (migrations.Migration,), { - "dependencies": dependencies, - "operations": new_operations, - "replaces": replaces, - }) + subclass = type( + "Migration", + (migrations.Migration,), + { + "dependencies": dependencies, + "operations": new_operations, + "replaces": replaces, + }, + ) if start_migration_name: if squashed_name: # Use the name from --squashed-name. - prefix, _ = start_migration.name.split('_', 1) - name = '%s_%s' % (prefix, squashed_name) + prefix, _ = start_migration.name.split("_", 1) + name = "%s_%s" % (prefix, squashed_name) else: # Generate a name. - name = '%s_squashed_%s' % (start_migration.name, migration.name) + name = "%s_squashed_%s" % (start_migration.name, migration.name) new_migration = subclass(name, app_label) else: - name = '0001_%s' % (squashed_name or 'squashed_%s' % migration.name) + name = "0001_%s" % (squashed_name or "squashed_%s" % migration.name) new_migration = subclass(name, app_label) new_migration.initial = True @@ -188,25 +210,28 @@ class Command(BaseCommand): writer = MigrationWriter(new_migration, include_header) if os.path.exists(writer.path): raise CommandError( - f'Migration {new_migration.name} already exists. Use a different name.' + f"Migration {new_migration.name} already exists. Use a different name." ) - with open(writer.path, "w", encoding='utf-8') as fh: + with open(writer.path, "w", encoding="utf-8") as fh: fh.write(writer.as_string()) if self.verbosity > 0: self.stdout.write( - self.style.MIGRATE_HEADING('Created new squashed migration %s' % writer.path) + '\n' - ' You should commit this migration but leave the old ones in place;\n' - ' the new migration will be used for new installs. Once you are sure\n' - ' all instances of the codebase have applied the migrations you squashed,\n' - ' you can delete them.' + self.style.MIGRATE_HEADING( + "Created new squashed migration %s" % writer.path + ) + + "\n" + " You should commit this migration but leave the old ones in place;\n" + " the new migration will be used for new installs. Once you are sure\n" + " all instances of the codebase have applied the migrations you squashed,\n" + " you can delete them." ) if writer.needs_manual_porting: self.stdout.write( - self.style.MIGRATE_HEADING('Manual porting required') + '\n' - ' Your migrations contained functions that must be manually copied over,\n' - ' as we could not safely copy their implementation.\n' - ' See the comment at the top of the squashed migration for details.' + self.style.MIGRATE_HEADING("Manual porting required") + "\n" + " Your migrations contained functions that must be manually copied over,\n" + " as we could not safely copy their implementation.\n" + " See the comment at the top of the squashed migration for details." ) def find_migration(self, loader, app_label, name): @@ -219,6 +244,6 @@ class Command(BaseCommand): ) except KeyError: raise CommandError( - "Cannot find a migration matching '%s' from app '%s'." % - (name, app_label) + "Cannot find a migration matching '%s' from app '%s'." + % (name, app_label) ) diff --git a/django/core/management/commands/startapp.py b/django/core/management/commands/startapp.py index bba9f3dee0..e85833b9a8 100644 --- a/django/core/management/commands/startapp.py +++ b/django/core/management/commands/startapp.py @@ -9,6 +9,6 @@ class Command(TemplateCommand): missing_args_message = "You must provide an application name." def handle(self, **options): - app_name = options.pop('name') - target = options.pop('directory') - super().handle('app', app_name, target, **options) + app_name = options.pop("name") + target = options.pop("directory") + super().handle("app", app_name, target, **options) diff --git a/django/core/management/commands/startproject.py b/django/core/management/commands/startproject.py index 164ccdffb5..ca17fa54cd 100644 --- a/django/core/management/commands/startproject.py +++ b/django/core/management/commands/startproject.py @@ -12,10 +12,10 @@ class Command(TemplateCommand): missing_args_message = "You must provide a project name." def handle(self, **options): - project_name = options.pop('name') - target = options.pop('directory') + project_name = options.pop("name") + target = options.pop("directory") # Create a random SECRET_KEY to put it in the main settings. - options['secret_key'] = SECRET_KEY_INSECURE_PREFIX + get_random_secret_key() + options["secret_key"] = SECRET_KEY_INSECURE_PREFIX + get_random_secret_key() - super().handle('project', project_name, target, **options) + super().handle("project", project_name, target, **options) diff --git a/django/core/management/commands/test.py b/django/core/management/commands/test.py index 7a76033424..e5660955cd 100644 --- a/django/core/management/commands/test.py +++ b/django/core/management/commands/test.py @@ -8,7 +8,7 @@ from django.test.utils import NullTimeKeeper, TimeKeeper, get_runner class Command(BaseCommand): - help = 'Discover and run tests in the specified modules or the current directory.' + help = "Discover and run tests in the specified modules or the current directory." # DiscoverRunner runs the checks after databases are set up. requires_system_checks = [] @@ -20,42 +20,48 @@ class Command(BaseCommand): option. This allows a test runner to define additional command line arguments. """ - self.test_runner = get_command_line_option(argv, '--testrunner') + self.test_runner = get_command_line_option(argv, "--testrunner") super().run_from_argv(argv) def add_arguments(self, parser): parser.add_argument( - 'args', metavar='test_label', nargs='*', - help='Module paths to test; can be modulename, modulename.TestCase or modulename.TestCase.test_method' + "args", + metavar="test_label", + nargs="*", + help="Module paths to test; can be modulename, modulename.TestCase or modulename.TestCase.test_method", ) parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '--failfast', action='store_true', - help='Tells Django to stop running the test suite after first failed test.', + "--failfast", + action="store_true", + help="Tells Django to stop running the test suite after first failed test.", ) parser.add_argument( - '--testrunner', - help='Tells Django to use specified test runner class instead of ' - 'the one specified by the TEST_RUNNER setting.', + "--testrunner", + help="Tells Django to use specified test runner class instead of " + "the one specified by the TEST_RUNNER setting.", ) test_runner_class = get_runner(settings, self.test_runner) - if hasattr(test_runner_class, 'add_arguments'): + if hasattr(test_runner_class, "add_arguments"): test_runner_class.add_arguments(parser) def handle(self, *test_labels, **options): - TestRunner = get_runner(settings, options['testrunner']) + TestRunner = get_runner(settings, options["testrunner"]) - time_keeper = TimeKeeper() if options.get('timing', False) else NullTimeKeeper() - parallel = options.get('parallel') - if parallel == 'auto': - options['parallel'] = get_max_test_processes() + time_keeper = TimeKeeper() if options.get("timing", False) else NullTimeKeeper() + parallel = options.get("parallel") + if parallel == "auto": + options["parallel"] = get_max_test_processes() test_runner = TestRunner(**options) - with time_keeper.timed('Total run'): + with time_keeper.timed("Total run"): failures = test_runner.run_tests(test_labels) time_keeper.print_results() if failures: diff --git a/django/core/management/commands/testserver.py b/django/core/management/commands/testserver.py index ee8709af8b..caff6c65cd 100644 --- a/django/core/management/commands/testserver.py +++ b/django/core/management/commands/testserver.py @@ -4,51 +4,62 @@ from django.db import connection class Command(BaseCommand): - help = 'Runs a development server with data from the given fixture(s).' + help = "Runs a development server with data from the given fixture(s)." requires_system_checks = [] def add_arguments(self, parser): parser.add_argument( - 'args', metavar='fixture', nargs='*', - help='Path(s) to fixtures to load before running the server.', + "args", + metavar="fixture", + nargs="*", + help="Path(s) to fixtures to load before running the server.", ) parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help='Tells Django to NOT prompt the user for input of any kind.', + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Tells Django to NOT prompt the user for input of any kind.", ) parser.add_argument( - '--addrport', default='', - help='Port number or ipaddr:port to run the server on.', + "--addrport", + default="", + help="Port number or ipaddr:port to run the server on.", ) parser.add_argument( - '--ipv6', '-6', action='store_true', dest='use_ipv6', - help='Tells Django to use an IPv6 address.', + "--ipv6", + "-6", + action="store_true", + dest="use_ipv6", + help="Tells Django to use an IPv6 address.", ) def handle(self, *fixture_labels, **options): - verbosity = options['verbosity'] - interactive = options['interactive'] + verbosity = options["verbosity"] + interactive = options["interactive"] # Create a test database. - db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False) + db_name = connection.creation.create_test_db( + verbosity=verbosity, autoclobber=not interactive, serialize=False + ) # Import the fixture data into the test database. - call_command('loaddata', *fixture_labels, **{'verbosity': verbosity}) + call_command("loaddata", *fixture_labels, **{"verbosity": verbosity}) # Run the development server. Turn off auto-reloading because it causes # a strange error -- it causes this handle() method to be called # multiple times. shutdown_message = ( - '\nServer stopped.\nNote that the test database, %r, has not been ' - 'deleted. You can explore it on your own.' % db_name + "\nServer stopped.\nNote that the test database, %r, has not been " + "deleted. You can explore it on your own." % db_name ) use_threading = connection.features.test_db_allows_multiple_connections call_command( - 'runserver', - addrport=options['addrport'], + "runserver", + addrport=options["addrport"], shutdown_message=shutdown_message, use_reloader=False, - use_ipv6=options['use_ipv6'], - use_threading=use_threading + use_ipv6=options["use_ipv6"], + use_threading=use_threading, ) diff --git a/django/core/management/sql.py b/django/core/management/sql.py index a7e122a15f..2375cc23ab 100644 --- a/django/core/management/sql.py +++ b/django/core/management/sql.py @@ -8,7 +8,9 @@ def sql_flush(style, connection, reset_sequences=True, allow_cascade=False): """ Return a list of the SQL statements used to flush the database. """ - tables = connection.introspection.django_table_names(only_existing=True, include_views=False) + tables = connection.introspection.django_table_names( + only_existing=True, include_views=False + ) return connection.ops.sql_flush( style, tables, @@ -23,15 +25,17 @@ def emit_pre_migrate_signal(verbosity, interactive, db, **kwargs): if app_config.models_module is None: continue if verbosity >= 2: - stdout = kwargs.get('stdout', sys.stdout) - stdout.write('Running pre-migrate handlers for application %s' % app_config.label) + stdout = kwargs.get("stdout", sys.stdout) + stdout.write( + "Running pre-migrate handlers for application %s" % app_config.label + ) models.signals.pre_migrate.send( sender=app_config, app_config=app_config, verbosity=verbosity, interactive=interactive, using=db, - **kwargs + **kwargs, ) @@ -41,13 +45,15 @@ def emit_post_migrate_signal(verbosity, interactive, db, **kwargs): if app_config.models_module is None: continue if verbosity >= 2: - stdout = kwargs.get('stdout', sys.stdout) - stdout.write('Running post-migrate handlers for application %s' % app_config.label) + stdout = kwargs.get("stdout", sys.stdout) + stdout.write( + "Running post-migrate handlers for application %s" % app_config.label + ) models.signals.post_migrate.send( sender=app_config, app_config=app_config, verbosity=verbosity, interactive=interactive, using=db, - **kwargs + **kwargs, ) diff --git a/django/core/management/templates.py b/django/core/management/templates.py index cfcaff7c0f..58005c23ed 100644 --- a/django/core/management/templates.py +++ b/django/core/management/templates.py @@ -29,46 +29,61 @@ class TemplateCommand(BaseCommand): :param directory: The directory to which the template should be copied. :param options: The additional variables passed to project or app templates """ + requires_system_checks = [] # The supported URL schemes - url_schemes = ['http', 'https', 'ftp'] + url_schemes = ["http", "https", "ftp"] # Rewrite the following suffixes when determining the target filename. rewrite_template_suffixes = ( # Allow shipping invalid .py files without byte-compilation. - ('.py-tpl', '.py'), + (".py-tpl", ".py"), ) def add_arguments(self, parser): - parser.add_argument('name', help='Name of the application or project.') - parser.add_argument('directory', nargs='?', help='Optional destination directory') - parser.add_argument('--template', help='The path or URL to load the template from.') + parser.add_argument("name", help="Name of the application or project.") + parser.add_argument( + "directory", nargs="?", help="Optional destination directory" + ) + parser.add_argument( + "--template", help="The path or URL to load the template from." + ) parser.add_argument( - '--extension', '-e', dest='extensions', - action='append', default=['py'], + "--extension", + "-e", + dest="extensions", + action="append", + default=["py"], help='The file extension(s) to render (default: "py"). ' - 'Separate multiple extensions with commas, or use ' - '-e multiple times.' + "Separate multiple extensions with commas, or use " + "-e multiple times.", ) parser.add_argument( - '--name', '-n', dest='files', - action='append', default=[], - help='The file name(s) to render. Separate multiple file names ' - 'with commas, or use -n multiple times.' + "--name", + "-n", + dest="files", + action="append", + default=[], + help="The file name(s) to render. Separate multiple file names " + "with commas, or use -n multiple times.", ) parser.add_argument( - '--exclude', '-x', - action='append', default=argparse.SUPPRESS, nargs='?', const='', + "--exclude", + "-x", + action="append", + default=argparse.SUPPRESS, + nargs="?", + const="", help=( - 'The directory name(s) to exclude, in addition to .git and ' - '__pycache__. Can be used multiple times.' + "The directory name(s) to exclude, in addition to .git and " + "__pycache__. Can be used multiple times." ), ) def handle(self, app_or_project, name, target=None, **options): self.app_or_project = app_or_project - self.a_or_an = 'an' if app_or_project == 'app' else 'a' + self.a_or_an = "an" if app_or_project == "app" else "a" self.paths_to_remove = [] - self.verbosity = options['verbosity'] + self.verbosity = options["verbosity"] self.validate_name(name) @@ -83,51 +98,55 @@ class TemplateCommand(BaseCommand): raise CommandError(e) else: top_dir = os.path.abspath(os.path.expanduser(target)) - if app_or_project == 'app': - self.validate_name(os.path.basename(top_dir), 'directory') + if app_or_project == "app": + self.validate_name(os.path.basename(top_dir), "directory") if not os.path.exists(top_dir): - raise CommandError("Destination directory '%s' does not " - "exist, please create it first." % top_dir) + raise CommandError( + "Destination directory '%s' does not " + "exist, please create it first." % top_dir + ) - extensions = tuple(handle_extensions(options['extensions'])) + extensions = tuple(handle_extensions(options["extensions"])) extra_files = [] - excluded_directories = ['.git', '__pycache__'] - for file in options['files']: - extra_files.extend(map(lambda x: x.strip(), file.split(','))) - if exclude := options.get('exclude'): + excluded_directories = [".git", "__pycache__"] + for file in options["files"]: + extra_files.extend(map(lambda x: x.strip(), file.split(","))) + if exclude := options.get("exclude"): for directory in exclude: excluded_directories.append(directory.strip()) if self.verbosity >= 2: self.stdout.write( - 'Rendering %s template files with extensions: %s' - % (app_or_project, ', '.join(extensions)) + "Rendering %s template files with extensions: %s" + % (app_or_project, ", ".join(extensions)) ) self.stdout.write( - 'Rendering %s template files with filenames: %s' - % (app_or_project, ', '.join(extra_files)) + "Rendering %s template files with filenames: %s" + % (app_or_project, ", ".join(extra_files)) ) - base_name = '%s_name' % app_or_project - base_subdir = '%s_template' % app_or_project - base_directory = '%s_directory' % app_or_project - camel_case_name = 'camel_case_%s_name' % app_or_project - camel_case_value = ''.join(x for x in name.title() if x != '_') + base_name = "%s_name" % app_or_project + base_subdir = "%s_template" % app_or_project + base_directory = "%s_directory" % app_or_project + camel_case_name = "camel_case_%s_name" % app_or_project + camel_case_value = "".join(x for x in name.title() if x != "_") - context = Context({ - **options, - base_name: name, - base_directory: top_dir, - camel_case_name: camel_case_value, - 'docs_version': get_docs_version(), - 'django_version': django.__version__, - }, autoescape=False) + context = Context( + { + **options, + base_name: name, + base_directory: top_dir, + camel_case_name: camel_case_value, + "docs_version": get_docs_version(), + "django_version": django.__version__, + }, + autoescape=False, + ) # Setup a stub settings environment for template rendering if not settings.configured: settings.configure() django.setup() - template_dir = self.handle_template(options['template'], - base_subdir) + template_dir = self.handle_template(options["template"], base_subdir) prefix_length = len(template_dir) + 1 for root, dirs, files in os.walk(template_dir): @@ -139,14 +158,14 @@ class TemplateCommand(BaseCommand): os.makedirs(target_dir, exist_ok=True) for dirname in dirs[:]: - if 'exclude' not in options: - if dirname.startswith('.') or dirname == '__pycache__': + if "exclude" not in options: + if dirname.startswith(".") or dirname == "__pycache__": dirs.remove(dirname) elif dirname in excluded_directories: dirs.remove(dirname) for filename in files: - if filename.endswith(('.pyo', '.pyc', '.py.class')): + if filename.endswith((".pyo", ".pyc", ".py.class")): # Ignore some files as they cause various breakages. continue old_path = os.path.join(root, filename) @@ -155,31 +174,34 @@ class TemplateCommand(BaseCommand): ) for old_suffix, new_suffix in self.rewrite_template_suffixes: if new_path.endswith(old_suffix): - new_path = new_path[:-len(old_suffix)] + new_suffix + new_path = new_path[: -len(old_suffix)] + new_suffix break # Only rewrite once if os.path.exists(new_path): raise CommandError( "%s already exists. Overlaying %s %s into an existing " - "directory won't replace conflicting files." % ( - new_path, self.a_or_an, app_or_project, + "directory won't replace conflicting files." + % ( + new_path, + self.a_or_an, + app_or_project, ) ) # Only render the Python files, as we don't want to # accidentally render Django templates files if new_path.endswith(extensions) or filename in extra_files: - with open(old_path, encoding='utf-8') as template_file: + with open(old_path, encoding="utf-8") as template_file: content = template_file.read() template = Engine().from_string(content) content = template.render(context) - with open(new_path, 'w', encoding='utf-8') as new_file: + with open(new_path, "w", encoding="utf-8") as new_file: new_file.write(content) else: shutil.copyfile(old_path, new_path) if self.verbosity >= 2: - self.stdout.write('Creating %s' % new_path) + self.stdout.write("Creating %s" % new_path) try: self.apply_umask(old_path, new_path) self.make_writeable(new_path) @@ -187,11 +209,13 @@ class TemplateCommand(BaseCommand): self.stderr.write( "Notice: Couldn't set permission bits on %s. You're " "probably using an uncommon filesystem setup. No " - "problem." % new_path, self.style.NOTICE) + "problem." % new_path, + self.style.NOTICE, + ) if self.paths_to_remove: if self.verbosity >= 2: - self.stdout.write('Cleaning up temporary files.') + self.stdout.write("Cleaning up temporary files.") for path_to_remove in self.paths_to_remove: if os.path.isfile(path_to_remove): os.remove(path_to_remove) @@ -205,9 +229,9 @@ class TemplateCommand(BaseCommand): directory isn't known. """ if template is None: - return os.path.join(django.__path__[0], 'conf', subdir) + return os.path.join(django.__path__[0], "conf", subdir) else: - if template.startswith('file://'): + if template.startswith("file://"): template = template[7:] expanded_template = os.path.expanduser(template) expanded_template = os.path.normpath(expanded_template) @@ -221,15 +245,18 @@ class TemplateCommand(BaseCommand): if os.path.exists(absolute_path): return self.extract(absolute_path) - raise CommandError("couldn't handle %s template %s." % - (self.app_or_project, template)) + raise CommandError( + "couldn't handle %s template %s." % (self.app_or_project, template) + ) - def validate_name(self, name, name_or_dir='name'): + def validate_name(self, name, name_or_dir="name"): if name is None: - raise CommandError('you must provide {an} {app} name'.format( - an=self.a_or_an, - app=self.app_or_project, - )) + raise CommandError( + "you must provide {an} {app} name".format( + an=self.a_or_an, + app=self.app_or_project, + ) + ) # Check it's a valid directory name. if not name.isidentifier(): raise CommandError( @@ -261,47 +288,49 @@ class TemplateCommand(BaseCommand): """ Download the given URL and return the file name. """ + def cleanup_url(url): - tmp = url.rstrip('/') - filename = tmp.split('/')[-1] - if url.endswith('/'): - display_url = tmp + '/' + tmp = url.rstrip("/") + filename = tmp.split("/")[-1] + if url.endswith("/"): + display_url = tmp + "/" else: display_url = url return filename, display_url - prefix = 'django_%s_template_' % self.app_or_project - tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_download') + prefix = "django_%s_template_" % self.app_or_project + tempdir = tempfile.mkdtemp(prefix=prefix, suffix="_download") self.paths_to_remove.append(tempdir) filename, display_url = cleanup_url(url) if self.verbosity >= 2: - self.stdout.write('Downloading %s' % display_url) + self.stdout.write("Downloading %s" % display_url) the_path = os.path.join(tempdir, filename) opener = build_opener() - opener.addheaders = [('User-Agent', f'Django/{django.__version__}')] + opener.addheaders = [("User-Agent", f"Django/{django.__version__}")] try: - with opener.open(url) as source, open(the_path, 'wb') as target: + with opener.open(url) as source, open(the_path, "wb") as target: headers = source.info() target.write(source.read()) except OSError as e: - raise CommandError("couldn't download URL %s to %s: %s" % - (url, filename, e)) + raise CommandError( + "couldn't download URL %s to %s: %s" % (url, filename, e) + ) - used_name = the_path.split('/')[-1] + used_name = the_path.split("/")[-1] # Trying to get better name from response headers - content_disposition = headers['content-disposition'] + content_disposition = headers["content-disposition"] if content_disposition: _, params = cgi.parse_header(content_disposition) - guessed_filename = params.get('filename') or used_name + guessed_filename = params.get("filename") or used_name else: guessed_filename = used_name # Falling back to content type guessing ext = self.splitext(guessed_filename)[1] - content_type = headers['content-type'] + content_type = headers["content-type"] if not ext and content_type: ext = mimetypes.guess_extension(content_type) if ext: @@ -322,7 +351,7 @@ class TemplateCommand(BaseCommand): Like os.path.splitext, but takes off .tar, too """ base, ext = posixpath.splitext(the_path) - if base.lower().endswith('.tar'): + if base.lower().endswith(".tar"): ext = base[-4:] + ext base = base[:-4] return base, ext @@ -332,23 +361,24 @@ class TemplateCommand(BaseCommand): Extract the given file to a temporary directory and return the path of the directory with the extracted content. """ - prefix = 'django_%s_template_' % self.app_or_project - tempdir = tempfile.mkdtemp(prefix=prefix, suffix='_extract') + prefix = "django_%s_template_" % self.app_or_project + tempdir = tempfile.mkdtemp(prefix=prefix, suffix="_extract") self.paths_to_remove.append(tempdir) if self.verbosity >= 2: - self.stdout.write('Extracting %s' % filename) + self.stdout.write("Extracting %s" % filename) try: archive.extract(filename, tempdir) return tempdir except (archive.ArchiveException, OSError) as e: - raise CommandError("couldn't extract file %s to %s: %s" % - (filename, tempdir, e)) + raise CommandError( + "couldn't extract file %s to %s: %s" % (filename, tempdir, e) + ) def is_url(self, template): """Return True if the name looks like a URL.""" - if ':' not in template: + if ":" not in template: return False - scheme = template.split(':', 1)[0].lower() + scheme = template.split(":", 1)[0].lower() return scheme in self.url_schemes def apply_umask(self, old_path, new_path): diff --git a/django/core/management/utils.py b/django/core/management/utils.py index c6901aa3d5..c12d90f6ae 100644 --- a/django/core/management/utils.py +++ b/django/core/management/utils.py @@ -10,20 +10,20 @@ from django.utils.encoding import DEFAULT_LOCALE_ENCODING from .base import CommandError, CommandParser -def popen_wrapper(args, stdout_encoding='utf-8'): +def popen_wrapper(args, stdout_encoding="utf-8"): """ Friendly wrapper around Popen. Return stdout output, stderr output, and OS status code. """ try: - p = run(args, capture_output=True, close_fds=os.name != 'nt') + p = run(args, capture_output=True, close_fds=os.name != "nt") except OSError as err: - raise CommandError('Error executing %s' % args[0]) from err + raise CommandError("Error executing %s" % args[0]) from err return ( p.stdout.decode(stdout_encoding), - p.stderr.decode(DEFAULT_LOCALE_ENCODING, errors='replace'), - p.returncode + p.stderr.decode(DEFAULT_LOCALE_ENCODING, errors="replace"), + p.returncode, ) @@ -42,25 +42,25 @@ def handle_extensions(extensions): """ ext_list = [] for ext in extensions: - ext_list.extend(ext.replace(' ', '').split(',')) + ext_list.extend(ext.replace(" ", "").split(",")) for i, ext in enumerate(ext_list): - if not ext.startswith('.'): - ext_list[i] = '.%s' % ext_list[i] + if not ext.startswith("."): + ext_list[i] = ".%s" % ext_list[i] return set(ext_list) def find_command(cmd, path=None, pathext=None): if path is None: - path = os.environ.get('PATH', '').split(os.pathsep) + path = os.environ.get("PATH", "").split(os.pathsep) if isinstance(path, str): path = [path] # check if there are funny path extensions for executables, e.g. Windows if pathext is None: - pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep) + pathext = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(os.pathsep) # don't use extensions if the command ends with one of them for ext in pathext: if cmd.endswith(ext): - pathext = [''] + pathext = [""] break # check if we find the command on PATH for p in path: @@ -78,7 +78,7 @@ def get_random_secret_key(): """ Return a 50 character random string usable as a SECRET_KEY setting value. """ - chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' + chars = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)" return get_random_string(50, chars) @@ -93,11 +93,11 @@ def parse_apps_and_model_labels(labels): models = set() for label in labels: - if '.' in label: + if "." in label: try: model = installed_apps.get_model(label) except LookupError: - raise CommandError('Unknown model: %s' % label) + raise CommandError("Unknown model: %s" % label) models.add(model) else: try: @@ -116,7 +116,7 @@ def get_command_line_option(argv, option): option wasn't passed or if the argument list couldn't be parsed. """ parser = CommandParser(add_help=False, allow_abbrev=False) - parser.add_argument(option, dest='value') + parser.add_argument(option, dest="value") try: options, _ = parser.parse_known_args(argv[2:]) except CommandError: @@ -128,12 +128,12 @@ def get_command_line_option(argv, option): def normalize_path_patterns(patterns): """Normalize an iterable of glob style patterns based on OS.""" patterns = [os.path.normcase(p) for p in patterns] - dir_suffixes = {'%s*' % path_sep for path_sep in {'/', os.sep}} + dir_suffixes = {"%s*" % path_sep for path_sep in {"/", os.sep}} norm_patterns = [] for pattern in patterns: for dir_suffix in dir_suffixes: if pattern.endswith(dir_suffix): - norm_patterns.append(pattern[:-len(dir_suffix)]) + norm_patterns.append(pattern[: -len(dir_suffix)]) break else: norm_patterns.append(pattern) @@ -148,6 +148,8 @@ def is_ignored_path(path, ignore_patterns): path = Path(path) def ignore(pattern): - return fnmatch.fnmatchcase(path.name, pattern) or fnmatch.fnmatchcase(str(path), pattern) + return fnmatch.fnmatchcase(path.name, pattern) or fnmatch.fnmatchcase( + str(path), pattern + ) return any(ignore(pattern) for pattern in normalize_path_patterns(ignore_patterns)) diff --git a/django/core/paginator.py b/django/core/paginator.py index 7db64913d9..568445607e 100644 --- a/django/core/paginator.py +++ b/django/core/paginator.py @@ -27,10 +27,9 @@ class EmptyPage(InvalidPage): class Paginator: # Translators: String used to replace omitted page numbers in elided page # range generated by paginators, e.g. [1, 2, '…', 5, 6, 7, '…', 9, 10]. - ELLIPSIS = _('…') + ELLIPSIS = _("…") - def __init__(self, object_list, per_page, orphans=0, - allow_empty_first_page=True): + def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True): self.object_list = object_list self._check_object_list_is_ordered() self.per_page = int(per_page) @@ -48,14 +47,14 @@ class Paginator: raise ValueError number = int(number) except (TypeError, ValueError): - raise PageNotAnInteger(_('That page number is not an integer')) + raise PageNotAnInteger(_("That page number is not an integer")) if number < 1: - raise EmptyPage(_('That page number is less than 1')) + raise EmptyPage(_("That page number is less than 1")) if number > self.num_pages: if number == 1 and self.allow_empty_first_page: pass else: - raise EmptyPage(_('That page contains no results')) + raise EmptyPage(_("That page contains no results")) return number def get_page(self, number): @@ -92,7 +91,7 @@ class Paginator: @cached_property def count(self): """Return the total number of objects, across all pages.""" - c = getattr(self.object_list, 'count', None) + c = getattr(self.object_list, "count", None) if callable(c) and not inspect.isbuiltin(c) and method_has_no_args(c): return c() return len(self.object_list) @@ -117,18 +116,20 @@ class Paginator: """ Warn if self.object_list is unordered (typically a QuerySet). """ - ordered = getattr(self.object_list, 'ordered', None) + ordered = getattr(self.object_list, "ordered", None) if ordered is not None and not ordered: obj_list_repr = ( - '{} {}'.format(self.object_list.model, self.object_list.__class__.__name__) - if hasattr(self.object_list, 'model') - else '{!r}'.format(self.object_list) + "{} {}".format( + self.object_list.model, self.object_list.__class__.__name__ + ) + if hasattr(self.object_list, "model") + else "{!r}".format(self.object_list) ) warnings.warn( - 'Pagination may yield inconsistent results with an unordered ' - 'object_list: {}.'.format(obj_list_repr), + "Pagination may yield inconsistent results with an unordered " + "object_list: {}.".format(obj_list_repr), UnorderedObjectListWarning, - stacklevel=3 + stacklevel=3, ) def get_elided_page_range(self, number=1, *, on_each_side=3, on_ends=2): @@ -164,14 +165,13 @@ class Paginator: class Page(collections.abc.Sequence): - def __init__(self, object_list, number, paginator): self.object_list = object_list self.number = number self.paginator = paginator def __repr__(self): - return '<Page %s of %s>' % (self.number, self.paginator.num_pages) + return "<Page %s of %s>" % (self.number, self.paginator.num_pages) def __len__(self): return len(self.object_list) @@ -179,7 +179,7 @@ class Page(collections.abc.Sequence): def __getitem__(self, index): if not isinstance(index, (int, slice)): raise TypeError( - 'Page indices must be integers or slices, not %s.' + "Page indices must be integers or slices, not %s." % type(index).__name__ ) # The object_list is converted to a list so that if it was a QuerySet diff --git a/django/core/serializers/__init__.py b/django/core/serializers/__init__.py index 793f6dc2bd..480c54b79b 100644 --- a/django/core/serializers/__init__.py +++ b/django/core/serializers/__init__.py @@ -42,6 +42,7 @@ class BadSerializer: is an error raised in the process of creating a serializer it will be raised and passed along to the caller when the serializer is used. """ + internal_use_only = False def __init__(self, exception): @@ -72,10 +73,14 @@ def register_serializer(format, serializer_module, serializers=None): except ImportError as exc: bad_serializer = BadSerializer(exc) - module = type('BadSerializerModule', (), { - 'Deserializer': bad_serializer, - 'Serializer': bad_serializer, - }) + module = type( + "BadSerializerModule", + (), + { + "Deserializer": bad_serializer, + "Serializer": bad_serializer, + }, + ) if serializers is None: _serializers[format] = module @@ -153,7 +158,9 @@ def _load_serializers(): register_serializer(format, BUILTIN_SERIALIZERS[format], serializers) if hasattr(settings, "SERIALIZATION_MODULES"): for format in settings.SERIALIZATION_MODULES: - register_serializer(format, settings.SERIALIZATION_MODULES[format], serializers) + register_serializer( + format, settings.SERIALIZATION_MODULES[format], serializers + ) _serializers = serializers @@ -177,8 +184,8 @@ def sort_dependencies(app_list, allow_cycles=False): for model in model_list: models.add(model) # Add any explicitly defined dependencies - if hasattr(model, 'natural_key'): - deps = getattr(model.natural_key, 'dependencies', []) + if hasattr(model, "natural_key"): + deps = getattr(model.natural_key, "dependencies", []) if deps: deps = [apps.get_model(dep) for dep in deps] else: @@ -189,7 +196,7 @@ def sort_dependencies(app_list, allow_cycles=False): for field in model._meta.fields: if field.remote_field: rel_model = field.remote_field.model - if hasattr(rel_model, 'natural_key') and rel_model != model: + if hasattr(rel_model, "natural_key") and rel_model != model: deps.append(rel_model) # Also add a dependency for any simple M2M relation with a model # that defines a natural key. M2M relations with explicit through @@ -197,7 +204,7 @@ def sort_dependencies(app_list, allow_cycles=False): for field in model._meta.many_to_many: if field.remote_field.through._meta.auto_created: rel_model = field.remote_field.model - if hasattr(rel_model, 'natural_key') and rel_model != model: + if hasattr(rel_model, "natural_key") and rel_model != model: deps.append(rel_model) model_dependencies.append((model, deps)) @@ -235,9 +242,11 @@ def sort_dependencies(app_list, allow_cycles=False): else: raise RuntimeError( "Can't resolve dependencies for %s in serialized app list." - % ', '.join( + % ", ".join( model._meta.label - for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__) + for model, deps in sorted( + skipped, key=lambda obj: obj[0].__name__ + ) ), ) model_dependencies = skipped diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py index 45c43a77d6..da85cb4b92 100644 --- a/django/core/serializers/base.py +++ b/django/core/serializers/base.py @@ -17,10 +17,11 @@ class PickleSerializer: Simple wrapper around pickle to be used in signing.dumps()/loads() and cache backends. """ + def __init__(self, protocol=None): warnings.warn( - 'PickleSerializer is deprecated due to its security risk. Use ' - 'JSONSerializer instead.', + "PickleSerializer is deprecated due to its security risk. Use " + "JSONSerializer instead.", RemovedInDjango50Warning, ) self.protocol = pickle.HIGHEST_PROTOCOL if protocol is None else protocol @@ -34,11 +35,13 @@ class PickleSerializer: class SerializerDoesNotExist(KeyError): """The requested serializer was not found.""" + pass class SerializationError(Exception): """Something bad happened during serialization.""" + pass @@ -51,11 +54,15 @@ class DeserializationError(Exception): Factory method for creating a deserialization error which has a more explanatory message. """ - return cls("%s: (%s:pk=%s) field_value was '%s'" % (original_exc, model, fk, field_value)) + return cls( + "%s: (%s:pk=%s) field_value was '%s'" + % (original_exc, model, fk, field_value) + ) class M2MDeserializationError(Exception): """Something bad happened during deserialization of a ManyToManyField.""" + def __init__(self, original_exc, pk): self.original_exc = original_exc self.pk = pk @@ -77,10 +84,12 @@ class ProgressBar: if self.prev_done >= done: return self.prev_done = done - cr = '' if self.total_count == 1 else '\r' - self.output.write(cr + '[' + '.' * done + ' ' * (self.progress_width - done) + ']') + cr = "" if self.total_count == 1 else "\r" + self.output.write( + cr + "[" + "." * done + " " * (self.progress_width - done) + "]" + ) if done == self.progress_width: - self.output.write('\n') + self.output.write("\n") self.output.flush() @@ -95,8 +104,18 @@ class Serializer: progress_class = ProgressBar stream_class = StringIO - def serialize(self, queryset, *, stream=None, fields=None, use_natural_foreign_keys=False, - use_natural_primary_keys=False, progress_output=None, object_count=0, **options): + def serialize( + self, + queryset, + *, + stream=None, + fields=None, + use_natural_foreign_keys=False, + use_natural_primary_keys=False, + progress_output=None, + object_count=0, + **options, + ): """ Serialize a queryset. """ @@ -120,20 +139,31 @@ class Serializer: # be serialized, otherwise deserialization isn't possible. if self.use_natural_primary_keys: pk = concrete_model._meta.pk - pk_parent = pk if pk.remote_field and pk.remote_field.parent_link else None + pk_parent = ( + pk if pk.remote_field and pk.remote_field.parent_link else None + ) else: pk_parent = None for field in concrete_model._meta.local_fields: if field.serialize or field is pk_parent: if field.remote_field is None: - if self.selected_fields is None or field.attname in self.selected_fields: + if ( + self.selected_fields is None + or field.attname in self.selected_fields + ): self.handle_field(obj, field) else: - if self.selected_fields is None or field.attname[:-3] in self.selected_fields: + if ( + self.selected_fields is None + or field.attname[:-3] in self.selected_fields + ): self.handle_fk_field(obj, field) for field in concrete_model._meta.local_many_to_many: if field.serialize: - if self.selected_fields is None or field.attname in self.selected_fields: + if ( + self.selected_fields is None + or field.attname in self.selected_fields + ): self.handle_m2m_field(obj, field) self.end_object(obj) progress_bar.update(count) @@ -145,7 +175,9 @@ class Serializer: """ Called when serializing of the queryset starts. """ - raise NotImplementedError('subclasses of Serializer must provide a start_serialization() method') + raise NotImplementedError( + "subclasses of Serializer must provide a start_serialization() method" + ) def end_serialization(self): """ @@ -157,7 +189,9 @@ class Serializer: """ Called when serializing of an object starts. """ - raise NotImplementedError('subclasses of Serializer must provide a start_object() method') + raise NotImplementedError( + "subclasses of Serializer must provide a start_object() method" + ) def end_object(self, obj): """ @@ -169,26 +203,32 @@ class Serializer: """ Called to handle each individual (non-relational) field on an object. """ - raise NotImplementedError('subclasses of Serializer must provide a handle_field() method') + raise NotImplementedError( + "subclasses of Serializer must provide a handle_field() method" + ) def handle_fk_field(self, obj, field): """ Called to handle a ForeignKey field. """ - raise NotImplementedError('subclasses of Serializer must provide a handle_fk_field() method') + raise NotImplementedError( + "subclasses of Serializer must provide a handle_fk_field() method" + ) def handle_m2m_field(self, obj, field): """ Called to handle a ManyToManyField. """ - raise NotImplementedError('subclasses of Serializer must provide a handle_m2m_field() method') + raise NotImplementedError( + "subclasses of Serializer must provide a handle_m2m_field() method" + ) def getvalue(self): """ Return the fully serialized queryset (or None if the output stream is not seekable). """ - if callable(getattr(self.stream, 'getvalue', None)): + if callable(getattr(self.stream, "getvalue", None)): return self.stream.getvalue() @@ -212,7 +252,9 @@ class Deserializer: def __next__(self): """Iteration interface -- return the next item in the stream""" - raise NotImplementedError('subclasses of Deserializer must provide a __next__() method') + raise NotImplementedError( + "subclasses of Deserializer must provide a __next__() method" + ) class DeserializedObject: @@ -256,18 +298,26 @@ class DeserializedObject: self.m2m_data = {} for field, field_value in self.deferred_fields.items(): opts = self.object._meta - label = opts.app_label + '.' + opts.model_name + label = opts.app_label + "." + opts.model_name if isinstance(field.remote_field, models.ManyToManyRel): try: - values = deserialize_m2m_values(field, field_value, using, handle_forward_references=False) + values = deserialize_m2m_values( + field, field_value, using, handle_forward_references=False + ) except M2MDeserializationError as e: - raise DeserializationError.WithData(e.original_exc, label, self.object.pk, e.pk) + raise DeserializationError.WithData( + e.original_exc, label, self.object.pk, e.pk + ) self.m2m_data[field.name] = values elif isinstance(field.remote_field, models.ManyToOneRel): try: - value = deserialize_fk_value(field, field_value, using, handle_forward_references=False) + value = deserialize_fk_value( + field, field_value, using, handle_forward_references=False + ) except Exception as e: - raise DeserializationError.WithData(e, label, self.object.pk, field_value) + raise DeserializationError.WithData( + e, label, self.object.pk, field_value + ) setattr(self.object, field.attname, value) self.save() @@ -281,8 +331,11 @@ def build_instance(Model, data, db): """ default_manager = Model._meta.default_manager pk = data.get(Model._meta.pk.attname) - if (pk is None and hasattr(default_manager, 'get_by_natural_key') and - hasattr(Model, 'natural_key')): + if ( + pk is None + and hasattr(default_manager, "get_by_natural_key") + and hasattr(Model, "natural_key") + ): natural_key = Model(**data).natural_key() try: data[Model._meta.pk.attname] = Model._meta.pk.to_python( @@ -295,13 +348,20 @@ def build_instance(Model, data, db): def deserialize_m2m_values(field, field_value, using, handle_forward_references): model = field.remote_field.model - if hasattr(model._default_manager, 'get_by_natural_key'): + if hasattr(model._default_manager, "get_by_natural_key"): + def m2m_convert(value): - if hasattr(value, '__iter__') and not isinstance(value, str): - return model._default_manager.db_manager(using).get_by_natural_key(*value).pk + if hasattr(value, "__iter__") and not isinstance(value, str): + return ( + model._default_manager.db_manager(using) + .get_by_natural_key(*value) + .pk + ) else: return model._meta.pk.to_python(value) + else: + def m2m_convert(v): return model._meta.pk.to_python(v) @@ -327,8 +387,11 @@ def deserialize_fk_value(field, field_value, using, handle_forward_references): model = field.remote_field.model default_manager = model._default_manager field_name = field.remote_field.field_name - if (hasattr(default_manager, 'get_by_natural_key') and - hasattr(field_value, '__iter__') and not isinstance(field_value, str)): + if ( + hasattr(default_manager, "get_by_natural_key") + and hasattr(field_value, "__iter__") + and not isinstance(field_value, str) + ): try: obj = default_manager.db_manager(using).get_by_natural_key(*field_value) except ObjectDoesNotExist: diff --git a/django/core/serializers/json.py b/django/core/serializers/json.py index 886e8f894c..59d7318409 100644 --- a/django/core/serializers/json.py +++ b/django/core/serializers/json.py @@ -8,9 +8,8 @@ import json import uuid from django.core.serializers.base import DeserializationError -from django.core.serializers.python import ( - Deserializer as PythonDeserializer, Serializer as PythonSerializer, -) +from django.core.serializers.python import Deserializer as PythonDeserializer +from django.core.serializers.python import Serializer as PythonSerializer from django.utils.duration import duration_iso_string from django.utils.functional import Promise from django.utils.timezone import is_aware @@ -18,18 +17,19 @@ from django.utils.timezone import is_aware class Serializer(PythonSerializer): """Convert a queryset to JSON.""" + internal_use_only = False def _init_options(self): self._current = None self.json_kwargs = self.options.copy() - self.json_kwargs.pop('stream', None) - self.json_kwargs.pop('fields', None) - if self.options.get('indent'): + self.json_kwargs.pop("stream", None) + self.json_kwargs.pop("fields", None) + if self.options.get("indent"): # Prevent trailing spaces - self.json_kwargs['separators'] = (',', ': ') - self.json_kwargs.setdefault('cls', DjangoJSONEncoder) - self.json_kwargs.setdefault('ensure_ascii', False) + self.json_kwargs["separators"] = (",", ": ") + self.json_kwargs.setdefault("cls", DjangoJSONEncoder) + self.json_kwargs.setdefault("ensure_ascii", False) def start_serialization(self): self._init_options() @@ -79,14 +79,15 @@ class DjangoJSONEncoder(json.JSONEncoder): JSONEncoder subclass that knows how to encode date/time, decimal types, and UUIDs. """ + def default(self, o): # See "Date Time String Format" in the ECMA-262 specification. if isinstance(o, datetime.datetime): r = o.isoformat() if o.microsecond: r = r[:23] + r[26:] - if r.endswith('+00:00'): - r = r[:-6] + 'Z' + if r.endswith("+00:00"): + r = r[:-6] + "Z" return r elif isinstance(o, datetime.date): return o.isoformat() diff --git a/django/core/serializers/jsonl.py b/django/core/serializers/jsonl.py index 4b3e46ed8e..c264c2ccaf 100644 --- a/django/core/serializers/jsonl.py +++ b/django/core/serializers/jsonl.py @@ -6,24 +6,24 @@ import json from django.core.serializers.base import DeserializationError from django.core.serializers.json import DjangoJSONEncoder -from django.core.serializers.python import ( - Deserializer as PythonDeserializer, Serializer as PythonSerializer, -) +from django.core.serializers.python import Deserializer as PythonDeserializer +from django.core.serializers.python import Serializer as PythonSerializer class Serializer(PythonSerializer): """Convert a queryset to JSON Lines.""" + internal_use_only = False def _init_options(self): self._current = None self.json_kwargs = self.options.copy() - self.json_kwargs.pop('stream', None) - self.json_kwargs.pop('fields', None) - self.json_kwargs.pop('indent', None) - self.json_kwargs['separators'] = (',', ': ') - self.json_kwargs.setdefault('cls', DjangoJSONEncoder) - self.json_kwargs.setdefault('ensure_ascii', False) + self.json_kwargs.pop("stream", None) + self.json_kwargs.pop("fields", None) + self.json_kwargs.pop("indent", None) + self.json_kwargs["separators"] = (",", ": ") + self.json_kwargs.setdefault("cls", DjangoJSONEncoder) + self.json_kwargs.setdefault("ensure_ascii", False) def start_serialization(self): self._init_options() diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py index 0ceb676e90..a3918bf9d2 100644 --- a/django/core/serializers/python.py +++ b/django/core/serializers/python.py @@ -32,10 +32,10 @@ class Serializer(base.Serializer): self._current = None def get_dump_object(self, obj): - data = {'model': str(obj._meta)} - if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'): + data = {"model": str(obj._meta)} + if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"): data["pk"] = self._value_from_field(obj, obj._meta.pk) - data['fields'] = self._current + data["fields"] = self._current return data def _value_from_field(self, obj, field): @@ -49,7 +49,9 @@ class Serializer(base.Serializer): self._current[field.name] = self._value_from_field(obj, field) def handle_fk_field(self, obj, field): - if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'): + if self.use_natural_foreign_keys and hasattr( + field.remote_field.model, "natural_key" + ): related = getattr(obj, field.name) if related: value = related.natural_key() @@ -61,13 +63,19 @@ class Serializer(base.Serializer): def handle_m2m_field(self, obj, field): if field.remote_field.through._meta.auto_created: - if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'): + if self.use_natural_foreign_keys and hasattr( + field.remote_field.model, "natural_key" + ): + def m2m_value(value): return value.natural_key() + else: + def m2m_value(value): return self._value_from_field(value, value._meta.pk) - m2m_iter = getattr(obj, '_prefetched_objects_cache', {}).get( + + m2m_iter = getattr(obj, "_prefetched_objects_cache", {}).get( field.name, getattr(obj, field.name).iterator(), ) @@ -77,14 +85,16 @@ class Serializer(base.Serializer): return self.objects -def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False, **options): +def Deserializer( + object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False, **options +): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ - handle_forward_references = options.pop('handle_forward_references', False) + handle_forward_references = options.pop("handle_forward_references", False) field_names_cache = {} # Model: <list of field_names> for d in object_list: @@ -97,11 +107,13 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False else: raise data = {} - if 'pk' in d: + if "pk" in d: try: - data[Model._meta.pk.attname] = Model._meta.pk.to_python(d.get('pk')) + data[Model._meta.pk.attname] = Model._meta.pk.to_python(d.get("pk")) except Exception as e: - raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), None) + raise base.DeserializationError.WithData( + e, d["model"], d.get("pk"), None + ) m2m_data = {} deferred_fields = {} @@ -119,21 +131,33 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False field = Model._meta.get_field(field_name) # Handle M2M relations - if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): + if field.remote_field and isinstance( + field.remote_field, models.ManyToManyRel + ): try: - values = base.deserialize_m2m_values(field, field_value, using, handle_forward_references) + values = base.deserialize_m2m_values( + field, field_value, using, handle_forward_references + ) except base.M2MDeserializationError as e: - raise base.DeserializationError.WithData(e.original_exc, d['model'], d.get('pk'), e.pk) + raise base.DeserializationError.WithData( + e.original_exc, d["model"], d.get("pk"), e.pk + ) if values == base.DEFER_FIELD: deferred_fields[field] = field_value else: m2m_data[field.name] = values # Handle FK fields - elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): + elif field.remote_field and isinstance( + field.remote_field, models.ManyToOneRel + ): try: - value = base.deserialize_fk_value(field, field_value, using, handle_forward_references) + value = base.deserialize_fk_value( + field, field_value, using, handle_forward_references + ) except Exception as e: - raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value) + raise base.DeserializationError.WithData( + e, d["model"], d.get("pk"), field_value + ) if value == base.DEFER_FIELD: deferred_fields[field] = field_value else: @@ -143,7 +167,9 @@ def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False try: data[field.name] = field.to_python(field_value) except Exception as e: - raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value) + raise base.DeserializationError.WithData( + e, d["model"], d.get("pk"), field_value + ) obj = base.build_instance(Model, data, using) yield base.DeserializedObject(obj, m2m_data, deferred_fields) @@ -154,4 +180,6 @@ def _get_model(model_identifier): try: return apps.get_model(model_identifier) except (LookupError, TypeError): - raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier) + raise base.DeserializationError( + "Invalid model identifier: '%s'" % model_identifier + ) diff --git a/django/core/serializers/pyyaml.py b/django/core/serializers/pyyaml.py index 9719f6e1b4..9a20b6658f 100644 --- a/django/core/serializers/pyyaml.py +++ b/django/core/serializers/pyyaml.py @@ -11,28 +11,30 @@ from io import StringIO import yaml from django.core.serializers.base import DeserializationError -from django.core.serializers.python import ( - Deserializer as PythonDeserializer, Serializer as PythonSerializer, -) +from django.core.serializers.python import Deserializer as PythonDeserializer +from django.core.serializers.python import Serializer as PythonSerializer from django.db import models # Use the C (faster) implementation if possible try: - from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader + from yaml import CSafeDumper as SafeDumper + from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeDumper, SafeLoader class DjangoSafeDumper(SafeDumper): def represent_decimal(self, data): - return self.represent_scalar('tag:yaml.org,2002:str', str(data)) + return self.represent_scalar("tag:yaml.org,2002:str", str(data)) def represent_ordered_dict(self, data): - return self.represent_mapping('tag:yaml.org,2002:map', data.items()) + return self.represent_mapping("tag:yaml.org,2002:map", data.items()) DjangoSafeDumper.add_representer(decimal.Decimal, DjangoSafeDumper.represent_decimal) -DjangoSafeDumper.add_representer(collections.OrderedDict, DjangoSafeDumper.represent_ordered_dict) +DjangoSafeDumper.add_representer( + collections.OrderedDict, DjangoSafeDumper.represent_ordered_dict +) # Workaround to represent dictionaries in insertion order. # See https://github.com/yaml/pyyaml/pull/143. DjangoSafeDumper.add_representer(dict, DjangoSafeDumper.represent_ordered_dict) @@ -56,7 +58,7 @@ class Serializer(PythonSerializer): super().handle_field(obj, field) def end_serialization(self): - self.options.setdefault('allow_unicode', True) + self.options.setdefault("allow_unicode", True) yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options) def getvalue(self): diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index 88bfa59032..8d3918cfaa 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -11,23 +11,25 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.serializers import base from django.db import DEFAULT_DB_ALIAS, models -from django.utils.xmlutils import ( - SimplerXMLGenerator, UnserializableContentError, -) +from django.utils.xmlutils import SimplerXMLGenerator, UnserializableContentError class Serializer(base.Serializer): """Serialize a QuerySet to XML.""" def indent(self, level): - if self.options.get('indent') is not None: - self.xml.ignorableWhitespace('\n' + ' ' * self.options.get('indent') * level) + if self.options.get("indent") is not None: + self.xml.ignorableWhitespace( + "\n" + " " * self.options.get("indent") * level + ) def start_serialization(self): """ Start serialization -- open the XML document and the root element. """ - self.xml = SimplerXMLGenerator(self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET)) + self.xml = SimplerXMLGenerator( + self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET) + ) self.xml.startDocument() self.xml.startElement("django-objects", {"version": "1.0"}) @@ -44,14 +46,16 @@ class Serializer(base.Serializer): Called as each object is handled. """ if not hasattr(obj, "_meta"): - raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj)) + raise base.SerializationError( + "Non-model object (%s) encountered during serialization" % type(obj) + ) self.indent(1) - attrs = {'model': str(obj._meta)} - if not self.use_natural_primary_keys or not hasattr(obj, 'natural_key'): + attrs = {"model": str(obj._meta)} + if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"): obj_pk = obj.pk if obj_pk is not None: - attrs['pk'] = str(obj_pk) + attrs["pk"] = str(obj_pk) self.xml.startElement("object", attrs) @@ -68,23 +72,28 @@ class Serializer(base.Serializer): ManyToManyFields). """ self.indent(2) - self.xml.startElement('field', { - 'name': field.name, - 'type': field.get_internal_type(), - }) + self.xml.startElement( + "field", + { + "name": field.name, + "type": field.get_internal_type(), + }, + ) # Get a "string version" of the object's data. if getattr(obj, field.name) is not None: value = field.value_to_string(obj) - if field.get_internal_type() == 'JSONField': + if field.get_internal_type() == "JSONField": # Dump value since JSONField.value_to_string() doesn't output # strings. value = json.dumps(value, cls=field.encoder) try: self.xml.characters(value) except UnserializableContentError: - raise ValueError("%s.%s (pk:%s) contains unserializable characters" % ( - obj.__class__.__name__, field.name, obj.pk)) + raise ValueError( + "%s.%s (pk:%s) contains unserializable characters" + % (obj.__class__.__name__, field.name, obj.pk) + ) else: self.xml.addQuickElement("None") @@ -98,7 +107,9 @@ class Serializer(base.Serializer): self._start_relational_field(field) related_att = getattr(obj, field.get_attname()) if related_att is not None: - if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'): + if self.use_natural_foreign_keys and hasattr( + field.remote_field.model, "natural_key" + ): related = getattr(obj, field.name) # If related object has a natural key, use it related = related.natural_key() @@ -121,7 +132,9 @@ class Serializer(base.Serializer): """ if field.remote_field.through._meta.auto_created: self._start_relational_field(field) - if self.use_natural_foreign_keys and hasattr(field.remote_field.model, 'natural_key'): + if self.use_natural_foreign_keys and hasattr( + field.remote_field.model, "natural_key" + ): # If the objects in the m2m have a natural key, use it def handle_m2m(value): natural = value.natural_key() @@ -132,12 +145,13 @@ class Serializer(base.Serializer): self.xml.characters(str(key_value)) self.xml.endElement("natural") self.xml.endElement("object") + else: + def handle_m2m(value): - self.xml.addQuickElement("object", attrs={ - 'pk': str(value.pk) - }) - m2m_iter = getattr(obj, '_prefetched_objects_cache', {}).get( + self.xml.addQuickElement("object", attrs={"pk": str(value.pk)}) + + m2m_iter = getattr(obj, "_prefetched_objects_cache", {}).get( field.name, getattr(obj, field.name).iterator(), ) @@ -149,19 +163,29 @@ class Serializer(base.Serializer): def _start_relational_field(self, field): """Output the <field> element for relational fields.""" self.indent(2) - self.xml.startElement('field', { - 'name': field.name, - 'rel': field.remote_field.__class__.__name__, - 'to': str(field.remote_field.model._meta), - }) + self.xml.startElement( + "field", + { + "name": field.name, + "rel": field.remote_field.__class__.__name__, + "to": str(field.remote_field.model._meta), + }, + ) class Deserializer(base.Deserializer): """Deserialize XML.""" - def __init__(self, stream_or_string, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False, **options): + def __init__( + self, + stream_or_string, + *, + using=DEFAULT_DB_ALIAS, + ignorenonexistent=False, + **options, + ): super().__init__(stream_or_string, **options) - self.handle_forward_references = options.pop('handle_forward_references', False) + self.handle_forward_references = options.pop("handle_forward_references", False) self.event_stream = pulldom.parse(self.stream, self._make_parser()) self.db = using self.ignore = ignorenonexistent @@ -185,9 +209,10 @@ class Deserializer(base.Deserializer): # Start building a data dictionary from the object. data = {} - if node.hasAttribute('pk'): + if node.hasAttribute("pk"): data[Model._meta.pk.attname] = Model._meta.pk.to_python( - node.getAttribute('pk')) + node.getAttribute("pk") + ) # Also start building a dict of m2m data (this is saved as # {m2m_accessor_attribute : [list_of_related_objects]}) @@ -201,7 +226,9 @@ class Deserializer(base.Deserializer): # sensing a pattern here?) field_name = field_node.getAttribute("name") if not field_name: - raise base.DeserializationError("<field> node is missing the 'name' attribute") + raise base.DeserializationError( + "<field> node is missing the 'name' attribute" + ) # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will @@ -211,34 +238,38 @@ class Deserializer(base.Deserializer): field = Model._meta.get_field(field_name) # As is usually the case, relation fields get the special treatment. - if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): + if field.remote_field and isinstance( + field.remote_field, models.ManyToManyRel + ): value = self._handle_m2m_field_node(field_node, field) if value == base.DEFER_FIELD: deferred_fields[field] = [ [ getInnerText(nat_node).strip() - for nat_node in obj_node.getElementsByTagName('natural') + for nat_node in obj_node.getElementsByTagName("natural") ] - for obj_node in field_node.getElementsByTagName('object') + for obj_node in field_node.getElementsByTagName("object") ] else: m2m_data[field.name] = value - elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): + elif field.remote_field and isinstance( + field.remote_field, models.ManyToOneRel + ): value = self._handle_fk_field_node(field_node, field) if value == base.DEFER_FIELD: deferred_fields[field] = [ getInnerText(k).strip() - for k in field_node.getElementsByTagName('natural') + for k in field_node.getElementsByTagName("natural") ] else: data[field.attname] = value else: - if field_node.getElementsByTagName('None'): + if field_node.getElementsByTagName("None"): value = None else: value = field.to_python(getInnerText(field_node).strip()) # Load value since JSONField.to_python() outputs strings. - if field.get_internal_type() == 'JSONField': + if field.get_internal_type() == "JSONField": value = json.loads(value, cls=field.decoder) data[field.name] = value @@ -252,17 +283,19 @@ class Deserializer(base.Deserializer): Handle a <field> node for a ForeignKey """ # Check if there is a child node named 'None', returning None if so. - if node.getElementsByTagName('None'): + if node.getElementsByTagName("None"): return None else: model = field.remote_field.model - if hasattr(model._default_manager, 'get_by_natural_key'): - keys = node.getElementsByTagName('natural') + if hasattr(model._default_manager, "get_by_natural_key"): + keys = node.getElementsByTagName("natural") if keys: # If there are 'natural' subelements, it must be a natural key field_value = [getInnerText(k).strip() for k in keys] try: - obj = model._default_manager.db_manager(self.db).get_by_natural_key(*field_value) + obj = model._default_manager.db_manager( + self.db + ).get_by_natural_key(*field_value) except ObjectDoesNotExist: if self.handle_forward_references: return base.DEFER_FIELD @@ -276,11 +309,15 @@ class Deserializer(base.Deserializer): else: # Otherwise, treat like a normal PK field_value = getInnerText(node).strip() - obj_pk = model._meta.get_field(field.remote_field.field_name).to_python(field_value) + obj_pk = model._meta.get_field( + field.remote_field.field_name + ).to_python(field_value) return obj_pk else: field_value = getInnerText(node).strip() - return model._meta.get_field(field.remote_field.field_name).to_python(field_value) + return model._meta.get_field(field.remote_field.field_name).to_python( + field_value + ) def _handle_m2m_field_node(self, node, field): """ @@ -288,23 +325,31 @@ class Deserializer(base.Deserializer): """ model = field.remote_field.model default_manager = model._default_manager - if hasattr(default_manager, 'get_by_natural_key'): + if hasattr(default_manager, "get_by_natural_key"): + def m2m_convert(n): - keys = n.getElementsByTagName('natural') + keys = n.getElementsByTagName("natural") if keys: # If there are 'natural' subelements, it must be a natural key field_value = [getInnerText(k).strip() for k in keys] - obj_pk = default_manager.db_manager(self.db).get_by_natural_key(*field_value).pk + obj_pk = ( + default_manager.db_manager(self.db) + .get_by_natural_key(*field_value) + .pk + ) else: # Otherwise, treat like a normal PK value. - obj_pk = model._meta.pk.to_python(n.getAttribute('pk')) + obj_pk = model._meta.pk.to_python(n.getAttribute("pk")) return obj_pk + else: + def m2m_convert(n): - return model._meta.pk.to_python(n.getAttribute('pk')) + return model._meta.pk.to_python(n.getAttribute("pk")) + values = [] try: - for c in node.getElementsByTagName('object'): + for c in node.getElementsByTagName("object"): values.append(m2m_convert(c)) except Exception as e: if isinstance(e, ObjectDoesNotExist) and self.handle_forward_references: @@ -323,13 +368,15 @@ class Deserializer(base.Deserializer): if not model_identifier: raise base.DeserializationError( "<%s> node is missing the required '%s' attribute" - % (node.nodeName, attr)) + % (node.nodeName, attr) + ) try: return apps.get_model(model_identifier) except (LookupError, TypeError): raise base.DeserializationError( "<%s> node has invalid model identifier: '%s'" - % (node.nodeName, model_identifier)) + % (node.nodeName, model_identifier) + ) def getInnerText(node): @@ -337,7 +384,10 @@ def getInnerText(node): # inspired by https://mail.python.org/pipermail/xml-sig/2005-March/011022.html inner_text = [] for child in node.childNodes: - if child.nodeType == child.TEXT_NODE or child.nodeType == child.CDATA_SECTION_NODE: + if ( + child.nodeType == child.TEXT_NODE + or child.nodeType == child.CDATA_SECTION_NODE + ): inner_text.append(child.data) elif child.nodeType == child.ELEMENT_NODE: inner_text.extend(getInnerText(child)) @@ -355,6 +405,7 @@ class DefusedExpatParser(_ExpatParser): Forbid DTDs, external entity references """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.setFeature(handler.feature_external_ges, False) @@ -363,8 +414,9 @@ class DefusedExpatParser(_ExpatParser): def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): raise DTDForbidden(name, sysid, pubid) - def entity_decl(self, name, is_parameter_entity, value, base, - sysid, pubid, notation_name): + def entity_decl( + self, name, is_parameter_entity, value, base, sysid, pubid, notation_name + ): raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name) def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): @@ -385,12 +437,14 @@ class DefusedExpatParser(_ExpatParser): class DefusedXmlException(ValueError): """Base exception.""" + def __repr__(self): return str(self) class DTDForbidden(DefusedXmlException): """Document type definition is forbidden.""" + def __init__(self, name, sysid, pubid): super().__init__() self.name = name @@ -404,6 +458,7 @@ class DTDForbidden(DefusedXmlException): class EntitiesForbidden(DefusedXmlException): """Entity definition is forbidden.""" + def __init__(self, name, value, base, sysid, pubid, notation_name): super().__init__() self.name = name @@ -420,6 +475,7 @@ class EntitiesForbidden(DefusedXmlException): class ExternalReferenceForbidden(DefusedXmlException): """Resolving an external reference is forbidden.""" + def __init__(self, context, base, sysid, pubid): super().__init__() self.context = context diff --git a/django/core/servers/basehttp.py b/django/core/servers/basehttp.py index 6cc8a46778..440e7bc9cb 100644 --- a/django/core/servers/basehttp.py +++ b/django/core/servers/basehttp.py @@ -19,9 +19,9 @@ from django.core.wsgi import get_wsgi_application from django.db import connections from django.utils.module_loading import import_string -__all__ = ('WSGIServer', 'WSGIRequestHandler') +__all__ = ("WSGIServer", "WSGIRequestHandler") -logger = logging.getLogger('django.server') +logger = logging.getLogger("django.server") def get_internal_wsgi_application(): @@ -38,7 +38,8 @@ def get_internal_wsgi_application(): whatever ``django.core.wsgi.get_wsgi_application`` returns. """ from django.conf import settings - app_path = getattr(settings, 'WSGI_APPLICATION') + + app_path = getattr(settings, "WSGI_APPLICATION") if app_path is None: return get_wsgi_application() @@ -53,11 +54,14 @@ def get_internal_wsgi_application(): def is_broken_pipe_error(): exc_type, _, _ = sys.exc_info() - return issubclass(exc_type, ( - BrokenPipeError, - ConnectionAbortedError, - ConnectionResetError, - )) + return issubclass( + exc_type, + ( + BrokenPipeError, + ConnectionAbortedError, + ConnectionResetError, + ), + ) class WSGIServer(simple_server.WSGIServer): @@ -80,6 +84,7 @@ class WSGIServer(simple_server.WSGIServer): class ThreadedWSGIServer(socketserver.ThreadingMixIn, WSGIServer): """A threaded version of the WSGIServer""" + daemon_threads = True def __init__(self, *args, connections_override=None, **kwargs): @@ -106,7 +111,7 @@ class ThreadedWSGIServer(socketserver.ThreadingMixIn, WSGIServer): class ServerHandler(simple_server.ServerHandler): - http_version = '1.1' + http_version = "1.1" def __init__(self, stdin, stdout, stderr, environ, **kwargs): """ @@ -116,24 +121,26 @@ class ServerHandler(simple_server.ServerHandler): This fix applies only for testserver/runserver. """ try: - content_length = int(environ.get('CONTENT_LENGTH')) + content_length = int(environ.get("CONTENT_LENGTH")) except (ValueError, TypeError): content_length = 0 - super().__init__(LimitedStream(stdin, content_length), stdout, stderr, environ, **kwargs) + super().__init__( + LimitedStream(stdin, content_length), stdout, stderr, environ, **kwargs + ) def cleanup_headers(self): super().cleanup_headers() # HTTP/1.1 requires support for persistent connections. Send 'close' if # the content length is unknown to prevent clients from reusing the # connection. - if 'Content-Length' not in self.headers: - self.headers['Connection'] = 'close' + if "Content-Length" not in self.headers: + self.headers["Connection"] = "close" # Persistent connections require threading server. elif not isinstance(self.request_handler.server, socketserver.ThreadingMixIn): - self.headers['Connection'] = 'close' + self.headers["Connection"] = "close" # Mark the connection for closing if it's set as such above or if the # application sent the header. - if self.headers.get('Connection') == 'close': + if self.headers.get("Connection") == "close": self.request_handler.close_connection = True def close(self): @@ -142,7 +149,7 @@ class ServerHandler(simple_server.ServerHandler): class WSGIRequestHandler(simple_server.WSGIRequestHandler): - protocol_version = 'HTTP/1.1' + protocol_version = "HTTP/1.1" def address_string(self): # Short-circuit parent method to not call socket.getfqdn @@ -150,22 +157,23 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler): def log_message(self, format, *args): extra = { - 'request': self.request, - 'server_time': self.log_date_time_string(), + "request": self.request, + "server_time": self.log_date_time_string(), } - if args[1][0] == '4': + if args[1][0] == "4": # 0x16 = Handshake, 0x03 = SSL 3.0 or TLS 1.x - if args[0].startswith('\x16\x03'): - extra['status_code'] = 500 + if args[0].startswith("\x16\x03"): + extra["status_code"] = 500 logger.error( "You're accessing the development server over HTTPS, but " - "it only supports HTTP.\n", extra=extra, + "it only supports HTTP.\n", + extra=extra, ) return if args[1].isdigit() and len(args[1]) == 3: status_code = int(args[1]) - extra['status_code'] = status_code + extra["status_code"] = status_code if status_code >= 500: level = logger.error @@ -184,7 +192,7 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler): # between underscores and dashes both normalized to underscores in WSGI # env vars. Nginx and Apache 2.4+ both do this as well. for k in self.headers: - if '_' in k: + if "_" in k: del self.headers[k] return super().get_environ() @@ -203,9 +211,9 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler): """Copy of WSGIRequestHandler.handle() but with different ServerHandler""" self.raw_requestline = self.rfile.readline(65537) if len(self.raw_requestline) > 65536: - self.requestline = '' - self.request_version = '' - self.command = '' + self.requestline = "" + self.request_version = "" + self.command = "" self.send_error(414) return @@ -215,14 +223,14 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler): handler = ServerHandler( self.rfile, self.wfile, self.get_stderr(), self.get_environ() ) - handler.request_handler = self # backpointer for logging & connection closing + handler.request_handler = self # backpointer for logging & connection closing handler.run(self.server.get_app()) def run(addr, port, wsgi_handler, ipv6=False, threading=False, server_cls=WSGIServer): server_address = (addr, port) if threading: - httpd_cls = type('WSGIServer', (socketserver.ThreadingMixIn, server_cls), {}) + httpd_cls = type("WSGIServer", (socketserver.ThreadingMixIn, server_cls), {}) else: httpd_cls = server_cls httpd = httpd_cls(server_address, WSGIRequestHandler, ipv6=ipv6) diff --git a/django/core/signing.py b/django/core/signing.py index cd86fdfab6..916885abb3 100644 --- a/django/core/signing.py +++ b/django/core/signing.py @@ -45,26 +45,28 @@ from django.utils.encoding import force_bytes from django.utils.module_loading import import_string from django.utils.regex_helper import _lazy_re_compile -_SEP_UNSAFE = _lazy_re_compile(r'^[A-z0-9-_=]*$') -BASE62_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' +_SEP_UNSAFE = _lazy_re_compile(r"^[A-z0-9-_=]*$") +BASE62_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" class BadSignature(Exception): """Signature does not match.""" + pass class SignatureExpired(BadSignature): """Signature timestamp is older than required max_age.""" + pass def b62_encode(s): if s == 0: - return '0' - sign = '-' if s < 0 else '' + return "0" + sign = "-" if s < 0 else "" s = abs(s) - encoded = '' + encoded = "" while s > 0: s, remainder = divmod(s, 62) encoded = BASE62_ALPHABET[remainder] + encoded @@ -72,10 +74,10 @@ def b62_encode(s): def b62_decode(s): - if s == '0': + if s == "0": return 0 sign = 1 - if s[0] == '-': + if s[0] == "-": s = s[1:] sign = -1 decoded = 0 @@ -85,24 +87,26 @@ def b62_decode(s): def b64_encode(s): - return base64.urlsafe_b64encode(s).strip(b'=') + return base64.urlsafe_b64encode(s).strip(b"=") def b64_decode(s): - pad = b'=' * (-len(s) % 4) + pad = b"=" * (-len(s) % 4) return base64.urlsafe_b64decode(s + pad) -def base64_hmac(salt, value, key, algorithm='sha1'): - return b64_encode(salted_hmac(salt, value, key, algorithm=algorithm).digest()).decode() +def base64_hmac(salt, value, key, algorithm="sha1"): + return b64_encode( + salted_hmac(salt, value, key, algorithm=algorithm).digest() + ).decode() def _cookie_signer_key(key): # SECRET_KEYS items may be str or bytes. - return b'django.http.cookies' + force_bytes(key) + return b"django.http.cookies" + force_bytes(key) -def get_cookie_signer(salt='django.core.signing.get_cookie_signer'): +def get_cookie_signer(salt="django.core.signing.get_cookie_signer"): Signer = import_string(settings.SIGNING_BACKEND) return Signer( key=_cookie_signer_key(settings.SECRET_KEY), @@ -116,14 +120,17 @@ class JSONSerializer: Simple wrapper around json to be used in signing.dumps and signing.loads. """ + def dumps(self, obj): - return json.dumps(obj, separators=(',', ':')).encode('latin-1') + return json.dumps(obj, separators=(",", ":")).encode("latin-1") def loads(self, data): - return json.loads(data.decode('latin-1')) + return json.loads(data.decode("latin-1")) -def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, compress=False): +def dumps( + obj, key=None, salt="django.core.signing", serializer=JSONSerializer, compress=False +): """ Return URL-safe, hmac signed base64 compressed JSON string. If key is None, use settings.SECRET_KEY instead. The hmac algorithm is the default @@ -140,13 +147,15 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, The serializer is expected to return a bytestring. """ - return TimestampSigner(key, salt=salt).sign_object(obj, serializer=serializer, compress=compress) + return TimestampSigner(key, salt=salt).sign_object( + obj, serializer=serializer, compress=compress + ) def loads( s, key=None, - salt='django.core.signing', + salt="django.core.signing", serializer=JSONSerializer, max_age=None, fallback_keys=None, @@ -167,7 +176,7 @@ class Signer: def __init__( self, key=None, - sep=':', + sep=":", salt=None, algorithm=None, fallback_keys=None, @@ -181,18 +190,21 @@ class Signer: self.sep = sep if _SEP_UNSAFE.match(self.sep): raise ValueError( - 'Unsafe Signer separator: %r (cannot be empty or consist of ' - 'only A-z0-9-_=)' % sep, + "Unsafe Signer separator: %r (cannot be empty or consist of " + "only A-z0-9-_=)" % sep, ) - self.salt = salt or '%s.%s' % (self.__class__.__module__, self.__class__.__name__) - self.algorithm = algorithm or 'sha256' + self.salt = salt or "%s.%s" % ( + self.__class__.__module__, + self.__class__.__name__, + ) + self.algorithm = algorithm or "sha256" def signature(self, value, key=None): key = key or self.key - return base64_hmac(self.salt + 'signer', value, key, algorithm=self.algorithm) + return base64_hmac(self.salt + "signer", value, key, algorithm=self.algorithm) def sign(self, value): - return '%s%s%s' % (value, self.sep, self.signature(value)) + return "%s%s%s" % (value, self.sep, self.signature(value)) def unsign(self, signed_value): if self.sep not in signed_value: @@ -225,14 +237,14 @@ class Signer: is_compressed = True base64d = b64_encode(data).decode() if is_compressed: - base64d = '.' + base64d + base64d = "." + base64d return self.sign(base64d) def unsign_object(self, signed_obj, serializer=JSONSerializer, **kwargs): # Signer.unsign() returns str but base64 and zlib compression operate # on bytes. base64d = self.unsign(signed_obj, **kwargs).encode() - decompress = base64d[:1] == b'.' + decompress = base64d[:1] == b"." if decompress: # It's compressed; uncompress it first. base64d = base64d[1:] @@ -243,12 +255,11 @@ class Signer: class TimestampSigner(Signer): - def timestamp(self): return b62_encode(int(time.time())) def sign(self, value): - value = '%s%s%s' % (value, self.sep, self.timestamp()) + value = "%s%s%s" % (value, self.sep, self.timestamp()) return super().sign(value) def unsign(self, value, max_age=None): @@ -265,6 +276,5 @@ class TimestampSigner(Signer): # Check timestamp is not older than max_age age = time.time() - timestamp if age > max_age: - raise SignatureExpired( - 'Signature age %s > %s seconds' % (age, max_age)) + raise SignatureExpired("Signature age %s > %s seconds" % (age, max_age)) return value diff --git a/django/core/validators.py b/django/core/validators.py index 9ad90f006f..5272258a77 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -8,21 +8,24 @@ from django.utils.deconstruct import deconstructible from django.utils.encoding import punycode from django.utils.ipv6 import is_valid_ipv6_address from django.utils.regex_helper import _lazy_re_compile -from django.utils.translation import gettext_lazy as _, ngettext_lazy +from django.utils.translation import gettext_lazy as _ +from django.utils.translation import ngettext_lazy # These values, if given to validate(), will trigger the self.required check. -EMPTY_VALUES = (None, '', [], (), {}) +EMPTY_VALUES = (None, "", [], (), {}) @deconstructible class RegexValidator: - regex = '' - message = _('Enter a valid value.') - code = 'invalid' + regex = "" + message = _("Enter a valid value.") + code = "invalid" inverse_match = False flags = 0 - def __init__(self, regex=None, message=None, code=None, inverse_match=None, flags=None): + def __init__( + self, regex=None, message=None, code=None, inverse_match=None, flags=None + ): if regex is not None: self.regex = regex if message is not None: @@ -34,7 +37,9 @@ class RegexValidator: if flags is not None: self.flags = flags if self.flags and not isinstance(self.regex, str): - raise TypeError("If the flags are set, regex must be a regular expression string.") + raise TypeError( + "If the flags are set, regex must be a regular expression string." + ) self.regex = _lazy_re_compile(self.regex, self.flags) @@ -46,54 +51,58 @@ class RegexValidator: regex_matches = self.regex.search(str(value)) invalid_input = regex_matches if self.inverse_match else not regex_matches if invalid_input: - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) def __eq__(self, other): return ( - isinstance(other, RegexValidator) and - self.regex.pattern == other.regex.pattern and - self.regex.flags == other.regex.flags and - (self.message == other.message) and - (self.code == other.code) and - (self.inverse_match == other.inverse_match) + isinstance(other, RegexValidator) + and self.regex.pattern == other.regex.pattern + and self.regex.flags == other.regex.flags + and (self.message == other.message) + and (self.code == other.code) + and (self.inverse_match == other.inverse_match) ) @deconstructible class URLValidator(RegexValidator): - ul = '\u00a1-\uffff' # Unicode letters range (must not be a raw string). + ul = "\u00a1-\uffff" # Unicode letters range (must not be a raw string). # IP patterns ipv4_re = ( - r'(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)' - r'(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}' + r"(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)" + r"(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}" ) - ipv6_re = r'\[[0-9a-f:.]+\]' # (simple regex, validated later) + ipv6_re = r"\[[0-9a-f:.]+\]" # (simple regex, validated later) # Host patterns - hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?' + hostname_re = ( + r"[a-z" + ul + r"0-9](?:[a-z" + ul + r"0-9-]{0,61}[a-z" + ul + r"0-9])?" + ) # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1 - domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?<!-))*' + domain_re = r"(?:\.(?!-)[a-z" + ul + r"0-9-]{1,63}(?<!-))*" tld_re = ( - r'\.' # dot - r'(?!-)' # can't start with a dash - r'(?:[a-z' + ul + '-]{2,63}' # domain label - r'|xn--[a-z0-9]{1,59})' # or punycode label - r'(?<!-)' # can't end with a dash - r'\.?' # may have a trailing dot + r"\." # dot + r"(?!-)" # can't start with a dash + r"(?:[a-z" + ul + "-]{2,63}" # domain label + r"|xn--[a-z0-9]{1,59})" # or punycode label + r"(?<!-)" # can't end with a dash + r"\.?" # may have a trailing dot ) - host_re = '(' + hostname_re + domain_re + tld_re + '|localhost)' + host_re = "(" + hostname_re + domain_re + tld_re + "|localhost)" regex = _lazy_re_compile( - r'^(?:[a-z0-9.+-]*)://' # scheme is validated separately - r'(?:[^\s:@/]+(?::[^\s:@/]*)?@)?' # user:pass authentication - r'(?:' + ipv4_re + '|' + ipv6_re + '|' + host_re + ')' - r'(?::[0-9]{1,5})?' # port - r'(?:[/?#][^\s]*)?' # resource path - r'\Z', re.IGNORECASE) - message = _('Enter a valid URL.') - schemes = ['http', 'https', 'ftp', 'ftps'] - unsafe_chars = frozenset('\t\r\n') + r"^(?:[a-z0-9.+-]*)://" # scheme is validated separately + r"(?:[^\s:@/]+(?::[^\s:@/]*)?@)?" # user:pass authentication + r"(?:" + ipv4_re + "|" + ipv6_re + "|" + host_re + ")" + r"(?::[0-9]{1,5})?" # port + r"(?:[/?#][^\s]*)?" # resource path + r"\Z", + re.IGNORECASE, + ) + message = _("Enter a valid URL.") + schemes = ["http", "https", "ftp", "ftps"] + unsafe_chars = frozenset("\t\r\n") def __init__(self, schemes=None, **kwargs): super().__init__(**kwargs) @@ -102,19 +111,19 @@ class URLValidator(RegexValidator): def __call__(self, value): if not isinstance(value, str): - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) if self.unsafe_chars.intersection(value): - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) # Check if the scheme is valid. - scheme = value.split('://')[0].lower() + scheme = value.split("://")[0].lower() if scheme not in self.schemes: - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) # Then check full URL try: splitted_url = urlsplit(value) except ValueError: - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) try: super().__call__(value) except ValidationError as e: @@ -131,26 +140,28 @@ class URLValidator(RegexValidator): raise else: # Now verify IPv6 in the netloc part - host_match = re.search(r'^\[(.+)\](?::[0-9]{1,5})?$', splitted_url.netloc) + host_match = re.search(r"^\[(.+)\](?::[0-9]{1,5})?$", splitted_url.netloc) if host_match: potential_ip = host_match[1] try: validate_ipv6_address(potential_ip) except ValidationError: - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError( + self.message, code=self.code, params={"value": value} + ) # The maximum length of a full host name is 253 characters per RFC 1034 # section 3.1. It's defined to be 255 bytes or less, but this includes # one byte for the length of the name and one byte for the trailing dot # that's used to indicate absolute names in DNS. if splitted_url.hostname is None or len(splitted_url.hostname) > 253: - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) integer_validator = RegexValidator( - _lazy_re_compile(r'^-?\d+\Z'), - message=_('Enter a valid integer.'), - code='invalid', + _lazy_re_compile(r"^-?\d+\Z"), + message=_("Enter a valid integer."), + code="invalid", ) @@ -160,21 +171,24 @@ def validate_integer(value): @deconstructible class EmailValidator: - message = _('Enter a valid email address.') - code = 'invalid' + message = _("Enter a valid email address.") + code = "invalid" user_regex = _lazy_re_compile( r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z" # dot-atom r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)', # quoted-string - re.IGNORECASE) + re.IGNORECASE, + ) domain_regex = _lazy_re_compile( # max length for domain name labels is 63 characters per RFC 1034 - r'((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z', - re.IGNORECASE) + r"((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+)(?:[A-Z0-9-]{2,63}(?<!-))\Z", + re.IGNORECASE, + ) literal_regex = _lazy_re_compile( # literal form, ipv4 or ipv6 address (SMTP 4.1.3) - r'\[([A-F0-9:.]+)\]\Z', - re.IGNORECASE) - domain_allowlist = ['localhost'] + r"\[([A-F0-9:.]+)\]\Z", + re.IGNORECASE, + ) + domain_allowlist = ["localhost"] def __init__(self, message=None, code=None, allowlist=None): if message is not None: @@ -185,16 +199,17 @@ class EmailValidator: self.domain_allowlist = allowlist def __call__(self, value): - if not value or '@' not in value: - raise ValidationError(self.message, code=self.code, params={'value': value}) + if not value or "@" not in value: + raise ValidationError(self.message, code=self.code, params={"value": value}) - user_part, domain_part = value.rsplit('@', 1) + user_part, domain_part = value.rsplit("@", 1) if not self.user_regex.match(user_part): - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) - if (domain_part not in self.domain_allowlist and - not self.validate_domain_part(domain_part)): + if domain_part not in self.domain_allowlist and not self.validate_domain_part( + domain_part + ): # Try for possible IDN domain-part try: domain_part = punycode(domain_part) @@ -203,7 +218,7 @@ class EmailValidator: else: if self.validate_domain_part(domain_part): return - raise ValidationError(self.message, code=self.code, params={'value': value}) + raise ValidationError(self.message, code=self.code, params={"value": value}) def validate_domain_part(self, domain_part): if self.domain_regex.match(domain_part): @@ -221,28 +236,30 @@ class EmailValidator: def __eq__(self, other): return ( - isinstance(other, EmailValidator) and - (self.domain_allowlist == other.domain_allowlist) and - (self.message == other.message) and - (self.code == other.code) + isinstance(other, EmailValidator) + and (self.domain_allowlist == other.domain_allowlist) + and (self.message == other.message) + and (self.code == other.code) ) validate_email = EmailValidator() -slug_re = _lazy_re_compile(r'^[-a-zA-Z0-9_]+\Z') +slug_re = _lazy_re_compile(r"^[-a-zA-Z0-9_]+\Z") validate_slug = RegexValidator( slug_re, # Translators: "letters" means latin letters: a-z and A-Z. - _('Enter a valid “slug” consisting of letters, numbers, underscores or hyphens.'), - 'invalid' + _("Enter a valid “slug” consisting of letters, numbers, underscores or hyphens."), + "invalid", ) -slug_unicode_re = _lazy_re_compile(r'^[-\w]+\Z') +slug_unicode_re = _lazy_re_compile(r"^[-\w]+\Z") validate_unicode_slug = RegexValidator( slug_unicode_re, - _('Enter a valid “slug” consisting of Unicode letters, numbers, underscores, or hyphens.'), - 'invalid' + _( + "Enter a valid “slug” consisting of Unicode letters, numbers, underscores, or hyphens." + ), + "invalid", ) @@ -250,25 +267,26 @@ def validate_ipv4_address(value): try: ipaddress.IPv4Address(value) except ValueError: - raise ValidationError(_('Enter a valid IPv4 address.'), code='invalid', params={'value': value}) + raise ValidationError( + _("Enter a valid IPv4 address."), code="invalid", params={"value": value} + ) else: # Leading zeros are forbidden to avoid ambiguity with the octal # notation. This restriction is included in Python 3.9.5+. # TODO: Remove when dropping support for PY39. - if any( - octet != '0' and octet[0] == '0' - for octet in value.split('.') - ): + if any(octet != "0" and octet[0] == "0" for octet in value.split(".")): raise ValidationError( - _('Enter a valid IPv4 address.'), - code='invalid', - params={'value': value}, + _("Enter a valid IPv4 address."), + code="invalid", + params={"value": value}, ) def validate_ipv6_address(value): if not is_valid_ipv6_address(value): - raise ValidationError(_('Enter a valid IPv6 address.'), code='invalid', params={'value': value}) + raise ValidationError( + _("Enter a valid IPv6 address."), code="invalid", params={"value": value} + ) def validate_ipv46_address(value): @@ -278,13 +296,17 @@ def validate_ipv46_address(value): try: validate_ipv6_address(value) except ValidationError: - raise ValidationError(_('Enter a valid IPv4 or IPv6 address.'), code='invalid', params={'value': value}) + raise ValidationError( + _("Enter a valid IPv4 or IPv6 address."), + code="invalid", + params={"value": value}, + ) ip_address_validator_map = { - 'both': ([validate_ipv46_address], _('Enter a valid IPv4 or IPv6 address.')), - 'ipv4': ([validate_ipv4_address], _('Enter a valid IPv4 address.')), - 'ipv6': ([validate_ipv6_address], _('Enter a valid IPv6 address.')), + "both": ([validate_ipv46_address], _("Enter a valid IPv4 or IPv6 address.")), + "ipv4": ([validate_ipv4_address], _("Enter a valid IPv4 address.")), + "ipv6": ([validate_ipv6_address], _("Enter a valid IPv6 address.")), } @@ -293,33 +315,39 @@ def ip_address_validators(protocol, unpack_ipv4): Depending on the given parameters, return the appropriate validators for the GenericIPAddressField. """ - if protocol != 'both' and unpack_ipv4: + if protocol != "both" and unpack_ipv4: raise ValueError( - "You can only use `unpack_ipv4` if `protocol` is set to 'both'") + "You can only use `unpack_ipv4` if `protocol` is set to 'both'" + ) try: return ip_address_validator_map[protocol.lower()] except KeyError: - raise ValueError("The protocol '%s' is unknown. Supported: %s" - % (protocol, list(ip_address_validator_map))) + raise ValueError( + "The protocol '%s' is unknown. Supported: %s" + % (protocol, list(ip_address_validator_map)) + ) -def int_list_validator(sep=',', message=None, code='invalid', allow_negative=False): - regexp = _lazy_re_compile(r'^%(neg)s\d+(?:%(sep)s%(neg)s\d+)*\Z' % { - 'neg': '(-)?' if allow_negative else '', - 'sep': re.escape(sep), - }) +def int_list_validator(sep=",", message=None, code="invalid", allow_negative=False): + regexp = _lazy_re_compile( + r"^%(neg)s\d+(?:%(sep)s%(neg)s\d+)*\Z" + % { + "neg": "(-)?" if allow_negative else "", + "sep": re.escape(sep), + } + ) return RegexValidator(regexp, message=message, code=code) validate_comma_separated_integer_list = int_list_validator( - message=_('Enter only digits separated by commas.'), + message=_("Enter only digits separated by commas."), ) @deconstructible class BaseValidator: - message = _('Ensure this value is %(limit_value)s (it is %(show_value)s).') - code = 'limit_value' + message = _("Ensure this value is %(limit_value)s (it is %(show_value)s).") + code = "limit_value" def __init__(self, limit_value, message=None): self.limit_value = limit_value @@ -328,8 +356,10 @@ class BaseValidator: def __call__(self, value): cleaned = self.clean(value) - limit_value = self.limit_value() if callable(self.limit_value) else self.limit_value - params = {'limit_value': limit_value, 'show_value': cleaned, 'value': value} + limit_value = ( + self.limit_value() if callable(self.limit_value) else self.limit_value + ) + params = {"limit_value": limit_value, "show_value": cleaned, "value": value} if self.compare(cleaned, limit_value): raise ValidationError(self.message, code=self.code, params=params) @@ -337,9 +367,9 @@ class BaseValidator: if not isinstance(other, self.__class__): return NotImplemented return ( - self.limit_value == other.limit_value and - self.message == other.message and - self.code == other.code + self.limit_value == other.limit_value + and self.message == other.message + and self.code == other.code ) def compare(self, a, b): @@ -351,8 +381,8 @@ class BaseValidator: @deconstructible class MaxValueValidator(BaseValidator): - message = _('Ensure this value is less than or equal to %(limit_value)s.') - code = 'max_value' + message = _("Ensure this value is less than or equal to %(limit_value)s.") + code = "max_value" def compare(self, a, b): return a > b @@ -360,8 +390,8 @@ class MaxValueValidator(BaseValidator): @deconstructible class MinValueValidator(BaseValidator): - message = _('Ensure this value is greater than or equal to %(limit_value)s.') - code = 'min_value' + message = _("Ensure this value is greater than or equal to %(limit_value)s.") + code = "min_value" def compare(self, a, b): return a < b @@ -370,10 +400,11 @@ class MinValueValidator(BaseValidator): @deconstructible class MinLengthValidator(BaseValidator): message = ngettext_lazy( - 'Ensure this value has at least %(limit_value)d character (it has %(show_value)d).', - 'Ensure this value has at least %(limit_value)d characters (it has %(show_value)d).', - 'limit_value') - code = 'min_length' + "Ensure this value has at least %(limit_value)d character (it has %(show_value)d).", + "Ensure this value has at least %(limit_value)d characters (it has %(show_value)d).", + "limit_value", + ) + code = "min_length" def compare(self, a, b): return a < b @@ -385,10 +416,11 @@ class MinLengthValidator(BaseValidator): @deconstructible class MaxLengthValidator(BaseValidator): message = ngettext_lazy( - 'Ensure this value has at most %(limit_value)d character (it has %(show_value)d).', - 'Ensure this value has at most %(limit_value)d characters (it has %(show_value)d).', - 'limit_value') - code = 'max_length' + "Ensure this value has at most %(limit_value)d character (it has %(show_value)d).", + "Ensure this value has at most %(limit_value)d characters (it has %(show_value)d).", + "limit_value", + ) + code = "max_length" def compare(self, a, b): return a > b @@ -403,22 +435,23 @@ class DecimalValidator: Validate that the input does not exceed the maximum number of digits expected, otherwise raise ValidationError. """ + messages = { - 'invalid': _('Enter a number.'), - 'max_digits': ngettext_lazy( - 'Ensure that there are no more than %(max)s digit in total.', - 'Ensure that there are no more than %(max)s digits in total.', - 'max' + "invalid": _("Enter a number."), + "max_digits": ngettext_lazy( + "Ensure that there are no more than %(max)s digit in total.", + "Ensure that there are no more than %(max)s digits in total.", + "max", ), - 'max_decimal_places': ngettext_lazy( - 'Ensure that there are no more than %(max)s decimal place.', - 'Ensure that there are no more than %(max)s decimal places.', - 'max' + "max_decimal_places": ngettext_lazy( + "Ensure that there are no more than %(max)s decimal place.", + "Ensure that there are no more than %(max)s decimal places.", + "max", ), - 'max_whole_digits': ngettext_lazy( - 'Ensure that there are no more than %(max)s digit before the decimal point.', - 'Ensure that there are no more than %(max)s digits before the decimal point.', - 'max' + "max_whole_digits": ngettext_lazy( + "Ensure that there are no more than %(max)s digit before the decimal point.", + "Ensure that there are no more than %(max)s digits before the decimal point.", + "max", ), } @@ -428,8 +461,10 @@ class DecimalValidator: def __call__(self, value): digit_tuple, exponent = value.as_tuple()[1:] - if exponent in {'F', 'n', 'N'}: - raise ValidationError(self.messages['invalid'], code='invalid', params={'value': value}) + if exponent in {"F", "n", "N"}: + raise ValidationError( + self.messages["invalid"], code="invalid", params={"value": value} + ) if exponent >= 0: # A positive exponent adds that many trailing zeros. digits = len(digit_tuple) + exponent @@ -449,43 +484,48 @@ class DecimalValidator: if self.max_digits is not None and digits > self.max_digits: raise ValidationError( - self.messages['max_digits'], - code='max_digits', - params={'max': self.max_digits, 'value': value}, + self.messages["max_digits"], + code="max_digits", + params={"max": self.max_digits, "value": value}, ) if self.decimal_places is not None and decimals > self.decimal_places: raise ValidationError( - self.messages['max_decimal_places'], - code='max_decimal_places', - params={'max': self.decimal_places, 'value': value}, + self.messages["max_decimal_places"], + code="max_decimal_places", + params={"max": self.decimal_places, "value": value}, ) - if (self.max_digits is not None and self.decimal_places is not None and - whole_digits > (self.max_digits - self.decimal_places)): + if ( + self.max_digits is not None + and self.decimal_places is not None + and whole_digits > (self.max_digits - self.decimal_places) + ): raise ValidationError( - self.messages['max_whole_digits'], - code='max_whole_digits', - params={'max': (self.max_digits - self.decimal_places), 'value': value}, + self.messages["max_whole_digits"], + code="max_whole_digits", + params={"max": (self.max_digits - self.decimal_places), "value": value}, ) def __eq__(self, other): return ( - isinstance(other, self.__class__) and - self.max_digits == other.max_digits and - self.decimal_places == other.decimal_places + isinstance(other, self.__class__) + and self.max_digits == other.max_digits + and self.decimal_places == other.decimal_places ) @deconstructible class FileExtensionValidator: message = _( - 'File extension “%(extension)s” is not allowed. ' - 'Allowed extensions are: %(allowed_extensions)s.' + "File extension “%(extension)s” is not allowed. " + "Allowed extensions are: %(allowed_extensions)s." ) - code = 'invalid_extension' + code = "invalid_extension" def __init__(self, allowed_extensions=None, message=None, code=None): if allowed_extensions is not None: - allowed_extensions = [allowed_extension.lower() for allowed_extension in allowed_extensions] + allowed_extensions = [ + allowed_extension.lower() for allowed_extension in allowed_extensions + ] self.allowed_extensions = allowed_extensions if message is not None: self.message = message @@ -494,23 +534,26 @@ class FileExtensionValidator: def __call__(self, value): extension = Path(value.name).suffix[1:].lower() - if self.allowed_extensions is not None and extension not in self.allowed_extensions: + if ( + self.allowed_extensions is not None + and extension not in self.allowed_extensions + ): raise ValidationError( self.message, code=self.code, params={ - 'extension': extension, - 'allowed_extensions': ', '.join(self.allowed_extensions), - 'value': value, - } + "extension": extension, + "allowed_extensions": ", ".join(self.allowed_extensions), + "value": value, + }, ) def __eq__(self, other): return ( - isinstance(other, self.__class__) and - self.allowed_extensions == other.allowed_extensions and - self.message == other.message and - self.code == other.code + isinstance(other, self.__class__) + and self.allowed_extensions == other.allowed_extensions + and self.message == other.message + and self.code == other.code ) @@ -525,14 +568,17 @@ def get_available_image_extensions(): def validate_image_file_extension(value): - return FileExtensionValidator(allowed_extensions=get_available_image_extensions())(value) + return FileExtensionValidator(allowed_extensions=get_available_image_extensions())( + value + ) @deconstructible class ProhibitNullCharactersValidator: """Validate that the string doesn't contain the null character.""" - message = _('Null characters are not allowed.') - code = 'null_characters_not_allowed' + + message = _("Null characters are not allowed.") + code = "null_characters_not_allowed" def __init__(self, message=None, code=None): if message is not None: @@ -541,12 +587,12 @@ class ProhibitNullCharactersValidator: self.code = code def __call__(self, value): - if '\x00' in str(value): - raise ValidationError(self.message, code=self.code, params={'value': value}) + if "\x00" in str(value): + raise ValidationError(self.message, code=self.code, params={"value": value}) def __eq__(self, other): return ( - isinstance(other, self.__class__) and - self.message == other.message and - self.code == other.code + isinstance(other, self.__class__) + and self.message == other.message + and self.code == other.code ) |
