Fixed #3711, #6734, #12581 -- Bounded connection.queries.

Prevented unlimited memory consumption when running background tasks
with DEBUG=True.

Thanks Rob, Alex, Baptiste, and others.

4 files changed, 20 insertions, 7 deletions

diff --git a/django/db/__init__.py b/django/db/__init__.py
index 84ef6e1400..a303a2c150 100644
--- a/django/db/__init__.py
+++ b/django/db/__init__.py
@@ -52,7 +52,7 @@ connection = DefaultConnectionProxy()
 # Register an event to reset saved queries when a Django request is started.
 def reset_queries(**kwargs):
     for conn in connections.all():
-        conn.queries = []
+        conn.queries_log.clear()
 signals.request_started.connect(reset_queries)
 
 
diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py
index bcc3979e2b..587cd874d9 100644
--- a/django/db/backends/__init__.py
+++ b/django/db/backends/__init__.py
@@ -1,3 +1,4 @@
+from collections import deque
 import datetime
 import time
 import warnings
@@ -30,17 +31,21 @@ class BaseDatabaseWrapper(object):
     ops = None
     vendor = 'unknown'
 
+    queries_limit = 9000
+
     def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
                  allow_thread_sharing=False):
         # Connection related attributes.
+        # The underlying database connection.
         self.connection = None
-        self.queries = []
         # `settings_dict` should be a dictionary containing keys such as
         # NAME, USER, etc. It's called `settings_dict` instead of `settings`
         # to disambiguate it from Django settings modules.
         self.settings_dict = settings_dict
         self.alias = alias
+        # Query logging in debug mode.
         self.use_debug_cursor = None
+        self.queries_log = deque(maxlen=self.queries_limit)
 
         # Transaction related attributes.
         # Tracks if the connection is in autocommit mode. Per PEP 249, by
@@ -79,6 +84,14 @@ class BaseDatabaseWrapper(object):
     def __hash__(self):
         return hash(self.alias)
 
+    @property
+    def queries(self):
+        if len(self.queries_log) == self.queries_log.maxlen:
+            warnings.warn(
+                "Limit for query logging exceeded, only the last {} queries "
+                "will be returned.".format(self.queries_log.maxlen))
+        return list(self.queries_log)
+
     ##### Backend-specific methods for creating connections and cursors #####
 
     def get_connection_params(self):
@@ -429,7 +442,7 @@ class BaseDatabaseWrapper(object):
 
     def make_debug_cursor(self, cursor):
         """
-        Creates a cursor that logs all queries in self.queries.
+        Creates a cursor that logs all queries in self.queries_log.
         """
         return utils.CursorDebugWrapper(cursor, self)
 
diff --git a/django/db/backends/utils.py b/django/db/backends/utils.py
index 8078af9f95..77132e3f25 100644
--- a/django/db/backends/utils.py
+++ b/django/db/backends/utils.py
@@ -80,7 +80,7 @@ class CursorDebugWrapper(CursorWrapper):
             stop = time()
             duration = stop - start
             sql = self.db.ops.last_executed_query(self.cursor, sql, params)
-            self.db.queries.append({
+            self.db.queries_log.append({
                 'sql': sql,
                 'time': "%.3f" % duration,
             })
@@ -99,7 +99,7 @@ class CursorDebugWrapper(CursorWrapper):
                 times = len(param_list)
             except TypeError:           # param_list could be an iterator
                 times = '?'
-            self.db.queries.append({
+            self.db.queries_log.append({
                 'sql': '%s times: %s' % (times, sql),
                 'time': "%.3f" % duration,
             })
diff --git a/django/test/utils.py b/django/test/utils.py
index fdbf920868..9a0d7a494c 100644
--- a/django/test/utils.py
+++ b/django/test/utils.py
@@ -509,7 +509,7 @@ class CaptureQueriesContext(object):
     def __enter__(self):
         self.use_debug_cursor = self.connection.use_debug_cursor
         self.connection.use_debug_cursor = True
-        self.initial_queries = len(self.connection.queries)
+        self.initial_queries = len(self.connection.queries_log)
         self.final_queries = None
         request_started.disconnect(reset_queries)
         return self
@@ -519,7 +519,7 @@ class CaptureQueriesContext(object):
         request_started.connect(reset_queries)
         if exc_type is not None:
             return
-        self.final_queries = len(self.connection.queries)
+        self.final_queries = len(self.connection.queries_log)
 
 
 class IgnoreDeprecationWarningsMixin(object):