Fixed #13795 -- Added a site-wide cache prefix and cache versioning. Thanks to bruth for the patch.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@14623 bcc190cf-cafb-0310-a4f2-bffc1f526a37

7 files changed, 174 insertions, 76 deletions

diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py
index 7c3f0927eb..d26eabd0c3 100644
--- a/django/core/cache/__init__.py
+++ b/django/core/cache/__init__.py
@@ -67,18 +67,30 @@ def parse_backend_uri(backend_uri):
 
     return scheme, host, params
 
-def get_cache(backend_uri):
+def get_cache(backend_uri, key_prefix=None, version=None, key_func=None):
+    if key_prefix is None:
+        key_prefix = settings.CACHE_KEY_PREFIX
+    if version is None:
+        version = settings.CACHE_VERSION
+    if key_func is None:
+        key_func = settings.CACHE_KEY_FUNCTION
+
+    if key_func is not None and not callable(key_func):
+        key_func_module_path, key_func_name = key_func.rsplit('.', 1)
+        key_func_module = importlib.import_module(key_func_module_path)
+        key_func = getattr(key_func_module, key_func_name)
+
     scheme, host, params = parse_backend_uri(backend_uri)
     if scheme in BACKENDS:
         name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
     else:
         name = scheme
     module = importlib.import_module(name)
-    return module.CacheClass(host, params)
+    return module.CacheClass(host, params, key_prefix=key_prefix, version=version, key_func=key_func)
 
 cache = get_cache(settings.CACHE_BACKEND)
 
-# Some caches -- pythont-memcached in particular -- need to do a cleanup at the
+# Some caches -- python-memcached in particular -- need to do a cleanup at the
 # end of a request cycle. If the cache provides a close() method, wire it up
 # here.
 if hasattr(cache, 'close'):
diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py
index 16f463285f..1296a853a5 100644
--- a/django/core/cache/backends/base.py
+++ b/django/core/cache/backends/base.py
@@ -3,6 +3,7 @@
 import warnings
 
 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
+from django.utils.encoding import smart_str
 
 class InvalidCacheBackendError(ImproperlyConfigured):
     pass
@@ -13,8 +14,17 @@ class CacheKeyWarning(DjangoRuntimeWarning):
 # Memcached does not accept keys longer than this.
 MEMCACHE_MAX_KEY_LENGTH = 250
 
+def default_key_func(key, key_prefix, version):
+    """Default function to generate keys.
+
+    Constructs the key used by all other methods. By default it prepends
+    the `key_prefix'. CACHE_KEY_FUNCTION can be used to specify an alternate
+    function with custom key making behavior.
+    """
+    return ':'.join([key_prefix, str(version), smart_str(key)])
+
 class BaseCache(object):
-    def __init__(self, params):
+    def __init__(self, params, key_prefix='', version=1, key_func=None):
         timeout = params.get('timeout', 300)
         try:
             timeout = int(timeout)
@@ -34,7 +44,25 @@ class BaseCache(object):
         except (ValueError, TypeError):
             self._cull_frequency = 3
 
-    def add(self, key, value, timeout=None):
+        self.key_prefix = smart_str(key_prefix)
+        self.version = version
+        self.key_func = key_func or default_key_func
+
+    def make_key(self, key, version=None):
+        """Constructs the key used by all other methods. By default it
+        uses the key_func to generate a key (which, by default,
+        prepends the `key_prefix' and 'version'). An different key
+        function can be provided at the time of cache construction;
+        alternatively, you can subclass the cache backend to provide
+        custom key making behavior.
+        """
+        if version is None:
+            version = self.version
+
+        new_key = self.key_func(key, self.key_prefix, version)
+        return new_key
+
+    def add(self, key, value, timeout=None, version=None):
         """
         Set a value in the cache if the key does not already exist. If
         timeout is given, that timeout will be used for the key; otherwise
@@ -44,27 +72,27 @@ class BaseCache(object):
         """
         raise NotImplementedError
 
-    def get(self, key, default=None):
+    def get(self, key, default=None, version=None):
         """
         Fetch a given key from the cache. If the key does not exist, return
         default, which itself defaults to None.
         """
         raise NotImplementedError
 
-    def set(self, key, value, timeout=None):
+    def set(self, key, value, timeout=None, version=None):
         """
         Set a value in the cache. If timeout is given, that timeout will be
         used for the key; otherwise the default cache timeout will be used.
         """
         raise NotImplementedError
 
-    def delete(self, key):
+    def delete(self, key, version=None):
         """
         Delete a key from the cache, failing silently.
         """
         raise NotImplementedError
 
-    def get_many(self, keys):
+    def get_many(self, keys, version=None):
         """
         Fetch a bunch of keys from the cache. For certain backends (memcached,
         pgsql) this can be *much* faster when fetching multiple values.
@@ -74,34 +102,35 @@ class BaseCache(object):
         """
         d = {}
         for k in keys:
-            val = self.get(k)
+            val = self.get(k, version=version)
             if val is not None:
                 d[k] = val
         return d
 
-    def has_key(self, key):
+    def has_key(self, key, version=None):
         """
         Returns True if the key is in the cache and has not expired.
         """
-        return self.get(key) is not None
+        return self.get(key, version=version) is not None
 
-    def incr(self, key, delta=1):
+    def incr(self, key, delta=1, version=None):
         """
         Add delta to value in the cache. If the key does not exist, raise a
         ValueError exception.
         """
-        if key not in self:
+        value = self.get(key, version=version)
+        if value is None:
             raise ValueError("Key '%s' not found" % key)
-        new_value = self.get(key) + delta
-        self.set(key, new_value)
+        new_value = value + delta
+        self.set(key, new_value, version=version)
         return new_value
 
-    def decr(self, key, delta=1):
+    def decr(self, key, delta=1, version=None):
         """
         Subtract delta from value in the cache. If the key does not exist, raise
         a ValueError exception.
         """
-        return self.incr(key, -delta)
+        return self.incr(key, -delta, version=version)
 
     def __contains__(self, key):
         """
@@ -112,7 +141,7 @@ class BaseCache(object):
         # if a subclass overrides it.
         return self.has_key(key)
 
-    def set_many(self, data, timeout=None):
+    def set_many(self, data, timeout=None, version=None):
         """
         Set a bunch of values in the cache at once from a dict of key/value
         pairs.  For certain backends (memcached), this is much more efficient
@@ -122,16 +151,16 @@ class BaseCache(object):
         the default cache timeout will be used.
         """
         for key, value in data.items():
-            self.set(key, value, timeout)
+            self.set(key, value, timeout=timeout, version=version)
 
-    def delete_many(self, keys):
+    def delete_many(self, keys, version=None):
         """
         Set a bunch of values in the cache at once.  For certain backends
         (memcached), this is much more efficient than calling delete() multiple
         times.
         """
         for key in keys:
-            self.delete(key)
+            self.delete(key, version=version)
 
     def clear(self):
         """Remove *all* values from the cache at once."""
@@ -154,3 +183,23 @@ class BaseCache(object):
                         'errors if used with memcached: %r' % key,
                               CacheKeyWarning)
 
+    def incr_version(self, key, delta=1, version=None):
+        """Adds delta to the cache version for the supplied key. Returns the
+        new version.
+        """
+        if version is None:
+            version = self.version
+
+        value = self.get(key, version=version)
+        if value is None:
+            raise ValueError("Key '%s' not found" % key)
+
+        self.set(key, value, version=version+delta)
+        self.delete(key, version=version)
+        return version+delta
+
+    def decr_version(self, key, delta=1, version=None):
+        """Substracts delta from the cache version for the supplied key. Returns
+        the new version.
+        """
+        return self.incr_version(key, -delta, version)
diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py
index 5569c60df6..68cd2e015e 100644
--- a/django/core/cache/backends/db.py
+++ b/django/core/cache/backends/db.py
@@ -26,8 +26,8 @@ class Options(object):
         self.proxy = False
 
 class BaseDatabaseCacheClass(BaseCache):
-    def __init__(self, table, params):
-        BaseCache.__init__(self, params)
+    def __init__(self, table, params, key_prefix='', version=1, key_func=None):
+        BaseCache.__init__(self, params, key_prefix, version, key_func)
         self._table = table
 
         class CacheEntry(object):
@@ -35,7 +35,8 @@ class BaseDatabaseCacheClass(BaseCache):
         self.cache_model_class = CacheEntry
 
 class CacheClass(BaseDatabaseCacheClass):
-    def get(self, key, default=None):
+    def get(self, key, default=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         db = router.db_for_read(self.cache_model_class)
         table = connections[db].ops.quote_name(self._table)
@@ -55,11 +56,13 @@ class CacheClass(BaseDatabaseCacheClass):
         value = connections[db].ops.process_clob(row[1])
         return pickle.loads(base64.decodestring(value))
 
-    def set(self, key, value, timeout=None):
+    def set(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._base_set('set', key, value, timeout)
 
-    def add(self, key, value, timeout=None):
+    def add(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         return self._base_set('add', key, value, timeout)
 
@@ -95,8 +98,10 @@ class CacheClass(BaseDatabaseCacheClass):
             transaction.commit_unless_managed(using=db)
             return True
 
-    def delete(self, key):
+    def delete(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
+
         db = router.db_for_write(self.cache_model_class)
         table = connections[db].ops.quote_name(self._table)
         cursor = connections[db].cursor()
@@ -104,8 +109,10 @@ class CacheClass(BaseDatabaseCacheClass):
         cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])
         transaction.commit_unless_managed(using=db)
 
-    def has_key(self, key):
+    def has_key(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
+
         db = router.db_for_read(self.cache_model_class)
         table = connections[db].ops.quote_name(self._table)
         cursor = connections[db].cursor()
diff --git a/django/core/cache/backends/dummy.py b/django/core/cache/backends/dummy.py
index f73b7408bc..7d90ddace4 100644
--- a/django/core/cache/backends/dummy.py
+++ b/django/core/cache/backends/dummy.py
@@ -3,34 +3,39 @@
 from django.core.cache.backends.base import BaseCache
 
 class CacheClass(BaseCache):
-    def __init__(self, *args, **kwargs):
-        pass
+    def __init__(self, host, *args, **kwargs):
+        BaseCache.__init__(self, *args, **kwargs)
 
-    def add(self, key, *args, **kwargs):
+    def add(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         return True
 
-    def get(self, key, default=None):
+    def get(self, key, default=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         return default
 
-    def set(self, key, *args, **kwargs):
+    def set(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
 
-    def delete(self, key, *args, **kwargs):
+    def delete(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
 
-    def get_many(self, *args, **kwargs):
+    def get_many(self, keys, version=None):
         return {}
 
-    def has_key(self, key, *args, **kwargs):
+    def has_key(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         return False
 
-    def set_many(self, *args, **kwargs):
+    def set_many(self, data, version=None):
         pass
 
-    def delete_many(self, *args, **kwargs):
+    def delete_many(self, keys, version=None):
         pass
 
     def clear(self):
diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py
index 4feacd385a..1f95faf5ee 100644
--- a/django/core/cache/backends/filebased.py
+++ b/django/core/cache/backends/filebased.py
@@ -12,22 +12,23 @@ from django.core.cache.backends.base import BaseCache
 from django.utils.hashcompat import md5_constructor
 
 class CacheClass(BaseCache):
-    def __init__(self, dir, params):
-        BaseCache.__init__(self, params)
+    def __init__(self, dir, params, key_prefix='', version=1, key_func=None):
+        BaseCache.__init__(self, params, key_prefix, version, key_func)
         self._dir = dir
         if not os.path.exists(self._dir):
             self._createdir()
 
-    def add(self, key, value, timeout=None):
-        self.validate_key(key)
-        if self.has_key(key):
+    def add(self, key, value, timeout=None, version=None):
+        if self.has_key(key, version=version):
             return False
 
-        self.set(key, value, timeout)
+        self.set(key, value, timeout, version=version)
         return True
 
-    def get(self, key, default=None):
+    def get(self, key, default=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
+
         fname = self._key_to_file(key)
         try:
             f = open(fname, 'rb')
@@ -44,8 +45,10 @@ class CacheClass(BaseCache):
             pass
         return default
 
-    def set(self, key, value, timeout=None):
+    def set(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
+
         fname = self._key_to_file(key)
         dirname = os.path.dirname(fname)
 
@@ -68,7 +71,8 @@ class CacheClass(BaseCache):
         except (IOError, OSError):
             pass
 
-    def delete(self, key):
+    def delete(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         try:
             self._delete(self._key_to_file(key))
@@ -85,7 +89,8 @@ class CacheClass(BaseCache):
         except (IOError, OSError):
             pass
 
-    def has_key(self, key):
+    def has_key(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         fname = self._key_to_file(key)
         try:
@@ -140,7 +145,7 @@ class CacheClass(BaseCache):
         Thus, a cache key of "foo" gets turnned into a file named
         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
         """
-        path = md5_constructor(key.encode('utf-8')).hexdigest()
+        path = md5_constructor(key).hexdigest()
         path = os.path.join(path[:2], path[2:4], path[4:])
         return os.path.join(self._dir, path)
 
diff --git a/django/core/cache/backends/locmem.py b/django/core/cache/backends/locmem.py
index 7e2dc07105..17fd8f33ce 100644
--- a/django/core/cache/backends/locmem.py
+++ b/django/core/cache/backends/locmem.py
@@ -10,13 +10,14 @@ from django.core.cache.backends.base import BaseCache
 from django.utils.synch import RWLock
 
 class CacheClass(BaseCache):
-    def __init__(self, _, params):
-        BaseCache.__init__(self, params)
+    def __init__(self, _, params, key_prefix='', version=1, key_func=None):
+        BaseCache.__init__(self, params, key_prefix, version, key_func)
         self._cache = {}
         self._expire_info = {}
         self._lock = RWLock()
 
-    def add(self, key, value, timeout=None):
+    def add(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._lock.writer_enters()
         try:
@@ -31,7 +32,8 @@ class CacheClass(BaseCache):
         finally:
             self._lock.writer_leaves()
 
-    def get(self, key, default=None):
+    def get(self, key, default=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._lock.reader_enters()
         try:
@@ -64,7 +66,8 @@ class CacheClass(BaseCache):
         self._cache[key] = value
         self._expire_info[key] = time.time() + timeout
 
-    def set(self, key, value, timeout=None):
+    def set(self, key, value, timeout=None, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._lock.writer_enters()
         # Python 2.4 doesn't allow combined try-except-finally blocks.
@@ -76,7 +79,8 @@ class CacheClass(BaseCache):
         finally:
             self._lock.writer_leaves()
 
-    def has_key(self, key):
+    def has_key(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._lock.reader_enters()
         try:
@@ -117,7 +121,8 @@ class CacheClass(BaseCache):
         except KeyError:
             pass
 
-    def delete(self, key):
+    def delete(self, key, version=None):
+        key = self.make_key(key, version=version)
         self.validate_key(key)
         self._lock.writer_enters()
         try:
diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py
index cd14fdaf95..4bd8547085 100644
--- a/django/core/cache/backends/memcached.py
+++ b/django/core/cache/backends/memcached.py
@@ -3,7 +3,6 @@
 import time
 
 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
-from django.utils.encoding import smart_unicode, smart_str
 
 try:
     import cmemcache as memcache
@@ -19,8 +18,8 @@ except ImportError:
         raise InvalidCacheBackendError("Memcached cache backend requires either the 'memcache' or 'cmemcache' library")
 
 class CacheClass(BaseCache):
-    def __init__(self, server, params):
-        BaseCache.__init__(self, params)
+    def __init__(self, server, params, key_prefix='', version=1, key_func=None):
+        BaseCache.__init__(self, params, key_prefix, version, key_func)
         self._cache = memcache.Client(server.split(';'))
 
     def _get_memcache_timeout(self, timeout):
@@ -39,30 +38,43 @@ class CacheClass(BaseCache):
             timeout += int(time.time())
         return timeout
 
-    def add(self, key, value, timeout=0):
+    def add(self, key, value, timeout=0, version=None):
+        key = self.make_key(key, version=version)
         if isinstance(value, unicode):
             value = value.encode('utf-8')
-        return self._cache.add(smart_str(key), value, self._get_memcache_timeout(timeout))
+        return self._cache.add(key, value, self._get_memcache_timeout(timeout))
 
-    def get(self, key, default=None):
-        val = self._cache.get(smart_str(key))
+    def get(self, key, default=None, version=None):
+        key = self.make_key(key, version=version)
+        val = self._cache.get(key)
         if val is None:
             return default
         return val
 
-    def set(self, key, value, timeout=0):
-        self._cache.set(smart_str(key), value, self._get_memcache_timeout(timeout))
+    def set(self, key, value, timeout=0, version=None):
+        key = self.make_key(key, version=version)
+        self._cache.set(key, value, self._get_memcache_timeout(timeout))
 
-    def delete(self, key):
-        self._cache.delete(smart_str(key))
+    def delete(self, key, version=None):
+        key = self.make_key(key, version=version)
+        self._cache.delete(key)
 
-    def get_many(self, keys):
-        return self._cache.get_multi(map(smart_str,keys))
+    def get_many(self, keys, version=None):
+        new_keys = map(lambda x: self.make_key(x, version=version), keys)
+        ret = self._cache.get_multi(new_keys)
+        if ret:
+            _ = {}
+            m = dict(zip(new_keys, keys))
+            for k, v in ret.items():
+                _[m[k]] = v
+            ret = _
+        return ret
 
     def close(self, **kwargs):
         self._cache.disconnect_all()
 
-    def incr(self, key, delta=1):
+    def incr(self, key, delta=1, version=None):
+        key = self.make_key(key, version=version)
         try:
             val = self._cache.incr(key, delta)
 
@@ -76,7 +88,8 @@ class CacheClass(BaseCache):
 
         return val
 
-    def decr(self, key, delta=1):
+    def decr(self, key, delta=1, version=None):
+        key = self.make_key(key, version=version)
         try:
             val = self._cache.decr(key, delta)
 
@@ -89,16 +102,18 @@ class CacheClass(BaseCache):
             raise ValueError("Key '%s' not found" % key)
         return val
 
-    def set_many(self, data, timeout=0):
+    def set_many(self, data, timeout=0, version=None):
         safe_data = {}
         for key, value in data.items():
+            key = self.make_key(key, version=version)
             if isinstance(value, unicode):
                 value = value.encode('utf-8')
-            safe_data[smart_str(key)] = value
+            safe_data[key] = value
         self._cache.set_multi(safe_data, self._get_memcache_timeout(timeout))
 
-    def delete_many(self, keys):
-        self._cache.delete_multi(map(smart_str, keys))
+    def delete_many(self, keys, version=None):
+        l = lambda x: self.make_key(x, version=version)
+        self._cache.delete_multi(map(l, keys))
 
     def clear(self):
         self._cache.flush_all()