Fixed #26093 -- Allowed escape sequences extraction by gettext on Python 3

Thanks Sylvain Fankhauser for the report and Tim Graham for the review.

3 files changed, 33 insertions, 10 deletions

diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py
index 5208e7a96f..a51d761145 100644
--- a/django/utils/translation/trans_real.py
+++ b/django/utils/translation/trans_real.py
@@ -566,6 +566,9 @@ def templatize(src, origin=None):
     comment = []
     lineno_comment_map = {}
     comment_lineno_cache = None
+    # Adding the u prefix allows gettext to recognize the Unicode string
+    # (#26093).
+    raw_prefix = 'u' if six.PY3 else ''
 
     def join_tokens(tokens, trim=False):
         message = ''.join(tokens)
@@ -597,26 +600,34 @@ def templatize(src, origin=None):
                 if endbmatch:
                     if inplural:
                         if message_context:
-                            out.write(' npgettext(%r, %r, %r,count) ' % (
+                            out.write(' npgettext({p}{!r}, {p}{!r}, {p}{!r},count) '.format(
                                 message_context,
                                 join_tokens(singular, trimmed),
-                                join_tokens(plural, trimmed)))
+                                join_tokens(plural, trimmed),
+                                p=raw_prefix,
+                            ))
                         else:
-                            out.write(' ngettext(%r, %r, count) ' % (
+                            out.write(' ngettext({p}{!r}, {p}{!r}, count) '.format(
                                 join_tokens(singular, trimmed),
-                                join_tokens(plural, trimmed)))
+                                join_tokens(plural, trimmed),
+                                p=raw_prefix,
+                            ))
                         for part in singular:
                             out.write(blankout(part, 'S'))
                         for part in plural:
                             out.write(blankout(part, 'P'))
                     else:
                         if message_context:
-                            out.write(' pgettext(%r, %r) ' % (
+                            out.write(' pgettext({p}{!r}, {p}{!r}) '.format(
                                 message_context,
-                                join_tokens(singular, trimmed)))
+                                join_tokens(singular, trimmed),
+                                p=raw_prefix,
+                            ))
                         else:
-                            out.write(' gettext(%r) ' % join_tokens(singular,
-                                                                    trimmed))
+                            out.write(' gettext({p}{!r}) '.format(
+                                join_tokens(singular, trimmed),
+                                p=raw_prefix,
+                            ))
                         for part in singular:
                             out.write(blankout(part, 'S'))
                     message_context = None
@@ -685,10 +696,12 @@ def templatize(src, origin=None):
                             message_context = message_context.strip('"')
                         elif message_context[0] == "'":
                             message_context = message_context.strip("'")
-                        out.write(' pgettext(%r, %r) ' % (message_context, g))
+                        out.write(' pgettext({p}{!r}, {p}{!r}) '.format(
+                            message_context, g, p=raw_prefix
+                        ))
                         message_context = None
                     else:
-                        out.write(' gettext(%r) ' % g)
+                        out.write(' gettext({p}{!r}) '.format(g, p=raw_prefix))
                 elif bmatch:
                     for fmatch in constant_re.findall(t.contents):
                         out.write(' _(%s) ' % fmatch)
diff --git a/tests/i18n/commands/templates/test.html b/tests/i18n/commands/templates/test.html
index dde42d3fdd..e7f7ba93eb 100644
--- a/tests/i18n/commands/templates/test.html
+++ b/tests/i18n/commands/templates/test.html
@@ -98,3 +98,5 @@ First `trans`, then `blocktrans` with a plural
 {% plural %}
 Plural for a `trans` and `blocktrans` collision case
 {% endblocktrans %}
+
+{% trans "Non-breaking space :" %}
diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py
index f72aa00771..dd28a76c4f 100644
--- a/tests/i18n/test_extraction.py
+++ b/tests/i18n/test_extraction.py
@@ -204,6 +204,14 @@ class BasicExtractorTests(ExtractorTests):
                 po_contents
             )
 
+    def test_special_char_extracted(self):
+        os.chdir(self.test_dir)
+        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
+        self.assertTrue(os.path.exists(self.PO_FILE))
+        with open(self.PO_FILE, 'r') as fp:
+            po_contents = force_text(fp.read())
+            self.assertMsgId("Non-breaking space\xa0:", po_contents)
+
     def test_blocktrans_trimmed(self):
         os.chdir(self.test_dir)
         management.call_command('makemessages', locale=[LOCALE], verbosity=0)