Fixed #19357 -- Allow non-ASCII chars in filesystem paths

Thanks kujiu for the report and Aymeric Augustin for the review.

2 files changed, 21 insertions, 17 deletions

diff --git a/tests/regressiontests/i18n/commands/compilation.py b/tests/regressiontests/i18n/commands/compilation.py
index c6ab77941b..2944469110 100644
--- a/tests/regressiontests/i18n/commands/compilation.py
+++ b/tests/regressiontests/i18n/commands/compilation.py
@@ -4,9 +4,10 @@ from django.core.management import call_command, CommandError
 from django.test import TestCase
 from django.test.utils import override_settings
 from django.utils import translation, six
+from django.utils._os import upath
 from django.utils.six import StringIO
 
-test_dir = os.path.abspath(os.path.dirname(__file__))
+test_dir = os.path.abspath(os.path.dirname(upath(__file__)))
 
 
 class MessageCompilationTests(TestCase):
@@ -25,9 +26,9 @@ class PoFileTests(MessageCompilationTests):
 
     def test_bom_rejection(self):
         os.chdir(test_dir)
-        with six.assertRaisesRegex(self, CommandError,
-                "file has a BOM \(Byte Order Mark\)"):
+        with self.assertRaises(CommandError) as cm:
             call_command('compilemessages', locale=self.LOCALE, stderr=StringIO())
+        self.assertIn("file has a BOM (Byte Order Mark)", cm.exception.args[0])
         self.assertFalse(os.path.exists(self.MO_FILE))
 
 
diff --git a/tests/regressiontests/i18n/commands/extraction.py b/tests/regressiontests/i18n/commands/extraction.py
index ca2c3cc026..aa5efe1967 100644
--- a/tests/regressiontests/i18n/commands/extraction.py
+++ b/tests/regressiontests/i18n/commands/extraction.py
@@ -1,4 +1,5 @@
 # -*- encoding: utf-8 -*-
+from __future__ import unicode_literals
 
 import os
 import re
@@ -6,6 +7,8 @@ import shutil
 
 from django.core import management
 from django.test import TestCase
+from django.utils.encoding import force_text
+from django.utils._os import upath
 from django.utils.six import StringIO
 
 
@@ -17,7 +20,7 @@ class ExtractorTests(TestCase):
 
     def setUp(self):
         self._cwd = os.getcwd()
-        self.test_dir = os.path.abspath(os.path.dirname(__file__))
+        self.test_dir = os.path.abspath(os.path.dirname(upath(__file__)))
 
     def _rmrf(self, dname):
         if os.path.commonprefix([self.test_dir, os.path.abspath(dname)]) != self.test_dir:
@@ -55,7 +58,7 @@ class BasicExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertTrue('#. Translators: This comment should be extracted' in po_contents)
             self.assertTrue('This comment should not be extracted' not in po_contents)
             # Comments in templates
@@ -83,7 +86,7 @@ class BasicExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertMsgId('Literal with a percent symbol at the end %%', po_contents)
             self.assertMsgId('Literal with a percent %% symbol in the middle', po_contents)
             self.assertMsgId('Completed 50%% of all the tasks', po_contents)
@@ -99,7 +102,7 @@ class BasicExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertMsgId('I think that 100%% is more that 50%% of anything.', po_contents)
             self.assertMsgId('I think that 100%% is more that 50%% of %(obj)s.', po_contents)
             self.assertMsgId("Blocktrans extraction shouldn't double escape this: %%, a=%(a)s", po_contents)
@@ -123,7 +126,7 @@ class BasicExtractorTests(ExtractorTests):
         stdout = StringIO()
         management.call_command('makemessages', locale=LOCALE, stdout=stdout)
         os.remove('./code_sample.py')
-        self.assertIn("code_sample.py:4", stdout.getvalue())
+        self.assertIn("code_sample.py:4", force_text(stdout.getvalue()))
 
     def test_template_message_context_extractor(self):
         """
@@ -135,7 +138,7 @@ class BasicExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             # {% trans %}
             self.assertTrue('msgctxt "Special trans context #1"' in po_contents)
             self.assertTrue("Translatable literal #7a" in po_contents)
@@ -161,7 +164,7 @@ class BasicExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             # {% trans %}
             self.assertTrue('msgctxt "Context wrapped in double quotes"' in po_contents)
             self.assertTrue('msgctxt "Context wrapped in single quotes"' in po_contents)
@@ -216,7 +219,7 @@ class SymlinkExtractorTests(ExtractorTests):
 
     def setUp(self):
         self._cwd = os.getcwd()
-        self.test_dir = os.path.abspath(os.path.dirname(__file__))
+        self.test_dir = os.path.abspath(os.path.dirname(upath(__file__)))
         self.symlinked_dir = os.path.join(self.test_dir, 'templates_symlinked')
 
     def tearDown(self):
@@ -238,7 +241,7 @@ class SymlinkExtractorTests(ExtractorTests):
             management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)
             self.assertTrue(os.path.exists(self.PO_FILE))
             with open(self.PO_FILE, 'r') as fp:
-                po_contents = fp.read()
+                po_contents = force_text(fp.read())
                 self.assertMsgId('This literal should be included.', po_contents)
                 self.assertTrue('templates_symlinked/test.html' in po_contents)
 
@@ -250,7 +253,7 @@ class CopyPluralFormsExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertTrue('Plural-Forms: nplurals=2; plural=(n != 1)' in po_contents)
 
 
@@ -261,7 +264,7 @@ class NoWrapExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=True)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertMsgId('This literal should also be included wrapped or not wrapped depending on the use of the --no-wrap option.', po_contents)
 
     def test_no_wrap_disabled(self):
@@ -269,7 +272,7 @@ class NoWrapExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=False)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertMsgId('""\n"This literal should also be included wrapped or not wrapped depending on the "\n"use of the --no-wrap option."', po_contents, use_quotes=False)
 
 
@@ -280,7 +283,7 @@ class NoLocationExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=True)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertFalse('#: templates/test.html:55' in po_contents)
 
     def test_no_location_disabled(self):
@@ -288,5 +291,5 @@ class NoLocationExtractorTests(ExtractorTests):
         management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=False)
         self.assertTrue(os.path.exists(self.PO_FILE))
         with open(self.PO_FILE, 'r') as fp:
-            po_contents = fp.read()
+            po_contents = force_text(fp.read())
             self.assertTrue('#: templates/test.html:55' in po_contents)