[1.8.x] Fixed #23271 -- Fixed makemessages crash/test failure for some locales.

Backport of 57202a112a966593857725071ecd652a87c157fb from master

3 files changed, 15 insertions, 10 deletions

diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
index d1287bd433..0e0130eb41 100644
--- a/django/core/management/commands/makemessages.py
+++ b/django/core/management/commands/makemessages.py
@@ -36,14 +36,17 @@ def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="u
     """
     Makes sure text obtained from stdout of gettext utilities is Unicode.
     """
-    stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
-    if os.name == 'nt' and six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING:
-        # This looks weird because it's undoing what
-        # subprocess.Popen(universal_newlines=True).communicate()
-        # does when capturing PO files contents from stdout of gettext command
-        # line programs. No need to do anything on Python 2 because it's
-        # already a byte-string there (#23271).
-        stdout = stdout.encode(DEFAULT_LOCALE_ENCODING).decode(stdout_encoding)
+    # This both decodes utf-8 and cleans line endings. Simply using
+    # popen_wrapper(universal_newlines=True) doesn't properly handle the
+    # encoding. This goes back to popen's flaky support for encoding:
+    # https://bugs.python.org/issue6135. This is a solution for #23271, #21928.
+    # No need to do anything on Python 2 because it's already a byte-string there.
+    manual_io_wrapper = six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING
+
+    stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type,
+                                                universal_newlines=not manual_io_wrapper)
+    if manual_io_wrapper:
+        stdout = io.TextIOWrapper(io.BytesIO(stdout), encoding=stdout_encoding).read()
     if six.PY2:
         stdout = stdout.decode(stdout_encoding)
     return stdout, stderr, status_code
diff --git a/django/core/management/utils.py b/django/core/management/utils.py
index 06ed04e377..53290e36d7 100644
--- a/django/core/management/utils.py
+++ b/django/core/management/utils.py
@@ -10,7 +10,7 @@ from django.utils.encoding import DEFAULT_LOCALE_ENCODING, force_text
 from .base import CommandError
 
 
-def popen_wrapper(args, os_err_exc_type=CommandError):
+def popen_wrapper(args, os_err_exc_type=CommandError, universal_newlines=True):
     """
     Friendly wrapper around Popen.
 
@@ -18,7 +18,7 @@ def popen_wrapper(args, os_err_exc_type=CommandError):
     """
     try:
         p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
-                close_fds=os.name != 'nt', universal_newlines=True)
+                close_fds=os.name != 'nt', universal_newlines=universal_newlines)
     except OSError as e:
         strerror = force_text(e.strerror, DEFAULT_LOCALE_ENCODING,
                               strings_only=True)
diff --git a/docs/releases/1.8.1.txt b/docs/releases/1.8.1.txt
index 31e66d8c1a..39277b8f2f 100644
--- a/docs/releases/1.8.1.txt
+++ b/docs/releases/1.8.1.txt
@@ -78,6 +78,8 @@ Bugfixes
 * Removed flushing of the test database with :djadminopt:`--keepdb`, which
   prevented apps with data migrations from using the option (:ticket:`24729`).
 
+* Fixed ``makemessages`` crash in some locales (:ticket:`23271`).
+
 Optimizations
 =============