[1.9.x] Fixed #25968 -- Changed project/app templates to use a "py-tpl" suffix.

Debian packages unconditionally byte-compile .py files on installation and
do not silence errors by design. Therefore, we need a way of shipping these
invalid .py files without a .py extension but ensuring that when we
template them, they end up as .py.

We don't special-case .py files so that the all the TemplateCommand
command-line options (eg. extra_files and extensions) still work entirely
as expected and it may even be useful for other formats too.

Backport of abc0777b63057e2ff97eee2ff184356051e14c47 from master

20 files changed, 58 insertions, 7 deletions

diff --git a/django/conf/app_template/__init__.py b/django/conf/app_template/__init__.py-tpl
index e69de29bb2..e69de29bb2 100644
--- a/django/conf/app_template/__init__.py
+++ b/django/conf/app_template/__init__.py-tpl
diff --git a/django/conf/app_template/admin.py b/django/conf/app_template/admin.py-tpl
index 8c38f3f3da..8c38f3f3da 100644
--- a/django/conf/app_template/admin.py
+++ b/django/conf/app_template/admin.py-tpl
diff --git a/django/conf/app_template/apps.py b/django/conf/app_template/apps.py-tpl
index 8d1a017751..8d1a017751 100644
--- a/django/conf/app_template/apps.py
+++ b/django/conf/app_template/apps.py-tpl
diff --git a/django/conf/app_template/migrations/__init__.py b/django/conf/app_template/migrations/__init__.py-tpl
index e69de29bb2..e69de29bb2 100644
--- a/django/conf/app_template/migrations/__init__.py
+++ b/django/conf/app_template/migrations/__init__.py-tpl
diff --git a/django/conf/app_template/models.py b/django/conf/app_template/models.py-tpl
index 7a54b3e371..7a54b3e371 100644
--- a/django/conf/app_template/models.py
+++ b/django/conf/app_template/models.py-tpl
diff --git a/django/conf/app_template/tests.py b/django/conf/app_template/tests.py-tpl
index 7ce503c2dd..7ce503c2dd 100644
--- a/django/conf/app_template/tests.py
+++ b/django/conf/app_template/tests.py-tpl
diff --git a/django/conf/app_template/views.py b/django/conf/app_template/views.py-tpl
index 91ea44a218..91ea44a218 100644
--- a/django/conf/app_template/views.py
+++ b/django/conf/app_template/views.py-tpl
diff --git a/django/conf/project_template/manage.py b/django/conf/project_template/manage.py-tpl
index 391dd88ba4..391dd88ba4 100755
--- a/django/conf/project_template/manage.py
+++ b/django/conf/project_template/manage.py-tpl
diff --git a/django/conf/project_template/project_name/__init__.py b/django/conf/project_template/project_name/__init__.py-tpl
index e69de29bb2..e69de29bb2 100644
--- a/django/conf/project_template/project_name/__init__.py
+++ b/django/conf/project_template/project_name/__init__.py-tpl
diff --git a/django/conf/project_template/project_name/settings.py b/django/conf/project_template/project_name/settings.py-tpl
index aa9bae99ac..aa9bae99ac 100644
--- a/django/conf/project_template/project_name/settings.py
+++ b/django/conf/project_template/project_name/settings.py-tpl
diff --git a/django/conf/project_template/project_name/urls.py b/django/conf/project_template/project_name/urls.py-tpl
index 30ddffb876..30ddffb876 100644
--- a/django/conf/project_template/project_name/urls.py
+++ b/django/conf/project_template/project_name/urls.py-tpl
diff --git a/django/conf/project_template/project_name/wsgi.py b/django/conf/project_template/project_name/wsgi.py-tpl
index 0d68b95645..0d68b95645 100644
--- a/django/conf/project_template/project_name/wsgi.py
+++ b/django/conf/project_template/project_name/wsgi.py-tpl
diff --git a/django/core/management/templates.py b/django/core/management/templates.py
index 3addb068f7..9cb4d952df 100644
--- a/django/core/management/templates.py
+++ b/django/core/management/templates.py
@@ -42,6 +42,11 @@ class TemplateCommand(BaseCommand):
     # Can't perform any active locale changes during this command, because
     # setting might not be available at all.
     leave_locale_alone = True
+    # Rewrite the following suffixes when determining the target filename.
+    rewrite_template_suffixes = (
+        # Allow shipping invalid .py files without byte-compilation.
+        ('.py-tpl', '.py'),
+    )
 
     def add_arguments(self, parser):
         parser.add_argument('name', help='Name of the application or project.')
@@ -139,6 +144,11 @@ class TemplateCommand(BaseCommand):
                 old_path = path.join(root, filename)
                 new_path = path.join(top_dir, relative_dir,
                                      filename.replace(base_name, name))
+                for old_suffix, new_suffix in self.rewrite_template_suffixes:
+                    if new_path.endswith(old_suffix):
+                        new_path = new_path[:-len(old_suffix)] + new_suffix
+                        break  # Only rewrite once
+
                 if path.exists(new_path):
                     raise CommandError("%s already exists, overlaying a "
                                        "project or app into an existing "
@@ -149,7 +159,7 @@ class TemplateCommand(BaseCommand):
                 # accidentally render Django templates files
                 with open(old_path, 'rb') as template_file:
                     content = template_file.read()
-                if filename.endswith(extensions) or filename in extra_files:
+                if new_path.endswith(extensions) or filename in extra_files:
                     content = content.decode('utf-8')
                     template = Engine().from_string(content)
                     content = template.render(context)
diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt
index 06a0374846..06fd1bfd8c 100644
--- a/docs/ref/django-admin.txt
+++ b/docs/ref/django-admin.txt
@@ -1147,6 +1147,15 @@ with the ``--name`` option. The :class:`template context
     To work around this problem, you can use the :ttag:`templatetag`
     templatetag to "escape" the various parts of the template syntax.
 
+    In addition, to allow Python template files that contain Django template
+    language syntax while also preventing packaging systems from trying to
+    byte-compile invalid ``*.py`` files, template files ending with ``.py-tpl``
+    will be renamed to ``.py``.
+
+    .. versionchanged:: 1.9.2
+
+        Renaming of ``.py-tpl`` to ``.py`` was added.
+
 .. _source: https://github.com/django/django/tree/master/django/conf/app_template/
 
 startproject <projectname> [destination]
diff --git a/docs/releases/1.9.2.txt b/docs/releases/1.9.2.txt
index f977b02ac3..e64c62e3da 100644
--- a/docs/releases/1.9.2.txt
+++ b/docs/releases/1.9.2.txt
@@ -4,7 +4,24 @@ Django 1.9.2 release notes
 
 *Under development*
 
-Django 1.9.2 fixes several bugs in 1.9.1.
+Django 1.9.2 fixes several bugs in 1.9.1 and makes a small backwards
+incompatible change that hopefully doesn't affect any users.
+
+Backwards incompatible change: ``.py-tpl`` files rewritten in project/app templates
+===================================================================================
+
+The addition of some Django template language syntax to the default app
+template in Django 1.9 means those files now have some invalid Python syntax.
+This causes difficulties for packaging systems that unconditionally
+byte-compile ``*.py`` files.
+
+To remedy this, a ``.py-tpl`` suffix is now used for the project and app
+template files included in Django. The ``.py-tpl`` suffix is replaced with
+``.py`` by the ``startproject`` and ``startapp`` commands. For example, a
+template with the filename ``manage.py-tpl`` will be created as ``manage.py``.
+
+Please file a ticket if you have a custom project template containing
+``.py-tpl`` files and find this behavior problematic.
 
 Bugfixes
 ========
diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt
index 039171634c..0742b52e88 100644
--- a/docs/releases/1.9.txt
+++ b/docs/releases/1.9.txt
@@ -1016,7 +1016,7 @@ a Django application with this structure::
 ``SyntaxError`` when installing Django setuptools 5.5.x
 -------------------------------------------------------
 
-When installing Django 1.9+ with setuptools 5.5.x, you'll see::
+When installing Django 1.9 or 1.9.1 with setuptools 5.5.x, you'll see::
 
     Compiling django/conf/app_template/apps.py ...
       File "django/conf/app_template/apps.py", line 4
@@ -1033,7 +1033,8 @@ When installing Django 1.9+ with setuptools 5.5.x, you'll see::
 It's safe to ignore these errors (Django will still install just fine), but you
 can avoid them by upgrading setuptools to a more recent version. If you're
 using pip, you can upgrade pip using ``pip install -U pip`` which will also
-upgrade setuptools.
+upgrade setuptools. This is resolved in later versions of Django as described
+in the :doc:`/releases/1.9.2`.
 
 Miscellaneous
 -------------
diff --git a/docs/topics/install.txt b/docs/topics/install.txt
index f0fbc47881..b5195bea57 100644
--- a/docs/topics/install.txt
+++ b/docs/topics/install.txt
@@ -165,8 +165,7 @@ This is the recommended way to install Django.
 1. Install pip_. The easiest is to use the `standalone pip installer`_. If your
    distribution already has ``pip`` installed, you might need to update it if
    it's outdated. If it's outdated, you'll know because installation won't
-   work. If you're using an old version of setuptools, you might see some
-   :ref:`harmless SyntaxErrors <syntax-error-old-setuptools-django-19>` also.
+   work.
 
 2. Take a look at virtualenv_ and virtualenvwrapper_. These tools provide
    isolated Python environments, which are more practical than installing
diff --git a/tests/admin_scripts/custom_templates/project_template/manage.py b/tests/admin_scripts/custom_templates/project_template/manage.py-tpl
index d9843c433f..d9843c433f 100755
--- a/tests/admin_scripts/custom_templates/project_template/manage.py
+++ b/tests/admin_scripts/custom_templates/project_template/manage.py-tpl
diff --git a/tests/admin_scripts/tests.py b/tests/admin_scripts/tests.py
index 71885a4641..839981767e 100644
--- a/tests/admin_scripts/tests.py
+++ b/tests/admin_scripts/tests.py
@@ -181,7 +181,7 @@ class AdminScriptTestCase(unittest.TestCase):
                 pass
 
         conf_dir = os.path.dirname(upath(conf.__file__))
-        template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py')
+        template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py-tpl')
 
         test_manage_py = os.path.join(self.test_dir, 'manage.py')
         shutil.copyfile(template_manage_py, test_manage_py)
diff --git a/tests/project_template/test_settings.py b/tests/project_template/test_settings.py
index ac115f7dc2..25a95717a9 100644
--- a/tests/project_template/test_settings.py
+++ b/tests/project_template/test_settings.py
@@ -1,13 +1,28 @@
+import os
+import shutil
 import unittest
 
+from django import conf
 from django.test import TestCase
 from django.utils import six
+from django.utils._os import upath
 
 
 @unittest.skipIf(six.PY2,
     'Python 2 cannot import the project template because '
     'django/conf/project_template doesn\'t have an __init__.py file.')
 class TestStartProjectSettings(TestCase):
+    def setUp(self):
+        # Ensure settings.py exists
+        project_dir = os.path.join(
+            os.path.dirname(upath(conf.__file__)),
+            'project_template',
+            'project_name',
+        )
+        template_settings_py = os.path.join(project_dir, 'settings.py-tpl')
+        test_settings_py = os.path.join(project_dir, 'settings.py')
+        shutil.copyfile(template_settings_py, test_settings_py)
+        self.addCleanup(os.remove, test_settings_py)
 
     def test_middleware_classes_headers(self):
         """