""" Update and build the documentation into files for display with the djangodocs app. """ import json import multiprocessing import os import shutil import subprocess import sys import zipfile from contextlib import closing from datetime import datetime from pathlib import Path from django.conf import settings from django.core.management import BaseCommand, call_command from django.db.models import Q from django.utils.translation import to_locale from sphinx.application import Sphinx from sphinx.config import Config from sphinx.errors import SphinxError from sphinx.testing.util import _clean_up_global_state from sphinx.util.docutils import docutils_namespace, patch_docutils from ...models import DocumentRelease class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( "--language", help="Only build docs for this specific language", ) parser.add_argument( "--force", action="store_true", default=False, help=( "Force docs update even if docs in git didn't change or the " "version is no longer supported." ), ) parser.add_argument( "--interactive", action="store_true", help="Ask before building each version", ) parser.add_argument( "--purge-cache", action="store_true", dest="purge_cache", default=False, help="Also invalidate downstream caches for any changed doc versions.", ) parser.add_argument( "args", metavar="versions", nargs="*", help="Which version to rebuild (all by default)", ) def _get_doc_releases(self, versions, options): """ Return a DocumentRelease queryset of all the versions that should be built, based on the arguments received on the command line. """ default_docs_version = DocumentRelease.objects.get( is_default=True ).release.version # Somehow, bizarely, there's a bug in Sphinx such that if I try to # build 1.0 before other versions, things fail in weird ways. However, # building newer versions first works. I suspect Sphinx is hanging onto # some global state. Anyway, we can work around it by making sure that # "dev" builds before "1.0". This is ugly, but oh well. queryset = DocumentRelease.objects.order_by("-release") # Skip translated non-stable versions to avoid a crash: # https://github.com/django/djangoproject.com/issues/627 queryset = queryset.filter( Q(lang=settings.DEFAULT_LANGUAGE_CODE) | Q(release=default_docs_version) ) if options["language"]: queryset = queryset.filter(lang=options["language"]) if versions: queryset = queryset.by_versions(*versions) return queryset def handle(self, *versions, **kwargs): self.verbosity = kwargs["verbosity"] self.purge_cache = kwargs["purge_cache"] self.default_builders = ["json", "djangohtml"] # Keep track of which Git sources have been updated, e.g., # {'1.8': True} if the 1.8 docs updated. self.release_docs_changed = {} for release in self._get_doc_releases(versions, kwargs): self.build_doc_release( release, force=kwargs["force"], interactive=kwargs["interactive"] ) if self.purge_cache: changed_versions = { version for version, changed in self.release_docs_changed.items() if changed } if changed_versions or kwargs["force"]: call_command( "purge_docs_cache", **{"doc_versions": changed_versions, "verbosity": self.verbosity}, ) else: if self.verbosity >= 1: self.stdout.write("No docs changes; skipping cache purge.") def build_doc_release(self, release, force=False, interactive=False): # Skip not supported releases. if not release.is_supported and not force: return if interactive: prompt = ( f"About to start building docs for release {release}. Continue? Y/n " ) if input(prompt).upper() not in {"", "Y", "YES", "OUI"}: return if self.verbosity >= 1: self.stdout.write(f"Starting update for {release} at {datetime.now()}...") release.sync_from_sitemap(force=force) # checkout_dir is shared for all languages. checkout_dir = settings.DOCS_BUILD_ROOT / "sources" / release.version parent_build_dir = settings.DOCS_BUILD_ROOT / release.lang / release.version if not checkout_dir.exists(): checkout_dir.mkdir(parents=True) if not parent_build_dir.exists(): parent_build_dir.mkdir(parents=True) # # Update the release from SCM. # # Make a git checkout/update into the destination directory. git_changed = self.update_git( release.scm_url, checkout_dir, changed_dir="docs/" ) if git_changed: self.release_docs_changed[release.version] = True version_changed = git_changed or self.release_docs_changed.get(release.version) if not force and not version_changed: if self.verbosity >= 1: self.stdout.write( "No docs changes for %s, skipping docs building." % release ) return source_dir = checkout_dir / "docs" if release.lang != settings.DEFAULT_LANGUAGE_CODE: scm_url = release.scm_url.replace( "django.git", "django-docs-translations.git" ) trans_dir = checkout_dir / "django-docs-translation" if not trans_dir.exists(): trans_dir.mkdir() self.update_git(scm_url, trans_dir) locale_dir = source_dir / "locale" if not locale_dir.exists(): locale_dir.symlink_to(trans_dir / "translations") extra_kwargs = {"stdout": subprocess.DEVNULL} if self.verbosity == 0 else {} subprocess.check_call( "cd %s && make translations" % trans_dir, shell=True, **extra_kwargs ) if release.is_default: # Build the pot files (later retrieved by Transifex) builders = self.default_builders[:] + ["gettext"] else: builders = self.default_builders # # Use Sphinx to build the release docs into JSON and HTML documents. # for builder in builders: # Wipe and re-create the build directory. See #18930. build_dir = parent_build_dir / "_build" / builder if build_dir.exists(): shutil.rmtree(str(build_dir)) build_dir.mkdir(parents=True) if self.verbosity >= 2: self.stdout.write(f" building {builder} ({source_dir} -> {build_dir})") # Retrieve the extensions from the conf.py so we can append to them. conf_extensions = Config.read(source_dir.resolve()).extensions extensions = [*conf_extensions, "docs.builder"] try: # Prevent global state persisting between builds # https://github.com/sphinx-doc/sphinx/issues/12130 with patch_docutils(source_dir), docutils_namespace(): Sphinx( srcdir=source_dir, confdir=source_dir, outdir=build_dir, doctreedir=build_dir / ".doctrees", buildername=builder, # Translated docs builds generate a lot of warnings, so send # stderr to stdout to be logged (rather than generating an email) warning=sys.stdout, parallel=multiprocessing.cpu_count(), verbosity=0, confoverrides={ "language": to_locale(release.lang), "extensions": extensions, }, ).build() # Clean up global state after building each language. _clean_up_global_state() except SphinxError as e: self.stderr.write( "sphinx-build returned an error (release %s, builder %s): %s" % (release, builder, str(e)) ) return # # Create a zip file of the HTML build for offline reading. # This gets moved into MEDIA_ROOT for downloading. # html_build_dir = parent_build_dir / "_build" / "djangohtml" zipfile_name = f"django-docs-{release.version}-{release.lang}.zip" zipfile_path = settings.MEDIA_ROOT / "docs" / zipfile_name if not zipfile_path.parent.exists(): zipfile_path.parent.mkdir(parents=True) if self.verbosity >= 2: self.stdout.write(" build zip (into %s)" % zipfile_path) def zipfile_inclusion_filter(file_path): return ".doctrees" not in file_path.parts with closing( zipfile.ZipFile(str(zipfile_path), "w", compression=zipfile.ZIP_DEFLATED) ) as zf: for root, dirs, files in os.walk(str(html_build_dir)): for f in files: file_path = Path(os.path.join(root, f)) if zipfile_inclusion_filter(file_path): rel_path = str(file_path.relative_to(html_build_dir)) zf.write(str(file_path), rel_path) # # Copy the build results to the directory used for serving # the documentation in the least disruptive way possible. # build_dir = parent_build_dir / "_build" built_dir = parent_build_dir / "_built" subprocess.check_call( [ "rsync", "--archive", "--delete", f"--link-dest={build_dir}", f"{build_dir}/", str(built_dir), ] ) if release.is_default: self._setup_stable_symlink(release, built_dir) json_built_dir = parent_build_dir / "_built" / "json" documents = gen_decoded_documents(json_built_dir) with open(json_built_dir / "globalcontext.json") as context: release.global_context = json.load(context) release.save(update_fields=["global_context"]) release.sync_to_db(documents) def update_git(self, url, destdir, changed_dir="."): """ Update a source checkout and return True if any docs were changed, False otherwise. """ quiet = "--quiet" if self.verbosity == 0 else "--" if "@" in url: repo, branch = url.rsplit("@", 1) else: repo, branch = url, "main" if (destdir / ".git").exists(): remote = "origin" branch_with_remote = f"{remote}/{branch}" try: cwd = os.getcwd() os.chdir(str(destdir)) # Git writes all output to stderr, so redirect it to stdout for # logging (so we don't get emailed with all Git output). subprocess.check_call( ["git", "reset", "--hard", "HEAD", quiet], stderr=sys.stdout ) subprocess.check_call( ["git", "clean", "-fdx", quiet], stderr=sys.stdout ) subprocess.check_call( [ "git", "fetch", remote, f"{branch}:refs/remotes/{branch_with_remote}", quiet, ], stderr=sys.stdout, ) docs_changed = ( subprocess.call( [ "git", "diff", branch_with_remote, "--quiet", "--exit-code", changed_dir, ], stderr=sys.stdout, ) == 1 ) if not docs_changed: return False subprocess.check_call( ["git", "merge", branch_with_remote, quiet], stderr=sys.stdout ) finally: os.chdir(cwd) else: subprocess.check_call( [ "git", "clone", "--depth", "1", "--branch", branch, repo, str(destdir), quiet, ], stderr=sys.stdout, ) return True def _setup_stable_symlink(self, release, built_dir): """ Setup a symbolic link called "stable" pointing to the given release build """ stable = built_dir / "stable" target = built_dir / release.version if stable.resolve() != target: # Symlink is either missing or has changed stable.unlink(missing_ok=True) stable.symlink_to(target, target_is_directory=True) def gen_decoded_documents(directory): """ Walk the given directory looking for fjson files and yield their data. """ for root, dirs, files in os.walk(str(directory)): for f in files: f = Path(root, f) if not f.suffix == ".fjson": continue with f.open() as fp: yield json.load(fp)